From 956942bba93ac390fcaee84c2e5c25f252214b3f Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Thu, 22 Oct 2026 07:37:16 +0100 Subject: [PATCH] Add a files browser for every folder on the data volume A bot's deliverables and the raw material behind them only existed as paths in a message, so they couldn't be opened from the app at all. This browses them read-only at /admin/files, one volume at a time: a bot's own folder, a workspace's shared one, a computer's home (where a bot's ~/... actually is), the media library, an MCP server's install folder and the database. Access is the whole gate: resolveStorageSpec checks the viewer against the volume first (a bot through grabUserBot, a workspace through grabUserWorkspace, a computer through grabUserComputer, media only their own unless they manage the crew) and no path is built before it passes. resolveStoragePath then applies the resolveBotPath rules to any root. The picker lists folders through reachableBotsSql, so it never offers one the resolver would refuse. Reading is listStorageEntries (folders first, paginated, symlinks skipped, catalog rows joined so a bot file still opens through its media row) and grabStorageFile for bytes under a cap, both behind an ETag. --- src/dict/admin-sections-dict.ts | 1 + src/dict/rail-pages-dict.ts | 4 +- .../backend/storage/grab-storage-file.ts | 62 ++++ .../backend/storage/list-storage-entries.ts | 223 ++++++++++++++ .../storage/resolve-storage-path.test.ts | 78 +++++ .../backend/storage/resolve-storage-path.ts | 128 ++++++++ .../backend/storage/resolve-storage-spec.ts | 281 ++++++++++++++++++ .../admin/files/(hooks)/use-storage-folder.ts | 94 ++++++ .../files/(partials)/files-breadcrumb-row.tsx | 59 ++++ .../files/(partials)/files-search-input.tsx | 28 ++ .../files/(partials)/files-volume-row.tsx | 40 +++ .../files/(partials)/storage-entry-row.tsx | 76 +++++ .../admin/files/(sections)/files-browser.tsx | 127 ++++++++ .../admin/files/(sections)/files-pane.tsx | 73 +++++ src/pages/admin/files/index.server.ts | 84 ++++++ src/pages/admin/files/index.tsx | 23 ++ src/pages/api/admin/storage/file.ts | 63 ++++ src/pages/api/admin/storage/list.ts | 69 +++++ src/schemas/storage-schema.ts | 41 +++ src/types/index.ts | 59 ++++ src/utils/crew-access/crew-access-sql.test.ts | 13 + src/utils/crew-access/reachable-bots-sql.ts | 28 ++ src/utils/path-ref.test.ts | 72 +++++ src/utils/path-ref.ts | 85 ++++++ src/utils/storage-volumes.test.ts | 89 ++++++ src/utils/storage-volumes.ts | 165 ++++++++++ 26 files changed, 2064 insertions(+), 1 deletion(-) create mode 100644 src/functions/backend/storage/grab-storage-file.ts create mode 100644 src/functions/backend/storage/list-storage-entries.ts create mode 100644 src/functions/backend/storage/resolve-storage-path.test.ts create mode 100644 src/functions/backend/storage/resolve-storage-path.ts create mode 100644 src/functions/backend/storage/resolve-storage-spec.ts create mode 100644 src/pages/admin/files/(hooks)/use-storage-folder.ts create mode 100644 src/pages/admin/files/(partials)/files-breadcrumb-row.tsx create mode 100644 src/pages/admin/files/(partials)/files-search-input.tsx create mode 100644 src/pages/admin/files/(partials)/files-volume-row.tsx create mode 100644 src/pages/admin/files/(partials)/storage-entry-row.tsx create mode 100644 src/pages/admin/files/(sections)/files-browser.tsx create mode 100644 src/pages/admin/files/(sections)/files-pane.tsx create mode 100644 src/pages/admin/files/index.server.ts create mode 100644 src/pages/admin/files/index.tsx create mode 100644 src/pages/api/admin/storage/file.ts create mode 100644 src/pages/api/admin/storage/list.ts create mode 100644 src/schemas/storage-schema.ts create mode 100644 src/utils/crew-access/reachable-bots-sql.ts create mode 100644 src/utils/path-ref.test.ts create mode 100644 src/utils/path-ref.ts create mode 100644 src/utils/storage-volumes.test.ts create mode 100644 src/utils/storage-volumes.ts diff --git a/src/dict/admin-sections-dict.ts b/src/dict/admin-sections-dict.ts index 8e36863..bd1cde3 100644 --- a/src/dict/admin-sections-dict.ts +++ b/src/dict/admin-sections-dict.ts @@ -6,6 +6,7 @@ export const AdminSections = [ "/admin/bots", "/admin/workspaces", "/admin/routines", + "/admin/files", "/admin/logs", "/admin/help", "/admin/settings", diff --git a/src/dict/rail-pages-dict.ts b/src/dict/rail-pages-dict.ts index 5377389..06981bb 100644 --- a/src/dict/rail-pages-dict.ts +++ b/src/dict/rail-pages-dict.ts @@ -1,4 +1,4 @@ -import { Activity, CircleHelp, Clock, type LucideIcon } from "lucide-react"; +import { Activity, CircleHelp, Clock, FolderOpen, type LucideIcon } from "lucide-react"; import { HelpPath } from "./help-dict"; export type RailPage = { @@ -12,6 +12,7 @@ export type RailPage = { }; export const RoutinesPath = "/admin/routines"; +export const FilesPath = "/admin/files"; export const ActivityPath = "/admin/logs"; /** @@ -20,6 +21,7 @@ export const ActivityPath = "/admin/logs"; */ export const RailPages: RailPage[] = [ { title: "Routines", url: RoutinesPath, Icon: Clock }, + { title: "Files", url: FilesPath, Icon: FolderOpen }, { title: "Activity", url: ActivityPath, Icon: Activity, super_admin_only: true }, { title: "Help", url: HelpPath, Icon: CircleHelp }, ]; diff --git a/src/functions/backend/storage/grab-storage-file.ts b/src/functions/backend/storage/grab-storage-file.ts new file mode 100644 index 0000000..550c0c5 --- /dev/null +++ b/src/functions/backend/storage/grab-storage-file.ts @@ -0,0 +1,62 @@ +import { statSync } from "node:fs"; +import path from "node:path"; +import { AppData } from "@/src/data/app-data"; +import { mimeTypeForFile } from "../db/bot-files/media-type-for-file"; +import resolveStoragePath from "./resolve-storage-path"; +import type { ResolvedStorageSpec } from "./resolve-storage-spec"; + +type Params = { + resolved: ResolvedStorageSpec; + /** + * The file to open, relative to the volume's root. + */ + path?: string | null; +}; + +export type StorageFile = { + /** + * The absolute path on disk, inside the volume. + */ + absolute: string; + /** + * Its path inside the volume, e.g. `seo/coderank/raw/crawl.json`. + */ + relative: string; + name: string; + size: number; + mime_type: string; + modified_at: number; +}; + +/** + * One file for the browser to open: checked to be inside the volume (the + * same rules a bot's own path gets), to exist, and to be a regular file + * rather than a folder. Null when it isn't one, so a caller can answer + * "not found" without leaking what is there. + */ +export default function grabStorageFile({ resolved, path: requested }: Params): StorageFile | null { + const resolved_path = resolveStoragePath({ root: resolved.root, path: requested }); + + if (!resolved_path.name) return null; + + let stats; + + try { + stats = statSync(resolved_path.absolute); + } catch { + return null; + } + + if (!stats.isFile()) return null; + + if (stats.size > AppData.StorageBrowserReadMaxBytes) return null; + + return { + absolute: resolved_path.absolute, + relative: resolved_path.relative, + name: path.basename(resolved_path.absolute), + size: stats.size, + mime_type: mimeTypeForFile({ name: resolved_path.name }), + modified_at: Math.round(stats.mtimeMs), + }; +} diff --git a/src/functions/backend/storage/list-storage-entries.ts b/src/functions/backend/storage/list-storage-entries.ts new file mode 100644 index 0000000..3adab30 --- /dev/null +++ b/src/functions/backend/storage/list-storage-entries.ts @@ -0,0 +1,223 @@ +import { existsSync, lstatSync, readdirSync } from "node:fs"; +import path from "node:path"; +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db"; +import { AppData } from "@/src/data/app-data"; +import grabMediaURL from "../media/grab-media-url"; +import { isInsideRoot } from "./resolve-storage-path"; +import type { ResolvedStorageSpec } from "./resolve-storage-spec"; +import { + storageSpecHref, + type StorageSpec, +} from "@/src/utils/storage-volumes"; + +type Params = { + /** + * The folder, already access-checked (`resolveStorageSpec`). + */ + resolved: ResolvedStorageSpec; + /** + * The directory to list, already checked (`resolveStoragePath`). + */ + absolute: string; + /** + * The same directory, relative to the volume's root. Empty at the top. + */ + relative: string; + /** + * Match a name or path, case-insensitive. Folders are kept so a match + * deeper down is still reachable. + */ + search?: string | null; + offset?: number; +}; + +export type StorageEntry = { + name: string; + /** + * Where it is, inside the volume's root, e.g. `seo/coderank/raw`. + */ + path: string; + kind: "file" | "folder"; + size: number; + modified_at: number; + media_type?: string; + /** + * The `bot_files` row, when one of the bot's own files is that path, so + * it keeps opening through its own media record. + */ + file_id?: number; + /** + * Where to open it: a `bot_files` row's media URL, else the volume + * browser's own streaming route. + */ + url?: string; +}; + +/** + * One directory on the data volume, for the files browser: its entries + * (folders first), a page of them, and a `bot_files` row attached to any + * that has one. Entries a symlink leads out of the volume with are left + * out, so nothing outside the folder is ever listed. + */ +export default async function listStorageEntries({ + resolved, + absolute, + relative, + search, + offset = 0, +}: Params) { + /** + * A folder nothing has written to yet (a computer's home before it + * starts, a bot's before its first file) shows as empty rather than as + * an error. + */ + if (!existsSync(absolute)) return { entries: [], total: 0, offset }; + + const rows = readdirSync(absolute, { withFileTypes: true }); + const indexed = await botFileRows({ resolved, paths: rows.map((row) => joinPath({ relative, name: row.name })) }); + const needle = search?.trim().toLowerCase() || ""; + const entries: StorageEntry[] = []; + + for (const row of rows) { + if (row.name == ".DS_Store") continue; + + const entry_path = joinPath({ relative, name: row.name }); + + if (needle && !entry_path.toLowerCase().includes(needle)) continue; + + const entry_absolute = path.join(absolute, row.name); + + if (!isInsideRoot({ root: resolved.root, absolute: entry_absolute })) continue; + + let stats; + + try { + stats = lstatSync(entry_absolute); + } catch { + continue; + } + + /** + * A symlink is not followed: it may lead anywhere, and the + * volume's own folders are what this lists. + */ + if (stats.isSymbolicLink()) continue; + + if (stats.isDirectory()) { + entries.push({ + name: row.name, + path: entry_path, + kind: "folder", + size: 0, + modified_at: Math.round(stats.mtimeMs), + url: storageSpecHref({ spec: resolved.spec, path: entry_path }), + }); + continue; + } + + if (!stats.isFile()) continue; + + const file = indexed.get(entry_path); + const { media_url } = file?.media_id + ? grabMediaURL({ media: { id: file.media_id } as any }) + : {}; + + entries.push({ + name: row.name, + path: entry_path, + kind: "file", + size: stats.size, + modified_at: Math.round(stats.mtimeMs), + media_type: file?.media_type || undefined, + file_id: file?.id ? Number(file.id) : undefined, + url: media_url || fileUrl({ spec: resolved.spec, path: entry_path }), + }); + } + + entries.sort((a, b) => { + if (a.kind != b.kind) return a.kind == "folder" ? -1 : 1; + + return a.name.localeCompare(b.name, undefined, { numeric: true }); + }); + + return { + entries: entries.slice(offset, offset + AppData.StorageBrowserPageSize), + total: entries.length, + offset, + }; +} + +/** + * Where a folder's file opens when it has no `bot_files` row (a computer's + * home, the media library, an MCP server's install): the browser's own + * streaming route, which checks the same access again. + */ +export function fileUrl({ spec, path }: { spec: StorageSpec; path: string }) { + const query = new URLSearchParams({ + volume: spec.volume, + owner: String(spec.owner_id), + path, + }); + + return `/api/admin/storage/file?${query.toString()}`; +} + +/** + * The `bot_files` rows behind the paths of one start and workspace folder, + * keyed by path. Nothing for the other volumes, which have no catalog. + */ +async function botFileRows({ + resolved, + paths, +}: { + resolved: ResolvedStorageSpec; + paths: string[]; +}) { + const rows = new Map(); + + if (resolved.spec.volume != "bots" && resolved.spec.volume != "workspaces") return rows; + if (!paths.length) return rows; + + const scope = resolved.spec.volume == "bots" ? "bot" : "workspace"; + const res = await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE scope = ? AND owner_id = ? AND path IN (${paths + .map(() => "?") + .join(",")})`, + values: [scope, resolved.spec.owner_id, ...paths], + }); + + for (const row of res.payload || []) { + if (row.path) rows.set(String(row.path), row); + } + + return rows; +} + +function joinPath({ relative, name }: { relative: string; name: string }) { + return relative ? `${relative}/${name}` : name; +} + +/** + * The breadcrumb trail of a folder, from the volume's top: one crumb per + * segment, each with the address that opens it. + */ +export function storageBreadcrumbs({ + spec, + path: relative, +}: { + spec: StorageSpec; + path?: string | null; +}) { + const segments = (relative || "").split("/").filter(Boolean); + const crumbs = [{ name: "", href: storageSpecHref({ spec }) }]; + + for (let i = 1; i <= segments.length; i++) { + crumbs.push({ + name: segments[i - 1] || "", + href: storageSpecHref({ spec, path: segments.slice(0, i).join("/") }), + }); + } + + return crumbs; +} diff --git a/src/functions/backend/storage/resolve-storage-path.test.ts b/src/functions/backend/storage/resolve-storage-path.test.ts new file mode 100644 index 0000000..c31cff0 --- /dev/null +++ b/src/functions/backend/storage/resolve-storage-path.test.ts @@ -0,0 +1,78 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import resolveStoragePath, { isInsideRoot } from "./resolve-storage-path"; + +const base = mkdtempSync(path.join(tmpdir(), "cobble-storage-path-")); +const root = path.join(base, "root"); +const outside = path.join(base, "outside"); + +mkdirSync(path.join(root, "reports"), { recursive: true }); +mkdirSync(outside, { recursive: true }); +writeFileSync(path.join(root, "reports", "q3.md"), "# Q3"); +writeFileSync(path.join(outside, "secret.md"), "secret"); +symlinkSync(outside, path.join(root, "escape")); + +afterAll(() => rmSync(base, { recursive: true, force: true })); + +describe("resolveStoragePath", () => { + test("keeps a path inside the root", () => { + expect(resolveStoragePath({ root, path: "reports/q3.md" })).toEqual({ + relative: "reports/q3.md", + absolute: path.join(root, "reports", "q3.md"), + folder: "reports", + name: "q3.md", + }); + }); + + test("normalizes slashes and trims", () => { + expect(resolveStoragePath({ root, path: " reports// /q3.md " }).relative).toBe( + "reports/q3.md", + ); + }); + + test("reads the root itself", () => { + expect(resolveStoragePath({ root, path: "" })).toEqual({ + relative: "", + absolute: path.resolve(root), + folder: "", + name: "", + }); + expect(resolveStoragePath({ root, path: null }).name).toBe(""); + }); + + test("refuses an absolute path", () => { + expect(() => resolveStoragePath({ root, path: "/etc/passwd" })).toThrow(); + expect(() => resolveStoragePath({ root, path: "C:/windows" })).toThrow(); + }); + + test("refuses climbing out", () => { + expect(() => resolveStoragePath({ root, path: "../outside/secret.md" })).toThrow(); + expect(() => resolveStoragePath({ root, path: "reports/../../outside" })).toThrow(); + expect(() => resolveStoragePath({ root, path: "./q3.md" })).toThrow(); + }); + + test("refuses a NUL byte", () => { + expect(() => resolveStoragePath({ root, path: "reports/q3.md\0" })).toThrow(); + }); + + test("refuses a symlink that leads out", () => { + expect(() => resolveStoragePath({ root, path: "escape/secret.md" })).toThrow(); + }); + + test("refuses a path that is too deep", () => { + expect(() => + resolveStoragePath({ root, path: Array.from({ length: 12 }, (_, i) => `f${i}`).join("/") }), + ).toThrow(); + }); +}); + +describe("isInsideRoot", () => { + test("tells a child from a sibling", () => { + expect(isInsideRoot({ root, absolute: path.join(root, "reports") })).toBe(true); + expect(isInsideRoot({ root, absolute: root })).toBe(true); + expect(isInsideRoot({ root, absolute: path.join(base, "rooted") })).toBe(false); + expect(isInsideRoot({ root, absolute: outside })).toBe(false); + }); +}); diff --git a/src/functions/backend/storage/resolve-storage-path.ts b/src/functions/backend/storage/resolve-storage-path.ts new file mode 100644 index 0000000..080af20 --- /dev/null +++ b/src/functions/backend/storage/resolve-storage-path.ts @@ -0,0 +1,128 @@ +import { existsSync, realpathSync } from "node:fs"; +import path from "node:path"; +import { AppData } from "@/src/data/app-data"; +import AppError from "@/src/utils/app-error"; + +type Params = { + /** + * The volume's absolute root (`resolveStorageSpec`'s `root`), already + * checked against the user. + */ + root: string; + /** + * The path a request asked for, relative to that root, e.g. + * `seo/coderank/raw`. Never absolute, and may be empty for the root + * itself. + */ + path?: string | null; +}; + +export type ResolvedStoragePath = { + /** + * The normalized path inside the root, forward slashes, no leading or + * trailing slash. Empty at the root. + */ + relative: string; + /** + * The absolute path on disk, inside `root`. + */ + absolute: string; + /** + * The folder part of `relative`, empty at the top. + */ + folder: string; + /** + * The last segment, empty at the root. + */ + name: string; +}; + +/** + * Turns a requested path into one safely inside a volume's root: rejects + * absolute paths, `..`, NUL bytes, anything too long or too deep, and + * anything a symlink points outside the root. The same rules a bot's own + * file path gets (`resolveBotPath`), for a root the browser picks rather + * than a bot's folders. + * + * A path that doesn't exist yet is allowed (a folder is made on demand by + * the bot's tools); a caller that needs an existing file checks it + * afterwards. + */ +export default function resolveStoragePath({ root, path: requested }: Params): ResolvedStoragePath { + const resolved_root = path.resolve(root); + const raw = (requested || "").trim().replace(/\\/g, "/"); + + if (raw.includes("\0")) { + throw new AppError(`That path isn't valid.`); + } + + if (raw.startsWith("/") || /^[a-zA-Z]:/.test(raw)) { + throw new AppError(`Use a path inside the folder, not an absolute one.`); + } + + const segments = raw.split("/").map((segment) => segment.trim()).filter(Boolean); + + for (const segment of segments) { + if (segment == "." || segment == "..") { + throw new AppError(`That path isn't valid: it must stay inside the folder.`); + } + } + + if (segments.length > AppData.BotFilePathMaxDepth) { + throw new AppError(`That path is too deep: at most ${AppData.BotFilePathMaxDepth} folders.`); + } + + const relative = segments.join("/"); + + if (relative.length > AppData.BotFilePathMaxChars) { + throw new AppError(`That path is too long.`); + } + + const absolute = path.join(resolved_root, ...segments); + + if (!isInsideRoot({ root: resolved_root, absolute })) { + throw new AppError(`That path isn't valid: it must stay inside the folder.`); + } + + guardRealPath({ root: resolved_root, absolute }); + + return { + relative, + absolute, + folder: segments.slice(0, -1).join("/"), + name: segments.at(-1) || "", + }; +} + +/** + * Refuses a path whose existing part resolves outside the root through a + * symlink (or any other reason). + */ +function guardRealPath({ root, absolute }: { root: string; absolute: string }) { + const real_root = existsSync(root) ? realpathSync(root) : path.resolve(root); + let probe = absolute; + + while (!existsSync(probe)) { + const parent = path.dirname(probe); + + if (parent == probe) break; + + probe = parent; + } + + const real_probe = realpathSync(probe); + + if (!isInsideRoot({ root: real_root, absolute: real_probe })) { + throw new AppError(`That path isn't valid: it must stay inside the folder.`); + } +} + +/** + * Whether a path is inside a root, before it exists. + */ +export function isInsideRoot({ root, absolute }: { root: string; absolute: string }) { + const resolved_root = path.resolve(root); + const resolved = path.resolve(absolute); + + return resolved == resolved_root || resolved.startsWith(`${resolved_root}${path.sep}`); +} diff --git a/src/functions/backend/storage/resolve-storage-spec.ts b/src/functions/backend/storage/resolve-storage-spec.ts new file mode 100644 index 0000000..ab3ad92 --- /dev/null +++ b/src/functions/backend/storage/resolve-storage-spec.ts @@ -0,0 +1,281 @@ +import path from "node:path"; +import BunSQLite from "@moduletrace/bun-sqlite"; +import { AppData } from "@/src/data/app-data"; +import computersSql from "@/src/utils/crew-access/computers-sql"; +import reachableBotsSql from "@/src/utils/crew-access/reachable-bots-sql"; +import workspacesSql from "@/src/utils/crew-access/workspaces-sql"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import type { CrewRole } from "@/src/utils/rbac/crew-role"; +import { storageVolumeLabel, type StorageSpec } from "@/src/utils/storage-volumes"; +import grabCrewRole from "../db/access/grab-crew-role"; +import grabUserBot from "../db/bots/grab-user-bot"; +import grabUserComputer from "../db/computers/grab-user-computer"; +import grabUserWorkspace from "../db/workspaces/grab-user-workspace"; + +type Params = { + user_id: number | string; + spec: StorageSpec; +}; + +/** + * One browsable folder, only if the user reaches it: its absolute root on + * the data volume and a label for the page. Null when the folder is not + * theirs to open, so a request never turns into a path before this has + * passed. + * + * Who reaches what: a bot, workspace or computer through the same SQL the + * crew uses (`grabUserBot`, `grabUserWorkspace`, `grabUserComputer`, so a + * member sees only their own grants); their own media folder (super + * admins and admins anyone's); an MCP server's install folder for crew + * managers; the database for the super admin alone. + */ +export default async function resolveStorageSpec({ user_id, spec }: Params) { + const role = await grabCrewRole({ user_id }); + const root = await storageRoot({ spec, role, user_id }); + + if (!root) return null; + + return { + spec, + root, + label: await storageLabel({ spec, role, user_id }), + }; +} + +export type ResolvedStorageSpec = NonNullable>>; + +/** + * The folder's absolute root, when the user reaches it. + */ +async function storageRoot({ + spec, + role, + user_id, +}: { + spec: StorageSpec; + role: CrewRole; + user_id: number | string; +}) { + const { DATA_DIR } = grabDirNames(); + + switch (spec.volume) { + case "bots": + return (await grabUserBot({ bot_id: spec.owner_id, user_id })) + ? path.join(DATA_DIR, AppData.BotDataDir, String(spec.owner_id)) + : null; + + case "workspaces": + return (await grabUserWorkspace({ workspace_id: spec.owner_id, user_id })) + ? path.join(DATA_DIR, AppData.WorkspaceDataDir, String(spec.owner_id)) + : null; + + case "computers": + return (await grabUserComputer({ computer_id: spec.owner_id, user_id })) + ? path.join(DATA_DIR, AppData.ComputerDataDir, String(spec.owner_id)) + : null; + + case "media": + if (role == "member" && Number(spec.owner_id) != Number(user_id)) return null; + + return (await hasUser({ user_id: spec.owner_id })) + ? path.join(DATA_DIR, "media", String(spec.owner_id)) + : null; + + case "mcp": + if (role == "member") return null; + + return (await hasMcpServer({ server_id: spec.owner_id })) + ? path.join(DATA_DIR, AppData.McpDataDir, String(spec.owner_id)) + : null; + + case "db": + if (role != "super_admin") return null; + + return dbRoot(); + } +} + +/** + * What the page calls the folder: the bot's, workspace's or computer's + * own name where one is known. + */ +async function storageLabel({ + spec, + role, + user_id, +}: { + spec: StorageSpec; + role: CrewRole; + user_id: number | string; +}) { + switch (spec.volume) { + case "bots": + return (await grabUserBot({ bot_id: spec.owner_id, user_id }))?.name || `Bot ${spec.owner_id}`; + case "workspaces": + return ( + (await grabUserWorkspace({ workspace_id: spec.owner_id, user_id }))?.name || + `Workspace ${spec.owner_id}` + ); + case "computers": + return ( + (await grabUserComputer({ computer_id: spec.owner_id, user_id }))?.name || + `Computer ${spec.owner_id}` + ); + case "media": { + if (role == "member" || Number(spec.owner_id) == Number(user_id)) return "My media"; + + const person = await grabPersonName({ user_id: spec.owner_id }); + + return person ? `${person}'s media` : `Media of user ${spec.owner_id}`; + } + case "mcp": + return (await grabMcpServerName({ server_id: spec.owner_id })) || `MCP server ${spec.owner_id}`; + case "db": + return storageVolumeLabel({ volume: spec.volume }); + } +} + +/** + * Where the database is kept: `DB_DIR` on the data volume in the app's + * container, else the folder bun-sqlite uses from the working directory. + */ +function dbRoot() { + const dir = process.env.DB_DIR || "./db"; + + return path.isAbsolute(dir) ? dir : path.join(process.cwd(), dir); +} + +async function hasUser({ user_id }: { user_id: number | string }) { + const res = await BunSQLite.sql<{ id: number }>({ + sql: `SELECT id FROM users WHERE id = ? LIMIT 1`, + values: [Number(user_id)], + }); + + return Boolean(res.payload?.[0]?.id); +} + +async function grabPersonName({ user_id }: { user_id: number | string }) { + const res = await BunSQLite.sql<{ + first_name?: string | null; + last_name?: string | null; + username?: string | null; + }>({ + sql: `SELECT first_name, last_name, username FROM users WHERE id = ? LIMIT 1`, + values: [Number(user_id)], + }); + const person = res.payload?.[0]; + + if (!person) return ""; + + return [person.first_name, person.last_name].filter(Boolean).join(" ") || person.username || ""; +} + +async function hasMcpServer({ server_id }: { server_id: number | string }) { + const res = await BunSQLite.sql<{ id: number }>({ + sql: `SELECT id FROM mcp_servers WHERE id = ? LIMIT 1`, + values: [Number(server_id)], + }); + + return Boolean(res.payload?.[0]?.id); +} + +async function grabMcpServerName({ server_id }: { server_id: number | string }) { + const res = await BunSQLite.sql<{ name?: string | null }>({ + sql: `SELECT name FROM mcp_servers WHERE id = ? LIMIT 1`, + values: [Number(server_id)], + }); + + return res.payload?.[0]?.name || ""; +} + +export type StorageSpecOption = { + spec: StorageSpec; + label: string; +}; + +/** + * Every folder a user can open, for the page's picker: their bots, + * workspaces and computers, their own media folder, and (for crew + * managers) every MCP server's folder. The database comes last, for the + * super admin alone. The crew SQL is the same as the rest of the app, so + * the picker never lists a folder the resolver would refuse. + */ +export async function grabStorageSpecs({ + user_id, +}: { + user_id: number | string; +}): Promise { + const role = await grabCrewRole({ user_id }); + const options: StorageSpecOption[] = []; + + const bot_access = reachableBotsSql({ role, user_id }); + const bots = await rows<{ id: number; name?: string | null }>({ + sql: `SELECT b.id, b.name FROM bots b WHERE ${bot_access.sql} ORDER BY b.name`, + values: bot_access.values, + }); + + for (const bot of bots) { + options.push({ + spec: { volume: "bots", owner_id: Number(bot.id) }, + label: bot.name || `Bot ${bot.id}`, + }); + } + + const workspace_access = workspacesSql({ role, user_id }); + const workspaces = await rows<{ id: number; name?: string | null }>({ + sql: `SELECT w.id, w.name FROM workspaces w WHERE ${workspace_access.sql} AND w.archived = 0 ORDER BY w.name`, + values: workspace_access.values, + }); + + for (const workspace of workspaces) { + options.push({ + spec: { volume: "workspaces", owner_id: Number(workspace.id) }, + label: workspace.name || `Workspace ${workspace.id}`, + }); + } + + const computer_access = computersSql({ role, user_id }); + const computers = await rows<{ id: number; name?: string | null }>({ + sql: `SELECT c.id, c.name FROM computers c WHERE ${computer_access.sql} ORDER BY c.name`, + values: computer_access.values, + }); + + for (const computer of computers) { + options.push({ + spec: { volume: "computers", owner_id: Number(computer.id) }, + label: computer.name || `Computer ${computer.id}`, + }); + } + + options.push({ spec: { volume: "media", owner_id: Number(user_id) }, label: "My media" }); + + if (role != "member") { + const servers = await rows<{ id: number; name?: string | null }>({ + sql: `SELECT id, name FROM mcp_servers ORDER BY name`, + values: [], + }); + + for (const server of servers) { + options.push({ + spec: { volume: "mcp", owner_id: Number(server.id) }, + label: server.name || `MCP server ${server.id}`, + }); + } + } + + if (role == "super_admin") { + options.push({ spec: { volume: "db", owner_id: 0 }, label: storageVolumeLabel({ volume: "db" }) }); + } + + return options; +} + +async function rows({ + sql, + values, +}: { + sql: string; + values: (string | number)[]; +}) { + return (await BunSQLite.sql({ sql, values })).payload || []; +} diff --git a/src/pages/admin/files/(hooks)/use-storage-folder.ts b/src/pages/admin/files/(hooks)/use-storage-folder.ts new file mode 100644 index 0000000..a5cd3fa --- /dev/null +++ b/src/pages/admin/files/(hooks)/use-storage-folder.ts @@ -0,0 +1,94 @@ +import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi"; +import type { APIResponseObject } from "@moduletrace/bunext/types"; +import { useEffect, useState } from "react"; +import type { StorageFolderPayload } from "@/src/types"; + +type Params = { + spec: { volume: string; owner_id: number } | null; + /** + * The folder's path inside the volume; empty at the top. + */ + path: string; + /** + * The search the user submitted (not what is typed). + */ + search?: string; + /** + * The folder the page already carries, when this is that one. + */ + page_folder?: StorageFolderPayload | null; +}; + +/** + * One folder of the files browser. The page's own copy is used when it is + * the folder asked for and nothing is searched; anything else loads from + * the API, so a page never carries the whole volume. + */ +export default function useStorageFolder({ spec, path, search, page_folder }: Params) { + const from_page = Boolean( + page_folder && + spec && + page_folder.spec.volume == spec.volume && + page_folder.spec.owner_id == spec.owner_id && + page_folder.path == path && + !search, + ); + const [folder, set_folder] = useState( + from_page ? page_folder || null : null, + ); + const [loading, set_loading] = useState(!from_page && Boolean(spec)); + const [error, set_error] = useState(); + + useEffect(() => { + if (!spec) { + set_folder(null); + set_loading(false); + return; + } + + if (from_page) { + set_folder(page_folder || null); + set_loading(false); + set_error(undefined); + return; + } + + const controller = new AbortController(); + + const load = async () => { + set_loading(true); + set_error(undefined); + + const query = new URLSearchParams({ + volume: spec.volume, + owner: String(spec.owner_id), + }); + + if (path) query.set("path", path); + if (search) query.set("search", search); + + const res = await fetchApi<{}, APIResponseObject>( + `/api/admin/storage/list?${query.toString()}`, + { method: "GET" }, + ).catch(() => null); + + if (controller.signal.aborted) return; + + set_loading(false); + + if (!res?.success || !res.singleRes) { + set_folder(null); + set_error(res?.msg || `Couldn't open that folder.`); + return; + } + + set_folder(res.singleRes); + }; + + load().catch(() => {}); + + return () => controller.abort(); + }, [spec?.volume, spec?.owner_id, path, search, from_page, page_folder]); + + return { folder, loading, error }; +} diff --git a/src/pages/admin/files/(partials)/files-breadcrumb-row.tsx b/src/pages/admin/files/(partials)/files-breadcrumb-row.tsx new file mode 100644 index 0000000..c87dc5d --- /dev/null +++ b/src/pages/admin/files/(partials)/files-breadcrumb-row.tsx @@ -0,0 +1,59 @@ +import Stack from "@moduletrace/twui/layout/Stack"; +import Button from "@moduletrace/twui/layout/Button"; +import Span from "@moduletrace/twui/layout/Span"; +import { ChevronRight } from "lucide-react"; + +type Props = { + crumbs: { name: string; href: string }[]; + onOpenFolder: (path: string) => void; +}; + +/** + * The trail back up the open folder: the volume's top, then one step per + * folder. Every crumb but the one being shown opens where it points. + */ +export default function FilesBreadcrumbRow({ crumbs, onOpenFolder }: Props) { + if (!crumbs.length) return null; + + return ( + + {crumbs.map((crumb, index) => { + const last = index == crumbs.length - 1; + + return ( + + {index ? ( + + + + ) : null} + + + ); + })} + + ); +} + +/** + * The path a crumb's address opens, read back from its query. + */ +function pathOfHref({ href }: { href: string }) { + const query = href.split("?")[1]; + + return new URLSearchParams(query).get("path") || ""; +} diff --git a/src/pages/admin/files/(partials)/files-search-input.tsx b/src/pages/admin/files/(partials)/files-search-input.tsx new file mode 100644 index 0000000..de11813 --- /dev/null +++ b/src/pages/admin/files/(partials)/files-search-input.tsx @@ -0,0 +1,28 @@ +import Input from "@moduletrace/twui/form/Input"; + +type Props = { + value: string; + loading?: boolean; + onChange: (search: string) => void; + onSubmit: (search: string) => void; +}; + +/** + * Narrows the open folder by name or path. Enter runs the search; clearing + * the field runs it again empty. + */ +export default function FilesSearchInput({ value, loading, onChange, onSubmit }: Props) { + return ( + onChange(event.target.value)} + onKeyDown={(event) => { + if (event.key == "Enter") onSubmit(event.currentTarget.value); + }} + clearInputProps={{ onClick: () => onSubmit("") }} + wrapperWrapperProps={{ className: "w-full mt-3" }} + /> + ); +} diff --git a/src/pages/admin/files/(partials)/files-volume-row.tsx b/src/pages/admin/files/(partials)/files-volume-row.tsx new file mode 100644 index 0000000..b5a3737 --- /dev/null +++ b/src/pages/admin/files/(partials)/files-volume-row.tsx @@ -0,0 +1,40 @@ +import Link from "@moduletrace/twui/layout/Link"; +import Span from "@moduletrace/twui/layout/Span"; +import { twMerge } from "tailwind-merge"; +import type { StorageSpecOption } from "@/src/types"; +import { storageSpecHref, storageVolumeLabel } from "@/src/utils/storage-volumes"; + +type Props = { + option: StorageSpecOption; + selected: boolean; + onOpen: () => void; +}; + +/** + * One folder in the files browser's picker: the bot, workspace, computer, + * media library, MCP server or database it is, named the way the crew + * names it. + */ +export default function FilesVolumeRow({ option, selected, onOpen }: Props) { + return ( + { + event.preventDefault(); + onOpen(); + }} + className={twMerge( + "flex w-full min-w-0 items-center gap-2 rounded-default px-2 py-1.5 border-0 text-ink", + "hover:opacity-100 hover:bg-sunken transition-colors duration-[var(--m-fast)]", + selected && "bg-ink/5 hover:bg-ink/5", + )} + > + {option.label} + + {storageVolumeLabel({ volume: option.spec.volume })} + + + ); +} diff --git a/src/pages/admin/files/(partials)/storage-entry-row.tsx b/src/pages/admin/files/(partials)/storage-entry-row.tsx new file mode 100644 index 0000000..082c1bc --- /dev/null +++ b/src/pages/admin/files/(partials)/storage-entry-row.tsx @@ -0,0 +1,76 @@ +import Link from "@moduletrace/twui/layout/Link"; +import Row from "@moduletrace/twui/layout/Row"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import { twMerge } from "tailwind-merge"; +import { FolderOpen } from "lucide-react"; +import type { MediaType, StorageEntry } from "@/src/types"; +import formatBytes from "@/src/utils/format-bytes"; +import relativeTime from "@/src/utils/relative-time"; +import AttachmentIcon from "@/src/pages/admin/bots/[bot_id]/(partials)/attachment-icon"; + +type Props = { + entry: StorageEntry; + selected?: boolean; + onOpenFolder: (path: string) => void; +}; + +/** + * One entry in the open folder: a folder opens in place, a file opens in a + * new tab (its own media record when it is one of the bot's files, else + * the browser's streaming route). + */ +export default function StorageEntryRow({ entry, selected, onOpenFolder }: Props) { + const is_folder = entry.kind == "folder"; + + return ( + + + {is_folder ? ( + + ) : ( + + )} + + + {is_folder ? ( + { + event.preventDefault(); + onOpenFolder(entry.path); + }} + className="text-[13px] font-bold leading-5 text-ink break-words hover:opacity-80" + > + {entry.name} + + ) : ( + + {entry.name} + + )} + + {[ + is_folder ? "Folder" : formatBytes({ bytes: entry.size }), + entry.modified_at ? relativeTime({ at: entry.modified_at }) : "", + entry.file_id ? "Saved by the bot" : "", + ] + .filter(Boolean) + .join(" ยท ")} + + + + ); +} diff --git a/src/pages/admin/files/(sections)/files-browser.tsx b/src/pages/admin/files/(sections)/files-browser.tsx new file mode 100644 index 0000000..bf8fae3 --- /dev/null +++ b/src/pages/admin/files/(sections)/files-browser.tsx @@ -0,0 +1,127 @@ +import H2 from "@moduletrace/twui/layout/H2"; +import Row from "@moduletrace/twui/layout/Row"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import type { StorageFolderPayload, StorageSpecOption } from "@/src/types"; +import type { StorageSpec } from "@/src/utils/storage-volumes"; +import FilesBreadcrumbRow from "../(partials)/files-breadcrumb-row"; +import FilesSearchInput from "../(partials)/files-search-input"; +import FilesVolumeRow from "../(partials)/files-volume-row"; +import StorageEntryRow from "../(partials)/storage-entry-row"; +import Divider from "@moduletrace/twui/layout/Divider"; +import Paper from "@moduletrace/twui/elements/Paper"; + +type Props = { + folders: StorageSpecOption[]; + folder: StorageFolderPayload | null; + spec: StorageSpec | null; + path: string; + search: string; + loading?: boolean; + error?: string; + onSearch: (search: string) => void; + onSubmitSearch: () => void; + onOpenFolder: (path: string) => void; + onOpenVolume: (option: StorageSpecOption) => void; +}; + +/** + * The files browser: the folders the user reaches on the left, the open + * folder's entries on the right. Read-only: a bot changes its own files + * with its tools, and the chat's files modal can delete them. + */ +export default function FilesBrowser({ + folders, + folder, + spec, + path, + search, + loading, + error, + onSearch, + onSubmitSearch, + onOpenFolder, + onOpenVolume, +}: Props) { + return ( + + +

+ Folders +

+ + + {folders.length ? ( + folders.map((option) => ( + onOpenVolume(option)} + /> + )) + ) : ( + + Nothing to show yet: give a bot a file, or let one + work on a computer. + + )} + + + + + + +

+ {folder?.breadcrumbs.at(-1)?.name || + folder?.label || + "Files"} +

+ + {folder + ? `${folder.label}: ${folder.total} item(s). Everything a bot, a workspace or a computer keeps for you.` + : "Pick a folder to look inside."} + + + + + + {folder?.entries.length ? ( + folder.entries.map((entry) => ( + + )) + ) : ( + + {loading + ? "Loading..." + : error || + (search + ? `Nothing matches "${search}".` + : "This folder is empty.")} + + )} + + + + + ); +} diff --git a/src/pages/admin/files/(sections)/files-pane.tsx b/src/pages/admin/files/(sections)/files-pane.tsx new file mode 100644 index 0000000..6005484 --- /dev/null +++ b/src/pages/admin/files/(sections)/files-pane.tsx @@ -0,0 +1,73 @@ +import { useContext, useState } from "react"; +import { AppContext } from "@/src/pages/__root"; +import type { StorageSpecOption } from "@/src/types"; +import { parseStorageSpec, type StorageSpec } from "@/src/utils/storage-volumes"; +import useStorageFolder from "../(hooks)/use-storage-folder"; +import FilesBrowser from "./files-browser"; + +/** + * The files page's state: which folder is open and where inside it, kept + * in the address so a link to a folder can be shared and the browser's own + * back button walks back up. The folder asked for comes with the page; a + * new folder loads through the API. + */ +export default function FilesPane() { + const { pageProps } = useContext(AppContext); + const folders = pageProps?.storage_folders || []; + const page_folder = pageProps?.storage_folder; + const [spec, set_spec] = useState( + parseStorageSpec({ volume: page_folder?.spec.volume, owner_id: page_folder?.spec.owner_id }), + ); + const [path, set_path] = useState(page_folder?.path || ""); + const [search, set_search] = useState(""); + const [query, set_query] = useState(""); + const { folder, loading, error } = useStorageFolder({ + spec, + path, + search: query, + page_folder, + }); + + const navigate = ({ + next_spec, + next_path, + }: { + next_spec: StorageSpec | null; + next_path: string; + }) => { + set_spec(next_spec); + set_path(next_path); + set_search(""); + set_query(""); + + const url = new URL(window.location.href); + + if (next_spec) { + url.searchParams.set("volume", next_spec.volume); + url.searchParams.set("owner", String(next_spec.owner_id)); + } + + if (next_path) url.searchParams.set("path", next_path); + else url.searchParams.delete("path"); + + history.pushState(null, "", url); + }; + + return ( + set_query(search)} + onOpenFolder={(next_path: string) => navigate({ next_spec: spec, next_path })} + onOpenVolume={(option: StorageSpecOption) => + navigate({ next_spec: option.spec, next_path: "" }) + } + /> + ); +} diff --git a/src/pages/admin/files/index.server.ts b/src/pages/admin/files/index.server.ts new file mode 100644 index 0000000..7815c2e --- /dev/null +++ b/src/pages/admin/files/index.server.ts @@ -0,0 +1,84 @@ +import userAuth from "@/src/functions/backend/auth/user-auth"; +import listStorageEntries, { storageBreadcrumbs } from "@/src/functions/backend/storage/list-storage-entries"; +import resolveStoragePath from "@/src/functions/backend/storage/resolve-storage-path"; +import resolveStorageSpec, { grabStorageSpecs } from "@/src/functions/backend/storage/resolve-storage-spec"; +import type { PagePropsType, StorageFolderPayload } from "@/src/types"; +import { parseStorageSpec, type StorageSpec } from "@/src/utils/storage-volumes"; +import type { BunextPageServerFn } from "@moduletrace/bunext/types"; + +/** + * Files: one folder on the app's data volume. The folders the user may + * open come with the page, and so does the folder asked for (or the first + * one); moving to another folder loads through the API, so a page stays + * small. + */ +export const server: BunextPageServerFn = async (params) => { + const query = params.query || {}; + const { user } = await userAuth({ req: params.req }); + + if (!user?.id) { + return { redirect: { destination: "/auth/login" } }; + } + + const folders = await grabStorageSpecs({ user_id: user.id }); + const asked = parseStorageSpec({ + volume: query.volume, + owner_id: query.owner, + }); + const known = + asked && + folders.some( + (folder) => + folder.spec.volume == asked.volume && folder.spec.owner_id == asked.owner_id, + ); + const spec = known ? asked : folders[0]?.spec; + + if (!spec) { + return { props: { storage_folders: [], storage_folder: null } }; + } + + const storage_folder = await folderPayload({ + user_id: Number(user.id), + spec, + path: query.path || "", + }); + + return { props: { storage_folders: folders, storage_folder } }; +}; + +/** + * One folder's payload, or null when the user doesn't reach it (or its + * path escapes the volume). + */ +async function folderPayload({ + user_id, + spec, + path, +}: { + user_id: number; + spec: StorageSpec; + path?: string; +}): Promise { + const resolved = await resolveStorageSpec({ user_id, spec }).catch(() => null); + + if (!resolved) return null; + + const target = resolveStoragePath({ root: resolved.root, path }); + const listing = await listStorageEntries({ + resolved, + absolute: target.absolute, + relative: target.relative, + }).catch(() => null); + + if (!listing) return null; + + return { + spec: resolved.spec, + label: resolved.label, + path: target.relative, + breadcrumbs: storageBreadcrumbs({ spec: resolved.spec, path: target.relative }), + entries: listing.entries, + total: listing.total, + offset: listing.offset, + }; +} diff --git a/src/pages/admin/files/index.tsx b/src/pages/admin/files/index.tsx new file mode 100644 index 0000000..5165603 --- /dev/null +++ b/src/pages/admin/files/index.tsx @@ -0,0 +1,23 @@ +import type { BunextPageModuleMetaFn } from "@moduletrace/bunext/types"; +import { SiteData } from "@/src/data/site-data"; +import SettingsShell from "@/src/pages/admin/settings/(partials)/settings-shell"; +import FilesPane from "./(sections)/files-pane"; + +export default function AdminFilesPage() { + return ( + + + + ); +} + +export const meta: BunextPageModuleMetaFn = async () => { + return { + title: `Files | ${SiteData["SiteName"]}`, + description: "Browse the files your bots, workspaces and computers keep.", + }; +}; diff --git a/src/pages/api/admin/storage/file.ts b/src/pages/api/admin/storage/file.ts new file mode 100644 index 0000000..7a2bfb4 --- /dev/null +++ b/src/pages/api/admin/storage/file.ts @@ -0,0 +1,63 @@ +import { statSync } from "node:fs"; +import userAuth from "@/src/functions/backend/auth/user-auth"; +import grabStorageFile from "@/src/functions/backend/storage/grab-storage-file"; +import resolveStorageSpec from "@/src/functions/backend/storage/resolve-storage-spec"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import { StorageFileRequestSchema } from "@/src/schemas/storage-schema"; +import { parseStorageSpec } from "@/src/utils/storage-volumes"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * `GET ?volume=..&owner=..&path=..`: one file on the data volume, streamed + * as it is. Access is checked twice, once for the folder + * (`resolveStorageSpec`) and once for the file's own path + * (`grabStorageFile`), so a folder the user reaches never becomes a way to + * read the rest of the disk. + */ +export const handler: BunextAPIRouteHandler = async ({ req, query }) => { + try { + if (req.method != "GET") { + return { success: false, msg: `Method not allowed` }; + } + + const { user } = await userAuth({ req }); + + if (!user?.logged_in_status || !user.id) { + return { success: false, msg: `Unauthorized`, logoutUser: true }; + } + + const { volume, owner, path } = parseWithSchema({ + schema: StorageFileRequestSchema, + data: query, + }); + const spec = parseStorageSpec({ volume, owner_id: owner }); + + if (!spec) return { success: false, msg: `Not found.` }; + + const resolved = await resolveStorageSpec({ user_id: user.id, spec }); + + if (!resolved) return { success: false, msg: `Not found.` }; + + const file = grabStorageFile({ resolved, path }); + + if (!file) return { success: false, msg: `Not found.` }; + + const etag = `"${file.size}-${Math.round(statSync(file.absolute).mtimeMs)}"`; + const headers = new Headers({ + "Content-Type": file.mime_type, + "Content-Disposition": `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`, + "X-Content-Type-Options": "nosniff", + ETag: etag, + "Cache-Control": "private, no-cache", + }); + + if (req.headers.get("if-none-match") === etag) { + return new Response(null, { status: 304, headers }); + } + + return new Response(Bun.file(file.absolute), { headers }); + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; diff --git a/src/pages/api/admin/storage/list.ts b/src/pages/api/admin/storage/list.ts new file mode 100644 index 0000000..a4286be --- /dev/null +++ b/src/pages/api/admin/storage/list.ts @@ -0,0 +1,69 @@ +import userAuth from "@/src/functions/backend/auth/user-auth"; +import listStorageEntries, { storageBreadcrumbs } from "@/src/functions/backend/storage/list-storage-entries"; +import resolveStoragePath from "@/src/functions/backend/storage/resolve-storage-path"; +import resolveStorageSpec from "@/src/functions/backend/storage/resolve-storage-spec"; +import type { StorageFolderPayload } from "@/src/types"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import { StorageListRequestSchema } from "@/src/schemas/storage-schema"; +import { parseStorageSpec } from "@/src/utils/storage-volumes"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * One folder on the data volume, for the files browser: its entries, the + * breadcrumb trail and the folder's name. Only a folder the user reaches + * (`resolveStorageSpec`) and only a path inside it + * (`resolveStoragePath`), so a request never turns into a read. + */ +export const handler: BunextAPIRouteHandler> = async ({ + req, + query, +}) => { + try { + if (req.method != "GET") { + return { success: false, msg: `Method not allowed` }; + } + + const { user } = await userAuth({ req }); + + if (!user?.logged_in_status || !user.id) { + return { success: false, msg: `Unauthorized`, logoutUser: true }; + } + + const { volume, owner, path, search, offset } = parseWithSchema({ + schema: StorageListRequestSchema, + data: query, + }); + const spec = parseStorageSpec({ volume, owner_id: owner }); + + if (!spec) return { success: false, msg: `Not found.` }; + + const resolved = await resolveStorageSpec({ user_id: user.id, spec }); + + if (!resolved) return { success: false, msg: `Not found.` }; + + const target = resolveStoragePath({ root: resolved.root, path }); + const listing = await listStorageEntries({ + resolved, + absolute: target.absolute, + relative: target.relative, + search, + offset, + }); + + return { + success: true, + singleRes: { + spec: resolved.spec, + label: resolved.label, + path: target.relative, + breadcrumbs: storageBreadcrumbs({ spec: resolved.spec, path: target.relative }), + entries: listing.entries, + total: listing.total, + offset: listing.offset, + }, + }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; diff --git a/src/schemas/storage-schema.ts b/src/schemas/storage-schema.ts new file mode 100644 index 0000000..40af135 --- /dev/null +++ b/src/schemas/storage-schema.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import { StorageVolumeDefs } from "@/src/utils/storage-volumes"; + +const VolumeField = z.enum( + StorageVolumeDefs.map((def) => def.value) as [ + (typeof StorageVolumeDefs)[number]["value"], + ...((typeof StorageVolumeDefs)[number]["value"])[], + ], +); + +/** + * The volume and owner a storage request asks for. The owner is left out + * for the database, which has one folder. + */ +const StorageTargetFields = { + volume: VolumeField, + owner: z.coerce.number().int().nonnegative().default(0), +}; + +/** + * `GET api/admin/storage/list`: one folder on the data volume. + */ +export const StorageListRequestSchema = z.object({ + ...StorageTargetFields, + path: z.string().max(AppData.BotFilePathMaxChars).optional(), + search: z.string().max(120).optional(), + offset: z.coerce.number().int().nonnegative().default(0), +}); + +export type StorageListRequest = z.infer; + +/** + * `GET api/admin/storage/file`: one file's bytes. + */ +export const StorageFileRequestSchema = z.object({ + ...StorageTargetFields, + path: z.string().min(1, "Pick a file").max(AppData.BotFilePathMaxChars), +}); + +export type StorageFileRequest = z.infer; diff --git a/src/types/index.ts b/src/types/index.ts index 91700bb..03a69f6 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -18,6 +18,7 @@ import type { BunSQLiteTables, } from "@/db/types/db"; import type { UserTypes } from "../dict/user-types-dict"; +import type { StorageVolume } from "../utils/storage-volumes"; import type { WebsocketRooms } from "../dict/websocket-rooms"; import type { LucideIcon } from "lucide-react"; import type { ServerQueryParam } from "@moduletrace/bun-sqlite"; @@ -712,6 +713,58 @@ export type BotFileScopesPayload = { quota_mb: number; }; +/** + * One entry in a folder the files browser shows: a file or a subfolder. + */ +export type StorageEntry = { + name: string; + /** + * Where it is inside the volume, e.g. `seo/coderank/raw`. + */ + path: string; + kind: "file" | "folder"; + size: number; + modified_at: number; + media_type?: string; + /** + * The `bot_files` row that holds it, when the folder is a bot's or a + * workspace's. + */ + file_id?: number; + /** + * Where it opens: a `bot_files` row's media URL, else the browser's own + * streaming route. + */ + url?: string; +}; + +/** + * One folder of the files browser: what it holds and how to get back up. + */ +export type StorageFolderPayload = { + spec: { volume: StorageVolume; owner_id: number }; + /** + * What the folder is called: the bot's, workspace's or computer's name. + */ + label: string; + /** + * The folder asked for, inside the volume (empty at the top). + */ + path: string; + breadcrumbs: { name: string; href: string }[]; + entries: StorageEntry[]; + total: number; + offset: number; +}; + +/** + * A folder in the files browser's picker. + */ +export type StorageSpecOption = { + spec: { volume: StorageVolume; owner_id: number }; + label: string; +}; + /** * A worker bot's model settings page data, also returned when it saves. */ @@ -907,6 +960,12 @@ export type PagePropsType = { mcps_overview?: McpServersOverviewPayload | null; tools_overview?: CustomToolsOverviewPayload | null; bot_extensions?: BotExtensionsPayload | null; + /** + * The files browser's folders (a user's own crew and media, and what a + * super admin may open) and the folder it is showing. + */ + storage_folders?: StorageSpecOption[] | null; + storage_folder?: StorageFolderPayload | null; /** * The open conversation's files (a bot's own, or a workspace's shared * ones), for the files modal and the chips that name one. diff --git a/src/utils/crew-access/crew-access-sql.test.ts b/src/utils/crew-access/crew-access-sql.test.ts index 887e1cf..32b73ab 100644 --- a/src/utils/crew-access/crew-access-sql.test.ts +++ b/src/utils/crew-access/crew-access-sql.test.ts @@ -2,6 +2,7 @@ import { Database } from "bun:sqlite"; import { beforeAll, describe, expect, test } from "bun:test"; import chatBotsSql from "./chat-bots-sql"; import computersSql from "./computers-sql"; +import reachableBotsSql from "./reachable-bots-sql"; import workspaceBotsSql from "./workspace-bots-sql"; import workspacesSql from "./workspaces-sql"; import type { SqlCondition } from "./types"; @@ -50,6 +51,18 @@ describe("chatBotsSql", () => { }); }); +describe("reachableBotsSql", () => { + test("a member also reaches the workers of a workspace they were added to", () => { + expect(ids({ table: "bots", alias: "b", condition: reachableBotsSql({ role: "member", user_id: 7 }) })).toEqual([2, 3]); + expect(ids({ table: "bots", alias: "b", condition: reachableBotsSql({ role: "member", user_id: 8 }) })).toEqual([]); + }); + + test("managers reach what they chat with", () => { + expect(ids({ table: "bots", alias: "b", condition: reachableBotsSql({ role: "super_admin", user_id: 1 }) })).toEqual([1, 2, 3]); + expect(ids({ table: "bots", alias: "b", condition: reachableBotsSql({ role: "admin", user_id: 5 }) })).toEqual([2, 3]); + }); +}); + describe("workspacesSql", () => { test("managers reach every workspace, members only theirs", () => { expect(ids({ table: "workspaces", alias: "w", condition: workspacesSql({ role: "admin", user_id: 5 }) })).toEqual([10, 11]); diff --git a/src/utils/crew-access/reachable-bots-sql.ts b/src/utils/crew-access/reachable-bots-sql.ts new file mode 100644 index 0000000..f2857b0 --- /dev/null +++ b/src/utils/crew-access/reachable-bots-sql.ts @@ -0,0 +1,28 @@ +import type { CrewAccessParams, SqlCondition } from "./types"; +import chatBotsSql from "./chat-bots-sql"; + +/** + * The bots a user reaches at all: the ones in their own chat + * (`chatBotsSql`) plus, outside it, every worker attached to a workspace + * they were added to. The same condition `grabUserBot` applies, so a + * picker built from it lists exactly the folders the resolver opens. The + * coordinator is the super admin's alone. + */ +export default function reachableBotsSql({ + role, + user_id, + alias = "b", +}: CrewAccessParams): SqlCondition { + const chat = chatBotsSql({ role, user_id, alias }); + + if (role != "member") return chat; + + return { + sql: `(${chat.sql} OR (${alias}.kind != 'coordinator' AND ${alias}.id IN ( + SELECT wb.bot_id FROM workspace_bots wb + JOIN workspace_users wu ON wu.workspace_id = wb.workspace_id AND wu.user_id = ? + JOIN workspaces w ON w.id = wb.workspace_id AND w.archived = 0 + )))`, + values: [...chat.values, Number(user_id)], + }; +} diff --git a/src/utils/path-ref.test.ts b/src/utils/path-ref.test.ts new file mode 100644 index 0000000..e0c3950 --- /dev/null +++ b/src/utils/path-ref.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test"; +import pathRef, { parsePathRefHref, pathRefHref, pathRefUrl } from "./path-ref"; +import linkComputerRefs from "./link-computer-refs"; + +describe("pathRef", () => { + test("writes a folder and a file", () => { + expect(pathRef({ volume: "computer", owner_id: 3, path: "seo/coderank/raw" })).toBe( + "[path:computer:3:seo/coderank/raw]", + ); + expect(pathRef({ volume: "computer", owner_id: 3, path: "/seo/raw.json " })).toBe( + "[path:computer:3:seo/raw.json]", + ); + expect(pathRef({ volume: "bot", owner_id: 1 })).toBe("[path:bot:1]"); + }); +}); + +describe("pathRefHref", () => { + test("encodes the path into the link", () => { + expect(pathRefHref({ volume: "computer", owner_id: 3, path: "seo/raw mes" })).toBe( + `#path-${encodeURIComponent("computer:3:seo/raw mes")}`, + ); + }); + + test("reads back what it wrote", () => { + const href = pathRefHref({ volume: "computer", owner_id: 3, path: "seo/raw mes/#1" }); + + expect(parsePathRefHref({ href })).toEqual({ + spec: { volume: "computers", owner_id: 3 }, + path: "seo/raw mes/#1", + }); + }); + + test("maps the ref's volume name to the browser's", () => { + expect(parsePathRefHref({ href: pathRefHref({ volume: "bot", owner_id: 1, path: "a" }) })).toEqual({ + spec: { volume: "bots", owner_id: 1 }, + path: "a", + }); + expect( + parsePathRefHref({ href: pathRefHref({ volume: "workspace", owner_id: 2 }) }), + ).toEqual({ + spec: { volume: "workspaces", owner_id: 2 }, + path: "", + }); + }); + + test("refuses an unknown volume or owner", () => { + expect(parsePathRefHref({ href: "#path-secrets:1:a" })).toBeNull(); + expect(parsePathRefHref({ href: "#path-computer:abc:a" })).toBeNull(); + expect(parsePathRefHref({ href: "#file-12" })).toBeNull(); + expect(parsePathRefHref({ href: undefined })).toBeNull(); + }); +}); + +describe("pathRefUrl", () => { + test("opens the files browser on the folder", () => { + expect(pathRefUrl({ volume: "computer", owner_id: 3, path: "seo/raw" })).toBe( + `/admin/files?volume=computers&owner=3&path=seo%2Fraw`, + ); + }); +}); + +describe("linkComputerRefs", () => { + test("turns a path ref into a link", () => { + const linked = linkComputerRefs({ text: "In [path:computer:3:seo/raw]." }); + + expect(linked).toContain("(#path-"); + expect(parsePathRefHref({ href: linked.match(/\((#path-[^)]+)\)/)?.[1] })).toEqual({ + spec: { volume: "computers", owner_id: 3 }, + path: "seo/raw", + }); + }); +}); diff --git a/src/utils/path-ref.ts b/src/utils/path-ref.ts new file mode 100644 index 0000000..2ec3a06 --- /dev/null +++ b/src/utils/path-ref.ts @@ -0,0 +1,85 @@ +/** + * Where a bot names a folder or file on the app's data volume in a + * message: `[path:computer:3:seo/coderank/raw]`. The chat turns it into a + * chip that opens it in the files browser (`/admin/files`), which is how a + * bot points the user at something it left on a computer (raw evidence, + * a crawl, a report it never copied) without importing it first. + */ +import { StorageBrowserPath, storageSpecHref, storageVolumeFromRef, type StorageSpec } from "./storage-volumes"; + +/** + * The link target a path ref becomes, e.g. `#path-computer%3A3%3Aseo%2Fraw`. + */ +export const PathRefHrefPrefix = "#path-"; + +type RefParams = { + /** + * The volume by the name a ref uses: `bot`, `workspace`, `computer`, + * `media`, `mcp` or `db`. + */ + volume: string; + owner_id: number | string; + path?: string | null; +}; + +/** + * How a bot names a folder or file in a message. The path is written as it + * is; the chat encodes it into the link. + */ +export default function pathRef({ volume, owner_id, path }: RefParams) { + const clean = (path || "").trim().replace(/^\/+|\/+$/g, ""); + + return clean + ? `[path:${volume}:${owner_id}:${clean}]` + : `[path:${volume}:${owner_id}]`; +} + +/** + * The href a path ref becomes: everything after `#path-`, URI encoded, so + * spaces, `#` and `:` in a path survive markdown. + */ +export function pathRefHref({ volume, owner_id, path }: RefParams) { + const clean = (path || "").trim().replace(/^\/+|\/+$/g, ""); + + return `${PathRefHrefPrefix}${encodeURIComponent( + clean ? `${volume}:${owner_id}:${clean}` : `${volume}:${owner_id}`, + )}`; +} + +/** + * Reads a path ref's link back into a folder and a path inside it, or null + * when it names a volume that isn't one (or has no owner). + */ +export function parsePathRefHref({ + href, +}: { + href?: string | null; +}): { spec: StorageSpec; path: string } | null { + if (!href?.startsWith(PathRefHrefPrefix)) return null; + + const decoded = decodeURIComponent(href.slice(PathRefHrefPrefix.length)); + const [name, raw_owner, ...rest] = decoded.split(":"); + const volume = storageVolumeFromRef({ name }); + const owner_id = Number(raw_owner); + + if (!volume) return null; + + if (!Number.isFinite(owner_id) || owner_id < 0) return null; + + return { + spec: { volume, owner_id: Math.trunc(owner_id) }, + path: rest.join("/").replace(/^\/+|\/+$/g, ""), + }; +} + +/** + * Where a path ref opens: the files browser, on that folder. A volume the + * browser doesn't know lands on its first folder. + */ +export function pathRefUrl({ volume, owner_id, path }: RefParams) { + const known = storageVolumeFromRef({ name: volume }); + + if (!known) return StorageBrowserPath; + + return storageSpecHref({ spec: { volume: known, owner_id: Number(owner_id) }, path }); +} diff --git a/src/utils/storage-volumes.test.ts b/src/utils/storage-volumes.test.ts new file mode 100644 index 0000000..5ff15a1 --- /dev/null +++ b/src/utils/storage-volumes.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, test } from "bun:test"; +import { + StorageBrowserPath, + isSameStorageSpec, + isStorageVolume, + parseStorageSpec, + storagePathName, + storageSpecHref, + storageVolumeFromRef, + storageVolumeLabel, + storageVolumeNeedsOwner, +} from "./storage-volumes"; + +describe("parseStorageSpec", () => { + test("reads a volume and its owner", () => { + expect(parseStorageSpec({ volume: "computers", owner_id: "3" })).toEqual({ + volume: "computers", + owner_id: 3, + }); + }); + + test("gives the database no owner", () => { + expect(parseStorageSpec({ volume: "db", owner_id: "999" })).toEqual({ + volume: "db", + owner_id: 0, + }); + }); + + test("refuses an unknown volume", () => { + expect(parseStorageSpec({ volume: "secrets", owner_id: 1 })).toBeNull(); + expect(parseStorageSpec({ volume: "", owner_id: 1 })).toBeNull(); + expect(parseStorageSpec({ volume: null, owner_id: 1 })).toBeNull(); + }); + + test("refuses a missing or bad owner", () => { + expect(parseStorageSpec({ volume: "bots" })).toBeNull(); + expect(parseStorageSpec({ volume: "bots", owner_id: 0 })).toBeNull(); + expect(parseStorageSpec({ volume: "bots", owner_id: -2 })).toBeNull(); + expect(parseStorageSpec({ volume: "bots", owner_id: "abc" })).toBeNull(); + expect(parseStorageSpec({ volume: "bots", owner_id: "../../etc" })).toBeNull(); + }); +}); + +describe("storageSpecHref", () => { + test("builds the browser's address", () => { + expect(storageSpecHref({ spec: { volume: "bots", owner_id: 3 } })).toBe( + `${StorageBrowserPath}?volume=bots&owner=3`, + ); + }); + + test("adds and encodes a path", () => { + expect( + storageSpecHref({ spec: { volume: "computers", owner_id: 1 }, path: "/seo/raw mes/" }), + ).toBe(`${StorageBrowserPath}?volume=computers&owner=1&path=seo%2Fraw+mes`); + }); +}); + +describe("volume names", () => { + test("maps a path ref's name to its volume", () => { + expect(storageVolumeFromRef({ name: "computer" })).toBe("computers"); + expect(storageVolumeFromRef({ name: "bot" })).toBe("bots"); + expect(storageVolumeFromRef({ name: "nope" })).toBeNull(); + expect(storageVolumeFromRef({ name: null })).toBeNull(); + }); + + test("knows which volumes need an owner", () => { + expect(storageVolumeNeedsOwner({ volume: "db" })).toBe(false); + expect(storageVolumeNeedsOwner({ volume: "media" })).toBe(true); + expect(isStorageVolume("mcp")).toBe(true); + expect(isStorageVolume("Media")).toBe(false); + }); + + test("labels a volume and a path", () => { + expect(storageVolumeLabel({ volume: "mcp" })).toBe("MCP servers"); + expect(storagePathName({ volume: "computers", path: "seo/coderank/raw" })).toBe("raw"); + expect(storagePathName({ volume: "computers", path: "" })).toBe("Computers"); + }); +}); + +describe("isSameStorageSpec", () => { + test("compares volume and owner", () => { + const a = { volume: "bots", owner_id: 1 } as const; + + expect(isSameStorageSpec({ a, b: { volume: "bots", owner_id: 1 } })).toBe(true); + expect(isSameStorageSpec({ a, b: { volume: "bots", owner_id: 2 } })).toBe(false); + expect(isSameStorageSpec({ a, b: { volume: "workspaces", owner_id: 1 } })).toBe(false); + expect(isSameStorageSpec({ a, b: null })).toBe(false); + }); +}); diff --git a/src/utils/storage-volumes.ts b/src/utils/storage-volumes.ts new file mode 100644 index 0000000..70de060 --- /dev/null +++ b/src/utils/storage-volumes.ts @@ -0,0 +1,165 @@ +/** + * The folders on the app's data volume (`DATA_DIR`) the files browser at + * `/admin/files` can open, one volume per kind of owner: a bot's own + * folder, a workspace's shared one, a computer's home, the media library, + * an MCP server's install folder, and the database. + * + * Client-safe: no `node:path` here, so a chip in a message can build the + * same link the page reads. + */ +export const StorageVolumeDefs = [ + { + value: "bots", + title: "Bots", + /** + * How a path ref names it (`[path:bot:3:reports]`). + */ + ref: "bot", + owner_kind: "bot", + }, + { + value: "workspaces", + title: "Workspaces", + ref: "workspace", + owner_kind: "workspace", + }, + { + value: "computers", + title: "Computers", + ref: "computer", + owner_kind: "computer", + }, + { + value: "media", + title: "Media", + ref: "media", + owner_kind: "user", + }, + { + value: "mcp", + title: "MCP servers", + ref: "mcp", + owner_kind: "mcp", + }, + { + value: "db", + title: "Database", + ref: "db", + owner_kind: "none", + }, +] as const; + +export const StorageVolumes = StorageVolumeDefs.map((def) => def.value); + +export type StorageVolume = (typeof StorageVolumeDefs)[number]["value"]; + +export type StorageOwnerKind = (typeof StorageVolumeDefs)[number]["owner_kind"]; + +/** + * One browsable folder: its volume and who owns it (a bot id, workspace + * id, computer id, user id or MCP server id; 0 for the database). + */ +export type StorageSpec = { + volume: StorageVolume; + owner_id: number; +}; + +/** + * Where the files browser lands when it has no folder asked for. + */ +export const StorageBrowserPath = "/admin/files"; + +/** + * A volume's definition, or undefined for an unknown one. + */ +export function storageVolumeDef({ volume }: { volume?: string | null }) { + return StorageVolumeDefs.find((def): def is (typeof StorageVolumeDefs)[number] => def.value == volume); +} + +/** + * Whether `volume` is one of `StorageVolumes`. + */ +export function isStorageVolume(volume?: string | null): volume is StorageVolume { + return Boolean(storageVolumeDef({ volume })); +} + +/** + * The volume a path ref names (`bot`, `computer`, ...), or null. + */ +export function storageVolumeFromRef({ name }: { name?: string | null }): StorageVolume | null { + const def = StorageVolumeDefs.find((it) => it.ref == name); + + return def ? def.value : null; +} + +/** + * Whether that volume's folder needs an owner id (everything but the + * database). + */ +export function storageVolumeNeedsOwner({ volume }: { volume: StorageVolume }) { + return storageVolumeDef({ volume })?.owner_kind != "none"; +} + +/** + * A volume's title, for the picker and a chip's tooltip. + */ +export function storageVolumeLabel({ volume }: { volume: StorageVolume }) { + return storageVolumeDef({ volume })?.title || volume; +} + +/** + * Reads a volume and owner from an untrusted source (a query string, a + * ref, a request body): null when either is wrong, so a bad id never + * becomes a path. + */ +export function parseStorageSpec({ + volume, + owner_id, +}: { + volume?: string | null; + owner_id?: number | string | null; +}): StorageSpec | null { + if (!isStorageVolume(volume)) return null; + + const wanted = storageVolumeNeedsOwner({ volume }); + const owner = Number(owner_id); + + if (!wanted) return { volume, owner_id: 0 }; + + if (!Number.isFinite(owner) || owner <= 0) return null; + + return { volume, owner_id: Math.trunc(owner) }; +} + +/** + * Whether two folders are the same one. + */ +export function isSameStorageSpec({ a, b }: { a?: StorageSpec | null; b?: StorageSpec | null }) { + return Boolean(a && b && a.volume == b.volume && a.owner_id == b.owner_id); +} + +/** + * The files browser's address for a folder and, optionally, a path inside + * it. What a chip links to, and what the page's breadcrumb builds. + */ +export function storageSpecHref({ spec, path }: { spec: StorageSpec; path?: string | null }) { + const query = new URLSearchParams({ + volume: spec.volume, + owner: String(spec.owner_id), + }); + const clean = (path || "").trim().replace(/^\/+|\/+$/g, ""); + + if (clean) query.set("path", clean); + + return `${StorageBrowserPath}?${query.toString()}`; +} + +/** + * A folder's label for a chip: the last path segment, or the volume's + * title at the top. + */ +export function storagePathName({ volume, path }: { volume: StorageVolume; path?: string | null }) { + const segments = (path || "").split("/").filter(Boolean); + + return segments.at(-1) || storageVolumeLabel({ volume }); +}