Add bots files handling logic

2026-10-21 17:56:09 +01:00
parent b0af4db1bf
commit d17a4a7bdd
97 changed files with 5843 additions and 41 deletions
@@ -0,0 +1,94 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file";
+import deleteBotFile from "@/src/functions/backend/db/bot-files/delete-bot-file";
+import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot";
+import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { DeleteBotFileRequestSchema } from "@/src/schemas/bot-file-schema";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import canManageCrew from "@/src/utils/rbac/can-manage-crew";
+import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { StorageScope } from "@/src/utils/storage-access";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * Deletes one of a conversation's files. A crew manager only, and never a
+ * read-only user.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") throw new AppError(`Method not allowed`);
+
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.id) return { success: false, msg: `Unauthorized`, logoutUser: true };
+        if (isReadOnlyUser({ user_types: user_types || [] })) {
+            throw new AppError(`This user can only read records.`);
+        }
+        if (!canManageCrew({ user_types: user_types || [] })) {
+            throw new AppError(`Only a crew manager can delete files.`);
+        }
+
+        const { file_id } = parseWithSchema({ schema: DeleteBotFileRequestSchema, data: body });
+
+        const file = await grabBotFile({
+            scopes: await reachableScopes({ user_id: user.id, file_id }),
+            file_id,
+        });
+
+        if (!file?.id) throw new AppError(`That file isn't here.`);
+
+        await deleteBotFile({
+            scopes: [{ scope: file.scope as any, owner_id: Number(file.owner_id) }],
+            file_id: file.id,
+        });
+
+        await logEvent({
+            title: "Bot file deleted",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} deleted ${file.path} from ${file.scope} ${file.owner_id}.`,
+            metadata: { file_id: file.id },
+            ip_address: grabRequestIP(req),
+        });
+
+        return { success: true, msg: `${file.name || "The file"} is deleted.` };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};
+
+/**
+ * The directories the user reaches: every worker's for a crew manager,
+ * the coordinator's for the super admin, and every workspace it's in.
+ */
+async function reachableScopes({
+    user_id,
+    file_id,
+}: {
+    user_id: number;
+    file_id: number;
+}): Promise<StorageScope[]> {
+    const { default: BunSQLite } = await import("@moduletrace/bun-sqlite");
+
+    const res = await BunSQLite.sql<{ scope: string; owner_id: number }>({
+        sql: `SELECT scope, owner_id FROM bot_files WHERE id = ? LIMIT 1`,
+        values: [file_id],
+    });
+
+    const file = res.payload?.[0];
+
+    if (!file?.owner_id) return [];
+
+    if (file.scope == "workspace") {
+        const workspace = await grabUserWorkspace({ workspace_id: file.owner_id, user_id });
+
+        return workspace?.id ? [{ scope: "workspace", owner_id: Number(file.owner_id) }] : [];
+    }
+
+    const bot = await grabUserBot({ bot_id: file.owner_id, user_id });
+
+    return bot?.id ? [{ scope: "bot", owner_id: Number(file.owner_id) }] : [];
+}
@@ -0,0 +1,131 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file";
+import grabBotFileQuotaBytes, {
+    grabBotFileQuotaMb,
+} from "@/src/functions/backend/db/bot-files/grab-bot-file-quota-bytes";
+import grabStorageUsage from "@/src/functions/backend/db/bot-files/grab-storage-usage";
+import listBotFiles, { listBotFolders } from "@/src/functions/backend/db/bot-files/list-bot-files";
+import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot";
+import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace";
+import grabMediaURL from "@/src/functions/backend/media/grab-media-url";
+import type { BotFileScopesPayload } from "@/src/types";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { BotFileScope } from "@/src/dict/bot-file-dict";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * A conversation's files: the bot's own, or a workspace's shared ones.
+ * `?file_id=` returns one (for a message's file chip), else a folder's
+ * files. Only a user who reaches the bot or workspace sees them.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject<BotFileScopesPayload>> = async ({
+    req,
+}) => {
+    try {
+        if (req.method != "GET") {
+            return { success: false, msg: `Method not allowed` };
+        }
+
+        const { user } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        const url = new URL(req.url);
+        const bot_id = Number(url.searchParams.get("bot_id"));
+        const workspace_id = Number(url.searchParams.get("workspace_id"));
+        const file_id = Number(url.searchParams.get("file_id"));
+        const folder = url.searchParams.get("folder") || "";
+        const search = url.searchParams.get("search") || "";
+
+        const scope: { scope: BotFileScope; owner_id: number } | null = workspace_id
+            ? (await grabUserWorkspace({ workspace_id, user_id: user.id }))
+                ? { scope: "workspace", owner_id: workspace_id }
+                : null
+            : bot_id
+              ? (await grabUserBot({ bot_id, user_id: user.id }))
+                  ? { scope: "bot", owner_id: bot_id }
+                  : null
+              : null;
+
+        if (!scope) {
+            return { success: false, msg: `Not found.` };
+        }
+
+        const scopes = [scope];
+
+        if (file_id) {
+            const file = await grabBotFile({ scopes, file_id });
+
+            if (!file?.id) return { success: false, msg: `Not found.` };
+
+            return {
+                success: true,
+                singleRes: {
+                    files: [toPublicFile({ file })],
+                    folders: [],
+                    usage: { bytes: 0, files: 0 },
+                    quota_bytes: 0,
+                    quota_mb: 0,
+                },
+            };
+        }
+
+        const [files, folders, usage, quota_bytes, quota_mb] = await Promise.all([
+            listBotFiles({ scopes, folder, search }),
+            listBotFolders({ scopes }),
+            grabStorageUsage(scope),
+            grabBotFileQuotaBytes(),
+            grabBotFileQuotaMb(),
+        ]);
+
+        return {
+            success: true,
+            singleRes: {
+                files: files.map((file) => toPublicFile({ file })),
+                folders,
+                usage,
+                quota_bytes,
+                quota_mb,
+            },
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};
+
+function toPublicFile({ file }: { file: BotFileWithMediaLite }) {
+    const { media_url } = grabMediaURL({ media: { ...(file.media || {}), id: file.media_id } as any });
+
+    return {
+        id: Number(file.id),
+        name: file.name || "",
+        path: file.path || "",
+        size: Number(file.size) || 0,
+        media_type: file.media_type || "document",
+        purpose: file.purpose || "writing",
+        scope: file.scope || "bot",
+        owner_id: Number(file.owner_id) || 0,
+        ingested_at: file.ingested_at ? Number(file.ingested_at) : undefined,
+        ingest_error: file.ingest_error || undefined,
+        created_at: Number(file.created_at) || 0,
+        url: media_url || "",
+    };
+}
+
+type BotFileWithMediaLite = {
+    id?: number | "";
+    media_id?: number | "";
+    name?: string;
+    path?: string;
+    size?: number | "";
+    media_type?: string;
+    purpose?: string;
+    scope?: string;
+    owner_id?: number | "";
+    ingested_at?: number | "";
+    ingest_error?: string;
+    created_at?: number | "";
+    media?: Record<string, unknown> | null;
+};
@@ -0,0 +1,82 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file";
+import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot";
+import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace";
+import { IngestBotFileRequestSchema } from "@/src/schemas/bot-file-schema";
+import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import canManageCrew from "@/src/utils/rbac/can-manage-crew";
+import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { StorageScope } from "@/src/utils/storage-access";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * Queues a conversation's file to be read again and re-embedded, for when
+ * a search misses something the user knows is in it.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") throw new AppError(`Method not allowed`);
+
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.id) return { success: false, msg: `Unauthorized`, logoutUser: true };
+        if (isReadOnlyUser({ user_types: user_types || [] })) {
+            throw new AppError(`This user can only read records.`);
+        }
+        if (!canManageCrew({ user_types: user_types || [] })) {
+            throw new AppError(`Only a crew manager can re-read files.`);
+        }
+
+        const { file_id, read_media } = parseWithSchema({
+            schema: IngestBotFileRequestSchema,
+            data: body,
+        });
+
+        const scopes = await reachableScopes({ user_id: user.id, file_id });
+
+        const file = await grabBotFile({ scopes, file_id });
+
+        if (!file?.id) throw new AppError(`That file isn't here.`);
+
+        await enqueueFileIngest({ file_id: file.id });
+
+        return { success: true, msg: `${file.name || "The file"} will be read again shortly.` };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};
+
+/**
+ * The directories the user reaches for this file.
+ */
+async function reachableScopes({
+    user_id,
+    file_id,
+}: {
+    user_id: number;
+    file_id: number;
+}): Promise<StorageScope[]> {
+    const { default: BunSQLite } = await import("@moduletrace/bun-sqlite");
+
+    const res = await BunSQLite.sql<{ scope: string; owner_id: number }>({
+        sql: `SELECT scope, owner_id FROM bot_files WHERE id = ? LIMIT 1`,
+        values: [file_id],
+    });
+
+    const file = res.payload?.[0];
+
+    if (!file?.owner_id) return [];
+
+    if (file.scope == "workspace") {
+        const workspace = await grabUserWorkspace({ workspace_id: file.owner_id, user_id });
+
+        return workspace?.id ? [{ scope: "workspace", owner_id: Number(file.owner_id) }] : [];
+    }
+
+    const bot = await grabUserBot({ bot_id: file.owner_id, user_id });
+
+    return bot?.id ? [{ scope: "bot", owner_id: Number(file.owner_id) }] : [];
+}
@@ -4,6 +4,7 @@ import ensureBotSession from "@/src/functions/backend/db/bot-sessions/ensure-bot
 import insertUserMessage from "@/src/functions/backend/db/bots/insert-user-message";
 import grabUserBotMessage from "@/src/functions/backend/db/bots/grab-user-bot-message";
 import grabAttachableMedia from "@/src/functions/backend/db/bots/grab-attachable-media";
+import storeChatAttachments from "@/src/functions/backend/db/bot-files/store-chat-attachments";
 import enqueueBotReply from "@/src/functions/backend/db/bots-queue/enqueue-bot-reply";
 import pushBotNotice from "@/src/bots-queue/push-bot-notice";
 import wakeBotsQueue from "@/src/bots-queue/wake-bots-queue";
@@ -93,6 +94,17 @@ export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
            media,
        });

+        /**
+         * The files join the conversation's folder, so a later run can
+         * search what the user sent now.
+         */
+        await storeChatAttachments({
+            bot_id: bot.id,
+            workspace_id: conversation.workspace_id,
+            user_id: user.id,
+            media,
+        });
+
        await enqueueBotReply({
            user_id: user.id,
            bot_id: bot.id,
@@ -0,0 +1,52 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import grabStorageQuotaOverview, {
+    type StorageQuotaOverview,
+} from "@/src/functions/backend/db/app-settings/grab-storage-quota-overview";
+import saveAppSetting from "@/src/functions/backend/db/app-settings/save-app-setting";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { BotFileQuotaSettingKey } from "@/src/dict/bot-file-dict";
+import { SaveStorageQuotaRequestSchema } from "@/src/schemas/bot-file-schema";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: the per-folder quota for what bots save, in MB. Zero
+ * puts the code's default back.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject<StorageQuotaOverview>> = async ({
+    req,
+    body,
+}) => {
+    try {
+        if (req.method != "POST") throw new AppError(`Method not allowed`);
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) throw new AppError(`Only a super admin can change the storage quota.`);
+
+        const { quota_mb } = parseWithSchema({
+            schema: SaveStorageQuotaRequestSchema,
+            data: body,
+        });
+
+        await saveAppSetting({
+            key: BotFileQuotaSettingKey,
+            value: quota_mb > 0 ? String(quota_mb) : null,
+        });
+
+        await logEvent({
+            title: "Bot file quota changed",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} set the bot file quota to ${quota_mb} MB per folder.`,
+            metadata: { quota_mb },
+            ip_address: grabRequestIP(req),
+        });
+
+        return { success: true, singleRes: await grabStorageQuotaOverview() };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};