First Commit. Duplicate bunext-sqlite preset

src/functions/backend/db/users/media/file-to-text.ts

@@ -0,0 +1,84 @@
+import TurndownService from "turndown";
+import { extractText, getDocumentProxy } from "unpdf";
+import mammoth from "mammoth";
+// import * as XLSX from "xlsx";
+import bufferFromUrl from "@/src/utils/buffer-from-url";
+
+const turndownService = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+});
+
+type Params = {
+    file_stream?: Buffer<ArrayBuffer>;
+    file_path?: string;
+    /**
+     * File extension, e.g. `pdf`, `docx`, `xlsx`
+     */
+    file_type?: string;
+    url?: string;
+};
+
+export default async function fileToText({
+    file_type,
+    file_stream,
+    file_path,
+    url,
+}: Params): Promise<string | undefined> {
+    let buffer: Buffer | undefined;
+    let ext: string;
+
+    if (url) {
+        const result = await bufferFromUrl(url);
+        buffer = result.buffer;
+        ext = result.ext;
+    } else {
+        buffer = file_path
+            ? Buffer.from(await Bun.file(file_path).arrayBuffer())
+            : file_stream;
+        ext = (file_type || file_path?.split(".").pop() || "txt")
+            .toString()
+            .toLowerCase();
+    }
+
+    if (!buffer) return undefined;
+
+    switch (ext) {
+        case "docx": {
+            const { value: html } = await mammoth.convertToHtml({ buffer });
+            return turndownService.turndown(html);
+        }
+
+        case "pdf": {
+            try {
+                const pdfUintBuffer = new Uint8Array(buffer);
+                const pdfDocProxy = await getDocumentProxy(pdfUintBuffer);
+                const { text } = await extractText(pdfDocProxy, {
+                    mergePages: true,
+                });
+                return text;
+            } catch (err: any) {
+                console.error(
+                    `PDF text extraction failed: ${err?.message ?? err}`,
+                );
+                throw new Error(
+                    `Failed to extract text from PDF: ${err?.message ?? "unknown error"}`,
+                );
+            }
+        }
+
+        // case "xlsx":
+        // case "csv": {
+        //     const workbook = XLSX.read(buffer, { type: "buffer" });
+        //     return workbook.SheetNames.map((name) => {
+        //         const sheet = workbook.Sheets[name];
+        //         return `### Sheet: ${name}\n\n${XLSX.utils.sheet_to_csv(sheet!)}`;
+        //     }).join("\n\n");
+        // }
+
+        case "md":
+        case "txt":
+        default:
+            return buffer.toString("utf-8");
+    }
+}

src/functions/backend/db/users/media/media-data-to-array-buffer.ts

@@ -0,0 +1,27 @@
+import type { MediaDataType } from "@/src/types";
+
+export default async function MediaDataToArrayBuffer(
+    body: MediaDataType,
+): Promise<ArrayBuffer> {
+    if (body == null) {
+        return new ArrayBuffer(0);
+    }
+    if (body instanceof ArrayBuffer) {
+        return body;
+    }
+    if (body instanceof SharedArrayBuffer) {
+        const copy = new ArrayBuffer(body.byteLength);
+        new Uint8Array(copy).set(new Uint8Array(body));
+        return copy;
+    }
+    if (typeof body === "string") {
+        return new TextEncoder().encode(body).buffer as ArrayBuffer;
+    }
+    if (body instanceof ReadableStream) {
+        return Bun.readableStreamToArrayBuffer(body);
+    }
+    // Blob, BunFile, Request, Response — wrap in Response and use Bun's reader
+    return new Response(body as BodyInit)
+        .bytes()
+        .then((b) => b.buffer as ArrayBuffer);
+}

src/functions/backend/db/users/media/meida-data-to-sharp-input.ts

@@ -0,0 +1,24 @@
+import type { MediaDataType } from "@/src/types";
+
+export default async function mediaDataToSharpInput(
+    body: MediaDataType,
+): Promise<Buffer> {
+    if (body instanceof Buffer) return body;
+    if (body instanceof SharedArrayBuffer)
+        return Buffer.from(new Uint8Array(body));
+    if (body instanceof ArrayBuffer) return Buffer.from(body);
+    if (body instanceof Uint8Array) return Buffer.from(body);
+    if (body instanceof Blob || body instanceof File)
+        return Buffer.from(await body.arrayBuffer());
+    if (body instanceof Response || body instanceof Request)
+        return Buffer.from(await body.arrayBuffer());
+    if (typeof body === "string") return Buffer.from(body, "base64");
+    if (ArrayBuffer.isView(body))
+        return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
+
+    if ("arrayBuffer" in body && typeof body.arrayBuffer === "function") {
+        return Buffer.from(await (body as any).arrayBuffer());
+    }
+
+    throw new Error(`Unsupported body type: ${typeof body}`);
+}

src/functions/backend/db/users/media/sniff-media-ext.ts

@@ -0,0 +1,50 @@
+import type { MediaDataType } from "@/src/types";
+import MediaDataToArrayBuffer from "./media-data-to-array-buffer";
+
+export default async function sniffMediaExtension(
+    data: MediaDataType,
+): Promise<string | undefined> {
+    try {
+        const bytes = new Uint8Array(await MediaDataToArrayBuffer(data));
+
+        if (bytes.length < 12) return undefined;
+
+        const has = (...signature: number[]) =>
+            signature.every((byte, i) => bytes[i] === byte);
+
+        if (has(0xff, 0xd8, 0xff)) return "jpg";
+        if (has(0x89, 0x50, 0x4e, 0x47)) return "png";
+        if (has(0x47, 0x49, 0x46, 0x38)) return "gif";
+        if (has(0x25, 0x50, 0x44, 0x46)) return "pdf";
+        if (has(0x50, 0x4b, 0x03, 0x04)) return "zip";
+        if (has(0x49, 0x44, 0x33)) return "mp3";
+        if (has(0x4f, 0x67, 0x67, 0x53)) return "ogg";
+
+        if (has(0x52, 0x49, 0x46, 0x46)) {
+            const riff_type = String.fromCharCode(
+                bytes[8]!,
+                bytes[9]!,
+                bytes[10]!,
+                bytes[11]!,
+            );
+
+            if (riff_type === "WEBP") return "webp";
+            if (riff_type === "WAVE") return "wav";
+        }
+
+        if (
+            String.fromCharCode(bytes[4]!, bytes[5]!, bytes[6]!, bytes[7]!) ===
+            "ftyp"
+        ) {
+            return "mp4";
+        }
+
+        const head = String.fromCharCode(...bytes.subarray(0, 8)).trimStart();
+
+        if (head.startsWith("<svg") || head.startsWith("<?xml")) return "svg";
+
+        return undefined;
+    } catch (error) {
+        return undefined;
+    }
+}

src/functions/backend/db/users/media/upload-and-record-media.ts

@@ -0,0 +1,300 @@
+import type {
+    BUN_SQLITE_BSQLITE_MEDIA,
+    BUN_SQLITE_BSQLITE_MEDIA_PARADIGMS,
+    BUN_SQLITE_BSQLITE_USERS,
+    BunSQLiteTables,
+} from "@/db/types/db";
+import { AppData } from "@/src/data/app-data";
+import type {
+    MediaDataType,
+    MediaParadigm,
+    MediaType,
+    TableType,
+    User,
+} from "@/src/types";
+import grabDirNames from "@/src/utils/grab-dir-names";
+import BunSQLite, { type APIResponseObject } from "@moduletrace/bun-sqlite";
+import { existsSync, mkdirSync } from "fs";
+import path from "path";
+import sharp from "sharp";
+import fileToText from "./file-to-text";
+import MediaDataToArrayBuffer from "./media-data-to-array-buffer";
+import sniffMediaExtension from "./sniff-media-ext";
+import mediaDataToSharpInput from "./meida-data-to-sharp-input";
+import deleteMedia from "../../media/delete-media";
+
+type Params = {
+    media_paradigms?: MediaParadigm[];
+    media_type?: MediaType;
+    user: BUN_SQLITE_BSQLITE_USERS | User;
+    data?: MediaDataType;
+    is_primary?: boolean;
+    update_on_duplicate?: boolean;
+    extract_media_text_content?: boolean;
+    media?: BUN_SQLITE_BSQLITE_MEDIA;
+    media_name?: string;
+    media_mime_type?: string;
+    is_private?: boolean;
+    image_width?: number;
+    image_height?: number;
+    thumbnail_width?: number;
+    existing_media_id?: string | number;
+    media_text_content?: string;
+};
+
+export default async function uploadAndRecordMedia({
+    media_paradigms,
+    user,
+    data,
+    media_type,
+    is_primary,
+    update_on_duplicate,
+    extract_media_text_content,
+    media,
+    media_mime_type,
+    media_name,
+    is_private,
+    image_width,
+    image_height,
+    thumbnail_width,
+    existing_media_id,
+    media_text_content: passed_media_text_content,
+}: Params): Promise<APIResponseObject<BUN_SQLITE_BSQLITE_MEDIA>> {
+    const now = Date.now();
+    let target_file_name = `${media_name || now}`;
+    let target_file_thumbnail_name = `${media_name || now}-thumb`;
+
+    const {
+        USER_MEDIA_PRIVATE_RELATIVE_DIR,
+        USER_MEDIA_PUBLIC_RELATIVE_DIR,
+        DATA_DIR,
+    } = grabDirNames({
+        user,
+    });
+
+    const MEDIA_RELATIVE_DIR = is_private
+        ? USER_MEDIA_PRIVATE_RELATIVE_DIR
+        : USER_MEDIA_PUBLIC_RELATIVE_DIR;
+
+    if (!MEDIA_RELATIVE_DIR) {
+        throw new Error(`No target media directory found!`);
+    }
+
+    let media_written_to_disk = false;
+    let media_text_content = passed_media_text_content;
+
+    let media_ext = media_mime_type
+        ? media_mime_type.replace(/\./g, "")
+        : undefined;
+
+    if (!media_ext && data) {
+        if (typeof data === "string" && data.startsWith("data:")) {
+            const base64_mime = data.match(/^data:([^;]+);base64,/)?.[1];
+
+            if (base64_mime) {
+                media_ext = base64_mime.split("/").pop() ?? undefined;
+            }
+        } else if (data instanceof File && data.name.includes(".")) {
+            media_ext = data.name.split(".").pop();
+        } else if (
+            data instanceof Blob &&
+            data.type &&
+            !data.type.includes("octet-stream")
+        ) {
+            media_ext = data.type.split("/").pop();
+        } else if (media_type === "image") {
+            media_ext = "jpeg";
+        } else if (typeof data !== "string") {
+            media_ext = await sniffMediaExtension(data);
+        }
+    }
+
+    const media_write_path = path.join(
+        MEDIA_RELATIVE_DIR,
+        `${target_file_name}.${media_ext}`,
+    );
+    const media_thumbnail_write_path = path.join(
+        MEDIA_RELATIVE_DIR,
+        `${target_file_thumbnail_name}.${media_ext}`,
+    );
+
+    const media_write_path_full = path.join(DATA_DIR, media_write_path);
+    const media_thumbnail_write_path_full = path.join(
+        DATA_DIR,
+        media_thumbnail_write_path,
+    );
+
+    mkdirSync(path.dirname(media_write_path_full), { recursive: true });
+
+    if (data && media_type == "image" && media_ext) {
+        const buffer = await mediaDataToSharpInput(data);
+        const is_profile_image =
+            media_paradigms?.includes("user-profile-image");
+        const thumb_width = thumbnail_width ?? AppData["ImageThumbnailSize"];
+
+        let full_width: number | undefined;
+        let full_height: number | undefined;
+        let full_options: {
+            fit?: "cover" | "inside";
+            withoutEnlargement?: boolean;
+        } = {};
+
+        if (is_profile_image) {
+            full_width = AppData["MainImageSize"];
+            full_height = AppData["MainImageSize"];
+            full_options = { fit: "cover" };
+        } else {
+            const meta = await sharp(buffer).metadata();
+            const source_width = image_width ?? meta.width;
+            const source_height = image_height ?? meta.height;
+            const max = AppData["MaxImageWidth"];
+
+            full_width =
+                source_width != null
+                    ? source_width > max
+                        ? max
+                        : source_width
+                    : max;
+            full_height =
+                image_height != null
+                    ? source_height != null && source_height > max
+                        ? max
+                        : source_height
+                    : undefined;
+            full_options = { fit: "inside", withoutEnlargement: true };
+        }
+
+        const [full, thumb] = await Promise.all([
+            sharp(buffer)
+                .resize(full_width, full_height, full_options)
+                .jpeg({ quality: AppData["ImageQuality"] })
+                .toBuffer(),
+            sharp(buffer)
+                .resize(
+                    thumb_width,
+                    is_profile_image ? thumb_width : undefined,
+                    is_profile_image
+                        ? { fit: "cover" }
+                        : { fit: "inside", withoutEnlargement: true },
+                )
+                .jpeg({ quality: AppData["ImageThumbnailQuality"] })
+                .toBuffer(),
+        ]);
+
+        await Bun.write(media_write_path_full, full);
+        await Bun.write(media_thumbnail_write_path_full, thumb);
+    } else if (data) {
+        const media_array_buffer = await MediaDataToArrayBuffer(data);
+        await Bun.write(media_write_path_full, media_array_buffer, {
+            mode: 1,
+        });
+
+        if (extract_media_text_content) {
+            const media_text = await fileToText({
+                file_type: media_ext,
+                file_stream: Buffer.from(media_array_buffer),
+            });
+
+            if (media_text) {
+                media_text_content = media_text;
+            }
+        }
+    }
+
+    if (existsSync(media_write_path)) {
+        media_written_to_disk = true;
+    }
+
+    if (is_primary) {
+        const existing_user_primary_media = await BunSQLite.select<
+            BUN_SQLITE_BSQLITE_MEDIA,
+            TableType
+        >({
+            table: "media",
+            query: {
+                query: {
+                    user_id: { value: user.id },
+                    is_primary: { value: 1 },
+                },
+            },
+        });
+
+        if (existing_user_primary_media.payload) {
+            await deleteMedia({
+                can_delete_all_media: true,
+                ids: existing_user_primary_media.payload.map((m) => m.id!),
+            });
+        }
+    }
+
+    const media_data: BUN_SQLITE_BSQLITE_MEDIA = {
+        ...media,
+        user_id: Number(user.id),
+        is_primary: is_primary ? 1 : 0,
+        media_write_path: media_write_path,
+        media_thumbnail_write_path:
+            media_type === "image" ? media_thumbnail_write_path : undefined,
+        media_type: media_type as any,
+        mime_type: media_type == "image" ? "jpeg" : media_ext,
+        text_content: media_text_content,
+    };
+
+    let media_record: BUN_SQLITE_BSQLITE_MEDIA | undefined = undefined;
+
+    if (existing_media_id) {
+        const update_media_record = await BunSQLite.update<
+            BUN_SQLITE_BSQLITE_MEDIA,
+            TableType
+        >({
+            data: media_data,
+            table: "media",
+            targetId: existing_media_id,
+        });
+
+        media_record = update_media_record.singleRes || undefined;
+    } else {
+        const new_media_record = await BunSQLite.insert<
+            BUN_SQLITE_BSQLITE_MEDIA,
+            TableType
+        >({
+            data: [media_data],
+            table: "media",
+            update_on_duplicate,
+        });
+
+        media_record = new_media_record.singleRes || undefined;
+    }
+
+    if (!media_record) {
+        throw new Error(`Couldn't record media!`);
+    }
+
+    if (media_paradigms?.[0] && media_record?.id) {
+        const delete_existing_paradigms = await BunSQLite.delete<
+            BUN_SQLITE_BSQLITE_MEDIA_PARADIGMS,
+            (typeof BunSQLiteTables)[number]
+        >({
+            table: "media_paradigms",
+            query: {
+                query: { media_id: { value: media_record.id } },
+            },
+        });
+
+        await BunSQLite.insert<
+            BUN_SQLITE_BSQLITE_MEDIA_PARADIGMS,
+            (typeof BunSQLiteTables)[number]
+        >({
+            data: media_paradigms.map((mp) => ({
+                media_id: media_record.id,
+                media_paradigm: mp,
+            })),
+            table: "media_paradigms",
+            update_on_duplicate,
+        });
+    }
+
+    return {
+        success: true,
+        singleRes: media_record,
+    };
+}