Stop storing raw file bytes in media.text_content

`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:

- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
  and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
  `attachmentFileKind` classes everything non-image/audio/video as a
  document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
  text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
  `SELECT m.*` pulled the column into page props and API responses.

On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.

Changes:

- `file-to-text.ts` lists the text extensions explicitly and returns
  undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
  `BotTextFileExtensions`), used by both upload routes and by
  `uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
  catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
  `cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
  re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
  now has to be asked for (`grab-messages-media`'s reply path and
  ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
  an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
  (rerunnable, with a VACUUM hint).

Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.

src/utils/can-extract-text.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, test } from "bun:test";
+import canExtractText from "./can-extract-text";
+
+describe("canExtractText", () => {
+    test("accepts the types the extractor really unpacks", () => {
+        for (const name of ["a.pdf", "b.docx", "c.txt", "d.md", "e.csv", "f.tsv", "g.json", "h.html", "i.xml", "j.yml", "k.yaml", "l.log", "m.markdown"]) {
+            expect(canExtractText({ name })).toBe(true);
+        }
+    });
+
+    test("refuses a binary that only has bytes, not text", () => {
+        for (const name of ["icon.svg", "bundle.zip", "sheet.xlsx", "slides.pptx", "app.bin", "noext", "archive.tar.gz"]) {
+            expect(canExtractText({ name })).toBe(false);
+        }
+    });
+
+    test("reads the type when the name has no extension", () => {
+        expect(canExtractText({ name: "noext", mime_type: "application/pdf" })).toBe(true);
+        expect(canExtractText({ name: "noext", mime_type: "image/svg+xml" })).toBe(false);
+    });
+
+    test("an octet-stream says nothing, so the name decides", () => {
+        expect(canExtractText({ name: "notes.md", mime_type: "application/octet-stream" })).toBe(true);
+        expect(canExtractText({ name: "logo.svg", mime_type: "application/octet-stream" })).toBe(false);
+    });
+});

src/utils/can-extract-text.ts

@@ -0,0 +1,30 @@
+import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
+import extensionOf from "./extension-of";
+
+type Params = {
+    /**
+     * The file's name, e.g. `invoice.pdf`.
+     */
+    name?: string | null;
+    /**
+     * Its MIME type, for a file with no extension in its name.
+     */
+    mime_type?: string | null;
+};
+
+/**
+ * Whether a file's text is worth extracting and keeping on its `media` row.
+ *
+ * Only the extensions the local extractor really unpacks count
+ * (`BotTextFileExtensions`). Everything else - a zip, an SVG, a
+ * spreadsheet, a binary with an unknown name - has no text to store, and
+ * decoding its bytes anyway is how raw file content ended up in the
+ * database.
+ */
+export default function canExtractText({ name, mime_type }: Params): boolean {
+    const ext = extensionOf({ name, mime_type });
+
+    if (!ext) return false;
+
+    return (BotTextFileExtensions as readonly string[]).includes(ext);
+}

src/utils/cap-text-content.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import capTextContent from "./cap-text-content";
+
+describe("capTextContent", () => {
+    test("keeps text under the ceiling as it is", () => {
+        expect(capTextContent({ text: "short" })).toBe("short");
+    });
+
+    test("keeps exactly the ceiling", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars);
+        expect(capTextContent({ text })).toBe(text);
+    });
+
+    test("cuts it and says so", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 500);
+        const capped = capTextContent({ text });
+
+        expect(capped.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+        expect(capped.startsWith("a")).toBe(true);
+        expect(capped).toContain("Truncated");
+        expect(capped).toContain(String(text.length));
+    });
+});

src/utils/cap-text-content.ts

@@ -0,0 +1,31 @@
+import { AppData } from "@/src/data/app-data";
+
+type Params = {
+    /**
+     * The text extracted from a file.
+     */
+    text: string;
+};
+
+/**
+ * Extracted document text, cut to what the `media` row keeps
+ * (`AppData.BotAttachmentTextStoreMaxChars`) with a line saying so.
+ *
+ * The column is a cache, not a copy of the file: a 20 MB document would
+ * otherwise put 20 MB of text into the database, and no reader takes more
+ * than a slice of it anyway.
+ */
+export default function capTextContent({ text }: Params) {
+    const max = AppData.BotAttachmentTextStoreMaxChars;
+
+    if (text.length <= max) return text;
+
+    const note = `\n\n[Truncated: this document is ${text.length} characters long, so only its beginning was kept. Read the file itself for the rest.]`;
+    /**
+     * The note counts against the ceiling too: the column is capped, not
+     * the text plus whatever was appended to it.
+     */
+    const kept = Math.max(0, max - note.length);
+
+    return `${text.slice(0, kept)}${note}`;
+}

src/utils/decide-text-content.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import decideTextContent from "./decide-text-content";
+
+describe("decideTextContent", () => {
+    test("keeps the text of a document the extractor reads", () => {
+        expect(decideTextContent({ name: "invoice.pdf", text: "Total: 12" })).toBe("Total: 12");
+        expect(decideTextContent({ name: "notes.md", text: "# Hi" })).toBe("# Hi");
+        expect(decideTextContent({ name: "rows.csv", text: "a,b" })).toBe("a,b");
+    });
+
+    test("drops the raw bytes of anything else", () => {
+        expect(decideTextContent({ name: "icon.svg", text: "<svg>...</svg>" })).toBeNull();
+        expect(decideTextContent({ name: "bundle.zip", text: "PK\u0003\u0004binary" })).toBeNull();
+        expect(decideTextContent({ name: "sheet.xlsx", text: "PK\u0003\u0004binary" })).toBeNull();
+        expect(decideTextContent({ name: "app.bin", text: "\u0000\u0001\u0002" })).toBeNull();
+    });
+
+    test("nothing to store is null, not an empty string", () => {
+        expect(decideTextContent({ name: "invoice.pdf", text: "" })).toBeNull();
+        expect(decideTextContent({ name: "invoice.pdf", text: "   \n  " })).toBeNull();
+        expect(decideTextContent({ name: "invoice.pdf" })).toBeNull();
+    });
+
+    test("cuts a document over the ceiling", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 10);
+        const decided = decideTextContent({ name: "long.txt", text });
+
+        expect(decided).toContain("Truncated");
+        expect(decided!.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+    });
+
+    test("a document over the ceiling is still cut when the overflow is small", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 10);
+        const decided = decideTextContent({ name: "long.txt", text })!;
+
+        expect(decided).toContain("Truncated");
+        expect(decided.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+    });
+});

src/utils/decide-text-content.ts

@@ -0,0 +1,36 @@
+import canExtractText from "./can-extract-text";
+import capTextContent from "./cap-text-content";
+
+type Params = {
+    /**
+     * The file's name, when the row kept it (`media.title`).
+     */
+    name?: string | null;
+    /**
+     * Its MIME type (`media.mime_type`). For old rows this is often just
+     * the extension.
+     */
+    mime_type?: string | null;
+};
+
+/**
+ * What a `media` row's `text_content` should be: the text itself when it
+ * belongs there, `null` when the row should not carry any.
+ *
+ * A row qualifies when its file is one the extractor really unpacks
+ * (`canExtractText`); its text is then cut to the store ceiling
+ * (`capTextContent`). Anything else - a zip, an SVG, a spreadsheet, a
+ * binary that only ever had its bytes decoded - loses its `text_content`,
+ * because that column is an extracted-text cache, not a copy of the file.
+ */
+export default function decideTextContent({
+    name,
+    mime_type,
+    text,
+}: Params & { text?: string | null }): string | null {
+    if (!text || !text.trim()) return null;
+
+    if (!canExtractText({ name, mime_type })) return null;
+
+    return capTextContent({ text });
+}

src/utils/extension-of.ts

@@ -0,0 +1,28 @@
+type Params = {
+    /**
+     * A file name, e.g. `report.pdf`.
+     */
+    name?: string | null;
+    /**
+     * Its MIME type, used when the name has no extension.
+     */
+    mime_type?: string | null;
+};
+
+/**
+ * A file's extension, lower case and without the dot: from its name, else
+ * from its MIME type. Anything but letters and digits is stripped, so a
+ * hostile name can't turn into a path.
+ */
+export default function extensionOf({ name, mime_type }: Params) {
+    const from_name = name?.includes(".") ? name.split(".").pop() : undefined;
+
+    if (from_name) return from_name.toLowerCase().replace(/[^a-z0-9]/g, "");
+
+    const from_mime =
+        mime_type && !mime_type.includes("octet-stream")
+            ? mime_type.split("/").pop()
+            : undefined;
+
+    return from_mime ? from_mime.toLowerCase().replace(/[^a-z0-9]/g, "") : undefined;
+}