Stop storing raw file bytes in media.text_content

`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:

- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
  and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
  `attachmentFileKind` classes everything non-image/audio/video as a
  document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
  text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
  `SELECT m.*` pulled the column into page props and API responses.

On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.

Changes:

- `file-to-text.ts` lists the text extensions explicitly and returns
  undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
  `BotTextFileExtensions`), used by both upload routes and by
  `uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
  catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
  `cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
  re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
  now has to be asked for (`grab-messages-media`'s reply path and
  ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
  an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
  (rerunnable, with a VACUUM hint).

Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.

src/bots-queue/load-attachment-parts.ts

@@ -27,7 +27,10 @@ export default async function loadAttachmentParts({ messages }: Params) {
         .slice(-AppData.BotReplyAttachmentMessages)
         .map((message) => Number(message.id));
 
-    const media = await grabMessagesMedia({ message_ids: recent });
+    const media = await grabMessagesMedia({
+        message_ids: recent,
+        with_text_content: true,
+    });
     const parts = new Map<number, HarnessContentPart[]>();
 
     for (const [message_id, files] of media) {

src/data/app-data.ts

@@ -388,6 +388,13 @@ export const AppData = {
     BotAttachmentMaxBytes: 20 * 1024 * 1024,
     BotAttachmentMaxFiles: 6,
     BotAttachmentMaxTextChars: 20000,
+    /**
+     * How much of a document's text is kept on its `media` row. A ceiling
+     * on the column itself: without it a 20 MB PDF left 20 MB of text in
+     * the database, even though a reply reads at most
+     * `BotAttachmentMaxTextChars` of it.
+     */
+    BotAttachmentTextStoreMaxChars: 200000,
     /**
      * Only this many of the latest messages with files hand the files
      * themselves to a reply; older ones are named.

src/functions/backend/db/bot-files/grab-bot-file.ts

@@ -3,6 +3,7 @@ import type {
     BUN_SQLITE_COBBLE_BOT_FILES,
     BUN_SQLITE_COBBLE_MEDIA,
 } from "@/db/types/db";
+import mediaColumns from "@/src/functions/backend/db/media/media-columns";
 import { canReachScope, type StorageScope } from "@/src/utils/storage-access";
 
 type Params = {
@@ -59,7 +60,7 @@ export default async function grabBotFile({
 
     const media = file.media_id
         ? await BunSQLite.sql<BUN_SQLITE_COBBLE_MEDIA>({
-              sql: `SELECT * FROM media WHERE id = ? LIMIT 1`,
+              sql: `SELECT ${mediaColumns({ alias: "m" })} FROM media m WHERE m.id = ? LIMIT 1`,
               values: [file.media_id],
           }).then((r) => r.payload?.[0] || null)
         : null;

src/functions/backend/db/bot-files/ingest-bot-file.ts

@@ -57,7 +57,13 @@ export default async function ingestBotFile({
           }).then((res) => res.payload?.[0])
         : undefined;
 
-    if (Number(file.size) > AppData.IngestMaxBytes && !media?.text_content) {
+    /**
+     * Decided on the file's size alone. A cached `text_content` is not a
+     * reason to read an oversized file: it was only ever a slice of it,
+     * and taking it as the whole document made a file too big to unpack
+     * look already read.
+     */
+    if (Number(file.size) > AppData.IngestMaxBytes) {
         await BunSQLite.sql({
             sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`,
             values: [`Too large to read (over ${AppData.IngestMaxBytes / (1024 * 1024)} MB).`, Date.now(), file.id],

src/functions/backend/db/bot-files/media-type-for-file.ts

@@ -1,5 +1,6 @@
 import type { MediaType } from "@/src/types";
 import { BotImageFileExtensions } from "@/src/dict/bot-file-dict";
+import extensionOf from "@/src/utils/extension-of";
 
 type Params = {
     /**
@@ -27,19 +28,10 @@ export default function mediaTypeForFile({ name, mime_type }: Params): MediaType
 }
 
 /**
- * Its extension, lower case, without the dot.
+ * Its extension, lower case, without the dot. Shared, so the storage
+ * catalog and the text extractor read a name the same way.
  */
-export function extensionOf({ name, mime_type }: Params) {
-    const from_name = name?.includes(".") ? name.split(".").pop() : undefined;
-
-    if (from_name) return from_name.toLowerCase().replace(/[^a-z0-9]/g, "");
-
-    const from_mime = mime_type && !mime_type.includes("octet-stream")
-        ? mime_type.split("/").pop()
-        : undefined;
-
-    return from_mime ? from_mime.toLowerCase().replace(/[^a-z0-9]/g, "") : undefined;
-}
+export { extensionOf };
 
 /**
  * A MIME type for the row, from the extension when there isn't one.

src/functions/backend/db/bot-files/store-chat-attachments.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, test } from "bun:test";
+
+/**
+ * The module's own source, since the regression it guards is what the code
+ * writes to the database, not what it returns.
+ */
+async function source() {
+    return await Bun.file(
+        `${import.meta.dir}/store-chat-attachments.ts`,
+    ).text();
+}
+
+describe("storeChatAttachments", () => {
+    /**
+     * The same text used to be copied onto the new `bot-file` media row,
+     * so a chat upload kept a second copy of its extracted text (and left
+     * it behind when the upload was pruned). The text belongs only on the
+     * message's own `bot-attachment` row, where a reply reads it;
+     * ingestion re-reads the bytes through `extractBotFileText`.
+     */
+    test("never writes text_content onto the file's media row", async () => {
+        const code = await source();
+
+        expect(code).not.toContain("SET text_content");
+        expect(code).not.toContain("text_content = ?");
+    });
+
+    test("still saves the bytes and queues the ingest", async () => {
+        const code = await source();
+
+        expect(code).toContain("recordBotFile(");
+        expect(code).toContain("enqueueFileIngest(");
+    });
+});

src/functions/backend/db/bot-files/store-chat-attachments.ts

@@ -1,4 +1,3 @@
-import BunSQLite from "@moduletrace/bun-sqlite";
 import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
 import { BotUploadFolder } from "@/src/dict/bot-file-dict";
 import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
@@ -44,6 +43,13 @@ export default async function storeChatAttachments({
              */
             const name = safeName({ name: file.title || `file-${file.id}` });
 
+            /**
+             * The text stays on the message's own `bot-attachment` row,
+             * where a reply reads it. Copying it onto the file's row kept
+             * the same text twice and left the copy behind when the
+             * upload was pruned; ingestion re-reads the bytes once
+             * through `extractBotFileText` instead.
+             */
             const saved = await recordBotFile({
                 bot_id,
                 scope: workspace_id ? "workspace" : "bot",
@@ -57,17 +63,6 @@ export default async function storeChatAttachments({
                 user_id,
             });
 
-            /**
-             * The same text the upload already extracted, so a PDF isn't
-             * unpacked a second time.
-             */
-            if (file.text_content && saved.media_id) {
-                await BunSQLite.sql({
-                    sql: `UPDATE media SET text_content = ? WHERE id = ?`,
-                    values: [file.text_content, saved.media_id],
-                }).catch(() => {});
-            }
-
             await enqueueFileIngest({ file_id: saved.id! });
         } catch (error) {
             await logError({

src/functions/backend/db/bots/grab-messages-media.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "bun:sqlite";
+
+const db = new Database(":memory:");
+
+db.run(
+    `CREATE TABLE media (
+        id INTEGER PRIMARY KEY,
+        created_at INTEGER,
+        updated_at INTEGER,
+        user_id INTEGER,
+        title TEXT,
+        description TEXT,
+        summary TEXT,
+        media_type TEXT,
+        media_write_path TEXT,
+        media_thumbnail_write_path TEXT,
+        mime_type TEXT,
+        text_content TEXT,
+        is_primary INTEGER,
+        is_rag_ingested INTEGER,
+        archived INTEGER
+    )`,
+);
+db.run(`CREATE TABLE bot_message_media (id INTEGER PRIMARY KEY, message_id INTEGER, media_id INTEGER)`);
+db.run(
+    `INSERT INTO media (id, title, media_type, media_write_path, mime_type, text_content)
+        VALUES (1, 'invoice.pdf', 'document', 'media/1.pdf', 'application/pdf', 'Total: 12')`,
+);
+db.run(`INSERT INTO bot_message_media (id, message_id, media_id) VALUES (1, 10, 1)`);
+
+/**
+ * The query `grabMessagesMedia` builds, with the text column included or
+ * left out. It is the largest column in the table, so only the reply path
+ * that hands a document to a model asks for it.
+ */
+function read({ with_text_content }: { with_text_content: boolean }) {
+    const columns = [
+        "m.id",
+        "m.title",
+        "m.media_type",
+        "m.media_write_path",
+        "m.mime_type",
+        ...(with_text_content ? ["m.text_content"] : []),
+    ].join(", ");
+
+    return db
+        .query(
+            `SELECT ${columns}, l.message_id FROM bot_message_media l
+            JOIN media m ON m.id = l.media_id
+            WHERE l.message_id = 10`,
+        )
+        .get() as Record<string, unknown>;
+}
+
+describe("grabMessagesMedia", () => {
+    test("a plain read never drags the extracted text along", () => {
+        expect(read({ with_text_content: false })).not.toHaveProperty("text_content");
+    });
+
+    test("the reply path can ask for it", () => {
+        expect(read({ with_text_content: true }).text_content).toBe("Total: 12");
+    });
+
+    test("the row's name and path are always there", () => {
+        expect(read({ with_text_content: false })).toMatchObject({
+            title: "invoice.pdf",
+            media_write_path: "media/1.pdf",
+            message_id: 10,
+        });
+    });
+});

src/functions/backend/db/bots/grab-messages-media.ts

@@ -1,8 +1,15 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
 import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
+import mediaColumns from "@/src/functions/backend/db/media/media-columns";
 
 type Params = {
     message_ids: number[];
+    /**
+     * Also read `text_content`, the extracted text a reply hands a
+     * document to the model as. Off by default: it is the largest column
+     * in the table and nothing else here uses it.
+     */
+    with_text_content?: boolean;
 };
 
 export type MessageMediaRow = BUN_SQLITE_COBBLE_MEDIA & { message_id: number };
@@ -11,16 +18,21 @@ export type MessageMediaRow = BUN_SQLITE_COBBLE_MEDIA & { message_id: number };
  * The files attached to these messages, by message id, in the order they
  * were attached.
  */
-export default async function grabMessagesMedia({ message_ids }: Params) {
+export default async function grabMessagesMedia({
+    message_ids,
+    with_text_content,
+}: Params) {
     const by_message = new Map<number, MessageMediaRow[]>();
     const ids = [...new Set(message_ids.filter(Boolean))];
 
     if (!ids.length) return by_message;
 
+    const columns = mediaColumns({ alias: "m", with_text_content });
+
     const rows =
         (
             await BunSQLite.sql<MessageMediaRow>({
-                sql: `SELECT m.*, l.message_id FROM bot_message_media l
+                sql: `SELECT ${columns}, l.message_id FROM bot_message_media l
                     JOIN media m ON m.id = l.media_id
                     WHERE l.message_id IN (${ids.map(() => "?").join(", ")})
                     ORDER BY l.id ASC`,

src/functions/backend/db/media/grab-media.ts

@@ -6,6 +6,7 @@ import BunSQLite, { type APIResponseObject } from "@moduletrace/bun-sqlite";
 import type { SQLInsertGenValueType } from "@moduletrace/bun-sqlite";
 import _ from "lodash";
 import appendUsersToMedia from "./append-users-to-media";
+import mediaColumns from "./media-columns";
 
 type Params = {
     query?: ApiReqParams;
@@ -23,7 +24,7 @@ export default async function grabMedia(
         const hasParadigms = Boolean(query?.media_paradigms?.length);
 
         let sql = `SELECT\n`;
-        sql += `    m.*,\n`;
+        sql += `    ${mediaColumns({ alias: "m" })},\n`;
         sql += `    GROUP_CONCAT(DISTINCT mp.media_paradigm) AS media_paradigms\n`;
         sql += `FROM media m\n`;
         sql += `LEFT JOIN media_paradigms mp ON m.id = mp.media_id\n`;

src/functions/backend/db/media/media-columns.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, test } from "bun:test";
+import schema from "@/db/schema";
+import mediaColumns from "./media-columns";
+
+/**
+ * The schema's own list, so a column added later can't quietly be left
+ * out of (or forgotten in) the select this guards.
+ */
+function mediaSchemaFields(): string[] {
+    const tables = (schema as { tables?: { tableName: string; fields: { fieldName: string }[] }[] }).tables || [];
+    const media = tables.find((table) => table.tableName == "media");
+
+    /**
+     * `id`, `created_at` and `updated_at` are added to every table by the
+     * ORM, so they aren't in the schema's own field list.
+     */
+    return [...(media?.fields || []).map((field) => field.fieldName), "id", "created_at", "updated_at"];
+}
+
+describe("mediaColumns", () => {
+    test("names every media column except the extracted text", () => {
+        const listed = mediaColumns({ alias: "m" })
+            .split(", ")
+            .map((column) => column.replace(/^m\./, ""))
+            .sort();
+
+        const expected = mediaSchemaFields()
+            .filter((field) => field != "text_content")
+            .sort();
+
+        expect(listed).toEqual(expected);
+    });
+
+    test("never selects text_content unless asked", () => {
+        expect(mediaColumns()).not.toContain("text_content");
+        expect(mediaColumns({ with_text_content: true })).toContain("m.text_content");
+    });
+
+    test("asking for the text adds it to the same full list", () => {
+        const with_text = mediaColumns({ with_text_content: true });
+
+        expect(with_text).toContain(mediaColumns());
+        expect(with_text.split(", ")).toHaveLength(mediaColumns().split(", ").length + 1);
+    });
+
+    test("qualifies every column, so a join can't make one ambiguous", () => {
+        for (const column of mediaColumns({ alias: "media" }).split(", ")) {
+            expect(column.startsWith("media.")).toBe(true);
+        }
+    });
+
+    test("the alias defaults to m", () => {
+        expect(mediaColumns().startsWith("m.id")).toBe(true);
+    });
+});

src/functions/backend/db/media/media-columns.ts

@@ -0,0 +1,40 @@
+/**
+ * The `media` columns a normal read takes, with a table alias.
+ *
+ * `text_content` is deliberately left out: it holds up to
+ * `AppData.BotAttachmentTextStoreMaxChars` of extracted document text, and
+ * every caller that only needs a file's name, type or path (`SELECT m.*`
+ * did) pulled it into memory, into page props and into API responses.
+ * Read it explicitly where it is really used:
+ * `src/bots-queue/load-attachment-parts.ts` and
+ * `src/functions/backend/db/bot-files/ingest-bot-file.ts`.
+ */
+export default function mediaColumns({
+    alias = "m",
+    with_text_content,
+}: { alias?: string; with_text_content?: boolean } = {}) {
+    const columns = [
+        "id",
+        "created_at",
+        "updated_at",
+        "user_id",
+        "title",
+        "description",
+        "summary",
+        "media_type",
+        "media_write_path",
+        "media_thumbnail_write_path",
+        "mime_type",
+        "is_primary",
+        "is_rag_ingested",
+        "archived",
+    ];
+
+    /**
+     * Only the two paths that really use it ask for it: the reply that
+     * hands a document to a model, and ingestion.
+     */
+    if (with_text_content) columns.push("text_content");
+
+    return columns.map((column) => `${alias}.${column}`).join(", ");
+}

src/functions/backend/db/media/plan-text-content-prune.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import planTextContentPrune from "./plan-text-content-prune";
+
+describe("planTextContentPrune", () => {
+    test("clears the raw bytes stored for a type the extractor can't read", () => {
+        const changes = planTextContentPrune({
+            rows: [
+                { id: 1, title: "logo.svg", mime_type: "image/svg+xml", text_content: "<svg><path/></svg>" },
+                { id: 2, title: "bundle.zip", mime_type: "application/zip", text_content: "PK\u0003\u0004" },
+                { id: 3, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", text_content: "PK\u0003\u0004" },
+            ],
+        });
+
+        expect(changes.map((change) => change.media_id)).toEqual([1, 2, 3]);
+        expect(changes.every((change) => change.text_content === null)).toBe(true);
+        expect(changes.every((change) => change.after_chars === 0)).toBe(true);
+    });
+
+    test("leaves a document's text alone", () => {
+        const changes = planTextContentPrune({
+            rows: [
+                { id: 4, title: "invoice.pdf", mime_type: "application/pdf", text_content: "Total: 12" },
+                { id: 5, title: "notes.md", mime_type: "text/markdown", text_content: "# Hi" },
+                { id: 6, title: "rows.csv", mime_type: "text/csv", text_content: "a,b\n1,2" },
+            ],
+        });
+
+        expect(changes).toEqual([]);
+    });
+
+    test("cuts a document over the store ceiling", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 100);
+        const changes = planTextContentPrune({
+            rows: [{ id: 7, title: "long.txt", mime_type: "text/plain", text_content: text }],
+        });
+
+        expect(changes).toHaveLength(1);
+        expect(changes[0]!.after_chars).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+        expect(changes[0]!.text_content).toContain("Truncated");
+        expect(changes[0]!.before_chars).toBe(text.length);
+    });
+
+    test("nothing to say about a row with no text or no id", () => {
+        expect(
+            planTextContentPrune({
+                rows: [
+                    { id: 8, title: "empty.pdf", text_content: "" },
+                    { id: 9, title: "blank.pdf", text_content: "   \n " },
+                    { id: 10, title: "none.pdf" },
+                    { title: "no id.pdf", text_content: "text" },
+                ],
+            }),
+        ).toEqual([]);
+    });
+
+    test("counting what each change reclaims is left to the caller", () => {
+        const changes = planTextContentPrune({
+            rows: [{ id: 11, title: "icon.svg", mime_type: "image/svg+xml", text_content: "12345" }],
+        });
+
+        expect(changes).toHaveLength(1);
+        expect(changes[0]!.before_chars - changes[0]!.after_chars).toBe(5);
+    });
+});

src/functions/backend/db/media/plan-text-content-prune.ts

@@ -0,0 +1,55 @@
+import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
+import decideTextContent from "@/src/utils/decide-text-content";
+
+export type TextContentChange = {
+    media_id: number;
+    name: string;
+    before_chars: number;
+    after_chars: number;
+    /**
+     * What the row's `text_content` should become: `null` clears it.
+     */
+    text_content: string | null;
+};
+
+type Params = {
+    /**
+     * The `media` rows that have any `text_content`.
+     */
+    rows: Partial<BUN_SQLITE_COBBLE_MEDIA>[];
+};
+
+/**
+ * What `pruneMediaTextContent` would change, worked out without touching
+ * the database: one entry per row whose text belongs nowhere or is over
+ * the store ceiling, and nothing at all for a row that is already right.
+ */
+export default function planTextContentPrune({ rows }: Params): TextContentChange[] {
+    const changes: TextContentChange[] = [];
+
+    for (const row of rows) {
+        if (!row.id) continue;
+
+        const before = row.text_content || "";
+
+        if (!before.trim()) continue;
+
+        const text_content = decideTextContent({
+            name: row.title,
+            mime_type: row.mime_type,
+            text: before,
+        });
+
+        if (text_content === before) continue;
+
+        changes.push({
+            media_id: Number(row.id),
+            name: row.title || "no name",
+            before_chars: before.length,
+            after_chars: text_content?.length || 0,
+            text_content,
+        });
+    }
+
+    return changes;
+}

src/functions/backend/db/media/prune-media-text-content.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "bun:sqlite";
+import { AppData } from "@/src/data/app-data";
+import pruneMediaTextContent, { type SqlRunner } from "./prune-media-text-content";
+
+type Row = { id: number; title: string; mime_type: string; media_type: string; text_content: string | null; updated_at?: number };
+
+function makeDb({ rows }: { rows: Row[] }) {
+    const db = new Database(":memory:");
+
+    db.run(
+        `CREATE TABLE media (
+            id INTEGER PRIMARY KEY,
+            title TEXT,
+            mime_type TEXT,
+            media_type TEXT,
+            text_content TEXT,
+            updated_at INTEGER
+        )`,
+    );
+
+    const insert = db.prepare(
+        `INSERT INTO media (id, title, mime_type, media_type, text_content) VALUES (?, ?, ?, ?, ?)`,
+    );
+
+    for (const row of rows) insert.run(row.id, row.title, row.mime_type, row.media_type, row.text_content);
+
+    const run: SqlRunner = async ({ sql, values }) => {
+        const statement = db.prepare(sql);
+
+        if (sql.trim().toUpperCase().startsWith("SELECT")) {
+            return { payload: statement.all(...((values || []) as never[])) as unknown[] };
+        }
+
+        statement.run(...((values || []) as never[]));
+
+        return { payload: [] };
+    };
+
+    const read = (id: number) =>
+        (db.query(`SELECT text_content FROM media WHERE id = ?`).get(id) as { text_content: string | null }).text_content;
+
+    return { db, run, read };
+}
+
+describe("pruneMediaTextContent", () => {
+    test("clears raw bytes from a type the extractor can't read, and keeps a document's text", async () => {
+        const { run, read } = makeDb({
+            rows: [
+                { id: 1, title: "logo.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "<svg><path d='M0 0'/></svg>" },
+                { id: 2, title: "bundle.zip", mime_type: "application/zip", media_type: "document", text_content: "PK\u0003\u0004binary" },
+                { id: 3, title: "invoice.pdf", mime_type: "application/pdf", media_type: "document", text_content: "Total: 12" },
+            ],
+        });
+
+        const result = await pruneMediaTextContent({ run });
+
+        expect(result.checked).toBe(3);
+        expect(result.cleared).toBe(2);
+        expect(result.truncated).toBe(0);
+        expect(read(1)).toBeNull();
+        expect(read(2)).toBeNull();
+        expect(read(3)).toBe("Total: 12");
+    });
+
+    test("cuts an oversized document to the store ceiling", async () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 1000);
+        const { run, read } = makeDb({
+            rows: [{ id: 4, title: "huge.txt", mime_type: "text/plain", media_type: "document", text_content: text }],
+        });
+
+        const result = await pruneMediaTextContent({ run });
+        const after = read(4)!;
+
+        expect(result.truncated).toBe(1);
+        expect(result.cleared).toBe(0);
+        expect(after.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+        expect(after).toContain("Truncated");
+        expect(result.reclaimed_bytes).toBeGreaterThan(0);
+    });
+
+    test("is safe to rerun: a second sweep changes nothing", async () => {
+        const { run, read } = makeDb({
+            rows: [{ id: 5, title: "icon.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "raw" }],
+        });
+
+        await pruneMediaTextContent({ run });
+        const second = await pruneMediaTextContent({ run });
+
+        expect(second.checked).toBe(0);
+        expect(second.changes).toEqual([]);
+        expect(second.reclaimed_bytes).toBe(0);
+        expect(read(5)).toBeNull();
+    });
+
+    test("stamps updated_at on the rows it rewrites", async () => {
+        const now = 1_700_000_000_000;
+        const { run, db } = makeDb({
+            rows: [{ id: 6, title: "x.zip", mime_type: "application/zip", media_type: "document", text_content: "raw" }],
+        });
+
+        await pruneMediaTextContent({ run, now });
+
+        expect(
+            (db.query(`SELECT updated_at FROM media WHERE id = 6`).get() as { updated_at: number }).updated_at,
+        ).toBe(now);
+    });
+
+    test("reports every change with its row, name and sizes", async () => {
+        const { run } = makeDb({
+            rows: [{ id: 7, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", media_type: "document", text_content: "PK\u0003\u0004" }],
+        });
+
+        const result = await pruneMediaTextContent({ run });
+
+        expect(result.changes).toEqual([
+            { media_id: 7, name: "sheet.xlsx", before_chars: 4, after_chars: 0, text_content: null },
+        ]);
+    });
+});

src/functions/backend/db/media/prune-media-text-content.ts

@@ -0,0 +1,75 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
+import planTextContentPrune, { type TextContentChange } from "./plan-text-content-prune";
+
+/**
+ * One statement against the database. `BunSQLite.sql` is the real one; a
+ * test passes its own, so the whole sweep can be run without the app's
+ * database.
+ */
+export type SqlRunner = (params: {
+    sql: string;
+    values?: (string | number | null)[];
+}) => Promise<{ payload?: unknown[] | null }>;
+
+export type PruneMediaTextResult = {
+    checked: number;
+    cleared: number;
+    truncated: number;
+    reclaimed_bytes: number;
+    changes: TextContentChange[];
+};
+
+type Params = {
+    run?: SqlRunner;
+    /**
+     * When the sweep runs, for the rows it writes.
+     */
+    now?: number;
+};
+
+/**
+ * Brings every `media.text_content` in line with what may be stored there
+ * (`planTextContentPrune`): no text on a type the extractor can't read,
+ * and no text longer than the store ceiling.
+ *
+ * Run once per install after the allowlist landed; safe to rerun, since a
+ * row that is already right is not written.
+ */
+export default async function pruneMediaTextContent({
+    run,
+    now = Date.now(),
+}: Params = {}): Promise<PruneMediaTextResult> {
+    const sql: SqlRunner =
+        run || (({ sql, values }) => BunSQLite.sql<BUN_SQLITE_COBBLE_MEDIA>({ sql, values }));
+
+    const res = await sql({
+        sql: `SELECT id, title, mime_type, media_type, text_content
+            FROM media
+            WHERE text_content IS NOT NULL AND TRIM(text_content) != ''`,
+    });
+
+    const rows = (res.payload || []) as BUN_SQLITE_COBBLE_MEDIA[];
+    const changes = planTextContentPrune({ rows });
+    const result: PruneMediaTextResult = {
+        checked: rows.length,
+        cleared: 0,
+        truncated: 0,
+        reclaimed_bytes: 0,
+        changes,
+    };
+
+    for (const change of changes) {
+        await sql({
+            sql: `UPDATE media SET text_content = ?, updated_at = ? WHERE id = ?`,
+            values: [change.text_content, now, change.media_id],
+        });
+
+        result.reclaimed_bytes += change.before_chars - change.after_chars;
+
+        if (change.text_content === null) result.cleared += 1;
+        else result.truncated += 1;
+    }
+
+    return result;
+}

src/functions/backend/db/media/sanitize-media-row-text-content.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import sanitizeMediaRowTextContent from "./sanitize-media-row-text-content";
+
+describe("sanitizeMediaRowTextContent", () => {
+    test("keeps a document's text", () => {
+        const row = sanitizeMediaRowTextContent({
+            row: { title: "invoice.pdf", mime_type: "application/pdf", text_content: "Total: 12" },
+        });
+
+        expect(row.text_content).toBe("Total: 12");
+    });
+
+    test("refuses raw bytes typed into a row that names a binary", () => {
+        const row = sanitizeMediaRowTextContent({
+            row: { title: "logo.svg", mime_type: "image/svg+xml", text_content: "<svg/>" },
+        });
+
+        expect(row.text_content).toBeUndefined();
+    });
+
+    test("an update is judged on the file it already is", () => {
+        const row = sanitizeMediaRowTextContent({
+            row: { text_content: "PK\u0003\u0004" },
+            existing: { title: "bundle.zip", mime_type: "application/zip" },
+        });
+
+        expect(row.text_content).toBeUndefined();
+    });
+
+    test("an update with no text is left as it is", () => {
+        const row = sanitizeMediaRowTextContent({
+            row: { title: "renamed.pdf" },
+            existing: { title: "invoice.pdf" },
+        });
+
+        expect(row).toEqual({ title: "renamed.pdf" });
+    });
+
+    test("a long text without a name is still cut, not dropped", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 10);
+        const row = sanitizeMediaRowTextContent({ row: { text_content: text } });
+
+        expect(row.text_content).toContain("Truncated");
+        expect(row.text_content!.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+    });
+
+    test("empty text becomes unset", () => {
+        const row = sanitizeMediaRowTextContent({ row: { title: "a.pdf", text_content: "" } });
+
+        expect(row.text_content).toBeUndefined();
+    });
+});

src/functions/backend/db/media/sanitize-media-row-text-content.ts

@@ -0,0 +1,49 @@
+import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
+import capTextContent from "@/src/utils/cap-text-content";
+import decideTextContent from "@/src/utils/decide-text-content";
+
+type Params = {
+    /**
+     * The row as the request sent it.
+     */
+    row: Partial<BUN_SQLITE_COBBLE_MEDIA>;
+    /**
+     * The row already in the database, for an update: its name and type
+     * decide whether the new text may be kept when the update doesn't
+     * repeat them.
+     */
+    existing?: Partial<BUN_SQLITE_COBBLE_MEDIA> | null;
+};
+
+/**
+ * A `media` row whose `text_content` is what the column may hold.
+ *
+ * The upload path decides this itself, but the CRUD API writes rows
+ * straight through, so a typed-in `text_content` could store raw bytes or
+ * megabytes of text. An update that carries no text is left alone.
+ */
+export default function sanitizeMediaRowTextContent({
+    row,
+    existing,
+}: Params): Partial<BUN_SQLITE_COBBLE_MEDIA> {
+    const text = row.text_content;
+
+    if (text === undefined) return row;
+
+    const name = row.title || existing?.title || null;
+    const mime_type = row.mime_type || existing?.mime_type || null;
+
+    /**
+     * With no name and no type there is nothing to judge it by, so it is
+     * only cut: dropping text that a rename would have kept would be
+     * worse than a slightly generous cache.
+     */
+    const text_content =
+        name || mime_type
+            ? decideTextContent({ name, mime_type, text })
+            : text
+              ? capTextContent({ text })
+              : null;
+
+    return { ...row, text_content: text_content ?? undefined };
+}

src/functions/backend/db/users/media/file-to-text.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from "bun:test";
+import fileToText from "./file-to-text";
+
+function stream({ text }: { text: string }) {
+    return Buffer.from(text, "utf-8") as Buffer<ArrayBuffer>;
+}
+
+describe("fileToText", () => {
+    test("reads the plain text types as they are", async () => {
+        for (const file_type of ["txt", "text", "md", "markdown", "csv", "tsv", "json", "html", "htm", "xml", "yml", "yaml", "log"]) {
+            expect(await fileToText({ file_type, file_stream: stream({ text: "hello" }) })).toBe("hello");
+        }
+    });
+
+    test("keeps the real case of the text", async () => {
+        expect(await fileToText({ file_type: "md", file_stream: stream({ text: "# Heading\n\nBody" }) })).toBe(
+            "# Heading\n\nBody",
+        );
+    });
+
+    /**
+     * The bug this guards: the switch used to fall through to
+     * `buffer.toString("utf-8")`, so a zip, an SVG or a spreadsheet was
+     * "extracted" as garbled bytes and stored in `media.text_content`.
+     */
+    test("refuses a binary instead of decoding its bytes", async () => {
+        for (const file_type of ["svg", "zip", "xlsx", "pptx", "bin", "exe", ""]) {
+            expect(await fileToText({ file_type, file_stream: stream({ text: "PK\u0003\u0004binary" }) })).toBeUndefined();
+        }
+    });
+
+    test("no buffer is no text", async () => {
+        expect(await fileToText({ file_type: "txt" })).toBeUndefined();
+    });
+});

src/functions/backend/db/users/media/file-to-text.ts

@@ -36,9 +36,15 @@ export default async function fileToText({
         buffer = file_path
             ? Buffer.from(await Bun.file(file_path).arrayBuffer())
             : file_stream;
-        ext = (file_type || file_path?.split(".").pop() || "txt")
-            .toString()
-            .toLowerCase();
+        const known_ext = file_type || file_path?.split(".").pop();
+
+        /**
+         * No extension and no type: nothing says this is text, so nothing
+         * gets decoded. Guessing "txt" is how a binary became text.
+         */
+        if (!known_ext) return undefined;
+
+        ext = known_ext.toString().toLowerCase();
     }
 
     if (!buffer) return undefined;
@@ -76,9 +82,32 @@ export default async function fileToText({
         //     }).join("\n\n");
         // }
 
-        case "md":
+        /**
+         * The text types read as they are. Listed one by one, never a
+         * catch-all: decoding a zip, an SVG or a spreadsheet as UTF-8
+         * fills the media row with raw bytes instead of text.
+         */
         case "txt":
-        default:
+        case "text":
+        case "md":
+        case "markdown":
+        case "csv":
+        case "tsv":
+        case "json":
+        case "html":
+        case "htm":
+        case "xml":
+        case "yml":
+        case "yaml":
+        case "log":
             return buffer.toString("utf-8");
+
+        default:
+            /**
+             * Nothing readable locally: the caller decides whether to read
+             * it another way (an image or audio through the harness) or to
+             * leave its text unset.
+             */
+            return undefined;
     }
 }

src/functions/backend/db/users/media/upload-and-record-media.ts

@@ -17,6 +17,8 @@ import BunSQLite, { type APIResponseObject } from "@moduletrace/bun-sqlite";
 import { existsSync, mkdirSync } from "fs";
 import path from "path";
 import sharp from "sharp";
+import canExtractText from "@/src/utils/can-extract-text";
+import capTextContent from "@/src/utils/cap-text-content";
 import fileToText from "./file-to-text";
 import MediaDataToArrayBuffer from "./media-data-to-array-buffer";
 import sniffMediaExtension from "./sniff-media-ext";
@@ -180,14 +182,22 @@ export default async function uploadAndRecordMedia({
             mode: 1,
         });
 
-        if (extract_media_text_content) {
+        /**
+         * Only an extension the extractor really unpacks is read: any
+         * other binary has no text, and decoding it anyway stored raw
+         * bytes in the row.
+         */
+        if (
+            extract_media_text_content &&
+            canExtractText({ name: media_name, mime_type: media_mime_type })
+        ) {
             const media_text = await fileToText({
                 file_type: media_ext,
                 file_stream: Buffer.from(media_array_buffer),
             });
 
             if (media_text) {
-                media_text_content = media_text;
+                media_text_content = capTextContent({ text: media_text });
             }
         }
     }

src/harness/media/read-media.ts

@@ -1,3 +1,4 @@
+import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
 import fileToText from "@/src/functions/backend/db/users/media/file-to-text";
 import AppError from "@/src/utils/app-error";
 import transcribeAudio from "../audio/transcribe-audio";
@@ -24,7 +25,11 @@ type Params = {
     workspace_id?: number | string | null;
 };
 
-const DocumentExtensions = ["pdf", "docx", "txt", "md", "csv", "json", "html", "xml", "yaml", "yml"];
+/**
+ * One list, the same one the local extractor reads: keeping a second copy
+ * here is how a type got read by one and silently missed by the other.
+ */
+const DocumentExtensions: readonly string[] = BotTextFileExtensions;
 
 const TypeExtensions: Record<string, string> = {
     "application/pdf": "pdf",

src/pages/api/admin/bots/upload-attachment.ts

@@ -8,6 +8,7 @@ import type { BotMessageAttachment } from "@/src/types";
 import AppError from "@/src/utils/app-error";
 import attachmentFileKind from "@/src/utils/attachment-file-kind";
 import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
+import canExtractText from "@/src/utils/can-extract-text";
 import parseWithSchema from "@/src/utils/parse-with-schema";
 import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
 import safeErrorMessage from "@/src/utils/safe-error-message";
@@ -64,7 +65,7 @@ export const handler: BunextAPIRouteHandler<
             media_mime_type: ext,
             media_paradigms: ["bot-attachment"],
             media: { title: name },
-            extract_media_text_content: media_type == "document",
+            extract_media_text_content: canExtractText({ name, mime_type: mime }),
             user: { id: Number(user.id) },
         }).catch(async (error) => {
             await logError({ title: `Attachment ${name} couldn't be saved`, error }).catch(() => {});

src/pages/api/admin/crud/(tables)/media/post.ts

@@ -1,5 +1,6 @@
 import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
 import checkUserAccess from "@/src/functions/backend/auth/check-user-access";
+import sanitizeMediaRowTextContent from "@/src/functions/backend/db/media/sanitize-media-row-text-content";
 import type { AdminCrudAPIParams } from "@/src/types";
 import BunSQLite from "@moduletrace/bun-sqlite";
 import type { APIResponseObject } from "@moduletrace/bunext/types";
@@ -10,9 +11,13 @@ export default async function (
 ): Promise<APIResponseObject> {
     const { table, body, user_types, user } = params;
 
+    /**
+     * A row written straight through the API still may not carry raw
+     * bytes or megabytes in `text_content`.
+     */
     let final_insert_data: BUN_SQLITE_COBBLE_MEDIA[] = [
         ...(body?.insert_data || []),
-    ].map((d) => ({ ...d, user_id: user.id }));
+    ].map((d) => ({ ...sanitizeMediaRowTextContent({ row: d }), user_id: user.id }));
 
     const is_user_allowed_to_post_any = checkUserAccess({
         includes: ["admin"],

src/pages/api/admin/crud/(tables)/media/put.ts

@@ -1,4 +1,6 @@
+import type { BunSQLiteTables, BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
 import checkUserAccess from "@/src/functions/backend/auth/check-user-access";
+import sanitizeMediaRowTextContent from "@/src/functions/backend/db/media/sanitize-media-row-text-content";
 import type { AdminCrudAPIParams } from "@/src/types";
 import BunSQLite from "@moduletrace/bun-sqlite";
 import type { APIResponseObject } from "@moduletrace/bunext/types";
@@ -29,9 +31,21 @@ export default async function (
         targetId = user.id;
     }
 
+    /**
+     * The row it replaces decides whether the new text belongs there, so
+     * an update that only sends `text_content` is judged on the file's
+     * real name and type.
+     */
+    const existing = targetId
+        ? await BunSQLite.select<BUN_SQLITE_COBBLE_MEDIA, (typeof BunSQLiteTables)[number]>({
+              table,
+              query: { query: { id: { value: targetId } } },
+          }).then((res) => res.payload?.[0] || null)
+        : null;
+
     const PUT = await BunSQLite.update({
         table,
-        data: body?.update_data,
+        data: sanitizeMediaRowTextContent({ row: body.update_data, existing }),
         targetId,
         query: final_sql_query,
     });

src/pages/api/admin/tasks/attach-file.ts

@@ -9,6 +9,7 @@ import type { TaskDetail } from "@/src/types";
 import AppError from "@/src/utils/app-error";
 import attachmentFileKind from "@/src/utils/attachment-file-kind";
 import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
+import canExtractText from "@/src/utils/can-extract-text";
 import parseWithSchema from "@/src/utils/parse-with-schema";
 import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
 import safeErrorMessage from "@/src/utils/safe-error-message";
@@ -65,7 +66,7 @@ export const handler: BunextAPIRouteHandler<APIResponseObject<TaskDetail>> = asy
                 media_mime_type: ext,
                 media_paradigms: ["task-media"],
                 media: { title: name },
-                extract_media_text_content: media_type == "document",
+                extract_media_text_content: canExtractText({ name, mime_type: mime }),
                 user: { id: Number(user.id) },
             });
 

src/scripts/prune-media-text-content.ts

@@ -0,0 +1,40 @@
+import { AppData } from "@/src/data/app-data";
+import pruneMediaTextContent from "@/src/functions/backend/db/media/prune-media-text-content";
+
+/**
+ * Cleans the `media.text_content` that should never have been stored.
+ *
+ * Before the extractor had an allowlist, every non-image/audio/video file
+ * was decoded as UTF-8 and its bytes were written to `text_content`: an
+ * SVG, a zip, a spreadsheet and an unknown binary all left raw (garbled)
+ * content in the database. Nothing capped the length either, so a large
+ * document could leave megabytes of text behind.
+ *
+ * Safe to rerun: it only writes a row whose text has to change.
+ *
+ * `bun src/scripts/prune-media-text-content.ts`
+ */
+async function main() {
+    const result = await pruneMediaTextContent();
+
+    for (const change of result.changes) {
+        console.log(
+            change.after_chars === 0
+                ? `Media ${change.media_id} (${change.name}): dropped ${change.before_chars} characters.`
+                : `Media ${change.media_id} (${change.name}): cut ${change.before_chars} characters to ${change.after_chars}.`,
+        );
+    }
+
+    console.log(
+        `Done: ${result.cleared} row(s) cleared, ${result.truncated} truncated, about ${Math.round(result.reclaimed_bytes / 1024)} KB reclaimed from ${result.checked} row(s) with text.`,
+    );
+
+    if (result.reclaimed_bytes > 0) {
+        console.log(
+            `The freed space stays in the database file until it is vacuumed: run "bunx bun-sqlite vacuum" (or restore from a fresh backup) to reclaim it on disk. Kept text is capped at AppData.BotAttachmentTextStoreMaxChars (${AppData.BotAttachmentTextStoreMaxChars}).`,
+        );
+    }
+}
+
+await main();
+process.exit(0);

src/utils/can-extract-text.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, test } from "bun:test";
+import canExtractText from "./can-extract-text";
+
+describe("canExtractText", () => {
+    test("accepts the types the extractor really unpacks", () => {
+        for (const name of ["a.pdf", "b.docx", "c.txt", "d.md", "e.csv", "f.tsv", "g.json", "h.html", "i.xml", "j.yml", "k.yaml", "l.log", "m.markdown"]) {
+            expect(canExtractText({ name })).toBe(true);
+        }
+    });
+
+    test("refuses a binary that only has bytes, not text", () => {
+        for (const name of ["icon.svg", "bundle.zip", "sheet.xlsx", "slides.pptx", "app.bin", "noext", "archive.tar.gz"]) {
+            expect(canExtractText({ name })).toBe(false);
+        }
+    });
+
+    test("reads the type when the name has no extension", () => {
+        expect(canExtractText({ name: "noext", mime_type: "application/pdf" })).toBe(true);
+        expect(canExtractText({ name: "noext", mime_type: "image/svg+xml" })).toBe(false);
+    });
+
+    test("an octet-stream says nothing, so the name decides", () => {
+        expect(canExtractText({ name: "notes.md", mime_type: "application/octet-stream" })).toBe(true);
+        expect(canExtractText({ name: "logo.svg", mime_type: "application/octet-stream" })).toBe(false);
+    });
+});

src/utils/can-extract-text.ts

@@ -0,0 +1,30 @@
+import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
+import extensionOf from "./extension-of";
+
+type Params = {
+    /**
+     * The file's name, e.g. `invoice.pdf`.
+     */
+    name?: string | null;
+    /**
+     * Its MIME type, for a file with no extension in its name.
+     */
+    mime_type?: string | null;
+};
+
+/**
+ * Whether a file's text is worth extracting and keeping on its `media` row.
+ *
+ * Only the extensions the local extractor really unpacks count
+ * (`BotTextFileExtensions`). Everything else - a zip, an SVG, a
+ * spreadsheet, a binary with an unknown name - has no text to store, and
+ * decoding its bytes anyway is how raw file content ended up in the
+ * database.
+ */
+export default function canExtractText({ name, mime_type }: Params): boolean {
+    const ext = extensionOf({ name, mime_type });
+
+    if (!ext) return false;
+
+    return (BotTextFileExtensions as readonly string[]).includes(ext);
+}

src/utils/cap-text-content.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import capTextContent from "./cap-text-content";
+
+describe("capTextContent", () => {
+    test("keeps text under the ceiling as it is", () => {
+        expect(capTextContent({ text: "short" })).toBe("short");
+    });
+
+    test("keeps exactly the ceiling", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars);
+        expect(capTextContent({ text })).toBe(text);
+    });
+
+    test("cuts it and says so", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 500);
+        const capped = capTextContent({ text });
+
+        expect(capped.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+        expect(capped.startsWith("a")).toBe(true);
+        expect(capped).toContain("Truncated");
+        expect(capped).toContain(String(text.length));
+    });
+});

src/utils/cap-text-content.ts

@@ -0,0 +1,31 @@
+import { AppData } from "@/src/data/app-data";
+
+type Params = {
+    /**
+     * The text extracted from a file.
+     */
+    text: string;
+};
+
+/**
+ * Extracted document text, cut to what the `media` row keeps
+ * (`AppData.BotAttachmentTextStoreMaxChars`) with a line saying so.
+ *
+ * The column is a cache, not a copy of the file: a 20 MB document would
+ * otherwise put 20 MB of text into the database, and no reader takes more
+ * than a slice of it anyway.
+ */
+export default function capTextContent({ text }: Params) {
+    const max = AppData.BotAttachmentTextStoreMaxChars;
+
+    if (text.length <= max) return text;
+
+    const note = `\n\n[Truncated: this document is ${text.length} characters long, so only its beginning was kept. Read the file itself for the rest.]`;
+    /**
+     * The note counts against the ceiling too: the column is capped, not
+     * the text plus whatever was appended to it.
+     */
+    const kept = Math.max(0, max - note.length);
+
+    return `${text.slice(0, kept)}${note}`;
+}

src/utils/decide-text-content.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import decideTextContent from "./decide-text-content";
+
+describe("decideTextContent", () => {
+    test("keeps the text of a document the extractor reads", () => {
+        expect(decideTextContent({ name: "invoice.pdf", text: "Total: 12" })).toBe("Total: 12");
+        expect(decideTextContent({ name: "notes.md", text: "# Hi" })).toBe("# Hi");
+        expect(decideTextContent({ name: "rows.csv", text: "a,b" })).toBe("a,b");
+    });
+
+    test("drops the raw bytes of anything else", () => {
+        expect(decideTextContent({ name: "icon.svg", text: "<svg>...</svg>" })).toBeNull();
+        expect(decideTextContent({ name: "bundle.zip", text: "PK\u0003\u0004binary" })).toBeNull();
+        expect(decideTextContent({ name: "sheet.xlsx", text: "PK\u0003\u0004binary" })).toBeNull();
+        expect(decideTextContent({ name: "app.bin", text: "\u0000\u0001\u0002" })).toBeNull();
+    });
+
+    test("nothing to store is null, not an empty string", () => {
+        expect(decideTextContent({ name: "invoice.pdf", text: "" })).toBeNull();
+        expect(decideTextContent({ name: "invoice.pdf", text: "   \n  " })).toBeNull();
+        expect(decideTextContent({ name: "invoice.pdf" })).toBeNull();
+    });
+
+    test("cuts a document over the ceiling", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 10);
+        const decided = decideTextContent({ name: "long.txt", text });
+
+        expect(decided).toContain("Truncated");
+        expect(decided!.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+    });
+
+    test("a document over the ceiling is still cut when the overflow is small", () => {
+        const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 10);
+        const decided = decideTextContent({ name: "long.txt", text })!;
+
+        expect(decided).toContain("Truncated");
+        expect(decided.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
+    });
+});

src/utils/decide-text-content.ts

@@ -0,0 +1,36 @@
+import canExtractText from "./can-extract-text";
+import capTextContent from "./cap-text-content";
+
+type Params = {
+    /**
+     * The file's name, when the row kept it (`media.title`).
+     */
+    name?: string | null;
+    /**
+     * Its MIME type (`media.mime_type`). For old rows this is often just
+     * the extension.
+     */
+    mime_type?: string | null;
+};
+
+/**
+ * What a `media` row's `text_content` should be: the text itself when it
+ * belongs there, `null` when the row should not carry any.
+ *
+ * A row qualifies when its file is one the extractor really unpacks
+ * (`canExtractText`); its text is then cut to the store ceiling
+ * (`capTextContent`). Anything else - a zip, an SVG, a spreadsheet, a
+ * binary that only ever had its bytes decoded - loses its `text_content`,
+ * because that column is an extracted-text cache, not a copy of the file.
+ */
+export default function decideTextContent({
+    name,
+    mime_type,
+    text,
+}: Params & { text?: string | null }): string | null {
+    if (!text || !text.trim()) return null;
+
+    if (!canExtractText({ name, mime_type })) return null;
+
+    return capTextContent({ text });
+}

src/utils/extension-of.ts

@@ -0,0 +1,28 @@
+type Params = {
+    /**
+     * A file name, e.g. `report.pdf`.
+     */
+    name?: string | null;
+    /**
+     * Its MIME type, used when the name has no extension.
+     */
+    mime_type?: string | null;
+};
+
+/**
+ * A file's extension, lower case and without the dot: from its name, else
+ * from its MIME type. Anything but letters and digits is stripped, so a
+ * hostile name can't turn into a path.
+ */
+export default function extensionOf({ name, mime_type }: Params) {
+    const from_name = name?.includes(".") ? name.split(".").pop() : undefined;
+
+    if (from_name) return from_name.toLowerCase().replace(/[^a-z0-9]/g, "");
+
+    const from_mime =
+        mime_type && !mime_type.includes("octet-stream")
+            ? mime_type.split("/").pop()
+            : undefined;
+
+    return from_mime ? from_mime.toLowerCase().replace(/[^a-z0-9]/g, "") : undefined;
+}