Stop storing raw file bytes in media.text_content

`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:

- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
  and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
  `attachmentFileKind` classes everything non-image/audio/video as a
  document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
  text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
  `SELECT m.*` pulled the column into page props and API responses.

On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.

Changes:

- `file-to-text.ts` lists the text extensions explicitly and returns
  undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
  `BotTextFileExtensions`), used by both upload routes and by
  `uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
  catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
  `cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
  re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
  now has to be asked for (`grab-messages-media`'s reply path and
  ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
  an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
  (rerunnable, with a VACUUM hint).

Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
This commit is contained in:
2026-10-23 05:22:04 +01:00
parent 5475241371
commit d3357cb7df
36 changed files with 1034 additions and 43 deletions
@@ -0,0 +1,120 @@
import { describe, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { AppData } from "@/src/data/app-data";
import pruneMediaTextContent, { type SqlRunner } from "./prune-media-text-content";
type Row = { id: number; title: string; mime_type: string; media_type: string; text_content: string | null; updated_at?: number };
function makeDb({ rows }: { rows: Row[] }) {
const db = new Database(":memory:");
db.run(
`CREATE TABLE media (
id INTEGER PRIMARY KEY,
title TEXT,
mime_type TEXT,
media_type TEXT,
text_content TEXT,
updated_at INTEGER
)`,
);
const insert = db.prepare(
`INSERT INTO media (id, title, mime_type, media_type, text_content) VALUES (?, ?, ?, ?, ?)`,
);
for (const row of rows) insert.run(row.id, row.title, row.mime_type, row.media_type, row.text_content);
const run: SqlRunner = async ({ sql, values }) => {
const statement = db.prepare(sql);
if (sql.trim().toUpperCase().startsWith("SELECT")) {
return { payload: statement.all(...((values || []) as never[])) as unknown[] };
}
statement.run(...((values || []) as never[]));
return { payload: [] };
};
const read = (id: number) =>
(db.query(`SELECT text_content FROM media WHERE id = ?`).get(id) as { text_content: string | null }).text_content;
return { db, run, read };
}
describe("pruneMediaTextContent", () => {
test("clears raw bytes from a type the extractor can't read, and keeps a document's text", async () => {
const { run, read } = makeDb({
rows: [
{ id: 1, title: "logo.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "<svg><path d='M0 0'/></svg>" },
{ id: 2, title: "bundle.zip", mime_type: "application/zip", media_type: "document", text_content: "PK\u0003\u0004binary" },
{ id: 3, title: "invoice.pdf", mime_type: "application/pdf", media_type: "document", text_content: "Total: 12" },
],
});
const result = await pruneMediaTextContent({ run });
expect(result.checked).toBe(3);
expect(result.cleared).toBe(2);
expect(result.truncated).toBe(0);
expect(read(1)).toBeNull();
expect(read(2)).toBeNull();
expect(read(3)).toBe("Total: 12");
});
test("cuts an oversized document to the store ceiling", async () => {
const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 1000);
const { run, read } = makeDb({
rows: [{ id: 4, title: "huge.txt", mime_type: "text/plain", media_type: "document", text_content: text }],
});
const result = await pruneMediaTextContent({ run });
const after = read(4)!;
expect(result.truncated).toBe(1);
expect(result.cleared).toBe(0);
expect(after.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
expect(after).toContain("Truncated");
expect(result.reclaimed_bytes).toBeGreaterThan(0);
});
test("is safe to rerun: a second sweep changes nothing", async () => {
const { run, read } = makeDb({
rows: [{ id: 5, title: "icon.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "raw" }],
});
await pruneMediaTextContent({ run });
const second = await pruneMediaTextContent({ run });
expect(second.checked).toBe(0);
expect(second.changes).toEqual([]);
expect(second.reclaimed_bytes).toBe(0);
expect(read(5)).toBeNull();
});
test("stamps updated_at on the rows it rewrites", async () => {
const now = 1_700_000_000_000;
const { run, db } = makeDb({
rows: [{ id: 6, title: "x.zip", mime_type: "application/zip", media_type: "document", text_content: "raw" }],
});
await pruneMediaTextContent({ run, now });
expect(
(db.query(`SELECT updated_at FROM media WHERE id = 6`).get() as { updated_at: number }).updated_at,
).toBe(now);
});
test("reports every change with its row, name and sizes", async () => {
const { run } = makeDb({
rows: [{ id: 7, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", media_type: "document", text_content: "PK\u0003\u0004" }],
});
const result = await pruneMediaTextContent({ run });
expect(result.changes).toEqual([
{ media_id: 7, name: "sheet.xlsx", before_chars: 4, after_chars: 0, text_content: null },
]);
});
});