Stop storing raw file bytes in media.text_content
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import pruneMediaTextContent, { type SqlRunner } from "./prune-media-text-content";
|
||||
|
||||
type Row = { id: number; title: string; mime_type: string; media_type: string; text_content: string | null; updated_at?: number };
|
||||
|
||||
function makeDb({ rows }: { rows: Row[] }) {
|
||||
const db = new Database(":memory:");
|
||||
|
||||
db.run(
|
||||
`CREATE TABLE media (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title TEXT,
|
||||
mime_type TEXT,
|
||||
media_type TEXT,
|
||||
text_content TEXT,
|
||||
updated_at INTEGER
|
||||
)`,
|
||||
);
|
||||
|
||||
const insert = db.prepare(
|
||||
`INSERT INTO media (id, title, mime_type, media_type, text_content) VALUES (?, ?, ?, ?, ?)`,
|
||||
);
|
||||
|
||||
for (const row of rows) insert.run(row.id, row.title, row.mime_type, row.media_type, row.text_content);
|
||||
|
||||
const run: SqlRunner = async ({ sql, values }) => {
|
||||
const statement = db.prepare(sql);
|
||||
|
||||
if (sql.trim().toUpperCase().startsWith("SELECT")) {
|
||||
return { payload: statement.all(...((values || []) as never[])) as unknown[] };
|
||||
}
|
||||
|
||||
statement.run(...((values || []) as never[]));
|
||||
|
||||
return { payload: [] };
|
||||
};
|
||||
|
||||
const read = (id: number) =>
|
||||
(db.query(`SELECT text_content FROM media WHERE id = ?`).get(id) as { text_content: string | null }).text_content;
|
||||
|
||||
return { db, run, read };
|
||||
}
|
||||
|
||||
describe("pruneMediaTextContent", () => {
|
||||
test("clears raw bytes from a type the extractor can't read, and keeps a document's text", async () => {
|
||||
const { run, read } = makeDb({
|
||||
rows: [
|
||||
{ id: 1, title: "logo.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "<svg><path d='M0 0'/></svg>" },
|
||||
{ id: 2, title: "bundle.zip", mime_type: "application/zip", media_type: "document", text_content: "PK\u0003\u0004binary" },
|
||||
{ id: 3, title: "invoice.pdf", mime_type: "application/pdf", media_type: "document", text_content: "Total: 12" },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await pruneMediaTextContent({ run });
|
||||
|
||||
expect(result.checked).toBe(3);
|
||||
expect(result.cleared).toBe(2);
|
||||
expect(result.truncated).toBe(0);
|
||||
expect(read(1)).toBeNull();
|
||||
expect(read(2)).toBeNull();
|
||||
expect(read(3)).toBe("Total: 12");
|
||||
});
|
||||
|
||||
test("cuts an oversized document to the store ceiling", async () => {
|
||||
const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 1000);
|
||||
const { run, read } = makeDb({
|
||||
rows: [{ id: 4, title: "huge.txt", mime_type: "text/plain", media_type: "document", text_content: text }],
|
||||
});
|
||||
|
||||
const result = await pruneMediaTextContent({ run });
|
||||
const after = read(4)!;
|
||||
|
||||
expect(result.truncated).toBe(1);
|
||||
expect(result.cleared).toBe(0);
|
||||
expect(after.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
|
||||
expect(after).toContain("Truncated");
|
||||
expect(result.reclaimed_bytes).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("is safe to rerun: a second sweep changes nothing", async () => {
|
||||
const { run, read } = makeDb({
|
||||
rows: [{ id: 5, title: "icon.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "raw" }],
|
||||
});
|
||||
|
||||
await pruneMediaTextContent({ run });
|
||||
const second = await pruneMediaTextContent({ run });
|
||||
|
||||
expect(second.checked).toBe(0);
|
||||
expect(second.changes).toEqual([]);
|
||||
expect(second.reclaimed_bytes).toBe(0);
|
||||
expect(read(5)).toBeNull();
|
||||
});
|
||||
|
||||
test("stamps updated_at on the rows it rewrites", async () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const { run, db } = makeDb({
|
||||
rows: [{ id: 6, title: "x.zip", mime_type: "application/zip", media_type: "document", text_content: "raw" }],
|
||||
});
|
||||
|
||||
await pruneMediaTextContent({ run, now });
|
||||
|
||||
expect(
|
||||
(db.query(`SELECT updated_at FROM media WHERE id = 6`).get() as { updated_at: number }).updated_at,
|
||||
).toBe(now);
|
||||
});
|
||||
|
||||
test("reports every change with its row, name and sizes", async () => {
|
||||
const { run } = makeDb({
|
||||
rows: [{ id: 7, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", media_type: "document", text_content: "PK\u0003\u0004" }],
|
||||
});
|
||||
|
||||
const result = await pruneMediaTextContent({ run });
|
||||
|
||||
expect(result.changes).toEqual([
|
||||
{ media_id: 7, name: "sheet.xlsx", before_chars: 4, after_chars: 0, text_content: null },
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user