tben
d3357cb7df
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
66 lines
2.8 KiB
TypeScript
66 lines
2.8 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { AppData } from "@/src/data/app-data";
|
|
import planTextContentPrune from "./plan-text-content-prune";
|
|
|
|
describe("planTextContentPrune", () => {
|
|
test("clears the raw bytes stored for a type the extractor can't read", () => {
|
|
const changes = planTextContentPrune({
|
|
rows: [
|
|
{ id: 1, title: "logo.svg", mime_type: "image/svg+xml", text_content: "<svg><path/></svg>" },
|
|
{ id: 2, title: "bundle.zip", mime_type: "application/zip", text_content: "PK\u0003\u0004" },
|
|
{ id: 3, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", text_content: "PK\u0003\u0004" },
|
|
],
|
|
});
|
|
|
|
expect(changes.map((change) => change.media_id)).toEqual([1, 2, 3]);
|
|
expect(changes.every((change) => change.text_content === null)).toBe(true);
|
|
expect(changes.every((change) => change.after_chars === 0)).toBe(true);
|
|
});
|
|
|
|
test("leaves a document's text alone", () => {
|
|
const changes = planTextContentPrune({
|
|
rows: [
|
|
{ id: 4, title: "invoice.pdf", mime_type: "application/pdf", text_content: "Total: 12" },
|
|
{ id: 5, title: "notes.md", mime_type: "text/markdown", text_content: "# Hi" },
|
|
{ id: 6, title: "rows.csv", mime_type: "text/csv", text_content: "a,b\n1,2" },
|
|
],
|
|
});
|
|
|
|
expect(changes).toEqual([]);
|
|
});
|
|
|
|
test("cuts a document over the store ceiling", () => {
|
|
const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 100);
|
|
const changes = planTextContentPrune({
|
|
rows: [{ id: 7, title: "long.txt", mime_type: "text/plain", text_content: text }],
|
|
});
|
|
|
|
expect(changes).toHaveLength(1);
|
|
expect(changes[0]!.after_chars).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
|
|
expect(changes[0]!.text_content).toContain("Truncated");
|
|
expect(changes[0]!.before_chars).toBe(text.length);
|
|
});
|
|
|
|
test("nothing to say about a row with no text or no id", () => {
|
|
expect(
|
|
planTextContentPrune({
|
|
rows: [
|
|
{ id: 8, title: "empty.pdf", text_content: "" },
|
|
{ id: 9, title: "blank.pdf", text_content: " \n " },
|
|
{ id: 10, title: "none.pdf" },
|
|
{ title: "no id.pdf", text_content: "text" },
|
|
],
|
|
}),
|
|
).toEqual([]);
|
|
});
|
|
|
|
test("counting what each change reclaims is left to the caller", () => {
|
|
const changes = planTextContentPrune({
|
|
rows: [{ id: 11, title: "icon.svg", mime_type: "image/svg+xml", text_content: "12345" }],
|
|
});
|
|
|
|
expect(changes).toHaveLength(1);
|
|
expect(changes[0]!.before_chars - changes[0]!.after_chars).toBe(5);
|
|
});
|
|
});
|