tben
d3357cb7df
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
56 lines
2.0 KiB
TypeScript
56 lines
2.0 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import schema from "@/db/schema";
|
|
import mediaColumns from "./media-columns";
|
|
|
|
/**
|
|
* The schema's own list, so a column added later can't quietly be left
|
|
* out of (or forgotten in) the select this guards.
|
|
*/
|
|
function mediaSchemaFields(): string[] {
|
|
const tables = (schema as { tables?: { tableName: string; fields: { fieldName: string }[] }[] }).tables || [];
|
|
const media = tables.find((table) => table.tableName == "media");
|
|
|
|
/**
|
|
* `id`, `created_at` and `updated_at` are added to every table by the
|
|
* ORM, so they aren't in the schema's own field list.
|
|
*/
|
|
return [...(media?.fields || []).map((field) => field.fieldName), "id", "created_at", "updated_at"];
|
|
}
|
|
|
|
describe("mediaColumns", () => {
|
|
test("names every media column except the extracted text", () => {
|
|
const listed = mediaColumns({ alias: "m" })
|
|
.split(", ")
|
|
.map((column) => column.replace(/^m\./, ""))
|
|
.sort();
|
|
|
|
const expected = mediaSchemaFields()
|
|
.filter((field) => field != "text_content")
|
|
.sort();
|
|
|
|
expect(listed).toEqual(expected);
|
|
});
|
|
|
|
test("never selects text_content unless asked", () => {
|
|
expect(mediaColumns()).not.toContain("text_content");
|
|
expect(mediaColumns({ with_text_content: true })).toContain("m.text_content");
|
|
});
|
|
|
|
test("asking for the text adds it to the same full list", () => {
|
|
const with_text = mediaColumns({ with_text_content: true });
|
|
|
|
expect(with_text).toContain(mediaColumns());
|
|
expect(with_text.split(", ")).toHaveLength(mediaColumns().split(", ").length + 1);
|
|
});
|
|
|
|
test("qualifies every column, so a join can't make one ambiguous", () => {
|
|
for (const column of mediaColumns({ alias: "media" }).split(", ")) {
|
|
expect(column.startsWith("media.")).toBe(true);
|
|
}
|
|
});
|
|
|
|
test("the alias defaults to m", () => {
|
|
expect(mediaColumns().startsWith("m.id")).toBe(true);
|
|
});
|
|
});
|