Stop storing raw file bytes in media.text_content

`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:

- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
  and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
  `attachmentFileKind` classes everything non-image/audio/video as a
  document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
  text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
  `SELECT m.*` pulled the column into page props and API responses.

On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.

Changes:

- `file-to-text.ts` lists the text extensions explicitly and returns
  undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
  `BotTextFileExtensions`), used by both upload routes and by
  `uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
  catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
  `cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
  re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
  now has to be asked for (`grab-messages-media`'s reply path and
  ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
  an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
  (rerunnable, with a VACUUM hint).

Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
This commit is contained in:
2026-10-23 05:22:04 +01:00
parent 5475241371
commit d3357cb7df
36 changed files with 1034 additions and 43 deletions
@@ -3,6 +3,7 @@ import type {
BUN_SQLITE_COBBLE_BOT_FILES,
BUN_SQLITE_COBBLE_MEDIA,
} from "@/db/types/db";
import mediaColumns from "@/src/functions/backend/db/media/media-columns";
import { canReachScope, type StorageScope } from "@/src/utils/storage-access";
type Params = {
@@ -59,7 +60,7 @@ export default async function grabBotFile({
const media = file.media_id
? await BunSQLite.sql<BUN_SQLITE_COBBLE_MEDIA>({
sql: `SELECT * FROM media WHERE id = ? LIMIT 1`,
sql: `SELECT ${mediaColumns({ alias: "m" })} FROM media m WHERE m.id = ? LIMIT 1`,
values: [file.media_id],
}).then((r) => r.payload?.[0] || null)
: null;
@@ -57,7 +57,13 @@ export default async function ingestBotFile({
}).then((res) => res.payload?.[0])
: undefined;
if (Number(file.size) > AppData.IngestMaxBytes && !media?.text_content) {
/**
* Decided on the file's size alone. A cached `text_content` is not a
* reason to read an oversized file: it was only ever a slice of it,
* and taking it as the whole document made a file too big to unpack
* look already read.
*/
if (Number(file.size) > AppData.IngestMaxBytes) {
await BunSQLite.sql({
sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`,
values: [`Too large to read (over ${AppData.IngestMaxBytes / (1024 * 1024)} MB).`, Date.now(), file.id],
@@ -1,5 +1,6 @@
import type { MediaType } from "@/src/types";
import { BotImageFileExtensions } from "@/src/dict/bot-file-dict";
import extensionOf from "@/src/utils/extension-of";
type Params = {
/**
@@ -27,19 +28,10 @@ export default function mediaTypeForFile({ name, mime_type }: Params): MediaType
}
/**
* Its extension, lower case, without the dot.
* Its extension, lower case, without the dot. Shared, so the storage
* catalog and the text extractor read a name the same way.
*/
export function extensionOf({ name, mime_type }: Params) {
const from_name = name?.includes(".") ? name.split(".").pop() : undefined;
if (from_name) return from_name.toLowerCase().replace(/[^a-z0-9]/g, "");
const from_mime = mime_type && !mime_type.includes("octet-stream")
? mime_type.split("/").pop()
: undefined;
return from_mime ? from_mime.toLowerCase().replace(/[^a-z0-9]/g, "") : undefined;
}
export { extensionOf };
/**
* A MIME type for the row, from the extension when there isn't one.
@@ -0,0 +1,34 @@
import { describe, expect, test } from "bun:test";
/**
* The module's own source, since the regression it guards is what the code
* writes to the database, not what it returns.
*/
async function source() {
return await Bun.file(
`${import.meta.dir}/store-chat-attachments.ts`,
).text();
}
describe("storeChatAttachments", () => {
/**
* The same text used to be copied onto the new `bot-file` media row,
* so a chat upload kept a second copy of its extracted text (and left
* it behind when the upload was pruned). The text belongs only on the
* message's own `bot-attachment` row, where a reply reads it;
* ingestion re-reads the bytes through `extractBotFileText`.
*/
test("never writes text_content onto the file's media row", async () => {
const code = await source();
expect(code).not.toContain("SET text_content");
expect(code).not.toContain("text_content = ?");
});
test("still saves the bytes and queues the ingest", async () => {
const code = await source();
expect(code).toContain("recordBotFile(");
expect(code).toContain("enqueueFileIngest(");
});
});
@@ -1,4 +1,3 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
import { BotUploadFolder } from "@/src/dict/bot-file-dict";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
@@ -44,6 +43,13 @@ export default async function storeChatAttachments({
*/
const name = safeName({ name: file.title || `file-${file.id}` });
/**
* The text stays on the message's own `bot-attachment` row,
* where a reply reads it. Copying it onto the file's row kept
* the same text twice and left the copy behind when the
* upload was pruned; ingestion re-reads the bytes once
* through `extractBotFileText` instead.
*/
const saved = await recordBotFile({
bot_id,
scope: workspace_id ? "workspace" : "bot",
@@ -57,17 +63,6 @@ export default async function storeChatAttachments({
user_id,
});
/**
* The same text the upload already extracted, so a PDF isn't
* unpacked a second time.
*/
if (file.text_content && saved.media_id) {
await BunSQLite.sql({
sql: `UPDATE media SET text_content = ? WHERE id = ?`,
values: [file.text_content, saved.media_id],
}).catch(() => {});
}
await enqueueFileIngest({ file_id: saved.id! });
} catch (error) {
await logError({