tben
d3357cb7df
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
89 lines
3.0 KiB
TypeScript
89 lines
3.0 KiB
TypeScript
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
|
|
import { BotUploadFolder } from "@/src/dict/bot-file-dict";
|
|
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
|
|
import { logError } from "@/src/functions/backend/logger";
|
|
import grabDirNames from "@/src/utils/grab-dir-names";
|
|
import recordBotFile from "./record-bot-file";
|
|
|
|
type Params = {
|
|
bot_id: number | string;
|
|
workspace_id?: number | string | null;
|
|
user_id: number | string;
|
|
media: BUN_SQLITE_COBBLE_MEDIA[];
|
|
};
|
|
|
|
/**
|
|
* Copies the files the user attached to a message into the conversation's
|
|
* folder (`uploads/<yyyy-mm>/<name>`), so they stay searchable after the
|
|
* message scrolls away and a later run can find them. Best effort: a
|
|
* failure is logged, never shown, and the message still goes out.
|
|
*/
|
|
export default async function storeChatAttachments({
|
|
bot_id,
|
|
workspace_id,
|
|
user_id,
|
|
media,
|
|
}: Params) {
|
|
if (!media.length) return;
|
|
|
|
const { DATA_DIR } = grabDirNames();
|
|
const folder = `${BotUploadFolder}/${new Date().toISOString().slice(0, 7)}`;
|
|
|
|
for (const file of media) {
|
|
try {
|
|
if (!file.media_write_path) continue;
|
|
|
|
const source = Bun.file(`${DATA_DIR}/${file.media_write_path}`);
|
|
|
|
if (!(await source.exists())) continue;
|
|
|
|
/**
|
|
* Its name, made safe and unique enough that a second upload
|
|
* of "invoice.pdf" doesn't overwrite the first.
|
|
*/
|
|
const name = safeName({ name: file.title || `file-${file.id}` });
|
|
|
|
/**
|
|
* The text stays on the message's own `bot-attachment` row,
|
|
* where a reply reads it. Copying it onto the file's row kept
|
|
* the same text twice and left the copy behind when the
|
|
* upload was pruned; ingestion re-reads the bytes once
|
|
* through `extractBotFileText` instead.
|
|
*/
|
|
const saved = await recordBotFile({
|
|
bot_id,
|
|
scope: workspace_id ? "workspace" : "bot",
|
|
owner_id: workspace_id || bot_id,
|
|
path: `${folder}/${name}`,
|
|
data: new Uint8Array(await source.arrayBuffer()),
|
|
purpose: "upload",
|
|
source: "chat-upload",
|
|
media_type: (file.media_type as any) || undefined,
|
|
mime_type: file.mime_type || undefined,
|
|
user_id,
|
|
});
|
|
|
|
await enqueueFileIngest({ file_id: saved.id! });
|
|
} catch (error) {
|
|
await logError({
|
|
title: `A chat attachment couldn't be saved to the bot's files`,
|
|
error,
|
|
metadata: { media_id: file.id, bot_id },
|
|
}).catch(() => {});
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A file name that is safe on disk.
|
|
*/
|
|
function safeName({ name }: { name: string }) {
|
|
const cleaned = name
|
|
.replace(/[<>:"|?*/\\]/g, "-")
|
|
.replace(/\s+/g, " ")
|
|
.trim()
|
|
.slice(0, 100);
|
|
|
|
return cleaned || "file";
|
|
}
|