tben
d3357cb7df
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
114 lines
3.3 KiB
TypeScript
114 lines
3.3 KiB
TypeScript
import TurndownService from "turndown";
|
|
import { extractText, getDocumentProxy } from "unpdf";
|
|
import mammoth from "mammoth";
|
|
// import * as XLSX from "xlsx";
|
|
import bufferFromUrl from "@/src/utils/buffer-from-url";
|
|
|
|
const turndownService = new TurndownService({
|
|
headingStyle: "atx",
|
|
codeBlockStyle: "fenced",
|
|
});
|
|
|
|
type Params = {
|
|
file_stream?: Buffer<ArrayBuffer>;
|
|
file_path?: string;
|
|
/**
|
|
* File extension, e.g. `pdf`, `docx`, `xlsx`
|
|
*/
|
|
file_type?: string;
|
|
url?: string;
|
|
};
|
|
|
|
export default async function fileToText({
|
|
file_type,
|
|
file_stream,
|
|
file_path,
|
|
url,
|
|
}: Params): Promise<string | undefined> {
|
|
let buffer: Buffer | undefined;
|
|
let ext: string;
|
|
|
|
if (url) {
|
|
const result = await bufferFromUrl(url);
|
|
buffer = result.buffer;
|
|
ext = result.ext;
|
|
} else {
|
|
buffer = file_path
|
|
? Buffer.from(await Bun.file(file_path).arrayBuffer())
|
|
: file_stream;
|
|
const known_ext = file_type || file_path?.split(".").pop();
|
|
|
|
/**
|
|
* No extension and no type: nothing says this is text, so nothing
|
|
* gets decoded. Guessing "txt" is how a binary became text.
|
|
*/
|
|
if (!known_ext) return undefined;
|
|
|
|
ext = known_ext.toString().toLowerCase();
|
|
}
|
|
|
|
if (!buffer) return undefined;
|
|
|
|
switch (ext) {
|
|
case "docx": {
|
|
const { value: html } = await mammoth.convertToHtml({ buffer });
|
|
return turndownService.turndown(html);
|
|
}
|
|
|
|
case "pdf": {
|
|
try {
|
|
const pdfUintBuffer = new Uint8Array(buffer);
|
|
const pdfDocProxy = await getDocumentProxy(pdfUintBuffer);
|
|
const { text } = await extractText(pdfDocProxy, {
|
|
mergePages: true,
|
|
});
|
|
return text;
|
|
} catch (err: any) {
|
|
console.error(
|
|
`PDF text extraction failed: ${err?.message ?? err}`,
|
|
);
|
|
throw new Error(
|
|
`Failed to extract text from PDF: ${err?.message ?? "unknown error"}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// case "xlsx":
|
|
// case "csv": {
|
|
// const workbook = XLSX.read(buffer, { type: "buffer" });
|
|
// return workbook.SheetNames.map((name) => {
|
|
// const sheet = workbook.Sheets[name];
|
|
// return `### Sheet: ${name}\n\n${XLSX.utils.sheet_to_csv(sheet!)}`;
|
|
// }).join("\n\n");
|
|
// }
|
|
|
|
/**
|
|
* The text types read as they are. Listed one by one, never a
|
|
* catch-all: decoding a zip, an SVG or a spreadsheet as UTF-8
|
|
* fills the media row with raw bytes instead of text.
|
|
*/
|
|
case "txt":
|
|
case "text":
|
|
case "md":
|
|
case "markdown":
|
|
case "csv":
|
|
case "tsv":
|
|
case "json":
|
|
case "html":
|
|
case "htm":
|
|
case "xml":
|
|
case "yml":
|
|
case "yaml":
|
|
case "log":
|
|
return buffer.toString("utf-8");
|
|
|
|
default:
|
|
/**
|
|
* Nothing readable locally: the caller decides whether to read
|
|
* it another way (an image or audio through the harness) or to
|
|
* leave its text unset.
|
|
*/
|
|
return undefined;
|
|
}
|
|
}
|