tben
d3357cb7df
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
87 lines
3.3 KiB
TypeScript
87 lines
3.3 KiB
TypeScript
import userAuth from "@/src/functions/backend/auth/user-auth";
|
|
import { logError } from "@/src/functions/backend/logger";
|
|
import mediaToAttachment from "@/src/functions/backend/db/bots/media-to-attachment";
|
|
import uploadAndRecordMedia from "@/src/functions/backend/db/users/media/upload-and-record-media";
|
|
import { AppData } from "@/src/data/app-data";
|
|
import { UploadBotAttachmentRequestSchema } from "@/src/schemas/bot-schema";
|
|
import type { BotMessageAttachment } from "@/src/types";
|
|
import AppError from "@/src/utils/app-error";
|
|
import attachmentFileKind from "@/src/utils/attachment-file-kind";
|
|
import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
|
|
import canExtractText from "@/src/utils/can-extract-text";
|
|
import parseWithSchema from "@/src/utils/parse-with-schema";
|
|
import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
|
|
import safeErrorMessage from "@/src/utils/safe-error-message";
|
|
import type {
|
|
APIResponseObject,
|
|
BunextAPIRouteHandler,
|
|
} from "@moduletrace/bunext/types";
|
|
|
|
/**
|
|
* Stores one file for a chat message, private to the session user, before
|
|
* the message is sent with its id. Images are resized; documents have
|
|
* their text pulled out now, so bots can read them later.
|
|
*/
|
|
export const handler: BunextAPIRouteHandler<
|
|
APIResponseObject<BotMessageAttachment>
|
|
> = async ({ req, body }) => {
|
|
try {
|
|
if (req.method != "POST") {
|
|
throw new AppError(`Method not allowed`);
|
|
}
|
|
|
|
const { user, user_types } = await userAuth({ req });
|
|
|
|
if (!user?.logged_in_status || !user.id) {
|
|
return { success: false, msg: `Unauthorized`, logoutUser: true };
|
|
}
|
|
|
|
if (isReadOnlyUser({ user_types: user_types || [] })) {
|
|
throw new AppError(`This user can only read records.`);
|
|
}
|
|
|
|
const { data_url, name } = parseWithSchema({
|
|
schema: UploadBotAttachmentRequestSchema,
|
|
data: body,
|
|
});
|
|
|
|
const mime = data_url.match(/^data:([^;,]*)/)?.[1] || "";
|
|
const { media_type, ext } = attachmentFileKind({ mime, name });
|
|
const { buffer } = await bufferFromBase64(data_url);
|
|
|
|
if (!buffer.length) {
|
|
throw new AppError(`${name} is empty.`);
|
|
}
|
|
|
|
if (buffer.length > AppData.BotAttachmentMaxBytes) {
|
|
throw new AppError(
|
|
`${name} is too big. Files can be up to ${Math.round(AppData.BotAttachmentMaxBytes / 1024 / 1024)} MB.`,
|
|
);
|
|
}
|
|
|
|
const res = await uploadAndRecordMedia({
|
|
data: buffer,
|
|
media_type,
|
|
media_mime_type: ext,
|
|
media_paradigms: ["bot-attachment"],
|
|
media: { title: name },
|
|
extract_media_text_content: canExtractText({ name, mime_type: mime }),
|
|
user: { id: Number(user.id) },
|
|
}).catch(async (error) => {
|
|
await logError({ title: `Attachment ${name} couldn't be saved`, error }).catch(() => {});
|
|
throw new AppError(`Couldn't save ${name}. If it's an image, try another format.`);
|
|
});
|
|
|
|
if (!res.singleRes?.id) {
|
|
throw new AppError(`Couldn't save ${name}.`);
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
singleRes: mediaToAttachment({ media: res.singleRes }),
|
|
};
|
|
} catch (error: unknown) {
|
|
return { success: false, msg: safeErrorMessage({ error }) };
|
|
}
|
|
};
|