tben
d3357cb7df
`text_content` is an extracted-text cache, but nothing stopped it from
holding a dump of bytes:
- `fileToText` ended its switch with `default: return buffer.toString("utf-8")`
and defaulted a missing extension to "txt", so any binary was decoded.
- Both extraction callers gated on `media_type == "document"`, and
`attachmentFileKind` classes everything non-image/audio/video as a
document: an SVG, a zip, a spreadsheet, a .bin.
- Nothing capped the length, so a 20 MB attachment could leave 20 MB of
text in the database.
- `storeChatAttachments` copied the text onto a second `media` row, and
`SELECT m.*` pulled the column into page props and API responses.
On a real install an SVG held 6852 characters of markup and a
`.designcraft` file held 142 KB of raw bytes.
Changes:
- `file-to-text.ts` lists the text extensions explicitly and returns
undefined for anything else; no extension is no text.
- `can-extract-text.ts` is the one allowlist (reusing
`BotTextFileExtensions`), used by both upload routes and by
`uploadAndRecordMedia`; `extension-of.ts` is shared with the storage
catalog instead of duplicated.
- `BotAttachmentTextStoreMaxChars` (200k) caps the column via
`cap-text-content.ts`.
- `store-chat-attachments.ts` no longer copies the text; ingestion
re-reads the bytes once.
- `media-columns.ts` selects every media column but `text_content`, which
now has to be asked for (`grab-messages-media`'s reply path and
ingestion do); the media CRUD routes sanitize rows they write.
- `ingest-bot-file.ts` decides on size alone, since a cached slice made
an oversized file look already read.
- `bun run media:prune-text` cleans rows that were already stored
(rerunnable, with a VACUUM hint).
Verified on the dev database: both bad rows cleared (146 KB), rerun
reports nothing to do.
87 lines
3.0 KiB
JSON
87 lines
3.0 KiB
JSON
{
|
|
"name": "cobble",
|
|
"version": "0.1.3",
|
|
"module": "index.ts",
|
|
"type": "module",
|
|
"private": true,
|
|
"scripts": {
|
|
"dev": "bun src/server.ts",
|
|
"gen:favicon": "bun src/scripts/generate-brand-assets.ts",
|
|
"build": "bunx bunext build",
|
|
"db:schema": "bunx bun-sqlite schema -t",
|
|
"db:admin": "bunx bun-sqlite admin",
|
|
"db:backup": "bunx bun-sqlite backup",
|
|
"seed": "bun src/scripts/seed-super-admin.tsx",
|
|
"seed:bots": "bun src/scripts/seed-bots.ts",
|
|
"media:prune-text": "bun src/scripts/prune-media-text-content.ts",
|
|
"start": "MAIN_SERVER=1 NODE_ENV=production bun src/server.ts",
|
|
"typecheck": "tsc --noEmit",
|
|
"test": "bun test",
|
|
"check:bundle": "bun src/scripts/check-bundle-size.ts",
|
|
"publish:github": "bash scripts/publish-github.sh",
|
|
"publish:gitea": "bash scripts/publish-gitea.sh"
|
|
},
|
|
"devDependencies": {
|
|
"@next/mdx": "^16.3.0",
|
|
"@types/ace": "^0.0.52",
|
|
"@types/bun": "latest",
|
|
"@types/lodash": "^4.17.25",
|
|
"@types/mdx": "^2.0.14",
|
|
"@types/node": "^26.2.0",
|
|
"@types/nodemailer": "^8.0.1",
|
|
"@types/react": "^19.2.18",
|
|
"@types/react-dom": "^19.2.4",
|
|
"@types/sanitize-html": "^2.16.2",
|
|
"@types/turndown": "^5.0.6",
|
|
"@types/web-push": "^3.6.4",
|
|
"tailwindcss": "^4.3.3"
|
|
},
|
|
"peerDependencies": {
|
|
"typescript": "^7.0.2"
|
|
},
|
|
"dependencies": {
|
|
"@floating-ui/dom": "1.8.0",
|
|
"@modelcontextprotocol/sdk": "^1.32.1",
|
|
"@moduletrace/bun-sqlite": "^1.3.0",
|
|
"@moduletrace/bunext": "^1.3.3",
|
|
"@moduletrace/twui": "^0.1.6",
|
|
"@tiptap/extension-bubble-menu": "3.32.0",
|
|
"@tiptap/extension-mention": "3.32.0",
|
|
"@tiptap/extension-placeholder": "3.32.0",
|
|
"@tiptap/markdown": "3.32.0",
|
|
"@tiptap/pm": "3.32.0",
|
|
"@tiptap/react": "3.32.0",
|
|
"@tiptap/starter-kit": "3.32.0",
|
|
"ace-builds": "^1.44.0",
|
|
"dayjs": "^1.11.23",
|
|
"gray-matter": "^4.0.3",
|
|
"html-to-react": "^1.7.0",
|
|
"lodash": "^4.18.1",
|
|
"lucide-react": "^1.59.0",
|
|
"mammoth": "^1.12.1",
|
|
"mdx": "^0.3.1",
|
|
"next-mdx-remote": "^6.0.0",
|
|
"nodemailer": "^9.0.5",
|
|
"openai": "^7.4.0",
|
|
"pdf-lib": "^1.17.1",
|
|
"react-code-blocks": "^0.1.6",
|
|
"react-markdown": "^10.1.0",
|
|
"react-responsive-modal": "^7.2.0",
|
|
"rehype-prism-plus": "^2.0.2",
|
|
"remark-breaks": "4.0.0",
|
|
"remark-gfm": "^4.0.1",
|
|
"sanitize-html": "^2.18.1",
|
|
"sharp": "^0.35.3",
|
|
"tailwind-merge": "^3.6.0",
|
|
"turndown": "^7.2.4",
|
|
"unpdf": "^1.8.0",
|
|
"use-sync-external-store": "file:./packages/use-sync-external-store-esm",
|
|
"web-push": "^3.6.7",
|
|
"xlsx": "^0.18.5",
|
|
"zod": "4.1.12"
|
|
},
|
|
"overrides": {
|
|
"use-sync-external-store": "file:./packages/use-sync-external-store-esm"
|
|
}
|
|
}
|