tbenandClaude Opus 5.5
cdb93ac1bc
- Media rows can no longer be created through the CRUD API, updates only take description fields on the caller's own row, and every media read or delete goes through mediaFilePath, which refuses paths outside DATA_DIR or inside the database folder. - Uploads strip the file name and extension to safe characters and keep only title, description and summary from the request's media object. - Folder imports no longer follow symlinks (walkRegularFiles), and the source path is quoted in the computer's shell. - User and bot files are served with nosniff, and HTML, SVG, XML and scripts are sandboxed by CSP. - Sandbox cookies are kept to the computer's prefix, and websocket upgrades on /ws and the computer proxy must come from the app's own address (README and Help note nginx's Host header). - user_types CRUD writes go through the assignment matrix; members only read their own user record; read-only users can read CRUD again; grabUsers builds valid SQL for array search terms and bad pages; an admin media DELETE needs an id; read-only users can't delete media. - Rate limits and the audit log use the connection's address, trusting forwarded headers only from a proxy, and login limits are split per address, per account from that address and per account overall. Co-Authored-By: Claude Opus 5.5 <[email protected]>
97 lines
3.3 KiB
TypeScript
97 lines
3.3 KiB
TypeScript
import mediaFilePath from "@/src/utils/media-file-path";
|
|
import { AppData } from "@/src/data/app-data";
|
|
import grabMessagesMedia, {
|
|
type MessageMediaRow,
|
|
} from "@/src/functions/backend/db/bots/grab-messages-media";
|
|
import type { HarnessContentPart } from "@/src/harness/types";
|
|
import type { BotMessageWithAttachments } from "@/src/types";
|
|
import describeAttachment from "./describe-attachment";
|
|
|
|
type Params = {
|
|
/**
|
|
* The conversation a reply reads, oldest first.
|
|
*/
|
|
messages: BotMessageWithAttachments[];
|
|
};
|
|
|
|
/**
|
|
* The files of the latest few messages that have any
|
|
* (`AppData.BotReplyAttachmentMessages`), as model input by message id:
|
|
* images and audio as themselves (a model that can't take them gets them
|
|
* read first), documents as their text, videos as a note. Older files are
|
|
* only named (`botMessagesToHarness`).
|
|
*/
|
|
export default async function loadAttachmentParts({ messages }: Params) {
|
|
const recent = messages
|
|
.filter((message) => message.attachments?.length)
|
|
.slice(-AppData.BotReplyAttachmentMessages)
|
|
.map((message) => Number(message.id));
|
|
|
|
const media = await grabMessagesMedia({
|
|
message_ids: recent,
|
|
with_text_content: true,
|
|
});
|
|
const parts = new Map<number, HarnessContentPart[]>();
|
|
|
|
for (const [message_id, files] of media) {
|
|
parts.set(
|
|
message_id,
|
|
(await Promise.all(files.map((file) => fileToParts({ file })))).flat(),
|
|
);
|
|
}
|
|
|
|
return parts;
|
|
}
|
|
|
|
async function fileToParts({ file }: { file: MessageMediaRow }): Promise<HarnessContentPart[]> {
|
|
const name = file.title || `file ${file.id}`;
|
|
/**
|
|
* The upload's own id, so a bot can name it back to a tool
|
|
* (`set_picture` and `update_workspace` take one) instead of guessing
|
|
* which file the user means.
|
|
*/
|
|
const label = describeAttachment({ name, id: file.id! });
|
|
|
|
if (file.media_type == "document") {
|
|
const text = file.text_content?.trim();
|
|
|
|
if (!text) {
|
|
return [{ type: "text", text: `[Attached file ${label}: its text couldn't be read.]` }];
|
|
}
|
|
|
|
const cut = text.length > AppData.BotAttachmentMaxTextChars;
|
|
|
|
return [
|
|
{
|
|
type: "text",
|
|
text: `Attached file ${label}:\n\n${text.slice(0, AppData.BotAttachmentMaxTextChars)}${cut ? `\n\n[The rest of "${name}" was cut: it's ${text.length} characters long.]` : ""}`,
|
|
},
|
|
];
|
|
}
|
|
|
|
if (file.media_type == "video") {
|
|
return [{ type: "text", text: `[Attached video ${label}: videos can't be read yet.]` }];
|
|
}
|
|
|
|
const absolute = mediaFilePath({ write_path: file.media_write_path });
|
|
const disk = absolute ? Bun.file(absolute) : null;
|
|
|
|
if (!disk || !(await disk.exists())) {
|
|
return [{ type: "text", text: `[Attached file ${label} is missing on the server.]` }];
|
|
}
|
|
|
|
const base64 = Buffer.from(await disk.arrayBuffer()).toString("base64");
|
|
|
|
if (file.media_type == "image") {
|
|
return [
|
|
{ type: "text", text: `[Attached image ${label}]` },
|
|
{ type: "image", url: `data:image/${file.mime_type || "jpeg"};base64,${base64}`, name },
|
|
];
|
|
}
|
|
|
|
return [
|
|
{ type: "text", text: `[Attached audio ${label}]` },
|
|
{ type: "audio", data: base64, format: file.mime_type || "mp3", name },
|
|
];
|
|
}
|