Let bots set a picture from a file they can see
A coordinator asked to set an uploaded logo as a workspace picture could not: the picture tools take a media id, and nothing ever gave the bot one. Attachments reached the model as "[Attached image \"logo.png\"]", every file tool answers with a [file:ID] ref, and an SVG is deliberately stored as a document rather than an image. - copyMediaAsPicture copies an approved image into a new picture row (bot-image/workspace-image). The source keeps its row and bytes, so a message that attached it still loads and changing a picture can never take a file away from a conversation. - grabPictureSource resolves a picture by [file:ID] (the same scope gate every file tool uses) or by the user's own upload; parseFileRef reads the [file:12] form. - The new set_picture tool ships on every run: a bot sets its own face, the coordinator can also dress a worker. update_workspace gained picture_file_id, and setBotPicture/setProfileImage share the copy and the delete. - removeReplacedPicture deletes a replaced picture only when it really is one, never a chat upload. - uploadAndRecordMedia writes images in their own format: PNG stays PNG with its transparency, WebP stays WebP, GIF becomes PNG, everything else JPEG, and the row records the real mime type (it used to say jpeg for every image). - Pictures change live: BotUpdatedData carries image_media_id and a new server:workspace-updated pushes workspace pictures, relayed between processes. - Attachments are labelled with their media id, and BotFilesNote tells a bot that a [file:ID] can become a picture. - grabRunScopes is now the single source of a run's reachable folders, shared by the file tools, imports and the picture tools.
This commit is contained in:
@@ -97,7 +97,7 @@ describe("botMessagesToHarness with a file-only reply", () => {
|
||||
).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: `[Replying to your earlier message: "Send me the logo."]\n\n[Attached earlier: "logo.png"]`,
|
||||
content: `[Replying to your earlier message: "Send me the logo."]\n\n[Attached earlier: "logo.png", media id 9]`,
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -130,7 +130,13 @@ describe("botMessagesToHarness with files", () => {
|
||||
botMessagesToHarness({
|
||||
messages: [{ id: 2, author: "user", text: "", attachments }],
|
||||
}),
|
||||
).toEqual([{ role: "user", content: `[Attached earlier: "shot.png"]` }]);
|
||||
).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
/** Its media id rides along, so a picture can be set from it. */
|
||||
content: `[Attached earlier: "shot.png", media id 9]`,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { HarnessContentPart, HarnessMessage } from "@/src/harness/types";
|
||||
import type { BotMessageWithAttachments } from "@/src/types";
|
||||
import describeMessageRefs, { type MessageRefNames } from "@/src/utils/describe-message-refs";
|
||||
import describeAttachment from "./describe-attachment";
|
||||
|
||||
type Params = {
|
||||
messages: Pick<
|
||||
@@ -87,7 +88,7 @@ function messageParts({
|
||||
...content,
|
||||
{
|
||||
type: "text",
|
||||
text: `[Attached earlier: ${message.attachments.map((file) => `"${file.name}"`).join(", ")}]`,
|
||||
text: `[Attached earlier: ${message.attachments.map((file) => describeAttachment({ name: file.name, id: file.id })).join(", ")}]`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import describeAttachment from "./describe-attachment";
|
||||
|
||||
describe("describeAttachment", () => {
|
||||
test("names the upload and its media id, so a tool can be given it", () => {
|
||||
expect(describeAttachment({ name: "summit-logo.png", id: 42 })).toBe(
|
||||
`"summit-logo.png", media id 42`,
|
||||
);
|
||||
});
|
||||
|
||||
test("takes the id as text as well as a number", () => {
|
||||
expect(describeAttachment({ name: "logo.svg", id: "7" })).toBe(`"logo.svg", media id 7`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
type Params = {
|
||||
/**
|
||||
* The name the file was uploaded with.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* Its `media` id.
|
||||
*/
|
||||
id: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* How an upload is described to the model: its name and its media id, e.g.
|
||||
* `"summit-logo.png", media id 42`.
|
||||
*
|
||||
* The id matters: the picture tools (`set_picture`, `update_workspace`)
|
||||
* take an upload by media id, so a bot that can read the id can act on the
|
||||
* image the user sent instead of asking them to upload it again.
|
||||
*/
|
||||
export default function describeAttachment({ name, id }: Params) {
|
||||
return `"${name}", media id ${id}`;
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import grabMessagesMedia, {
|
||||
import type { HarnessContentPart } from "@/src/harness/types";
|
||||
import type { BotMessageWithAttachments } from "@/src/types";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import describeAttachment from "./describe-attachment";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
@@ -45,12 +46,18 @@ export default async function loadAttachmentParts({ messages }: Params) {
|
||||
|
||||
async function fileToParts({ file }: { file: MessageMediaRow }): Promise<HarnessContentPart[]> {
|
||||
const name = file.title || `file ${file.id}`;
|
||||
/**
|
||||
* The upload's own id, so a bot can name it back to a tool
|
||||
* (`set_picture` and `update_workspace` take one) instead of guessing
|
||||
* which file the user means.
|
||||
*/
|
||||
const label = describeAttachment({ name, id: file.id! });
|
||||
|
||||
if (file.media_type == "document") {
|
||||
const text = file.text_content?.trim();
|
||||
|
||||
if (!text) {
|
||||
return [{ type: "text", text: `[Attached file "${name}": its text couldn't be read.]` }];
|
||||
return [{ type: "text", text: `[Attached file ${label}: its text couldn't be read.]` }];
|
||||
}
|
||||
|
||||
const cut = text.length > AppData.BotAttachmentMaxTextChars;
|
||||
@@ -58,32 +65,32 @@ async function fileToParts({ file }: { file: MessageMediaRow }): Promise<Harness
|
||||
return [
|
||||
{
|
||||
type: "text",
|
||||
text: `Attached file "${name}":\n\n${text.slice(0, AppData.BotAttachmentMaxTextChars)}${cut ? `\n\n[The rest of "${name}" was cut: it's ${text.length} characters long.]` : ""}`,
|
||||
text: `Attached file ${label}:\n\n${text.slice(0, AppData.BotAttachmentMaxTextChars)}${cut ? `\n\n[The rest of "${name}" was cut: it's ${text.length} characters long.]` : ""}`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (file.media_type == "video") {
|
||||
return [{ type: "text", text: `[Attached video "${name}": videos can't be read yet.]` }];
|
||||
return [{ type: "text", text: `[Attached video ${label}: videos can't be read yet.]` }];
|
||||
}
|
||||
|
||||
const disk = Bun.file(path.join(grabDirNames().DATA_DIR, file.media_write_path || ""));
|
||||
|
||||
if (!file.media_write_path || !(await disk.exists())) {
|
||||
return [{ type: "text", text: `[Attached file "${name}" is missing on the server.]` }];
|
||||
return [{ type: "text", text: `[Attached file ${label} is missing on the server.]` }];
|
||||
}
|
||||
|
||||
const base64 = Buffer.from(await disk.arrayBuffer()).toString("base64");
|
||||
|
||||
if (file.media_type == "image") {
|
||||
return [
|
||||
{ type: "text", text: `[Attached image "${name}"]` },
|
||||
{ type: "text", text: `[Attached image ${label}]` },
|
||||
{ type: "image", url: `data:image/${file.mime_type || "jpeg"};base64,${base64}`, name },
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
{ type: "text", text: `[Attached audio "${name}"]` },
|
||||
{ type: "text", text: `[Attached audio ${label}]` },
|
||||
{ type: "audio", data: base64, format: file.mime_type || "mp3", name },
|
||||
];
|
||||
}
|
||||
|
||||
@@ -63,10 +63,13 @@ async function push({
|
||||
workspace_ids,
|
||||
}: Params) {
|
||||
const audience = user_id ? [user_id] : await grabBotAudience({ bot_id });
|
||||
/** Its colour and stone, so a recolour reaches every open tab. */
|
||||
/**
|
||||
* Its colour, stone and picture, so a recolour or a new face reaches
|
||||
* every open tab.
|
||||
*/
|
||||
const identity = (
|
||||
await BunSQLite.sql<BUN_SQLITE_COBBLE_BOTS>({
|
||||
sql: `SELECT color, stone FROM bots WHERE id = ? LIMIT 1`,
|
||||
sql: `SELECT color, stone, image_media_id FROM bots WHERE id = ? LIMIT 1`,
|
||||
values: [bot_id],
|
||||
})
|
||||
).payload?.[0];
|
||||
@@ -83,6 +86,7 @@ async function push({
|
||||
...(await grabBotFace({ bot_id, user_id: viewer_id })),
|
||||
color: identity?.color || null,
|
||||
stone: identity?.stone || null,
|
||||
image_media_id: identity?.image_media_id ? Number(identity.image_media_id) : null,
|
||||
...(computer_id !== undefined ? { computer_id: computer_id || null } : {}),
|
||||
...(workspace_ids !== undefined ? { workspace_ids } : {}),
|
||||
task: await grabActiveBotTask({ bot_id, user_id: viewer_id }),
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_WORKSPACES } from "@/db/types/db";
|
||||
import grabWorkspaceAudience from "@/src/functions/backend/db/access/grab-workspace-audience";
|
||||
import { logError } from "@/src/functions/backend/logger";
|
||||
import { sendToUser } from "@/src/websocket/(functions)/send-to-user";
|
||||
|
||||
type Params = {
|
||||
workspace_id: number | string;
|
||||
/**
|
||||
* Only these users (the connected ones, for the live relay); else
|
||||
* everyone who sees the workspace.
|
||||
*/
|
||||
users?: Set<string>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pushes `server:workspace-updated` to everyone who sees a workspace whose
|
||||
* picture changed, so the rail and its header swap it without a reload.
|
||||
* Best effort: a push that fails is logged and never fails the change that
|
||||
* caused it.
|
||||
*/
|
||||
export default async function pushWorkspaceUpdated({ workspace_id, users }: Params) {
|
||||
if (!workspace_id) return;
|
||||
|
||||
try {
|
||||
const workspace = (
|
||||
await BunSQLite.sql<BUN_SQLITE_COBBLE_WORKSPACES>({
|
||||
sql: `SELECT id, image_media_id FROM workspaces WHERE id = ? LIMIT 1`,
|
||||
values: [Number(workspace_id)],
|
||||
})
|
||||
).payload?.[0];
|
||||
|
||||
if (!workspace?.id) return;
|
||||
|
||||
const data = {
|
||||
workspace_id: Number(workspace.id),
|
||||
image_media_id: workspace.image_media_id ? Number(workspace.image_media_id) : null,
|
||||
};
|
||||
|
||||
for (const user_id of await grabWorkspaceAudience({ workspace_id: workspace.id })) {
|
||||
if (users && !users.has(String(user_id))) continue;
|
||||
|
||||
sendToUser({
|
||||
user_id,
|
||||
data: { event: "server:workspace-updated", data },
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
await logError({
|
||||
title: "Workspace update wasn't pushed",
|
||||
error,
|
||||
metadata: { workspace_id },
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-comp
|
||||
import runTextLoop from "@/src/harness/text/run-text-loop";
|
||||
import grabBotChatTools from "@/src/harness/tools/bot/grab-bot-chat-tools";
|
||||
import grabBotFileTools from "@/src/harness/tools/storage/grab-bot-file-tools";
|
||||
import grabRunScopes from "@/src/functions/backend/db/bot-files/grab-run-scopes";
|
||||
import openBotExtensions from "@/src/harness/tools/extensions/open-bot-extensions";
|
||||
import openBotSkills from "@/src/harness/tools/skills/open-bot-skills";
|
||||
import botMessagesToHarness from "./bot-messages-to-harness";
|
||||
@@ -87,11 +88,18 @@ export default async function runBotReply({
|
||||
*/
|
||||
const attachments: number[] = [];
|
||||
|
||||
/**
|
||||
* The files this run reaches: its file tools and the pictures it may
|
||||
* set work from the same answer (`grabRunScopes`).
|
||||
*/
|
||||
const scopes = await grabRunScopes({ bot, user_id, workspace_id });
|
||||
|
||||
const file_tools = await grabBotFileTools({
|
||||
bot,
|
||||
user_id,
|
||||
workspace_id,
|
||||
session_id,
|
||||
scopes,
|
||||
onAttach: ({ media_id }) => attachments.push(media_id),
|
||||
});
|
||||
|
||||
@@ -101,6 +109,7 @@ export default async function runBotReply({
|
||||
computers,
|
||||
session_id,
|
||||
workspace_id,
|
||||
scopes,
|
||||
message_id: job.message_id || undefined,
|
||||
onTakeOver: ({ ref }) => take_overs.push(ref),
|
||||
});
|
||||
|
||||
@@ -24,7 +24,7 @@ import searchComputersTool from "@/src/harness/tools/computer/search-computers-t
|
||||
import grabBotFileTools from "@/src/harness/tools/storage/grab-bot-file-tools";
|
||||
import reportRunFailedTool from "@/src/harness/tools/bot/report-run-failed-tool";
|
||||
import computerRef from "@/src/utils/computer-ref";
|
||||
import grabStorageScopes from "@/src/utils/storage-access";
|
||||
import grabRunScopes from "@/src/functions/backend/db/bot-files/grab-run-scopes";
|
||||
import composeJobBrief from "./compose-job-brief";
|
||||
import grabSessionCarryOver from "./grab-session-carry-over";
|
||||
import jobSessionId from "@/src/functions/backend/db/bots-queue/job-session-id";
|
||||
@@ -108,6 +108,11 @@ export default async function runBotTask({
|
||||
...computers.filter((c) => c.id == job.computer_id),
|
||||
...computers.filter((c) => c.id != job.computer_id),
|
||||
];
|
||||
/**
|
||||
* The files this run reaches, for its file tools, an import and the
|
||||
* pictures it may set: one answer, where `grabRunScopes` decides.
|
||||
*/
|
||||
const scopes = await grabRunScopes({ bot, user_id, workspace_id });
|
||||
|
||||
const tools = [
|
||||
...(await grabBotTools({
|
||||
@@ -117,6 +122,7 @@ export default async function runBotTask({
|
||||
computers,
|
||||
session_id,
|
||||
is_coordinator: bot.kind == "coordinator",
|
||||
scopes,
|
||||
})),
|
||||
...(bot.kind == "coordinator"
|
||||
? [
|
||||
@@ -127,6 +133,7 @@ export default async function runBotTask({
|
||||
user_id,
|
||||
workspaces: coordinator_workspaces,
|
||||
workspace_id,
|
||||
scopes,
|
||||
}),
|
||||
]
|
||||
: []),
|
||||
@@ -148,11 +155,7 @@ export default async function runBotTask({
|
||||
? [
|
||||
importFileTool({
|
||||
computer_id: job.computer_id,
|
||||
scopes: grabStorageScopes({
|
||||
bot_id: bot.id,
|
||||
bot_kind: bot.kind,
|
||||
workspace_id,
|
||||
}),
|
||||
scopes,
|
||||
bot_id: bot.id,
|
||||
user_id,
|
||||
}),
|
||||
@@ -163,6 +166,7 @@ export default async function runBotTask({
|
||||
user_id,
|
||||
workspace_id,
|
||||
session_id,
|
||||
scopes,
|
||||
onAttach: ({ media_id }) => attachments.push(media_id),
|
||||
})),
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user