Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 11 Wiki Activity

Let bots set a picture from a file they can see

Browse Source 
A coordinator asked to set an uploaded logo as a workspace picture could
not: the picture tools take a media id, and nothing ever gave the bot one.
Attachments reached the model as "[Attached image \"logo.png\"]", every
file tool answers with a [file:ID] ref, and an SVG is deliberately stored
as a document rather than an image.

- copyMediaAsPicture copies an approved image into a new picture row
  (bot-image/workspace-image). The source keeps its row and bytes, so a
  message that attached it still loads and changing a picture can never
  take a file away from a conversation.
- grabPictureSource resolves a picture by [file:ID] (the same scope gate
  every file tool uses) or by the user's own upload; parseFileRef reads
  the [file:12] form.
- The new set_picture tool ships on every run: a bot sets its own face,
  the coordinator can also dress a worker. update_workspace gained
  picture_file_id, and setBotPicture/setProfileImage share the copy and
  the delete.
- removeReplacedPicture deletes a replaced picture only when it really is
  one, never a chat upload.
- uploadAndRecordMedia writes images in their own format: PNG stays PNG
  with its transparency, WebP stays WebP, GIF becomes PNG, everything
  else JPEG, and the row records the real mime type (it used to say jpeg
  for every image).
- Pictures change live: BotUpdatedData carries image_media_id and a new
  server:workspace-updated pushes workspace pictures, relayed between
  processes.
- Attachments are labelled with their media id, and BotFilesNote tells a
  bot that a [file:ID] can become a picture.
- grabRunScopes is now the single source of a run's reachable folders,
  shared by the file tools, imports and the picture tools.
This commit is contained in:
tben
2026-10-23 05:24:55 +01:00
parent d3357cb7df
commit f2802c1bf3
42 changed files with 1391 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/harness/tools/bot/grab-bot-chat-tools.ts
+9 -1
View File
@@ -1,6 +1,7 @@
import type { BUN_SQLITE_COBBLE_BOTS, BUN_SQLITE_COBBLE_COMPUTERS } from "@/db/types/db";
import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers";
import grabCrewWorkers from "@/src/functions/backend/db/bots/grab-crew-workers";
import type { StorageScope } from "@/src/utils/storage-access";
import type { HarnessTool } from "../../types";
import askToTakeOverTool from "../computer/ask-to-take-over-tool";
import searchComputersTool from "../computer/search-computers-tool";
@@ -41,6 +42,11 @@ type Params = {
* The user message being answered.
*/
message_id?: number | string;
/**
* The files the run reaches (`grabRunScopes`), so a picture it sets is
* copied from one it can already read.
*/
scopes?: StorageScope[];
/**
* Called when the bot asks the user to take over a computer.
*/
@@ -70,6 +76,7 @@ export default async function grabBotChatTools({
session_id,
workspace_id,
message_id,
scopes = [],
onTakeOver,
}: Params): Promise<HarnessTool[]> {
const bot_id = bot.id || 0;
@@ -81,6 +88,7 @@ export default async function grabBotChatTools({
computers,
session_id,
is_coordinator: bot.kind == "coordinator",
scopes,
})),
startTaskTool({
user_id,
@@ -104,7 +112,7 @@ export default async function grabBotChatTools({
tools.push(createWorkspaceTool({ user_id, workspaces }));
if (workspaces.length) {
tools.push(updateWorkspaceTool({ user_id, workspaces, workspace_id }));
tools.push(updateWorkspaceTool({ user_id, workspaces, workspace_id, scopes }));
}
tools.push(createBotTool({ user_id, workspaces, workspace_id }));
src/harness/tools/bot/grab-bot-tools.ts
+9
View File
@@ -9,6 +9,8 @@ import updateScheduleTool from "./update-schedule-tool";
import updateInstructionsTool from "./update-instructions-tool";
import updateMemoryTool from "./update-memory-tool";
import reactToMessageTool from "./react-to-message-tool";
import setPictureTool from "./set-picture-tool";
import type { StorageScope } from "@/src/utils/storage-access";
type Params = {
user_id: number | string;
@@ -31,6 +33,11 @@ type Params = {
* add to any bot's or workspace's list.
*/
is_coordinator?: boolean;
/**
* The files the run reaches (`grabRunScopes`), for the picture it may
* set from one.
*/
scopes?: StorageScope[];
};
/**
@@ -45,8 +52,10 @@ export default async function grabBotTools({
computers = [],
session_id,
is_coordinator,
scopes = [],
}: Params): Promise<HarnessTool[]> {
return [
setPictureTool({ user_id, bot_id, scopes, is_coordinator }),
updateMemoryTool({ user_id, bot_id, workspace_id }),
updateInstructionsTool({ user_id, bot_id, workspace_id }),
createScheduleTool({ user_id, bot_id, workspace_id, computers }),
src/harness/tools/bot/set-picture-tool.test.ts
+46
View File
@@ -0,0 +1,46 @@
import { describe, expect, test } from "bun:test";
import setPictureTool from "./set-picture-tool";
const signal = new AbortController().signal;
describe("setPictureTool", () => {
test("a worker sets its own picture, and is told it may not dress another", async () => {
const tool = setPictureTool({
user_id: 1,
bot_id: 5,
scopes: [{ scope: "bot", owner_id: 5 }],
});
await expect(
tool.run({ args: { bot_id: 6, remove: true }, signal }),
).rejects.toThrow(/only set your own picture/);
});
test("the coordinator may name a worker", async () => {
const tool = setPictureTool({
user_id: 1,
bot_id: 1,
scopes: [{ scope: "bot", owner_id: 1 }],
is_coordinator: true,
});
/** Its own bot id is accepted as well: nothing reaches the guard. */
await expect(
tool.run({ args: { bot_id: 1, file_id: "[file:0]" }, signal }),
).rejects.not.toThrow(/only set your own picture/);
});
test("needs a file, a media id or remove", () => {
const tool = setPictureTool({ user_id: 1, bot_id: 5, scopes: [] });
expect(tool.schema?.safeParse({}).success).toBe(false);
expect(tool.schema?.safeParse({ remove: true }).success).toBe(true);
expect(tool.schema?.safeParse({ file_id: "[file:12]" }).success).toBe(true);
expect(tool.schema?.safeParse({ file_id: 12 }).success).toBe(true);
expect(tool.schema?.safeParse({ media_id: 4 }).success).toBe(true);
});
test("is named as a built-in, so a custom tool can't shadow it", () => {
expect(setPictureTool({ user_id: 1, bot_id: 5, scopes: [] }).name).toBe("set_picture");
});
});
src/harness/tools/bot/set-picture-tool.ts
+115
View File
@@ -0,0 +1,115 @@
import { z } from "zod";
import pushBotUpdated from "@/src/bots-queue/push-bot-updated";
import grabPictureSource from "@/src/functions/backend/db/profile-images/grab-picture-source";
import setBotPicture from "@/src/functions/backend/db/profile-images/set-bot-picture";
import parseFileRef from "@/src/utils/parse-file-ref";
import type { StorageScope } from "@/src/utils/storage-access";
import type { HarnessTool } from "../../types";
type Params = {
user_id: number | string;
/**
* The bot running: it sets its own picture unless the coordinator names
* another worker.
*/
bot_id: number | string;
/**
* The files this run reaches (`grabRunScopes`), so a picture is only
* ever copied from a file the bot could already read.
*/
scopes: StorageScope[];
/**
* Whether the run is the coordinator, which may dress any worker.
*/
is_coordinator?: boolean;
};
type Args = {
file_id?: string | number;
media_id?: number;
bot_id?: number;
remove?: boolean;
};
/**
* Lets a bot set its own picture from a file it can see, and the
* coordinator set any worker's: an avatar in its place in the rail, in the
* thread and in the crew list. Only a super admin or an admin sets one
* (`setBotPicture` checks it), so a run for anyone else is told so.
*/
export default function setPictureTool({
user_id,
bot_id,
scopes,
is_coordinator,
}: Params): HarnessTool<Args> {
return {
name: "set_picture",
description: `Set a bot's picture (its avatar, shown in the rail and beside its messages) from an image in your files, or remove it. You set your own; ${
is_coordinator ? "as the coordinator you can pass bot_id to dress a worker" : "only the coordinator sets another bot's"
}. Give \`file_id\` with the image's [file:ID] ref, e.g. "[file:12]" (list_files shows them; what the user sends you lands in uploads/), or \`media_id\` for an image they uploaded. An SVG or another kind of file is turned into a PNG first, then that file is used. The picture has to be an image and at most 5 MB; a square one looks best.`,
schema: z
.object({
file_id: z
.union([z.string().max(60), z.number().int().positive()])
.optional()
.describe(`The image's [file:ID] ref, e.g. "[file:12]"`),
media_id: z
.number()
.int()
.positive()
.optional()
.describe(`An image the user uploaded, by media id`),
bot_id: is_coordinator
? z
.number()
.int()
.positive()
.optional()
.describe(`The worker to dress; leave it out for your own picture`)
: z.number().int().positive().optional(),
remove: z.boolean().optional().describe(`Take the picture away`),
})
.refine(
(args) =>
Boolean(args.remove) ||
Boolean(args.file_id) ||
Boolean(args.media_id),
"Give the image's file_id or media_id, or set remove.",
),
run: async ({ args }) => {
const target = args.bot_id ?? Number(bot_id);
if (Number(target) != Number(bot_id) && !is_coordinator) {
throw new Error(
`You can only set your own picture. The coordinator sets another bot's.`,
);
}
const media = args.remove
? null
: await grabPictureSource({
user_id,
scopes,
file_id: parseFileRef({ ref: args.file_id }),
media_id: args.media_id,
});
if (!args.remove && !media) {
throw new Error(
`Give the image's [file:ID] ref or its media_id. list_files lists the refs of the images you have.`,
);
}
const saved = await setBotPicture({ user_id, bot_id: target, media });
await pushBotUpdated({ bot_id: target });
const whose = Number(target) == Number(bot_id) ? "Your" : "That bot's";
return media
? `${whose} picture is saved. Tell the user it changed; they may need to reload to see it everywhere.`
: `${whose} picture is removed, so it shows its stone again.`;
},
};
}
src/harness/tools/bot/update-bot-tool.ts
+3 -2
View File
@@ -24,8 +24,9 @@ type Args = {
/**
* Lets the coordinator change an existing crew member: its name, its
* one-line job, and its core instructions (the start of its prompt in its
* own chat and in every workspace). Its colour, stone and picture stay as
* they are.
* own chat and in every workspace). Its colour and stone stay as they are,
* and its picture is set with `set_picture` (`set_picture_tool.ts`), which
* also lets a bot dress itself.
* Workspace-only instructions have their own tool
* (`set_bot_workspace_instructions`).
*/
src/harness/tools/bot/update-workspace-tool.ts
+81 -29
View File
@@ -1,7 +1,11 @@
import { z } from "zod";
import type { BUN_SQLITE_COBBLE_WORKSPACES } from "@/db/types/db";
import { AppData } from "@/src/data/app-data";
import pushWorkspaceUpdated from "@/src/bots-queue/push-workspace-updated";
import grabPictureSource from "@/src/functions/backend/db/profile-images/grab-picture-source";
import updateWorkspace from "@/src/functions/backend/db/workspaces/update-workspace";
import parseFileRef from "@/src/utils/parse-file-ref";
import type { StorageScope } from "@/src/utils/storage-access";
import type { HarnessTool } from "../../types";
type Params = {
@@ -15,6 +19,11 @@ type Params = {
* given.
*/
workspace_id?: number | string | null;
/**
* The files this run reaches (`grabRunScopes`): a picture is copied
* from one of them, so a bot can set the logo it was sent.
*/
scopes?: StorageScope[];
};
type Args = {
@@ -23,8 +32,13 @@ type Args = {
description?: string;
context?: string;
/**
* A picture for the workspace: the media id the user uploaded, or null
* to remove it.
* A picture for the workspace, from a file the run can see: its
* `[file:ID]` ref.
*/
picture_file_id?: string | number | null;
/**
* A picture for the workspace: the media id of an image the user
* uploaded, or null to remove it.
*/
picture_media_id?: number | null;
};
@@ -38,27 +52,43 @@ export default function updateWorkspaceTool({
user_id,
workspaces,
workspace_id,
scopes = [],
}: Params): HarnessTool<Args> {
const list = workspaces.map((w) => `${w.id}: ${w.name}`).join("; ");
return {
name: "update_workspace",
description: `Change a workspace: its name, its one-line description, its context (markdown every bot working there gets in its prompt), or its picture. Only the fields you give change; \`context\` replaces the whole text, so keep what still applies. Use it when the user asks, or to record something every bot in that workspace must know.${workspace_id ? " Leave out workspace_id for this workspace." : ""} Workspaces: ${list}.`,
schema: z.object({
workspace_id: z.number().int().optional(),
name: z.string().trim().min(1).max(60).optional(),
description: z.string().trim().max(240).optional(),
context: z.string().trim().max(AppData.PromptMaxLength).optional(),
picture_media_id: z
.number()
.int()
.positive()
.nullable()
.optional()
.describe(
`An image the user uploaded, by media id; null removes the picture. Leave it out to keep the one it has.`,
),
}),
description: `Change a workspace: its name, its one-line description, its context (markdown every bot working there gets in its prompt), or its picture. Only the fields you give change; \`context\` replaces the whole text, so keep what still applies. Use it when the user asks, or to record something every bot in that workspace must know.${workspace_id ? " Leave out workspace_id for this workspace." : ""} For the picture give \`picture_file_id\` with the image's [file:ID] ref, e.g. "[file:12]" (list_files shows them, and what the user sends you lands in uploads/), or \`picture_media_id\` for an image they uploaded; null removes it. An SVG or another kind of file is turned into a PNG first, then that file is used. Workspaces: ${list}.`,
schema: z
.object({
workspace_id: z.number().int().optional(),
name: z.string().trim().min(1).max(60).optional(),
description: z.string().trim().max(240).optional(),
context: z.string().trim().max(AppData.PromptMaxLength).optional(),
picture_file_id: z
.union([z.string().max(60), z.number().int().positive()])
.nullable()
.optional()
.describe(`The picture's [file:ID] ref, e.g. "[file:12]"`),
picture_media_id: z
.number()
.int()
.positive()
.nullable()
.optional()
.describe(
`An image the user uploaded, by media id; null removes the picture. Leave it out to keep the one it has.`,
),
})
.refine(
(args) =>
args.name !== undefined ||
args.description !== undefined ||
args.context !== undefined ||
args.picture_file_id !== undefined ||
args.picture_media_id !== undefined,
"Give a name, a description, a context or a picture to change.",
),
run: async ({ args }) => {
const target = workspaces.find((w) => w.id == (args.workspace_id || workspace_id));
@@ -68,16 +98,22 @@ export default function updateWorkspaceTool({
);
}
if (
args.name === undefined &&
args.description === undefined &&
args.context === undefined &&
args.picture_media_id === undefined
) {
throw new Error(
`Give a name, a description, a context or a picture to change.`,
);
}
const removing =
args.picture_file_id === null || args.picture_media_id === null;
/**
* A file ref is resolved against the run's directories
* (`grabPictureSource`), so a bot can only set what it could
* already read.
*/
const picture_source =
args.picture_file_id === undefined || args.picture_file_id === null
? null
: await grabPictureSource({
user_id,
scopes,
file_id: parseFileRef({ ref: args.picture_file_id }),
});
await updateWorkspace({
user_id,
@@ -87,10 +123,26 @@ export default function updateWorkspaceTool({
description: args.description,
system_prompt: args.context,
image_media_id: args.picture_media_id,
picture_source,
},
});
return `${args.name || target.name} is saved. Bots there get the change from their next run.`;
/**
* Its picture reaches every open tab on its own, whether the
* run is in this process or another (`pushWorkspaceUpdated`,
* `liveRelay`).
*/
if (removing || picture_source) {
await pushWorkspaceUpdated({ workspace_id: target.id });
}
const picture = removing
? ` Its picture is removed.`
: picture_source
? ` Its picture is set from that file.`
: "";
return `${args.name || target.name} is saved. Bots there get the change from their next run.${picture}`;
},
};
}
src/harness/tools/storage/grab-bot-file-tools.ts
+9 -7
View File
@@ -1,5 +1,6 @@
import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
import grabUserWorkspaces from "@/src/functions/backend/db/workspaces/grab-user-workspaces";
import grabRunScopes from "@/src/functions/backend/db/bot-files/grab-run-scopes";
import type { StorageScope } from "@/src/utils/storage-access";
import type { HarnessTool } from "../../types";
import generateImageTool from "./generate-image-tool";
import grabStorageTools from "./grab-storage-tools";
@@ -11,6 +12,11 @@ type Params = {
user_id: number | string;
workspace_id?: number | string | null;
session_id?: number | string;
/**
* The files the run reaches, when the caller already read them
* (`grabRunScopes`); read here otherwise.
*/
scopes?: StorageScope[];
/**
* Called for a file the bot wrote with "attach" set, so the reply
* carries it.
@@ -30,19 +36,15 @@ export default async function grabBotFileTools({
user_id,
workspace_id,
session_id,
scopes,
onAttach,
}: Params): Promise<HarnessTool[]> {
const all_workspace_ids =
bot.kind == "coordinator"
? (await grabUserWorkspaces({ user_id })).filter((w) => !w.archived).map((w) => Number(w.id))
: [];
const context = storageToolContext({
bot,
user_id,
workspace_id,
session_id,
all_workspace_ids,
scopes: scopes || (await grabRunScopes({ bot, user_id, workspace_id })),
onAttach,
});
src/harness/tools/storage/storage-tool-context.ts
+8 -13
View File
@@ -1,5 +1,5 @@
import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
import grabStorageScopes, { type StorageScope } from "@/src/utils/storage-access";
import type { StorageScope } from "@/src/utils/storage-access";
export type StorageToolContext = {
/**
@@ -32,34 +32,29 @@ type Params = {
workspace_id?: number | string | null;
session_id?: number | string;
/**
* Every workspace, for the coordinator (it reaches them all).
* The directories the run may touch (`grabRunScopes`): the bot's own
* first, then the workspace's or, for the coordinator, every one's.
*/
all_workspace_ids?: number[];
scopes: StorageScope[];
onAttach?: StorageToolContext["onAttach"];
};
/**
* The directories one run may touch, from the bot it's for. The
* coordinator reaches every bot's and workspace's files; a worker its own,
* plus its workspace's shared one while it works there.
* The context every storage tool works from: which bot it is, whose files it
* reaches (`grabRunScopes`, which owns that decision) and where it works.
*/
export default function storageToolContext({
bot,
user_id,
workspace_id,
session_id,
all_workspace_ids,
scopes,
onAttach,
}: Params): StorageToolContext {
return {
user_id,
bot_id: bot.id || 0,
scopes: grabStorageScopes({
bot_id: bot.id || 0,
bot_kind: bot.kind,
workspace_id,
all_workspace_ids,
}),
scopes,
session_id,
workspace_id,
onAttach,