Add the harness, envs, per-bot models and chat attachments
- Harness: providers, model catalog synced hourly, automatic free picks, cooldowns, tool loop with context trimming, MCP computer tools - Bots queue runs replies and tasks through the harness; jobs wait (attempt refunded) while every model is resting - Envs: Settings > Environment, scoped general/workspace/computer, written to computers and redacted from tool output - Models: Settings > Models, per-harness provider select defaulting to OpenRouter, and a models modal on each bot's chat (the coordinator's edits the defaults) - Chat: thinking row while a reply runs, file attachments (upload, paste, drop) that replies read as images, audio and document text - Fix media uploads failing after a user's first file, and private files being served from the public media path Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { HarnessMessage } from "../types";
|
||||
import fitTranscriptToContext from "./fit-transcript-to-context";
|
||||
|
||||
const big = "x".repeat(30000);
|
||||
|
||||
const run = (): HarnessMessage[] => [
|
||||
{ role: "user", content: "Do the thing." },
|
||||
{ role: "assistant", content: null, tool_calls: [{ id: "1", name: "sandbox_execute_bash", arguments: "{}" }] },
|
||||
{ role: "tool", tool_call_id: "1", content: big },
|
||||
{ role: "assistant", content: null, tool_calls: [{ id: "2", name: "sandbox_execute_bash", arguments: "{}" }] },
|
||||
{ role: "tool", tool_call_id: "2", content: big },
|
||||
{ role: "assistant", content: null, tool_calls: [{ id: "3", name: "sandbox_execute_bash", arguments: "{}" }] },
|
||||
{ role: "tool", tool_call_id: "3", content: "recent" },
|
||||
];
|
||||
|
||||
describe("fitTranscriptToContext", () => {
|
||||
test("leaves a transcript that fits alone", () => {
|
||||
const transcript = run();
|
||||
const result = fitTranscriptToContext({ system: "s", transcript, context_length: 200000 });
|
||||
|
||||
expect(result.trimmed).toBe(0);
|
||||
expect(transcript[2]!.content).toBe(big);
|
||||
});
|
||||
|
||||
test("trims the oldest tool output first, keeping the newest", () => {
|
||||
const transcript = run();
|
||||
const result = fitTranscriptToContext({ system: "s", transcript, context_length: 20000 });
|
||||
|
||||
expect(result.trimmed).toBe(1);
|
||||
expect(String(transcript[2]!.content)).toStartWith("[Trimmed: 30000 characters");
|
||||
expect(transcript[4]!.content).toBe(big);
|
||||
expect(transcript[6]!.content).toBe("recent");
|
||||
expect(result.estimated_tokens).toBeLessThanOrEqual(result.budget);
|
||||
});
|
||||
|
||||
test("never trims the newest messages, even over budget", () => {
|
||||
const transcript: HarnessMessage[] = [
|
||||
{ role: "user", content: "go" },
|
||||
{ role: "tool", tool_call_id: "1", content: big },
|
||||
];
|
||||
const result = fitTranscriptToContext({ system: "s", transcript, context_length: 1000 });
|
||||
|
||||
expect(result.trimmed).toBe(0);
|
||||
expect(transcript[1]!.content).toBe(big);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import type { HarnessMessage } from "../types";
|
||||
|
||||
type Params = {
|
||||
system: string;
|
||||
/**
|
||||
* The conversation after the system prompt. Trimmed in place, so the
|
||||
* run keeps the savings.
|
||||
*/
|
||||
transcript: HarnessMessage[];
|
||||
/**
|
||||
* The model's context window, in tokens.
|
||||
*/
|
||||
context_length: number;
|
||||
/**
|
||||
* The newest messages, never trimmed: the model needs them to go on.
|
||||
*/
|
||||
keep_last?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Share of the window the prompt may fill; the rest is for the answer.
|
||||
*/
|
||||
const PromptShare = 0.7;
|
||||
/**
|
||||
* Rough characters per token, on the low side so the estimate errs high.
|
||||
*/
|
||||
const CharsPerToken = 3;
|
||||
/**
|
||||
* What an image or audio part is counted as, in tokens.
|
||||
*/
|
||||
const MediaTokens = 1500;
|
||||
|
||||
/**
|
||||
* Keeps a long tool run inside the model's context window: while the
|
||||
* estimated prompt is over budget, the oldest tool results (then the
|
||||
* oldest assistant texts) are replaced with a note of what was cut.
|
||||
* Returns how many messages were trimmed.
|
||||
*/
|
||||
export default function fitTranscriptToContext({
|
||||
system,
|
||||
transcript,
|
||||
context_length,
|
||||
keep_last = 4,
|
||||
}: Params) {
|
||||
const budget = Math.floor(context_length * PromptShare);
|
||||
let total = estimateTokens({ text: system }) + transcript.reduce((sum, m) => sum + messageTokens({ message: m }), 0);
|
||||
let trimmed = 0;
|
||||
|
||||
const trimmable = transcript.length - keep_last;
|
||||
|
||||
for (const role of ["tool", "assistant"] as const) {
|
||||
for (let index = 0; index < trimmable && total > budget; index++) {
|
||||
const message = transcript[index]!;
|
||||
|
||||
if (message.role != role || typeof message.content != "string") continue;
|
||||
if (message.content.startsWith("[Trimmed:")) continue;
|
||||
|
||||
const before = messageTokens({ message });
|
||||
const note = `[Trimmed: ${message.content.length} characters of earlier ${role == "tool" ? "tool output" : "notes"} to fit the context.]`;
|
||||
|
||||
transcript[index] = { ...message, content: note } as HarnessMessage;
|
||||
total += messageTokens({ message: transcript[index]! }) - before;
|
||||
trimmed++;
|
||||
}
|
||||
}
|
||||
|
||||
return { trimmed, estimated_tokens: total, budget };
|
||||
}
|
||||
|
||||
function messageTokens({ message }: { message: HarnessMessage }) {
|
||||
let tokens = 4;
|
||||
|
||||
if (typeof message.content == "string") {
|
||||
tokens += estimateTokens({ text: message.content });
|
||||
} else if (Array.isArray(message.content)) {
|
||||
for (const part of message.content) {
|
||||
tokens += part.type == "text" ? estimateTokens({ text: part.text }) : MediaTokens;
|
||||
}
|
||||
}
|
||||
|
||||
if (message.role == "assistant" && message.tool_calls) {
|
||||
for (const call of message.tool_calls) {
|
||||
tokens += estimateTokens({ text: call.name + call.arguments });
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function estimateTokens({ text }: { text: string }) {
|
||||
return Math.ceil(text.length / CharsPerToken);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import HarnessError from "../harness-error";
|
||||
import grabHarnessAdapter from "../providers/grab-harness-adapter";
|
||||
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
|
||||
import withCandidates from "../resolve/with-candidates";
|
||||
import type {
|
||||
HarnessCandidate,
|
||||
HarnessChatResult,
|
||||
HarnessMessage,
|
||||
HarnessToolDefinition,
|
||||
} from "../types";
|
||||
import prepareMessagesForModel from "./prepare-messages-for-model";
|
||||
import stripThinkTags from "./strip-think-tags";
|
||||
|
||||
export type GenerateTextParams = {
|
||||
messages: HarnessMessage[];
|
||||
tools?: HarnessToolDefinition[];
|
||||
/**
|
||||
* Resolved for the text harness when not given.
|
||||
*/
|
||||
candidates?: HarnessCandidate[];
|
||||
/**
|
||||
* Tried first: the model the run already uses.
|
||||
*/
|
||||
prefer_key?: string;
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Media readings shared across calls of one run.
|
||||
*/
|
||||
media_cache?: Map<string, string>;
|
||||
/**
|
||||
* The bot the turn is for: its own models are used where it has them.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
};
|
||||
|
||||
export type GenerateTextResult = HarnessChatResult & {
|
||||
candidate: HarnessCandidate;
|
||||
};
|
||||
|
||||
/**
|
||||
* One model turn on the text harness, falling back across its models.
|
||||
* Media the model can't take is read by the helper harnesses first.
|
||||
* Reasoning tags are stripped from the answer.
|
||||
*/
|
||||
export default async function generateText({
|
||||
messages,
|
||||
tools,
|
||||
candidates,
|
||||
prefer_key,
|
||||
max_tokens,
|
||||
temperature,
|
||||
signal = new AbortController().signal,
|
||||
media_cache = new Map(),
|
||||
bot_id,
|
||||
}: GenerateTextParams): Promise<GenerateTextResult> {
|
||||
const { result, candidate } = await withCandidates({
|
||||
harness: "text",
|
||||
candidates:
|
||||
candidates || (await resolveHarnessCandidates({ harness: "text", bot_id })),
|
||||
prefer_key,
|
||||
run: async (candidate) => {
|
||||
const answer = await grabHarnessAdapter({
|
||||
kind: candidate.provider.kind,
|
||||
}).chat({
|
||||
candidate,
|
||||
messages: await prepareMessagesForModel({
|
||||
messages,
|
||||
model: candidate.model,
|
||||
cache: media_cache,
|
||||
signal,
|
||||
bot_id,
|
||||
}),
|
||||
tools,
|
||||
max_tokens,
|
||||
temperature,
|
||||
signal,
|
||||
});
|
||||
|
||||
const text = answer.text ? stripThinkTags({ text: answer.text }) : "";
|
||||
|
||||
if (!text && !answer.tool_calls.length) {
|
||||
throw new HarnessError({
|
||||
message: `The model returned an empty answer.`,
|
||||
retryable: true,
|
||||
candidate_key: candidate.key,
|
||||
});
|
||||
}
|
||||
|
||||
return { ...answer, text: text || null };
|
||||
},
|
||||
});
|
||||
|
||||
return { ...result, candidate };
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { MediaReadNote } from "@/src/data/harness-prompts";
|
||||
import type { HarnessMessage } from "../types";
|
||||
import prepareMessagesForModel from "./prepare-messages-for-model";
|
||||
|
||||
const messages: HarnessMessage[] = [
|
||||
{ role: "system", content: "You are a bot." },
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What's this?" },
|
||||
{ type: "image", url: "data:image/png;base64,AAAA", name: "a.png" },
|
||||
{ type: "audio", data: "AAAA", format: "mp3" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
describe("prepareMessagesForModel", () => {
|
||||
test("keeps media the model can take", async () => {
|
||||
const fitted = await prepareMessagesForModel({
|
||||
messages,
|
||||
model: { input_modalities: ["text", "image", "audio"] },
|
||||
cache: new Map(),
|
||||
signal,
|
||||
read_image: async () => {
|
||||
throw new Error("should not read");
|
||||
},
|
||||
});
|
||||
|
||||
expect(fitted).toEqual(messages);
|
||||
});
|
||||
|
||||
test("reads media the model can't take, once, and notes it", async () => {
|
||||
const cache = new Map<string, string>();
|
||||
let reads = 0;
|
||||
const run = () =>
|
||||
prepareMessagesForModel({
|
||||
messages,
|
||||
model: { input_modalities: ["text"] },
|
||||
cache,
|
||||
signal,
|
||||
read_image: async () => {
|
||||
reads++;
|
||||
return "A red square.";
|
||||
},
|
||||
transcribe: async () => "hello there",
|
||||
});
|
||||
|
||||
const fitted = await run();
|
||||
await run();
|
||||
|
||||
expect(reads).toBe(1);
|
||||
expect(fitted[0]).toEqual({
|
||||
role: "system",
|
||||
content: `You are a bot.\n\n${MediaReadNote}`,
|
||||
});
|
||||
expect(fitted[1]).toEqual({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What's this?" },
|
||||
{ type: "text", text: `[Image "a.png": A red square.]` },
|
||||
{ type: "text", text: "[Audio, transcribed: hello there]" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("says when a part can't be read instead of failing", async () => {
|
||||
const fitted = await prepareMessagesForModel({
|
||||
messages: [messages[1]!],
|
||||
model: { input_modalities: ["text", "audio"] },
|
||||
cache: new Map(),
|
||||
signal,
|
||||
read_image: async () => {
|
||||
throw new Error("No media reading model is set up.");
|
||||
},
|
||||
});
|
||||
|
||||
expect(fitted[0]).toMatchObject({
|
||||
content: [
|
||||
{},
|
||||
{
|
||||
type: "text",
|
||||
text: `[Image "a.png": couldn't be read. No media reading model is set up.]`,
|
||||
},
|
||||
{ type: "audio" },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
import { MediaReadNote } from "@/src/data/harness-prompts";
|
||||
import transcribeAudio from "../audio/transcribe-audio";
|
||||
import HarnessError from "../harness-error";
|
||||
import readImage from "../media/read-image";
|
||||
import type {
|
||||
HarnessContentPart,
|
||||
HarnessMessage,
|
||||
HarnessModelInfo,
|
||||
} from "../types";
|
||||
|
||||
type Params = {
|
||||
messages: HarnessMessage[];
|
||||
model: Pick<HarnessModelInfo, "input_modalities">;
|
||||
/**
|
||||
* Readings by media key, kept across candidates and steps so nothing is
|
||||
* read twice.
|
||||
*/
|
||||
cache: Map<string, string>;
|
||||
signal: AbortSignal;
|
||||
/**
|
||||
* Reads media with this bot's models when it has its own.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
read_image?: typeof readImage;
|
||||
transcribe?: typeof transcribeAudio;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fits messages to a model's inputs: images it can't see are read by the
|
||||
* media reading harness and audio it can't hear is transcribed, each
|
||||
* replaced with a `[bracketed]` reading. A part that can't be read says so
|
||||
* instead of failing the call.
|
||||
*/
|
||||
export default async function prepareMessagesForModel({
|
||||
messages,
|
||||
model,
|
||||
cache,
|
||||
signal,
|
||||
bot_id,
|
||||
read_image = readImage,
|
||||
transcribe = transcribeAudio,
|
||||
}: Params): Promise<HarnessMessage[]> {
|
||||
const sees = model.input_modalities.includes("image");
|
||||
const hears = model.input_modalities.includes("audio");
|
||||
let replaced = false;
|
||||
|
||||
const fitPart = async (
|
||||
part: HarnessContentPart,
|
||||
): Promise<HarnessContentPart> => {
|
||||
if (part.type == "image" && !sees) {
|
||||
replaced = true;
|
||||
return {
|
||||
type: "text",
|
||||
text: await readOnce({
|
||||
key: `image:${Bun.hash(part.url)}`,
|
||||
label: `Image${part.name ? ` "${part.name}"` : ""}`,
|
||||
cache,
|
||||
signal,
|
||||
read: () => read_image({ url: part.url, signal, bot_id }),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
if (part.type == "audio" && !hears) {
|
||||
replaced = true;
|
||||
return {
|
||||
type: "text",
|
||||
text: await readOnce({
|
||||
key: `audio:${Bun.hash(part.data)}`,
|
||||
label: `Audio${part.name ? ` "${part.name}"` : ""}, transcribed`,
|
||||
cache,
|
||||
signal,
|
||||
read: () =>
|
||||
transcribe({
|
||||
audio: {
|
||||
data: Buffer.from(part.data, "base64"),
|
||||
format: part.format,
|
||||
},
|
||||
signal,
|
||||
bot_id,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return part;
|
||||
};
|
||||
|
||||
const fitted: HarnessMessage[] = [];
|
||||
|
||||
for (const message of messages) {
|
||||
if (message.role == "user" && typeof message.content != "string") {
|
||||
const content: HarnessContentPart[] = [];
|
||||
|
||||
for (const part of message.content) {
|
||||
content.push(await fitPart(part));
|
||||
}
|
||||
|
||||
fitted.push({ ...message, content });
|
||||
} else {
|
||||
fitted.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
if (!replaced) return fitted;
|
||||
|
||||
const system = fitted.findIndex((message) => message.role == "system");
|
||||
|
||||
if (system >= 0) {
|
||||
const message = fitted[system] as { role: "system"; content: string };
|
||||
fitted[system] = {
|
||||
role: "system",
|
||||
content: `${message.content}\n\n${MediaReadNote}`,
|
||||
};
|
||||
}
|
||||
|
||||
return fitted;
|
||||
}
|
||||
|
||||
async function readOnce({
|
||||
key,
|
||||
label,
|
||||
cache,
|
||||
signal,
|
||||
read,
|
||||
}: {
|
||||
key: string;
|
||||
label: string;
|
||||
cache: Map<string, string>;
|
||||
signal: AbortSignal;
|
||||
read: () => Promise<string>;
|
||||
}) {
|
||||
const cached = cache.get(key);
|
||||
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
try {
|
||||
const text = `[${label}: ${await read()}]`;
|
||||
cache.set(key, text);
|
||||
return text;
|
||||
} catch (error: unknown) {
|
||||
if (signal.aborted || (error instanceof HarnessError && error.code == "aborted")) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
return `[${label}: couldn't be read. ${error instanceof Error ? error.message : String(error)}]`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { z } from "zod";
|
||||
import HarnessError from "../harness-error";
|
||||
import type { HarnessCandidate, HarnessTool } from "../types";
|
||||
import type { GenerateTextParams, GenerateTextResult } from "./generate-text";
|
||||
import runTextLoop from "./run-text-loop";
|
||||
|
||||
const candidate: HarnessCandidate = {
|
||||
key: "1:m",
|
||||
provider: { id: 1, kind: "openrouter", base_url: "https://x.test" },
|
||||
model: {
|
||||
slug: "m",
|
||||
input_modalities: ["text"],
|
||||
output_modalities: ["text"],
|
||||
supports_tools: true,
|
||||
},
|
||||
source: "auto",
|
||||
};
|
||||
|
||||
const memory: string[] = [];
|
||||
|
||||
const remember: HarnessTool<{ note: string }> = {
|
||||
name: "remember",
|
||||
description: "Keeps a note.",
|
||||
schema: z.object({ note: z.string() }),
|
||||
run: async ({ args }) => {
|
||||
memory.push(args.note);
|
||||
return "Saved.";
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* A fake model that plays back scripted turns and records what it saw.
|
||||
*/
|
||||
function scripted(turns: Partial<GenerateTextResult>[]) {
|
||||
const seen: GenerateTextParams[] = [];
|
||||
|
||||
const generate = async (params: GenerateTextParams) => {
|
||||
seen.push(structuredClone({ ...params, signal: undefined }));
|
||||
const turn = turns[seen.length - 1] || { text: "Done." };
|
||||
|
||||
return {
|
||||
text: null,
|
||||
tool_calls: [],
|
||||
candidate,
|
||||
...turn,
|
||||
} as GenerateTextResult;
|
||||
};
|
||||
|
||||
return { generate, seen };
|
||||
}
|
||||
|
||||
const resolveCandidates = async () => [candidate];
|
||||
|
||||
describe("runTextLoop", () => {
|
||||
test("runs tools and feeds their results back until the model answers", async () => {
|
||||
memory.length = 0;
|
||||
const { generate, seen } = scripted([
|
||||
{
|
||||
text: "Noting that.",
|
||||
tool_calls: [
|
||||
{ id: "c1", name: "remember", arguments: `{"note":"tea"}` },
|
||||
],
|
||||
usage: { prompt_tokens: 10, completion_tokens: 3, cost: 0.5 },
|
||||
},
|
||||
{
|
||||
text: "I'll remember you like tea.",
|
||||
usage: { prompt_tokens: 20, completion_tokens: 5 },
|
||||
},
|
||||
]);
|
||||
const steps: number[] = [];
|
||||
|
||||
const result = await runTextLoop({
|
||||
system: "You are a bot.",
|
||||
messages: [{ role: "user", content: "I like tea." }],
|
||||
tools: [remember],
|
||||
generate,
|
||||
resolveCandidates,
|
||||
onStep: ({ index }) => {
|
||||
steps.push(index);
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.text).toBe("I'll remember you like tea.");
|
||||
expect(result.steps).toBe(2);
|
||||
expect(steps).toEqual([0, 1]);
|
||||
expect(memory).toEqual(["tea"]);
|
||||
expect(result.usage).toEqual({
|
||||
prompt_tokens: 30,
|
||||
completion_tokens: 8,
|
||||
cost: 0.5,
|
||||
});
|
||||
expect(seen[0]!.messages[0]).toEqual({
|
||||
role: "system",
|
||||
content: "You are a bot.",
|
||||
});
|
||||
expect(seen[0]!.tools?.[0]?.name).toBe("remember");
|
||||
expect(seen[1]!.prefer_key).toBe("1:m");
|
||||
expect(seen[1]!.messages.slice(-2)).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: "Noting that.",
|
||||
tool_calls: [
|
||||
{ id: "c1", name: "remember", arguments: `{"note":"tea"}` },
|
||||
],
|
||||
},
|
||||
{ role: "tool", tool_call_id: "c1", content: "Saved." },
|
||||
]);
|
||||
});
|
||||
|
||||
test("asks the model to wrap up, without tools, after the last step", async () => {
|
||||
const loop_call = {
|
||||
tool_calls: [{ id: "c", name: "remember", arguments: `{"note":"x"}` }],
|
||||
};
|
||||
const { generate, seen } = scripted([
|
||||
loop_call,
|
||||
loop_call,
|
||||
{ text: "Ran out of room; here's where I got to.", ...loop_call },
|
||||
]);
|
||||
|
||||
const result = await runTextLoop({
|
||||
system: "s",
|
||||
messages: [{ role: "user", content: "go" }],
|
||||
tools: [remember],
|
||||
max_steps: 2,
|
||||
generate,
|
||||
resolveCandidates,
|
||||
});
|
||||
|
||||
expect(seen).toHaveLength(3);
|
||||
expect(seen[2]!.tools).toBe(undefined);
|
||||
expect(seen[2]!.messages.at(-1)).toMatchObject({ role: "user" });
|
||||
expect(result.text).toBe("Ran out of room; here's where I got to.");
|
||||
expect(result.messages.at(-1)).toEqual({
|
||||
role: "assistant",
|
||||
content: "Ran out of room; here's where I got to.",
|
||||
});
|
||||
});
|
||||
|
||||
test("stops when cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
const error = await runTextLoop({
|
||||
system: "s",
|
||||
messages: [{ role: "user", content: "go" }],
|
||||
signal: controller.signal,
|
||||
generate: scripted([]).generate,
|
||||
resolveCandidates,
|
||||
}).catch((e) => e);
|
||||
|
||||
expect(error).toBeInstanceOf(HarnessError);
|
||||
expect(error.code).toBe("aborted");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,167 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import HarnessError from "../harness-error";
|
||||
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
|
||||
import runToolCall from "../tools/run-tool-call";
|
||||
import toolDefinition from "../tools/tool-definition";
|
||||
import type {
|
||||
HarnessCandidate,
|
||||
HarnessMessage,
|
||||
HarnessTool,
|
||||
HarnessToolCall,
|
||||
HarnessUsage,
|
||||
} from "../types";
|
||||
import fitTranscriptToContext from "./fit-transcript-to-context";
|
||||
import generateText from "./generate-text";
|
||||
|
||||
export type TextLoopStep = {
|
||||
index: number;
|
||||
text: string | null;
|
||||
tool_calls: HarnessToolCall[];
|
||||
candidate: HarnessCandidate;
|
||||
};
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* The bot's full prompt (`grabBotPrompt`).
|
||||
*/
|
||||
system: string;
|
||||
/**
|
||||
* The conversation so far, ending with what to do now.
|
||||
*/
|
||||
messages: HarnessMessage[];
|
||||
tools?: HarnessTool[];
|
||||
max_steps?: number;
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Called after each model turn, before its tools run. Use it for
|
||||
* progress and checkpoints.
|
||||
*/
|
||||
onStep?: (step: TextLoopStep) => Promise<void> | void;
|
||||
/**
|
||||
* The bot the run is for: its own models are used where it has them.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
generate?: typeof generateText;
|
||||
resolveCandidates?: () => Promise<HarnessCandidate[]>;
|
||||
};
|
||||
|
||||
export type TextLoopResult = {
|
||||
/**
|
||||
* The final answer for the user.
|
||||
*/
|
||||
text: string;
|
||||
/**
|
||||
* Everything after the system prompt, including tool turns.
|
||||
*/
|
||||
messages: HarnessMessage[];
|
||||
steps: number;
|
||||
candidate: HarnessCandidate;
|
||||
usage: HarnessUsage;
|
||||
};
|
||||
|
||||
const WrapUpPrompt = `You've used every step this run allows. Don't call any more tools. Report what you did, what is left, and anything you need from the user.`;
|
||||
|
||||
/**
|
||||
* The text harness at work: the model answers or calls tools, tool results
|
||||
* go back to it, and so on until it answers without calling a tool. Tools
|
||||
* run one at a time, in order. Stays on one model unless it fails. After
|
||||
* `max_steps` turns the model is asked to wrap up without tools.
|
||||
*/
|
||||
export default async function runTextLoop({
|
||||
system,
|
||||
messages,
|
||||
tools = [],
|
||||
max_steps = AppData.HarnessMaxToolSteps,
|
||||
signal = new AbortController().signal,
|
||||
onStep,
|
||||
bot_id,
|
||||
generate = generateText,
|
||||
resolveCandidates = () => resolveHarnessCandidates({ harness: "text", bot_id }),
|
||||
}: Params): Promise<TextLoopResult> {
|
||||
const candidates = await resolveCandidates();
|
||||
const definitions = tools.map((tool) => toolDefinition({ tool }));
|
||||
const media_cache = new Map<string, string>();
|
||||
const transcript: HarnessMessage[] = [...messages];
|
||||
const usage: Required<HarnessUsage> = {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
cost: 0,
|
||||
};
|
||||
let prefer_key: string | undefined;
|
||||
|
||||
for (let index = 0; index <= max_steps; index++) {
|
||||
if (signal.aborted) {
|
||||
throw new HarnessError({ message: `Cancelled.`, code: "aborted" });
|
||||
}
|
||||
|
||||
const wrap_up = index == max_steps;
|
||||
|
||||
if (wrap_up) {
|
||||
transcript.push({ role: "user", content: WrapUpPrompt });
|
||||
}
|
||||
|
||||
/**
|
||||
* Sized for the model the run is on (or will start on). A fallback
|
||||
* with a smaller window gets a 400 and the next model is tried.
|
||||
*/
|
||||
const model = (candidates.find((c) => c.key == prefer_key) || candidates[0])?.model;
|
||||
|
||||
fitTranscriptToContext({
|
||||
system,
|
||||
transcript,
|
||||
context_length: model?.context_length || AppData.HarnessAutoMinContext,
|
||||
});
|
||||
|
||||
const answer = await generate({
|
||||
messages: [{ role: "system", content: system }, ...transcript],
|
||||
tools: wrap_up || !definitions.length ? undefined : definitions,
|
||||
candidates,
|
||||
prefer_key,
|
||||
signal,
|
||||
media_cache,
|
||||
bot_id,
|
||||
});
|
||||
|
||||
prefer_key = answer.candidate.key;
|
||||
usage.prompt_tokens += answer.usage?.prompt_tokens || 0;
|
||||
usage.completion_tokens += answer.usage?.completion_tokens || 0;
|
||||
usage.cost += answer.usage?.cost || 0;
|
||||
|
||||
const tool_calls = wrap_up ? [] : answer.tool_calls;
|
||||
|
||||
transcript.push({
|
||||
role: "assistant",
|
||||
content: answer.text,
|
||||
...(tool_calls.length ? { tool_calls } : {}),
|
||||
});
|
||||
|
||||
await onStep?.({
|
||||
index,
|
||||
text: answer.text,
|
||||
tool_calls,
|
||||
candidate: answer.candidate,
|
||||
});
|
||||
|
||||
if (!tool_calls.length) {
|
||||
return {
|
||||
text:
|
||||
answer.text ||
|
||||
(wrap_up ? `I ran out of steps before I could finish.` : ""),
|
||||
messages: transcript,
|
||||
steps: index + 1,
|
||||
candidate: answer.candidate,
|
||||
usage,
|
||||
};
|
||||
}
|
||||
|
||||
for (const call of tool_calls) {
|
||||
transcript.push({
|
||||
role: "tool",
|
||||
tool_call_id: call.id,
|
||||
content: await runToolCall({ call, tools, signal }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Unreachable: the text loop ended without an answer.`);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import stripThinkTags from "./strip-think-tags";
|
||||
|
||||
describe("stripThinkTags", () => {
|
||||
test("removes reasoning blocks", () => {
|
||||
expect(
|
||||
stripThinkTags({ text: "<think>plan\nmore</think>\n\nDone." }),
|
||||
).toBe("Done.");
|
||||
expect(
|
||||
stripThinkTags({ text: "A <THINK>x</THINK>B <think>y</think>C" }),
|
||||
).toBe("A B C");
|
||||
});
|
||||
|
||||
test("drops an unclosed block at the start", () => {
|
||||
expect(stripThinkTags({ text: "<think>never ends" })).toBe("");
|
||||
});
|
||||
|
||||
test("leaves plain text alone", () => {
|
||||
expect(stripThinkTags({ text: " Hello " })).toBe("Hello");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
type Params = {
|
||||
text: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Removes the `<think>...</think>` reasoning some models write into their
|
||||
* answer, including an unclosed one at the start.
|
||||
*/
|
||||
export default function stripThinkTags({ text }: Params) {
|
||||
return text
|
||||
.replace(/<think>[\s\S]*?<\/think>/gi, "")
|
||||
.replace(/^\s*<think>[\s\S]*$/i, "")
|
||||
.trim();
|
||||
}
|
||||
Reference in New Issue
Block a user