Show how each bot message was generated
Every bot message gets an info button that opens its details: the provider and model behind it, tokens and cost, the job that ran it, each model used and every call in order. Each model call now records how it was made in token_usages.settings (generation settings sent, whose settings they were, pinned or automatic model, fallbacks), shown per model in the modal. Live replies now carry their token usage, from the job's push and the live relay alike, so it shows without a reload. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import describeCallSettings from "./describe-call-settings";
|
||||
|
||||
describe("describeCallSettings", () => {
|
||||
test("names every setting by its title", () => {
|
||||
expect(
|
||||
describeCallSettings({
|
||||
harness: "text",
|
||||
settings: {
|
||||
model_source: "pinned",
|
||||
settings_source: "workspace",
|
||||
params: { temperature: 0.3, reasoning_effort: "high" },
|
||||
max_tokens: 4000,
|
||||
fell_back_from: ["2:openai/gpt-x"],
|
||||
},
|
||||
}),
|
||||
).toEqual([
|
||||
{ label: "Model pick", value: "Picked in the settings" },
|
||||
{ label: "Settings from", value: "The workspace's" },
|
||||
{ label: "Temperature", value: "0.3" },
|
||||
{ label: "Reasoning effort", value: "High" },
|
||||
{ label: "Max tokens", value: "4,000" },
|
||||
{ label: "Fell back from", value: "openai/gpt-x" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("says when nothing was sent", () => {
|
||||
expect(describeCallSettings({ harness: "text", settings: { model_source: "auto" } })).toEqual([
|
||||
{ label: "Model pick", value: "Automatic" },
|
||||
{ label: "Generation", value: "The provider's defaults" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("leaves generation out where it doesn't apply", () => {
|
||||
expect(describeCallSettings({ harness: "speech", settings: { model_source: "pinned", voice: "alloy" } })).toEqual([
|
||||
{ label: "Model pick", value: "Picked in the settings" },
|
||||
{ label: "Voice", value: "alloy" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import { HarnessParameterHarnesses, HarnessParameters } from "@/src/dict/harness-parameter-dict";
|
||||
import type { HarnessCallSettings } from "@/src/harness/usage/types";
|
||||
|
||||
const SettingsSources: Record<NonNullable<HarnessCallSettings["settings_source"]>, string> = {
|
||||
default: "The install's defaults",
|
||||
bot: "The bot's own",
|
||||
workspace: "The workspace's",
|
||||
};
|
||||
|
||||
type Params = {
|
||||
settings: HarnessCallSettings;
|
||||
/**
|
||||
* Generation settings only apply to the chat harnesses.
|
||||
*/
|
||||
harness: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A call's settings as label and value pairs, in the order Settings,
|
||||
* under Models, lists them: the model's pick, whose settings they were,
|
||||
* every generation setting sent (by its title), then the token limit, the
|
||||
* voice and any models that failed before it.
|
||||
*/
|
||||
export default function describeCallSettings({ settings, harness }: Params) {
|
||||
const chat = (HarnessParameterHarnesses as readonly string[]).includes(harness);
|
||||
const rows: { label: string; value: string }[] = [
|
||||
{ label: "Model pick", value: settings.model_source == "pinned" ? "Picked in the settings" : "Automatic" },
|
||||
];
|
||||
|
||||
if (settings.settings_source) {
|
||||
rows.push({ label: "Settings from", value: SettingsSources[settings.settings_source] });
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(settings.params || {})) {
|
||||
const parameter = HarnessParameters.find((p) => p.key == key);
|
||||
const option =
|
||||
parameter && "options" in parameter ? parameter.options.find((o) => o.value == value) : undefined;
|
||||
|
||||
rows.push({ label: parameter?.title || key, value: option?.title || String(value) });
|
||||
}
|
||||
|
||||
if (chat && (!settings.params || !Object.keys(settings.params).length)) {
|
||||
rows.push({ label: "Generation", value: "The provider's defaults" });
|
||||
}
|
||||
|
||||
if (settings.max_tokens) rows.push({ label: "Max tokens", value: settings.max_tokens.toLocaleString() });
|
||||
if (settings.voice) rows.push({ label: "Voice", value: settings.voice });
|
||||
|
||||
if (settings.fell_back_from?.length) {
|
||||
rows.push({
|
||||
label: "Fell back from",
|
||||
value: settings.fell_back_from.map((key) => key.slice(key.indexOf(":") + 1)).join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import formatDuration from "./format-duration";
|
||||
|
||||
describe("formatDuration", () => {
|
||||
test("reads at every scale", () => {
|
||||
expect(formatDuration({ ms: 850 })).toBe("850ms");
|
||||
expect(formatDuration({ ms: 4200 })).toBe("4.2s");
|
||||
expect(formatDuration({ ms: 3000 })).toBe("3s");
|
||||
expect(formatDuration({ ms: 125_000 })).toBe("2m 5s");
|
||||
expect(formatDuration({ ms: 120_000 })).toBe("2m");
|
||||
expect(formatDuration({ ms: 3_780_000 })).toBe("1h 3m");
|
||||
expect(formatDuration({ ms: 7_200_000 })).toBe("2h");
|
||||
});
|
||||
|
||||
test("is empty for nothing to show", () => {
|
||||
expect(formatDuration({ ms: null })).toBe("");
|
||||
expect(formatDuration({ ms: -5 })).toBe("");
|
||||
expect(formatDuration({ ms: Number.NaN })).toBe("");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
type Params = {
|
||||
/**
|
||||
* In milliseconds.
|
||||
*/
|
||||
ms: number | null | undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* A length of time as a person reads it: `850ms`, `4.2s`, `2m 5s`,
|
||||
* `1h 3m`. Empty for a missing or negative one.
|
||||
*/
|
||||
export default function formatDuration({ ms }: Params) {
|
||||
if (ms === null || ms === undefined) return "";
|
||||
|
||||
const value = Number(ms);
|
||||
|
||||
if (!Number.isFinite(value) || value < 0) return "";
|
||||
if (value < 1000) return `${Math.round(value)}ms`;
|
||||
if (value < 60_000) return `${(value / 1000).toFixed(1).replace(/\.0$/, "")}s`;
|
||||
|
||||
const seconds = Math.round(value / 1000);
|
||||
|
||||
if (seconds < 3600) {
|
||||
const rest = seconds % 60;
|
||||
return rest ? `${Math.floor(seconds / 60)}m ${rest}s` : `${Math.floor(seconds / 60)}m`;
|
||||
}
|
||||
|
||||
const minutes = Math.round(seconds / 60);
|
||||
const rest = minutes % 60;
|
||||
|
||||
return rest ? `${Math.floor(minutes / 60)}h ${rest}m` : `${Math.floor(minutes / 60)}h`;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { MessageDetailsCall } from "@/src/types";
|
||||
import groupMessageCalls from "./group-message-calls";
|
||||
|
||||
function call(overrides: Partial<MessageDetailsCall>): MessageDetailsCall {
|
||||
return {
|
||||
id: 1,
|
||||
created_at: 0,
|
||||
harness: "text",
|
||||
provider_id: 1,
|
||||
provider_name: "OpenRouter",
|
||||
provider_kind: "openrouter",
|
||||
model_slug: "deepseek/deepseek-v4",
|
||||
model_name: "DeepSeek V4",
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 10,
|
||||
cost: null,
|
||||
settings: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("groupMessageCalls", () => {
|
||||
test("sums calls of the same provider, model and harness", () => {
|
||||
const groups = groupMessageCalls({
|
||||
calls: [call({ id: 1, cost: 0.01 }), call({ id: 2, prompt_tokens: 200, cost: 0.02 })],
|
||||
});
|
||||
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]).toMatchObject({ prompt_tokens: 300, completion_tokens: 20, total_tokens: 320, calls: 2 });
|
||||
expect(groups[0]!.cost).toBeCloseTo(0.03);
|
||||
});
|
||||
|
||||
test("keeps harnesses and models apart, busiest first", () => {
|
||||
const groups = groupMessageCalls({
|
||||
calls: [
|
||||
call({ id: 1, harness: "vision", model_slug: "small", prompt_tokens: 5, completion_tokens: 0 }),
|
||||
call({ id: 2 }),
|
||||
call({ id: 3, model_slug: "other" }),
|
||||
call({ id: 4 }),
|
||||
],
|
||||
});
|
||||
|
||||
expect(groups.map((g) => [g.harness, g.model_slug, g.calls])).toEqual([
|
||||
["text", "deepseek/deepseek-v4", 2],
|
||||
["text", "other", 1],
|
||||
["vision", "small", 1],
|
||||
]);
|
||||
});
|
||||
|
||||
test("leaves the cost out when no call reported one", () => {
|
||||
const [group] = groupMessageCalls({ calls: [call({})] });
|
||||
|
||||
expect(group!.cost).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { MessageDetailsCall, MessageDetailsModel } from "@/src/types";
|
||||
|
||||
type Params = {
|
||||
calls: MessageDetailsCall[];
|
||||
};
|
||||
|
||||
/**
|
||||
* A message's calls summed per provider, model and harness, busiest
|
||||
* first: the "Models" list of its details modal. A call whose model or
|
||||
* provider was never recorded is grouped with the others like it.
|
||||
*/
|
||||
export default function groupMessageCalls({ calls }: Params): MessageDetailsModel[] {
|
||||
const groups = new Map<string, MessageDetailsModel>();
|
||||
|
||||
for (const call of calls) {
|
||||
const key = [call.harness, call.provider_id ?? call.provider_name ?? "", call.model_slug ?? ""].join("\u0000");
|
||||
const group = groups.get(key) || {
|
||||
harness: call.harness,
|
||||
provider_name: call.provider_name,
|
||||
provider_kind: call.provider_kind,
|
||||
model_slug: call.model_slug,
|
||||
model_name: call.model_name,
|
||||
settings: call.settings,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
calls: 0,
|
||||
};
|
||||
|
||||
group.prompt_tokens += call.prompt_tokens;
|
||||
group.completion_tokens += call.completion_tokens;
|
||||
group.total_tokens = group.prompt_tokens + group.completion_tokens;
|
||||
group.calls += 1;
|
||||
group.settings = call.settings || group.settings;
|
||||
|
||||
if (typeof call.cost == "number" && call.cost > 0) {
|
||||
group.cost = (group.cost || 0) + call.cost;
|
||||
}
|
||||
|
||||
groups.set(key, group);
|
||||
}
|
||||
|
||||
return [...groups.values()].sort((a, b) => b.total_tokens - a.total_tokens || b.calls - a.calls);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import parseCallSettings from "./parse-call-settings";
|
||||
|
||||
describe("parseCallSettings", () => {
|
||||
test("reads what was stored", () => {
|
||||
expect(
|
||||
parseCallSettings({
|
||||
json: JSON.stringify({
|
||||
model_source: "pinned",
|
||||
settings_source: "bot",
|
||||
params: { temperature: 0.3, reasoning_effort: "high", bad: { x: 1 } },
|
||||
max_tokens: 800,
|
||||
fell_back_from: ["2:a/b", 3],
|
||||
}),
|
||||
}),
|
||||
).toEqual({
|
||||
model_source: "pinned",
|
||||
settings_source: "bot",
|
||||
params: { temperature: 0.3, reasoning_effort: "high" },
|
||||
max_tokens: 800,
|
||||
fell_back_from: ["2:a/b"],
|
||||
});
|
||||
});
|
||||
|
||||
test("is null for an older call or a broken value", () => {
|
||||
expect(parseCallSettings({ json: null })).toBeNull();
|
||||
expect(parseCallSettings({ json: "{nope" })).toBeNull();
|
||||
expect(parseCallSettings({ json: "3" })).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { HarnessCallSettings } from "@/src/harness/usage/types";
|
||||
|
||||
type SettingsSource = NonNullable<HarnessCallSettings["settings_source"]>;
|
||||
|
||||
const SettingsSources: SettingsSource[] = ["default", "bot", "workspace"];
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* `token_usages.settings`, as stored.
|
||||
*/
|
||||
json?: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* A call's stored settings (`HarnessCallSettings`), or null for a call
|
||||
* recorded before they were, or one that doesn't parse. Only the known
|
||||
* fields of the right type are kept.
|
||||
*/
|
||||
export default function parseCallSettings({ json }: Params): HarnessCallSettings | null {
|
||||
if (!json) return null;
|
||||
|
||||
let raw: unknown;
|
||||
|
||||
try {
|
||||
raw = JSON.parse(json);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!raw || typeof raw != "object") return null;
|
||||
|
||||
const value = raw as Record<string, unknown>;
|
||||
const params = value.params && typeof value.params == "object" ? (value.params as Record<string, unknown>) : null;
|
||||
const kept_params = Object.fromEntries(
|
||||
Object.entries(params || {}).filter(([, v]) => typeof v == "number" || typeof v == "string"),
|
||||
) as Record<string, number | string>;
|
||||
|
||||
return {
|
||||
model_source: value.model_source == "pinned" ? "pinned" : "auto",
|
||||
...(SettingsSources.includes(value.settings_source as SettingsSource)
|
||||
? { settings_source: value.settings_source as SettingsSource }
|
||||
: {}),
|
||||
...(Object.keys(kept_params).length ? { params: kept_params } : {}),
|
||||
...(typeof value.max_tokens == "number" ? { max_tokens: value.max_tokens } : {}),
|
||||
...(typeof value.voice == "string" && value.voice ? { voice: value.voice } : {}),
|
||||
...(Array.isArray(value.fell_back_from)
|
||||
? { fell_back_from: value.fell_back_from.filter((k): k is string => typeof k == "string") }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user