From c13b9ea826b1a177159b4937a3a340d118335224 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Sun, 25 Oct 2026 04:33:47 +0100 Subject: [PATCH] Show how each bot message was generated Every bot message gets an info button that opens its details: the provider and model behind it, tokens and cost, the job that ran it, each model used and every call in order. Each model call now records how it was made in token_usages.settings (generation settings sent, whose settings they were, pinned or automatic model, fallbacks), shown per model in the modal. Live replies now carry their token usage, from the job's push and the live relay alike, so it shows without a reload. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- db/schema.ts | 5 + db/types/db.ts | 4 + src/data/app-data.ts | 7 +- .../backend/db/bots-queue/complete-bot-job.ts | 15 ++- .../backend/db/usage/grab-message-details.ts | 126 ++++++++++++++++++ .../backend/db/usage/with-message-usage.ts | 29 ++++ src/harness/audio/speak-text.ts | 21 +-- src/harness/audio/transcribe-audio.ts | 3 +- src/harness/embeddings/embed-text.ts | 3 +- src/harness/image/generate-image.ts | 3 +- src/harness/media/read-image.ts | 33 +++-- src/harness/resolve/with-candidates.ts | 10 +- src/harness/text/generate-text.ts | 18 ++- src/harness/text/grab-chat-params.ts | 14 +- src/harness/usage/build-usage-call.test.ts | 18 +++ src/harness/usage/build-usage-call.ts | 22 ++- src/harness/usage/flatten-sent-params.test.ts | 20 +++ src/harness/usage/flatten-sent-params.ts | 26 ++++ src/harness/usage/record-harness-usage.ts | 1 + src/harness/usage/types.ts | 28 ++++ src/live-relay/relay-new-bot-messages.ts | 33 +++-- .../[bot_id]/(hooks)/use-message-details.ts | 47 +++++++ .../(partials)/message-action-button.tsx | 5 +- .../[bot_id]/(partials)/message-actions.tsx | 4 +- .../(partials)/message-detail-row.tsx | 26 ++++ .../(partials)/message-details-body.tsx | 100 ++++++++++++++ .../(partials)/message-details-call-row.tsx | 58 ++++++++ .../(partials)/message-details-heading.tsx | 16 +++ .../(partials)/message-details-modal.tsx | 54 ++++++++ .../(partials)/message-details-model-row.tsx | 60 +++++++++ .../(partials)/message-details-setting.tsx | 19 +++ .../bots/[bot_id]/(sections)/conversation.tsx | 13 +- .../admin/bots/[bot_id]/(utils)/cost-label.ts | 9 ++ .../bots/[bot_id]/(utils)/harness-label.ts | 9 ++ .../[bot_id]/(utils)/provider-label.test.ts | 21 +++ .../bots/[bot_id]/(utils)/provider-label.ts | 20 +++ src/pages/api/admin/bots/message-details.ts | 41 ++++++ src/schemas/bot-schema.ts | 6 + src/types/index.ts | 84 +++++++++++- src/utils/describe-call-settings.test.ts | 40 ++++++ src/utils/describe-call-settings.ts | 57 ++++++++ src/utils/format-duration.test.ts | 20 +++ src/utils/format-duration.ts | 32 +++++ src/utils/group-message-calls.test.ts | 56 ++++++++ src/utils/group-message-calls.ts | 44 ++++++ src/utils/parse-call-settings.test.ts | 30 +++++ src/utils/parse-call-settings.ts | 50 +++++++ 48 files changed, 1307 insertions(+), 57 deletions(-) create mode 100644 src/functions/backend/db/usage/grab-message-details.ts create mode 100644 src/functions/backend/db/usage/with-message-usage.ts create mode 100644 src/harness/usage/flatten-sent-params.test.ts create mode 100644 src/harness/usage/flatten-sent-params.ts create mode 100644 src/pages/admin/bots/[bot_id]/(hooks)/use-message-details.ts create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-detail-row.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-details-body.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-details-call-row.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-details-heading.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-details-modal.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-details-model-row.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/message-details-setting.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(utils)/cost-label.ts create mode 100644 src/pages/admin/bots/[bot_id]/(utils)/harness-label.ts create mode 100644 src/pages/admin/bots/[bot_id]/(utils)/provider-label.test.ts create mode 100644 src/pages/admin/bots/[bot_id]/(utils)/provider-label.ts create mode 100644 src/pages/api/admin/bots/message-details.ts create mode 100644 src/utils/describe-call-settings.test.ts create mode 100644 src/utils/describe-call-settings.ts create mode 100644 src/utils/format-duration.test.ts create mode 100644 src/utils/format-duration.ts create mode 100644 src/utils/group-message-calls.test.ts create mode 100644 src/utils/group-message-calls.ts create mode 100644 src/utils/parse-call-settings.test.ts create mode 100644 src/utils/parse-call-settings.ts diff --git a/AGENTS.md b/AGENTS.md index a18037e..5ad7b82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,9 +211,9 @@ Every folder on the data volume, browsable read-only: a bot's own, a workspace's Every model call is written down in `token_usages` as it happens, so a message can show what it cost and `/admin/usage` can total the install's spending by day, by model and by provider. - **Whose call it was:** `src/harness/usage/harness-usage-scope.ts` is an `AsyncLocalStorage` scope (`user_id`, `bot_id`, `workspace_id`, `session_id`, `job_id`, `message_id`) that a run enters once, in `run-bot-job.ts` (`runWithHarnessUsageScope`), so everything underneath - the text loop's turns, a session summary, a transcription, an image, an embedding - is attributed without threading the ids through every harness function. The store is merged, not replaced: a nested run keeps the outer ids and overrides only what it knows better, and a `null` is a real value that clears a field (a bot's own chat has no workspace). `withUsageScope` (`with-usage-scope.ts`) wraps one call the app makes on a user's behalf outside any run (the chat's transcribe and speak routes), and `runWithFallbackScope` (`run-with-fallback-scope.ts`) attributes work whose run may be missing - a file the ingest queue embeds has no run, so it is charged to its owner (`grabUsageOwnerUserId`, the workspace's creator else the bot's) - while `mode: "own"` forces the given scope (the session namer, whose job is already complete and must not pay for it). Without an owner nothing is written: a row no one can be shown is not worth a row. -- **Recording:** `record-harness-usage.ts` writes one row and never throws (a model that answered must not lose its answer to a failed insert; it logs with `logError` instead). `build-usage-call.ts` turns the candidate that really answered into the row's provider and model (`provider_id`/`provider_name` and the catalog `model_id` plus the provider's `model_slug`), and providers that report no usage still get a row, so a call is never invisible. Every entry point calls it in `withCandidates`'s `on_success`: `generate-text.ts`, `read-image.ts`, `transcribe-audio.ts`, `speak-text.ts` (billed per character, so the text's length stands in for what was spoken), `generate-image.ts`, `embed-text.ts`. +- **Recording:** `record-harness-usage.ts` writes one row and never throws (a model that answered must not lose its answer to a failed insert; it logs with `logError` instead). `build-usage-call.ts` turns the candidate that really answered into the row's provider and model (`provider_id`/`provider_name` and the catalog `model_id` plus the provider's `model_slug`), and providers that report no usage still get a row, so a call is never invisible. Each row also keeps how the call was made in `token_usages.settings` (`HarnessCallSettings`, JSON): whether the model was picked or automatic, whose setting it used (`grabChatParams`' and `grabHarnessSetting`'s `source`), the generation settings actually sent (`flattenSentParams`: by setting key, so OpenRouter's `reasoning: { effort }` reads back as `reasoning_effort`), `max_tokens`, the speech voice, and the candidates that failed before it (`withCandidates` hands `failed` to `on_success`). Every entry point calls it in `withCandidates`'s `on_success`: `generate-text.ts`, `read-image.ts`, `transcribe-audio.ts`, `speak-text.ts` (billed per character, so the text's length stands in for what was spoken), `generate-image.ts`, `embed-text.ts`. - **Messages:** a reply or task job's calls carry the job's id as they happen, since its message doesn't exist yet; `complete-bot-job.ts` then runs `UPDATE token_usages SET message_id = ? WHERE job_id = ? AND message_id IS NULL` once the message is inserted, so the thread can show what the answer cost. Calls that produce no message (naming a session, embedding a file, transcribing a dictation) keep a null `message_id` and are still counted in the totals. -- **Message usage:** `grab-message-usage.ts` sums a batch of messages' calls (one `GROUP BY message_id`) and `grab-bot-messages.ts` attaches it as `BotMessageWithAttachments.usage` (`TokenUsageSummary`: prompt, completion, total, cost, calls). `formatTokens` (`src/utils/format-tokens.ts`) writes a count as a person reads it (842 / 1.2K / 842K / 1.2M). `MessageUsageChip` (`bots/[bot_id]/(partials)/message-usage-chip.tsx`) sits in the actions row under a bot message beside its timestamp and play button, showing the total with the breakdown in its `title`, and renders nothing when a message has no calls. +- **Message usage:** `grab-message-usage.ts` sums a batch of messages' calls (one `GROUP BY message_id`) and `grab-bot-messages.ts` attaches it as `BotMessageWithAttachments.usage` (`TokenUsageSummary`: prompt, completion, total, cost, calls). `formatTokens` (`src/utils/format-tokens.ts`) writes a count as a person reads it (842 / 1.2K / 842K / 1.2M). `MessageUsageChip` (`bots/[bot_id]/(partials)/message-usage-chip.tsx`) sits in the actions row under a bot message beside its timestamp and play button, showing the total with the breakdown in its `title`, and renders nothing when a message has no calls. Beside the reply and reaction actions, every bot message has an info button (`message-details-modal.tsx`, `use-message-details`) that opens its details, loaded when it opens: `GET api/admin/bots/message-details?message_id=` (`grabSpeakableMessage` for access, then `grabMessageDetails` in `src/functions/backend/db/usage/`) returns the provider and model of every call (`token_usages` joined to `harness_providers` and `harness_models` for the kind and display name), the calls summed per provider, model and harness (`groupMessageCalls`, `src/utils/group-message-calls.ts`), each model's settings (`parseCallSettings`, `describeCallSettings`, by their titles in `HarnessParameters`), the calls in order up to `AppData.MessageDetailsMaxCalls` (a fallback marked, settings on hover), the totals and the job that ran it (kind, status, attempts, how long it took and its steps, while `bots_queue` keeps it) (`MessageDetails` in `src/types/index.ts`). - **Usage page:** `/admin/usage`, a `RailPages` entry like Routines and Activity rather than a settings sub page, since it is the install's own record rather than a setting (super admins and admins only, `canManageCrew`; the dict entry's `can_view` hides it for anyone else, and `grab-usage-server-props.ts` redirects them to the bots). Helpers in `src/functions/backend/db/usage/`: `grab-usage-overview.ts` runs `grab-usage-totals.ts` (one total and the same split per harness), `grab-usage-days.ts` -> `bucket-usage-days.ts` (one bucket per day, empty days at zero), `grab-usage-models.ts` (per provider, model slug and harness, busiest first, joined to `harness_models` for a display name), plus the previous period of the same length and a rolling week. The window comes from `grab-usage-window.ts` (`UsageDefaultDays` 30, `UsageMaxDays` 365, clamped). API `api/admin/usage` (GET only) for another period, zod in `src/schemas/usage-schema.ts`. - **Days are local, not UTC:** `grabUsageDays` groups in SQL with `date(created_at / 1000, 'unixepoch', 'localtime')`, matching the local midnights `grabUsageWindow` builds and the `YYYY-MM-DD` `localDate` writes. Grouping by the bare UTC day put a late evening's calls in the previous bucket and silently dropped them whenever no bucket carried that name (the chart's total disagreed with the totals card); `bucket-usage-days.test.ts` pins this. Never name a local day with `toISOString().slice(0, 10)`. - **The page:** `index.server.ts` -> `(functions)/grab-usage-server-props.ts` (crew managers only, the period in the address), `index.tsx` in `SettingsShell` with `wide`, `(sections)/usage-section.tsx` (state in the address through `history.pushState`, `use-usage-overview.ts` loading another period), `(partials)/usage-totals.tsx` (tokens, model calls, cost, previous period with a change label), `usage-day-chart.tsx` + `usage-day-bar.tsx` (bars from static `h-*` classes in `(data)/usage-bar-heights.ts`, since inline styles are not allowed), `usage-harness-breakdown.tsx` + `usage-harness-row.tsx`, and `usage-models-table.tsx` + `usage-model-row.tsx` (calls, in, out, tokens, **price per thousand tokens** worked out from cost and tokens, which is what makes one model's cost comparable with another's). diff --git a/db/schema.ts b/db/schema.ts index 563771a..f561361 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -2985,6 +2985,11 @@ const schema: BUN_SQLITE_DatabaseSchemaType = { fieldDescription: `The provider's price in USD, when it reports one`, dataType: "REAL", }, + { + fieldName: "settings", + fieldDescription: `JSON (\`HarnessCallSettings\`): how the call was made. The generation settings sent (temperature, reasoning effort, max tokens, voice), where they came from, whether the model was picked or automatic, and the models that failed before it`, + dataType: "TEXT", + }, ], indexes: [ { diff --git a/db/types/db.ts b/db/types/db.ts index eff4364..9b8c0e0 100644 --- a/db/types/db.ts +++ b/db/types/db.ts @@ -1787,6 +1787,10 @@ export type BUN_SQLITE_COBBLE_TOKEN_USAGES = { * The provider's price in USD, when it reports one */ cost?: number | ""; + /** + * JSON (`HarnessCallSettings`): how the call was made. The generation settings sent (temperature, reasoning effort, max tokens, voice), where they came from, whether the model was picked or automatic, and the models that failed before it + */ + settings?: string; } export type BUN_SQLITE_COBBLE_BOT_FILE_VECTORS = { diff --git a/src/data/app-data.ts b/src/data/app-data.ts index 481c9be..7440d00 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -331,6 +331,11 @@ export const AppData = { * How many older messages each scroll to the top of the thread loads. */ ConversationOlderMessages: 30, + /** + * The most model calls a message's details modal lists one by one. A + * long task can take hundreds; the totals still count every one. + */ + MessageDetailsMaxCalls: 200, /** * Days of usage the usage page totals by default, and the most it will * total at once. @@ -396,7 +401,7 @@ export const AppData = { * setting, Settings > General), which wins over this one; a reply * reads it through `grabReplyMaxToolSteps`. */ - BotReplyMaxToolSteps: 8, + BotReplyMaxToolSteps: 20, /** * The most steps a saved run limit may ask for, for a task run or a * chat reply. A typo in the settings page can't turn one run into an diff --git a/src/functions/backend/db/bots-queue/complete-bot-job.ts b/src/functions/backend/db/bots-queue/complete-bot-job.ts index ebeaf00..b0a27ab 100644 --- a/src/functions/backend/db/bots-queue/complete-bot-job.ts +++ b/src/functions/backend/db/bots-queue/complete-bot-job.ts @@ -9,6 +9,7 @@ import relayJobToRequester from "./relay-job-to-requester"; import jobSessionId from "./job-session-id"; import grabJobUserId from "./grab-job-user-id"; import grabRunningBotJobFace from "./grab-running-bot-job-face"; +import withMessageUsage from "../usage/with-message-usage"; type Params = { job_id: number | string; @@ -78,7 +79,7 @@ export default async function completeBotJob({ }); } - const message = silent + const inserted = silent ? null : await insertBotMessage({ bot_id: job.bot_id, @@ -89,7 +90,7 @@ export default async function completeBotJob({ media_ids: attachments, }); - if (job.id && message?.id) { + if (job.id && inserted?.id) { /** * The job's model calls were written down as they happened, before * its message existed. Now that it does, they belong to it, so the @@ -97,10 +98,18 @@ export default async function completeBotJob({ */ await BunSQLite.sql({ sql: `UPDATE token_usages SET message_id = ? WHERE job_id = ? AND message_id IS NULL`, - values: [message.id, job.id], + values: [inserted.id, job.id], }).catch(() => {}); } + /** + * Pushed live as it is returned, so it carries its usage now: the + * thread shows the tokens without a reload. + */ + const message = inserted + ? ((await withMessageUsage({ messages: [inserted] }).catch(() => [inserted]))[0] ?? inserted) + : null; + const user_id = await grabJobUserId({ job }); const bot = !user_id diff --git a/src/functions/backend/db/usage/grab-message-details.ts b/src/functions/backend/db/usage/grab-message-details.ts new file mode 100644 index 0000000..8ed34d0 --- /dev/null +++ b/src/functions/backend/db/usage/grab-message-details.ts @@ -0,0 +1,126 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { AppData } from "@/src/data/app-data"; +import type { MessageDetails, MessageDetailsCall, MessageDetailsJob } from "@/src/types"; +import groupMessageCalls from "@/src/utils/group-message-calls"; +import parseCallSettings from "@/src/utils/parse-call-settings"; +import grabMessageUsage from "./grab-message-usage"; + +type Params = { + /** + * A message the caller already checked the user reaches. + */ + message_id: number; + created_at?: number | null; +}; + +type CallRow = { + id: number; + created_at: number; + harness: string | null; + provider_id: number | null; + provider_name: string | null; + provider_kind: string | null; + model_slug: string | null; + model_name: string | null; + prompt_tokens: number | null; + completion_tokens: number | null; + cost: number | null; + settings: string | null; + job_id: number | null; +}; + +type JobRow = { + id: number; + kind: string | null; + title: string | null; + status: string | null; + attempts: number | null; + started_at: number | null; + finished_at: number | null; + steps: number | null; +}; + +/** + * How one message was generated: every model call it took (`token_usages`) + * with the provider and model that answered, summed per model, and the + * queued job that ran it, while it is kept. The provider's kind and the + * model's display name come from their live rows, so a deleted provider + * or an unlisted model still reads by the name it was called with. + */ +export default async function grabMessageDetails({ message_id, created_at }: Params): Promise { + const rows = + ( + await BunSQLite.sql({ + sql: `SELECT t.id, t.created_at, t.harness, t.provider_id, t.provider_name, + p.kind AS provider_kind, t.model_slug, m.name AS model_name, + t.prompt_tokens, t.completion_tokens, t.cost, t.settings, t.job_id + FROM token_usages t + LEFT JOIN harness_providers p ON p.id = t.provider_id + LEFT JOIN harness_models m ON m.id = t.model_id + WHERE t.message_id = ? + ORDER BY t.created_at ASC, t.id ASC`, + values: [message_id], + }) + ).payload || []; + + const calls: MessageDetailsCall[] = rows.map((row) => ({ + id: Number(row.id), + created_at: Number(row.created_at) || 0, + harness: row.harness || "text", + provider_id: row.provider_id ? Number(row.provider_id) : null, + provider_name: row.provider_name || null, + provider_kind: row.provider_kind || null, + model_slug: row.model_slug || null, + model_name: row.model_name || null, + prompt_tokens: Number(row.prompt_tokens) || 0, + completion_tokens: Number(row.completion_tokens) || 0, + cost: Number.isFinite(Number(row.cost)) && row.cost !== null ? Number(row.cost) : null, + settings: parseCallSettings({ json: row.settings }), + })); + + const totals = (await grabMessageUsage({ message_ids: [message_id] })).get(message_id) || { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + calls: 0, + }; + + /** + * The job is the one its calls ran for: the latest, since reading a + * message aloud later is a call without one. + */ + const job_id = [...rows].reverse().find((row) => row.job_id)?.job_id; + + return { + message_id, + created_at: created_at || null, + totals, + models: groupMessageCalls({ calls }), + calls: calls.slice(0, AppData.MessageDetailsMaxCalls), + job: job_id ? await grabJob({ job_id: Number(job_id) }) : null, + }; +} + +async function grabJob({ job_id }: { job_id: number }): Promise { + const row = ( + await BunSQLite.sql({ + sql: `SELECT q.id, q.kind, q.title, q.status, q.attempts, q.started_at, q.finished_at, + (SELECT COUNT(*) FROM bots_queue_updates u WHERE u.bots_queue_id = q.id) AS steps + FROM bots_queue q WHERE q.id = ? LIMIT 1`, + values: [job_id], + }) + ).payload?.[0]; + + if (!row?.id) return null; + + return { + id: Number(row.id), + kind: row.kind || null, + title: row.title || null, + status: row.status || null, + attempts: Number(row.attempts) || 0, + started_at: Number(row.started_at) || null, + finished_at: Number(row.finished_at) || null, + steps: Number(row.steps) || 0, + }; +} diff --git a/src/functions/backend/db/usage/with-message-usage.ts b/src/functions/backend/db/usage/with-message-usage.ts new file mode 100644 index 0000000..14fa8ba --- /dev/null +++ b/src/functions/backend/db/usage/with-message-usage.ts @@ -0,0 +1,29 @@ +import type { BUN_SQLITE_COBBLE_BOT_MESSAGES } from "@/db/types/db"; +import type { TokenUsageSummary } from "@/src/harness/usage/types"; +import grabMessageUsage from "./grab-message-usage"; + +type Params = { + messages: T[]; +}; + +/** + * The messages with what their model calls spent (`usage`), for a live + * push: a pushed bot message must carry it like a loaded one + * (`grabBotMessages`), or the thread shows its tokens only after a reload. + * Messages with no calls (the user's own among them) are left as they are. + */ +export default async function withMessageUsage({ + messages, +}: Params): Promise<(T & { usage?: TokenUsageSummary })[]> { + const ids = messages.filter((m) => m.id && m.author == "bot").map((m) => Number(m.id)); + + if (!ids.length) return messages; + + const usage = await grabMessageUsage({ message_ids: ids }); + + return messages.map((message) => { + const spent = usage.get(Number(message.id)); + + return spent ? { ...message, usage: spent } : message; + }); +} diff --git a/src/harness/audio/speak-text.ts b/src/harness/audio/speak-text.ts index 6ff359e..6f6373c 100644 --- a/src/harness/audio/speak-text.ts +++ b/src/harness/audio/speak-text.ts @@ -41,6 +41,12 @@ export default async function speakText({ grabHarnessSetting({ harness: "speech", bot_id, workspace_id }), ]); + const voiceFor = (candidate: HarnessCandidate) => + pickVoice({ + candidate, + voice: typeof setting.options.voice == "string" ? setting.options.voice : undefined, + }); + const { result } = await withCandidates({ harness: "speech", candidates, @@ -49,25 +55,24 @@ export default async function speakText({ * completion count to read: the call is recorded as it is, with * the text's length standing in for what was spoken. */ - on_success: async ({ candidate }) => + on_success: async ({ candidate, failed }) => await recordHarnessUsage({ call: buildUsageCall({ harness: "speech", candidate, usage: { prompt_tokens: text.length, completion_tokens: 0 }, + failed, + settings: { + settings_source: setting.source, + voice: voiceFor(candidate), + }, }), }), run: async (candidate) => await grabHarnessAdapter({ kind: candidate.provider.kind }).speak({ candidate, text, - voice: pickVoice({ - candidate, - voice: - typeof setting.options.voice == "string" - ? setting.options.voice - : undefined, - }), + voice: voiceFor(candidate), format, signal: signal || new AbortController().signal, }), diff --git a/src/harness/audio/transcribe-audio.ts b/src/harness/audio/transcribe-audio.ts index fbaaf8a..4860a53 100644 --- a/src/harness/audio/transcribe-audio.ts +++ b/src/harness/audio/transcribe-audio.ts @@ -51,11 +51,12 @@ export default async function transcribeAudio({ const { result } = await withCandidates<{ text: string; usage?: HarnessUsage }>({ harness: "transcription", candidates, - on_success: async ({ candidate, result: transcript }): Promise => + on_success: async ({ candidate, result: transcript, failed }): Promise => await recordHarnessUsage({ call: buildUsageCall({ harness: "transcription", candidate, + failed, usage: transcript.usage, }), }), diff --git a/src/harness/embeddings/embed-text.ts b/src/harness/embeddings/embed-text.ts index 8caffdf..eb7ac89 100644 --- a/src/harness/embeddings/embed-text.ts +++ b/src/harness/embeddings/embed-text.ts @@ -58,11 +58,12 @@ export default async function embedText({ const { result } = await withCandidates({ harness: "embedding", candidates, - on_success: async ({ candidate, result: embedded }): Promise => + on_success: async ({ candidate, result: embedded, failed }): Promise => await recordHarnessUsage({ call: buildUsageCall({ harness: "embedding", candidate, + failed, usage: embedded.usage, }), }), diff --git a/src/harness/image/generate-image.ts b/src/harness/image/generate-image.ts index 20dca7d..85b0004 100644 --- a/src/harness/image/generate-image.ts +++ b/src/harness/image/generate-image.ts @@ -41,11 +41,12 @@ export default async function generateImage({ const { result } = await withCandidates<{ images: HarnessImageOutput[]; usage?: HarnessUsage }>({ harness: "image", candidates, - on_success: async ({ candidate, result: made }): Promise => + on_success: async ({ candidate, result: made, failed }): Promise => await recordHarnessUsage({ call: buildUsageCall({ harness: "image", candidate, + failed, usage: made.usage, }), }), diff --git a/src/harness/media/read-image.ts b/src/harness/media/read-image.ts index fb63bbd..7b2da85 100644 --- a/src/harness/media/read-image.ts +++ b/src/harness/media/read-image.ts @@ -5,6 +5,7 @@ import resolveHarnessCandidates from "../resolve/resolve-harness-candidates"; import withCandidates from "../resolve/with-candidates"; import type { HarnessUsage } from "../types"; import grabChatParams from "../text/grab-chat-params"; +import flattenSentParams from "../usage/flatten-sent-params"; import buildUsageCall from "../usage/build-usage-call"; import recordHarnessUsage from "../usage/record-harness-usage"; @@ -45,18 +46,35 @@ export default async function readImage({ url, focus, signal, bot_id, workspace_ workspace_id, }); - const { result } = await withCandidates<{ text: string; usage?: HarnessUsage }>({ + const { result } = await withCandidates<{ + text: string; + usage?: HarnessUsage; + sent_params: Record; + }>({ harness: "vision", candidates, - on_success: async ({ candidate, result: answer }): Promise => + on_success: async ({ candidate, result: answer, failed }): Promise => await recordHarnessUsage({ call: buildUsageCall({ harness: "vision", candidate, usage: answer.usage, + failed, + settings: { + settings_source: saved_params.source, + params: flattenSentParams({ params: answer.sent_params }), + }, }), }), run: async (candidate) => { + /** + * Reading is a lookup, so it stays literal unless a + * temperature was set for the vision harness. + */ + const sent_params = { + temperature: 0, + ...saved_params.paramsFor(candidate.model), + }; const answer = await grabHarnessAdapter({ kind: candidate.provider.kind, }).chat({ @@ -75,14 +93,7 @@ export default async function readImage({ url, focus, signal, bot_id, workspace_ ], }, ], - /** - * Reading is a lookup, so it stays literal unless a - * temperature was set for the vision harness. - */ - params: { - temperature: 0, - ...saved_params(candidate.model), - }, + params: sent_params, signal: signal || new AbortController().signal, }); @@ -94,7 +105,7 @@ export default async function readImage({ url, focus, signal, bot_id, workspace_ }); } - return { text: answer.text.trim(), usage: answer.usage }; + return { text: answer.text.trim(), usage: answer.usage, sent_params }; }, }); diff --git a/src/harness/resolve/with-candidates.ts b/src/harness/resolve/with-candidates.ts index 714e2ad..3a34621 100644 --- a/src/harness/resolve/with-candidates.ts +++ b/src/harness/resolve/with-candidates.ts @@ -22,6 +22,10 @@ type Params = { on_success?: (params: { candidate: HarnessCandidate; result: T; + /** + * The candidates that failed before this one answered, by key. + */ + failed: string[]; }) => Promise | void; cooldowns?: HarnessCooldowns; now?: () => number; @@ -105,7 +109,11 @@ export default async function withCandidates({ try { const result = await run(candidate); - await on_success?.({ candidate, result }); + await on_success?.({ + candidate, + result, + failed: errors.flatMap((e) => (e.candidate_key ? [e.candidate_key] : [])), + }); return { result, candidate }; } catch (error: unknown) { diff --git a/src/harness/text/generate-text.ts b/src/harness/text/generate-text.ts index 16cfcb5..4549cf2 100644 --- a/src/harness/text/generate-text.ts +++ b/src/harness/text/generate-text.ts @@ -11,6 +11,7 @@ import type { import buildUsageCall from "../usage/build-usage-call"; import recordHarnessUsage from "../usage/record-harness-usage"; import grabChatParams from "./grab-chat-params"; +import flattenSentParams from "../usage/flatten-sent-params"; import prepareMessagesForModel from "./prepare-messages-for-model"; import stripThinkTags from "./strip-think-tags"; @@ -55,6 +56,10 @@ export type GenerateTextResult = HarnessChatResult & { */ type ChatAnswer = Omit & { text: string | null; + /** + * The generation settings sent to this candidate, for its usage row. + */ + sent_params?: Record; }; /** @@ -85,15 +90,22 @@ export default async function generateText({ harness: "text", candidates: resolved, prefer_key, - on_success: async ({ candidate: answered, result: answer }): Promise => + on_success: async ({ candidate: answered, result: answer, failed }): Promise => await recordHarnessUsage({ call: buildUsageCall({ harness: "text", candidate: answered, usage: answer.usage, + failed, + settings: { + ...(saved_params ? { settings_source: saved_params.source } : {}), + params: flattenSentParams({ params: answer.sent_params }), + ...(max_tokens ? { max_tokens } : {}), + }, }), }), run: async (candidate) => { + const sent_params = params || saved_params?.paramsFor(candidate.model); const answer = await grabHarnessAdapter({ kind: candidate.provider.kind, }).chat({ @@ -108,7 +120,7 @@ export default async function generateText({ }), tools, max_tokens, - params: params || saved_params?.(candidate.model), + params: sent_params, signal, }); @@ -122,7 +134,7 @@ export default async function generateText({ }); } - return { ...answer, text: text || null }; + return { ...answer, text: text || null, sent_params }; }, }); diff --git a/src/harness/text/grab-chat-params.ts b/src/harness/text/grab-chat-params.ts index 092d3d6..bff59ab 100644 --- a/src/harness/text/grab-chat-params.ts +++ b/src/harness/text/grab-chat-params.ts @@ -18,22 +18,24 @@ type Params = { * harness's setting (the workspace's, the bot's, else the default), else * `HarnessParameterDefaults` so a fresh install is ready to go. Keyed by * the field sent on the wire, keeping only what the model says it takes - * (a default only where the model explicitly advertises it). + * (a default only where the model explicitly advertises it). `source` + * says whose setting it was, for the call's usage row. */ export default async function grabChatParams({ harness, bot_id, workspace_id, -}: Params): Promise< - (model: HarnessModelInfo) => Record -> { +}: Params): Promise<{ + paramsFor: (model: HarnessModelInfo) => Record; + source: "default" | "bot" | "workspace"; +}> { const setting = await grabHarnessSetting({ harness, bot_id, workspace_id, }); - return (model) => + const paramsFor = (model: HarnessModelInfo) => Object.fromEntries( HarnessParameters.flatMap((parameter) => { const { value, saved } = harnessParameterValue({ @@ -67,4 +69,6 @@ export default async function grabChatParams({ ]; }), ); + + return { paramsFor, source: setting.source }; } diff --git a/src/harness/usage/build-usage-call.test.ts b/src/harness/usage/build-usage-call.test.ts index f0e1f75..46fa95e 100644 --- a/src/harness/usage/build-usage-call.test.ts +++ b/src/harness/usage/build-usage-call.test.ts @@ -44,6 +44,24 @@ describe("buildUsageCall", () => { prompt_tokens: 1200, completion_tokens: 340, cost: 0.0042, + settings: { model_source: "auto" }, + }); + }); + + test("keeps the settings sent and the models that failed first", () => { + const call = buildUsageCall({ + harness: "text", + candidate: candidate({ source: "pinned" }), + settings: { settings_source: "bot", params: { temperature: 0.3 }, max_tokens: 800 }, + failed: ["2:other/model"], + }); + + expect(call.settings).toEqual({ + model_source: "pinned", + settings_source: "bot", + params: { temperature: 0.3 }, + max_tokens: 800, + fell_back_from: ["2:other/model"], }); }); diff --git a/src/harness/usage/build-usage-call.ts b/src/harness/usage/build-usage-call.ts index acfc194..e2810f7 100644 --- a/src/harness/usage/build-usage-call.ts +++ b/src/harness/usage/build-usage-call.ts @@ -1,18 +1,29 @@ import type { HarnessCandidate, HarnessUsage } from "../types"; -import type { HarnessUsageCall } from "./types"; +import type { HarnessCallSettings, HarnessUsageCall } from "./types"; type Params = { harness: HarnessUsageCall["harness"]; candidate: HarnessCandidate; usage?: HarnessUsage; + /** + * How the call was made beyond the model: the settings it sent and + * whose they were. The model's source and the fallback are filled in + * here. + */ + settings?: Omit; + /** + * The candidates that failed before this one (`withCandidates`). + */ + failed?: string[]; }; /** * The candidate that really answered a call, and what it reported, as the * row `recordHarnessUsage` writes. Providers that report no usage at all - * still get a row (no tokens), so a call is never invisible. + * still get a row (no tokens), so a call is never invisible. Its settings + * say how it was made, for a message's details. */ -export default function buildUsageCall({ harness, candidate, usage }: Params): HarnessUsageCall { +export default function buildUsageCall({ harness, candidate, usage, settings, failed }: Params): HarnessUsageCall { return { harness, provider_id: candidate.provider.id, @@ -22,6 +33,11 @@ export default function buildUsageCall({ harness, candidate, usage }: Params): H prompt_tokens: Number(usage?.prompt_tokens) || 0, completion_tokens: Number(usage?.completion_tokens) || 0, cost: typeof usage?.cost == "number" && Number.isFinite(usage.cost) ? usage.cost : undefined, + settings: { + model_source: candidate.source, + ...settings, + ...(failed?.length ? { fell_back_from: failed } : {}), + }, }; } diff --git a/src/harness/usage/flatten-sent-params.test.ts b/src/harness/usage/flatten-sent-params.test.ts new file mode 100644 index 0000000..af7573f --- /dev/null +++ b/src/harness/usage/flatten-sent-params.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; +import flattenSentParams from "./flatten-sent-params"; + +describe("flattenSentParams", () => { + test("reads the nested reasoning effort back by its setting key", () => { + expect(flattenSentParams({ params: { temperature: 0.3, reasoning: { effort: "high" } } })).toEqual({ + temperature: 0.3, + reasoning_effort: "high", + }); + }); + + test("keeps a zero and drops empty fields", () => { + expect(flattenSentParams({ params: { temperature: 0, seed: "" } })).toEqual({ temperature: 0 }); + }); + + test("is undefined when nothing was sent", () => { + expect(flattenSentParams({ params: {} })).toBeUndefined(); + expect(flattenSentParams({ params: null })).toBeUndefined(); + }); +}); diff --git a/src/harness/usage/flatten-sent-params.ts b/src/harness/usage/flatten-sent-params.ts new file mode 100644 index 0000000..9c68473 --- /dev/null +++ b/src/harness/usage/flatten-sent-params.ts @@ -0,0 +1,26 @@ +type Params = { + /** + * Keyed by the field sent on the wire (`grabChatParams`). + */ + params?: Record | null; +}; + +/** + * The generation settings a call sent, keyed by their setting + * (`HarnessParameters`) rather than the wire: OpenRouter's nested + * `reasoning: { effort }` reads back as `reasoning_effort`. Empty fields + * are left out; undefined when nothing was sent. + */ +export default function flattenSentParams({ params }: Params) { + const flat: Record = {}; + + for (const [key, value] of Object.entries(params || {})) { + if (value && typeof value == "object") { + if (value.effort) flat[key == "reasoning" ? "reasoning_effort" : key] = String(value.effort); + } else if (typeof value == "number" || (typeof value == "string" && value)) { + flat[key] = value; + } + } + + return Object.keys(flat).length ? flat : undefined; +} diff --git a/src/harness/usage/record-harness-usage.ts b/src/harness/usage/record-harness-usage.ts index 957de11..5f989cb 100644 --- a/src/harness/usage/record-harness-usage.ts +++ b/src/harness/usage/record-harness-usage.ts @@ -52,6 +52,7 @@ export default async function recordHarnessUsage({ scope, call }: Params) { prompt_tokens: Math.max(0, Math.round(call.prompt_tokens || 0)), completion_tokens: Math.max(0, Math.round(call.completion_tokens || 0)), cost: typeof call.cost == "number" && Number.isFinite(call.cost) ? call.cost : undefined, + settings: call.settings ? JSON.stringify(call.settings) : undefined, }; try { diff --git a/src/harness/usage/types.ts b/src/harness/usage/types.ts index c33289c..1ba1248 100644 --- a/src/harness/usage/types.ts +++ b/src/harness/usage/types.ts @@ -30,6 +30,33 @@ export type HarnessUsageScope = { * asked, and which candidate answered. Both are filled in by the call's * own entry point, never by the code that runs it. */ +export type HarnessCallSettings = { + /** + * `pinned`: the model picked in the setting. `auto`: an automatic pick. + */ + model_source: "pinned" | "auto"; + /** + * Whose setting the call used: the workspace's, the bot's own or the + * install's default. Left out when the call overrode it. + */ + settings_source?: "default" | "bot" | "workspace"; + /** + * The generation settings sent, by their setting key + * (`HarnessParameters`: `temperature`, `reasoning_effort`, ...). + */ + params?: Record; + max_tokens?: number; + /** + * The voice asked for (speech). + */ + voice?: string; + /** + * The candidates (`:`) that failed before this one + * answered, in the order they were tried. + */ + fell_back_from?: string[]; +}; + export type HarnessUsageCall = { harness: "text" | "vision" | "transcription" | "speech" | "image" | "embedding"; provider_id?: number; @@ -39,6 +66,7 @@ export type HarnessUsageCall = { prompt_tokens: number; completion_tokens: number; cost?: number; + settings?: HarnessCallSettings; }; /** diff --git a/src/live-relay/relay-new-bot-messages.ts b/src/live-relay/relay-new-bot-messages.ts index 902a213..3f95cc9 100644 --- a/src/live-relay/relay-new-bot-messages.ts +++ b/src/live-relay/relay-new-bot-messages.ts @@ -1,6 +1,7 @@ import BunSQLite from "@moduletrace/bun-sqlite"; import type { BUN_SQLITE_COBBLE_BOT_MESSAGES } from "@/db/types/db"; import grabMessagesReplies from "@/src/functions/backend/db/bots/grab-messages-replies"; +import withMessageUsage from "@/src/functions/backend/db/usage/with-message-usage"; import { sendToUser } from "@/src/websocket/(functions)/send-to-user"; type Params = { @@ -14,7 +15,7 @@ type Params = { * with the message a reply quotes. Returns the newest id read. */ export default async function relayNewBotMessages({ after_id, users }: Params) { - const rows = + const read = ( await BunSQLite.sql({ sql: `SELECT m.*, s.user_id FROM bot_messages m JOIN bot_sessions s ON s.id = m.session_id @@ -23,25 +24,31 @@ export default async function relayNewBotMessages({ after_id, users }: Params) { }) ).payload || []; + /** + * With what each one spent, like a loaded message, so the thread shows + * its tokens without a reload. + */ + const rows = await withMessageUsage({ + messages: read.filter((row) => users.has(String(row.user_id))), + }); + const replies = await grabMessagesReplies({ message_ids: rows - .filter((row) => row.reply_to_message_id && users.has(String(row.user_id))) + .filter((row) => row.reply_to_message_id) .map((row) => Number(row.id)), }); for (const { user_id, ...message } of rows) { - if (users.has(String(user_id))) { - const reply_to = replies.get(Number(message.id)); + const reply_to = replies.get(Number(message.id)); - sendToUser({ - user_id, - data: { - event: "server:bot-message", - data: reply_to ? { ...message, reply_to } : message, - }, - }); - } + sendToUser({ + user_id, + data: { + event: "server:bot-message", + data: reply_to ? { ...message, reply_to } : message, + }, + }); } - return Number(rows[rows.length - 1]?.id) || after_id; + return Number(read[read.length - 1]?.id) || after_id; } diff --git a/src/pages/admin/bots/[bot_id]/(hooks)/use-message-details.ts b/src/pages/admin/bots/[bot_id]/(hooks)/use-message-details.ts new file mode 100644 index 0000000..ff30753 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(hooks)/use-message-details.ts @@ -0,0 +1,47 @@ +import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi"; +import type { APIResponseObject } from "@moduletrace/bunext/types"; +import { useCallback, useEffect, useState } from "react"; +import type { MessageDetails } from "@/src/types"; + +type Params = { + message_id: number; + /** + * Load only once the modal opens, so the thread's messages stay small. + */ + enabled?: boolean; +}; + +/** + * How one bot message was generated (`api/admin/bots/message-details`), + * loaded when its details modal opens and kept while the thread is open. + */ +export default function useMessageDetails({ message_id, enabled = true }: Params) { + const [details, setDetails] = useState(); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(); + + const load = useCallback(async () => { + setLoading(true); + setError(undefined); + + const res = await fetchApi<{}, APIResponseObject>( + `/api/admin/bots/message-details?message_id=${message_id}`, + { method: "GET" }, + ).catch(() => null); + + setLoading(false); + + if (!res?.success || !res.singleRes) { + setError(res?.msg || `Couldn't load this message's details.`); + return; + } + + setDetails(res.singleRes); + }, [message_id]); + + useEffect(() => { + if (enabled) load().catch(() => {}); + }, [enabled, load]); + + return { details, loading, error }; +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-action-button.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-action-button.tsx index b225579..006c33a 100644 --- a/src/pages/admin/bots/[bot_id]/(partials)/message-action-button.tsx +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-action-button.tsx @@ -4,7 +4,10 @@ import type { ReactNode } from "react"; type Props = { title: string; aria_label: string; - onClick: () => void; + /** + * Left out when a wrapper (a modal's target) handles the click. + */ + onClick?: () => void; children: ReactNode; }; diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-actions.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-actions.tsx index 00d0344..0f8be50 100644 --- a/src/pages/admin/bots/[bot_id]/(partials)/message-actions.tsx +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-actions.tsx @@ -5,6 +5,7 @@ import type { BotMessageWithAttachments } from "@/src/types"; import { BotMessageActionsContext } from "../(data)/message-actions-context"; import CopyMessageButton from "./copy-message-button"; import MessageActionButton from "./message-action-button"; +import MessageDetailsModal from "./message-details-modal"; import MessageReactionPicker from "./message-reaction-picker"; type Props = { @@ -12,7 +13,7 @@ type Props = { }; /** - * A message's copy, reply and reaction actions, shown under it. A message + * A message's copy, reply, reaction and details actions, shown under it. A message * still on its way to the server has no id yet, so it can only be copied. */ export default function MessageActions({ message }: Props) { @@ -36,6 +37,7 @@ export default function MessageActions({ message }: Props) { message_id={message.id} onReact={onReact} /> + {message.author == "bot" ? : null} ) : null} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-detail-row.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-detail-row.tsx new file mode 100644 index 0000000..9e16820 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-detail-row.tsx @@ -0,0 +1,26 @@ +import Row from "@moduletrace/twui/layout/Row"; +import Span from "@moduletrace/twui/layout/Span"; + +type Props = { + label: string; + value: string; + /** + * Shown on hover, for the exact figure behind a rounded one. + */ + title?: string; +}; + +/** + * One fact about how a message was generated: a label on the left, its + * value on the right, hairline-separated like a settings list. + */ +export default function MessageDetailRow({ label, value, title }: Props) { + return ( + + {label} + + {value} + + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-details-body.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-details-body.tsx new file mode 100644 index 0000000..363775a --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-details-body.tsx @@ -0,0 +1,100 @@ +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import { BotJobKinds } from "@/src/dict/bot-queue-dict"; +import type { MessageDetails } from "@/src/types"; +import formatDuration from "@/src/utils/format-duration"; +import formatTokens from "@/src/utils/format-tokens"; +import relativeTime from "@/src/utils/relative-time"; +import costLabel from "../(utils)/cost-label"; +import providerLabel from "../(utils)/provider-label"; +import MessageDetailRow from "./message-detail-row"; +import MessageDetailsCallRow from "./message-details-call-row"; +import MessageDetailsHeading from "./message-details-heading"; +import MessageDetailsModelRow from "./message-details-model-row"; + +type Props = { + details: MessageDetails; +}; + +/** + * How a message was generated: the provider and model that answered and + * what it spent, then the job that ran it, every model it used and every + * call in order. + */ +export default function MessageDetailsBody({ details }: Props) { + const { totals, models, calls, job } = details; + const providers = [ + ...new Set(models.map((m) => providerLabel({ name: m.provider_name, kind: m.provider_kind }))), + ]; + const model_names = [...new Set(models.map((m) => m.model_name || m.model_slug || "Unknown model"))]; + const first_at = calls[0]?.created_at || 0; + const job_kind = BotJobKinds.find((k) => k.value == job?.kind)?.title || job?.kind; + const job_took = + job?.started_at && job.finished_at ? formatDuration({ ms: job.finished_at - job.started_at }) : ""; + + return ( + + {totals.calls ? null : ( + + No model calls were recorded for this message. It was written by the app itself (a stop, a + failure notice), or before usage was recorded. + + )} + + 1 ? "Providers" : "Provider"} value={providers.join(", ") || "-"} /> + 1 ? "Models" : "Model"} value={model_names.join(", ") || "-"} /> + + + + {details.created_at ? ( + + ) : null} + + {job ? ( + <> + Run + + {job.title ? : null} + {job.status ? : null} + {job_took ? : null} + + + + ) : null} + + {models.length ? ( + <> + {models.length > 1 ? "Models used" : "Model used"} + {models.map((model) => ( + + ))} + + ) : null} + + {calls.length > 1 ? ( + <> + Calls in order (tokens in / out) + {calls.map((call, index) => ( + + ))} + {totals.calls > calls.length ? ( + + {`Showing the first ${calls.length} of ${totals.calls} calls. The totals count every one.`} + + ) : null} + + ) : null} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-details-call-row.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-details-call-row.tsx new file mode 100644 index 0000000..06f9fac --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-details-call-row.tsx @@ -0,0 +1,58 @@ +import Row from "@moduletrace/twui/layout/Row"; +import Span from "@moduletrace/twui/layout/Span"; +import type { MessageDetailsCall } from "@/src/types"; +import describeCallSettings from "@/src/utils/describe-call-settings"; +import formatDuration from "@/src/utils/format-duration"; +import formatTokens from "@/src/utils/format-tokens"; +import costLabel from "../(utils)/cost-label"; +import harnessLabel from "../(utils)/harness-label"; + +type Props = { + call: MessageDetailsCall; + index: number; + /** + * When the first call was made, so each one reads as an offset from it. + */ + first_at: number; +}; + +/** + * One model call in the order it was made: its number, when it came after + * the first, which model and what for, and its tokens and cost. A model + * that answered after others failed is marked; the hover lists the + * call's settings. + */ +export default function MessageDetailsCallRow({ call, index, first_at }: Props) { + const offset = index ? `+${formatDuration({ ms: call.created_at - first_at })}` : "start"; + const model = call.model_name || call.model_slug || "Unknown model"; + const settings = call.settings + ? describeCallSettings({ settings: call.settings, harness: call.harness }) + .map(({ label, value }) => `${label}: ${value}`) + .join("\n") + : ""; + const fell_back = Boolean(call.settings?.fell_back_from?.length); + + return ( + + {`#${index + 1}`} + {offset} + + {fell_back ? `${model} (fallback)` : model} + + + {`${formatTokens({ tokens: call.prompt_tokens })} / ${formatTokens({ tokens: call.completion_tokens })}`} + + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-details-heading.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-details-heading.tsx new file mode 100644 index 0000000..934ffae --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-details-heading.tsx @@ -0,0 +1,16 @@ +import Span from "@moduletrace/twui/layout/Span"; + +type Props = { + children: string; +}; + +/** + * The small label over each part of a message's details. + */ +export default function MessageDetailsHeading({ children }: Props) { + return ( + + {children} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-details-modal.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-details-modal.tsx new file mode 100644 index 0000000..3642e1e --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-details-modal.tsx @@ -0,0 +1,54 @@ +import { useState } from "react"; +import Modal from "@moduletrace/twui/elements/Modal"; +import H2 from "@moduletrace/twui/layout/H2"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import { Info } from "lucide-react"; +import useMessageDetails from "../(hooks)/use-message-details"; +import MessageActionButton from "./message-action-button"; +import MessageDetailsBody from "./message-details-body"; + +type Props = { + message_id: number; +}; + +/** + * The info button under a bot message, and the modal it opens: which + * provider and model generated it, the run behind it, and every model + * call with its tokens and cost. Loaded when it opens. + */ +export default function MessageDetailsModal({ message_id }: Props) { + const [open, set_open] = useState(false); + const { details, loading, error } = useMessageDetails({ message_id, enabled: open }); + + return ( + + + + } + targetWrapperProps={{ className: "shrink-0" }} + open={open} + setOpen={set_open} + className="max-w-xl bg-paper p-6" + > + +

Message details

+ + How this answer was generated: the provider and model behind it, the settings they were called with, and what it spent. + + + + + {details ? ( + + ) : ( + + {loading || !error ? "Loading details..." : error} + + )} + + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-details-model-row.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-details-model-row.tsx new file mode 100644 index 0000000..df97e51 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-details-model-row.tsx @@ -0,0 +1,60 @@ +import Row from "@moduletrace/twui/layout/Row"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import type { MessageDetailsModel } from "@/src/types"; +import describeCallSettings from "@/src/utils/describe-call-settings"; +import formatTokens from "@/src/utils/format-tokens"; +import costLabel from "../(utils)/cost-label"; +import harnessLabel from "../(utils)/harness-label"; +import providerLabel from "../(utils)/provider-label"; +import MessageDetailsSetting from "./message-details-setting"; + +type Props = { + model: MessageDetailsModel; +}; + +/** + * One model a message used, summed over its calls: the model by name with + * its exact id under it, the provider that served it and what it was used + * for, then its calls, tokens and cost, and the settings it was called + * with (temperature, reasoning effort, whose settings they were, any + * fallback). Calls recorded before settings were show none. + */ +export default function MessageDetailsModelRow({ model }: Props) { + const name = model.model_name || model.model_slug || "Unknown model"; + const settings = model.settings ? describeCallSettings({ settings: model.settings, harness: model.harness }) : []; + + return ( + + + + {name} + + + {`${formatTokens({ tokens: model.total_tokens })} tokens`} + + + {model.model_slug && model.model_slug != name ? ( + + {model.model_slug} + + ) : null} + + {[ + providerLabel({ name: model.provider_name, kind: model.provider_kind }), + harnessLabel({ harness: model.harness }), + `${model.calls} ${model.calls == 1 ? "call" : "calls"}`, + `${model.prompt_tokens.toLocaleString()} in, ${model.completion_tokens.toLocaleString()} out`, + costLabel({ cost: model.cost }), + ].join(" · ")} + + {settings.length ? ( + + {settings.map((setting) => ( + + ))} + + ) : null} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/message-details-setting.tsx b/src/pages/admin/bots/[bot_id]/(partials)/message-details-setting.tsx new file mode 100644 index 0000000..62ac7ac --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/message-details-setting.tsx @@ -0,0 +1,19 @@ +import Span from "@moduletrace/twui/layout/Span"; + +type Props = { + label: string; + value: string; +}; + +/** + * One setting a model was called with, as a small label and value pair: + * "Temperature 0.3", "Reasoning effort High". + */ +export default function MessageDetailsSetting({ label, value }: Props) { + return ( + + {`${label} `} + {value} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(sections)/conversation.tsx b/src/pages/admin/bots/[bot_id]/(sections)/conversation.tsx index 3b207d8..4baa8bc 100644 --- a/src/pages/admin/bots/[bot_id]/(sections)/conversation.tsx +++ b/src/pages/admin/bots/[bot_id]/(sections)/conversation.tsx @@ -219,9 +219,20 @@ export default function Conversation() { return; } + /** + * A copy already shown keeps its place; one that arrives with + * its usage (the relay or the catch-up, after a push without + * it) fills in the tokens. + */ setMessages((prev) => prev.some((m) => m.id == message.id) - ? prev + ? message.usage + ? prev.map((m) => + m.id == message.id && !m.usage + ? { ...m, usage: message.usage } + : m, + ) + : prev : [...prev, message], ); diff --git a/src/pages/admin/bots/[bot_id]/(utils)/cost-label.ts b/src/pages/admin/bots/[bot_id]/(utils)/cost-label.ts new file mode 100644 index 0000000..32f25cc --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(utils)/cost-label.ts @@ -0,0 +1,9 @@ +/** + * A price in USD, precise enough for a single model call. "Not reported" + * when the provider gave none (free models, most local ones). + */ +export default function costLabel({ cost }: { cost?: number | null }) { + if (typeof cost != "number" || !(cost > 0)) return "Not reported"; + + return `$${cost < 0.01 ? cost.toFixed(6) : cost.toFixed(4)}`; +} diff --git a/src/pages/admin/bots/[bot_id]/(utils)/harness-label.ts b/src/pages/admin/bots/[bot_id]/(utils)/harness-label.ts new file mode 100644 index 0000000..2803a61 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(utils)/harness-label.ts @@ -0,0 +1,9 @@ +import { Harnesses } from "@/src/dict/harness-dict"; + +/** + * What a harness is called in a message's details: its title in Settings, + * under Models. + */ +export default function harnessLabel({ harness }: { harness: string }) { + return Harnesses.find((h) => h.value == harness)?.title || harness; +} diff --git a/src/pages/admin/bots/[bot_id]/(utils)/provider-label.test.ts b/src/pages/admin/bots/[bot_id]/(utils)/provider-label.test.ts new file mode 100644 index 0000000..7c33fcd --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(utils)/provider-label.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from "bun:test"; +import costLabel from "./cost-label"; +import providerLabel from "./provider-label"; + +describe("providerLabel", () => { + test("adds the kind only when it says more than the name", () => { + expect(providerLabel({ name: "OpenRouter", kind: "openrouter" })).toBe("OpenRouter"); + expect(providerLabel({ name: "Work key", kind: "openai" })).toBe("Work key · OpenAI"); + expect(providerLabel({ name: "Old one", kind: null })).toBe("Old one"); + expect(providerLabel({ name: null, kind: null })).toBe("Unknown provider"); + }); +}); + +describe("costLabel", () => { + test("keeps small prices readable", () => { + expect(costLabel({ cost: 0.000123 })).toBe("$0.000123"); + expect(costLabel({ cost: 0.25 })).toBe("$0.2500"); + expect(costLabel({ cost: 0 })).toBe("Not reported"); + expect(costLabel({ cost: null })).toBe("Not reported"); + }); +}); diff --git a/src/pages/admin/bots/[bot_id]/(utils)/provider-label.ts b/src/pages/admin/bots/[bot_id]/(utils)/provider-label.ts new file mode 100644 index 0000000..b2b5d40 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(utils)/provider-label.ts @@ -0,0 +1,20 @@ +import { HarnessProviderKinds } from "@/src/dict/harness-provider-dict"; + +type Params = { + name: string | null; + kind: string | null; +}; + +/** + * A provider as a message's details name it: the name it was saved under, + * with its kind beside it when that says something more ("Work key · + * OpenAI"). + */ +export default function providerLabel({ name, kind }: Params) { + const kind_title = kind ? HarnessProviderKinds.find((k) => k.value == kind)?.title : undefined; + + if (!name) return kind_title || "Unknown provider"; + if (!kind_title || kind_title.toLowerCase() == name.toLowerCase()) return name; + + return `${name} · ${kind_title}`; +} diff --git a/src/pages/api/admin/bots/message-details.ts b/src/pages/api/admin/bots/message-details.ts new file mode 100644 index 0000000..81072de --- /dev/null +++ b/src/pages/api/admin/bots/message-details.ts @@ -0,0 +1,41 @@ +import userAuth from "@/src/functions/backend/auth/user-auth"; +import grabSpeakableMessage from "@/src/functions/backend/db/bots/grab-speakable-message"; +import grabMessageDetails from "@/src/functions/backend/db/usage/grab-message-details"; +import { MessageDetailsRequestSchema } from "@/src/schemas/bot-schema"; +import type { MessageDetails } from "@/src/types"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * `GET ?message_id=`: how one of the bot's messages in the user's own + * conversation was generated (the job, every model call with its provider + * and model, and the tokens and cost). Loaded when its details modal + * opens, so the thread's messages stay small. + */ +export const handler: BunextAPIRouteHandler> = async ({ req, query }) => { + try { + if (req.method != "GET") { + return { success: false, msg: `Method not allowed` }; + } + + const { user } = await userAuth({ req }); + + if (!user?.logged_in_status || !user.id) { + return { success: false, msg: `Unauthorized`, logoutUser: true }; + } + + const { message_id } = parseWithSchema({ schema: MessageDetailsRequestSchema, data: query }); + const { message } = await grabSpeakableMessage({ user_id: Number(user.id), message_id }); + + return { + success: true, + singleRes: await grabMessageDetails({ + message_id: Number(message.id), + created_at: Number(message.created_at) || null, + }), + }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; diff --git a/src/schemas/bot-schema.ts b/src/schemas/bot-schema.ts index 97f4d7f..47412f0 100644 --- a/src/schemas/bot-schema.ts +++ b/src/schemas/bot-schema.ts @@ -176,3 +176,9 @@ export const SpeakBotMessageRequestSchema = z.object({ }); export type SpeakBotMessageRequest = z.infer; + +export const MessageDetailsRequestSchema = z.object({ + message_id: z.coerce.number().int().positive("Pick a message"), +}); + +export type MessageDetailsRequest = z.infer; diff --git a/src/types/index.ts b/src/types/index.ts index 6e7570a..5838234 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -32,7 +32,7 @@ import type { HarnessProviderKind } from "../dict/harness-provider-dict"; import type { EnvParadigm, KnownEnv } from "../dict/env-dict"; import type { ContainerStatus } from "../dict/computer-dict"; import type { BotFace, BotMessageReactionEmoji } from "../dict/bot-dict"; -import type { TokenUsageSummary } from "../harness/usage/types"; +import type { HarnessCallSettings, TokenUsageSummary } from "../harness/usage/types"; import type { McpServerStatus, McpTransport } from "../dict/mcp-dict"; import type { CustomToolHttpMethod, @@ -1026,6 +1026,88 @@ export type BotMessageWithAttachments = BUN_SQLITE_COBBLE_BOT_MESSAGES & { usage?: TokenUsageSummary | null; }; +/** + * One model call behind a message, as its details modal lists it: which + * provider and model answered, for which harness, and what it spent. + */ +export type MessageDetailsCall = { + id: number; + created_at: number; + harness: string; + provider_id: number | null; + provider_name: string | null; + /** + * The provider's kind (`HarnessProviderKinds`), while the provider + * still exists. + */ + provider_kind: string | null; + model_slug: string | null; + /** + * The catalog's display name, while the model is still in it. + */ + model_name: string | null; + prompt_tokens: number; + completion_tokens: number; + cost: number | null; + /** + * How it was made (`token_usages.settings`); null for a call recorded + * before settings were. + */ + settings: HarnessCallSettings | null; +}; + +/** + * The calls of one provider, model and harness, summed: what the modal's + * "Models" list shows. + */ +export type MessageDetailsModel = TokenUsageSummary & { + harness: string; + provider_name: string | null; + provider_kind: string | null; + model_slug: string | null; + model_name: string | null; + /** + * The latest call's settings: a run keeps its settings, so they are + * the same for every call but a fallback. + */ + settings: HarnessCallSettings | null; +}; + +/** + * The queued job that produced a message, while it is kept + * (`BotQueueRetentionDays`). + */ +export type MessageDetailsJob = { + id: number; + kind: string | null; + title: string | null; + status: string | null; + attempts: number; + started_at: number | null; + finished_at: number | null; + /** + * How many model turns and notes it recorded (`bots_queue_updates`). + */ + steps: number; +}; + +/** + * Everything known about how one bot message was generated: the job that + * ran it, every model call it took and their totals. + */ +export type MessageDetails = { + message_id: number; + created_at: number | null; + totals: TokenUsageSummary; + models: MessageDetailsModel[]; + /** + * The calls in the order they were made, up to + * `AppData.MessageDetailsMaxCalls`; `totals.calls` counts them all. + */ + calls: MessageDetailsCall[]; + job: MessageDetailsJob | null; +}; + /** * `server:bot-reaction` data: a message's reactions changed. */ diff --git a/src/utils/describe-call-settings.test.ts b/src/utils/describe-call-settings.test.ts new file mode 100644 index 0000000..d393868 --- /dev/null +++ b/src/utils/describe-call-settings.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import describeCallSettings from "./describe-call-settings"; + +describe("describeCallSettings", () => { + test("names every setting by its title", () => { + expect( + describeCallSettings({ + harness: "text", + settings: { + model_source: "pinned", + settings_source: "workspace", + params: { temperature: 0.3, reasoning_effort: "high" }, + max_tokens: 4000, + fell_back_from: ["2:openai/gpt-x"], + }, + }), + ).toEqual([ + { label: "Model pick", value: "Picked in the settings" }, + { label: "Settings from", value: "The workspace's" }, + { label: "Temperature", value: "0.3" }, + { label: "Reasoning effort", value: "High" }, + { label: "Max tokens", value: "4,000" }, + { label: "Fell back from", value: "openai/gpt-x" }, + ]); + }); + + test("says when nothing was sent", () => { + expect(describeCallSettings({ harness: "text", settings: { model_source: "auto" } })).toEqual([ + { label: "Model pick", value: "Automatic" }, + { label: "Generation", value: "The provider's defaults" }, + ]); + }); + + test("leaves generation out where it doesn't apply", () => { + expect(describeCallSettings({ harness: "speech", settings: { model_source: "pinned", voice: "alloy" } })).toEqual([ + { label: "Model pick", value: "Picked in the settings" }, + { label: "Voice", value: "alloy" }, + ]); + }); +}); diff --git a/src/utils/describe-call-settings.ts b/src/utils/describe-call-settings.ts new file mode 100644 index 0000000..407489e --- /dev/null +++ b/src/utils/describe-call-settings.ts @@ -0,0 +1,57 @@ +import { HarnessParameterHarnesses, HarnessParameters } from "@/src/dict/harness-parameter-dict"; +import type { HarnessCallSettings } from "@/src/harness/usage/types"; + +const SettingsSources: Record, string> = { + default: "The install's defaults", + bot: "The bot's own", + workspace: "The workspace's", +}; + +type Params = { + settings: HarnessCallSettings; + /** + * Generation settings only apply to the chat harnesses. + */ + harness: string; +}; + +/** + * A call's settings as label and value pairs, in the order Settings, + * under Models, lists them: the model's pick, whose settings they were, + * every generation setting sent (by its title), then the token limit, the + * voice and any models that failed before it. + */ +export default function describeCallSettings({ settings, harness }: Params) { + const chat = (HarnessParameterHarnesses as readonly string[]).includes(harness); + const rows: { label: string; value: string }[] = [ + { label: "Model pick", value: settings.model_source == "pinned" ? "Picked in the settings" : "Automatic" }, + ]; + + if (settings.settings_source) { + rows.push({ label: "Settings from", value: SettingsSources[settings.settings_source] }); + } + + for (const [key, value] of Object.entries(settings.params || {})) { + const parameter = HarnessParameters.find((p) => p.key == key); + const option = + parameter && "options" in parameter ? parameter.options.find((o) => o.value == value) : undefined; + + rows.push({ label: parameter?.title || key, value: option?.title || String(value) }); + } + + if (chat && (!settings.params || !Object.keys(settings.params).length)) { + rows.push({ label: "Generation", value: "The provider's defaults" }); + } + + if (settings.max_tokens) rows.push({ label: "Max tokens", value: settings.max_tokens.toLocaleString() }); + if (settings.voice) rows.push({ label: "Voice", value: settings.voice }); + + if (settings.fell_back_from?.length) { + rows.push({ + label: "Fell back from", + value: settings.fell_back_from.map((key) => key.slice(key.indexOf(":") + 1)).join(", "), + }); + } + + return rows; +} diff --git a/src/utils/format-duration.test.ts b/src/utils/format-duration.test.ts new file mode 100644 index 0000000..0768a3d --- /dev/null +++ b/src/utils/format-duration.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; +import formatDuration from "./format-duration"; + +describe("formatDuration", () => { + test("reads at every scale", () => { + expect(formatDuration({ ms: 850 })).toBe("850ms"); + expect(formatDuration({ ms: 4200 })).toBe("4.2s"); + expect(formatDuration({ ms: 3000 })).toBe("3s"); + expect(formatDuration({ ms: 125_000 })).toBe("2m 5s"); + expect(formatDuration({ ms: 120_000 })).toBe("2m"); + expect(formatDuration({ ms: 3_780_000 })).toBe("1h 3m"); + expect(formatDuration({ ms: 7_200_000 })).toBe("2h"); + }); + + test("is empty for nothing to show", () => { + expect(formatDuration({ ms: null })).toBe(""); + expect(formatDuration({ ms: -5 })).toBe(""); + expect(formatDuration({ ms: Number.NaN })).toBe(""); + }); +}); diff --git a/src/utils/format-duration.ts b/src/utils/format-duration.ts new file mode 100644 index 0000000..2807fba --- /dev/null +++ b/src/utils/format-duration.ts @@ -0,0 +1,32 @@ +type Params = { + /** + * In milliseconds. + */ + ms: number | null | undefined; +}; + +/** + * A length of time as a person reads it: `850ms`, `4.2s`, `2m 5s`, + * `1h 3m`. Empty for a missing or negative one. + */ +export default function formatDuration({ ms }: Params) { + if (ms === null || ms === undefined) return ""; + + const value = Number(ms); + + if (!Number.isFinite(value) || value < 0) return ""; + if (value < 1000) return `${Math.round(value)}ms`; + if (value < 60_000) return `${(value / 1000).toFixed(1).replace(/\.0$/, "")}s`; + + const seconds = Math.round(value / 1000); + + if (seconds < 3600) { + const rest = seconds % 60; + return rest ? `${Math.floor(seconds / 60)}m ${rest}s` : `${Math.floor(seconds / 60)}m`; + } + + const minutes = Math.round(seconds / 60); + const rest = minutes % 60; + + return rest ? `${Math.floor(minutes / 60)}h ${rest}m` : `${Math.floor(minutes / 60)}h`; +} diff --git a/src/utils/group-message-calls.test.ts b/src/utils/group-message-calls.test.ts new file mode 100644 index 0000000..a8b8084 --- /dev/null +++ b/src/utils/group-message-calls.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from "bun:test"; +import type { MessageDetailsCall } from "@/src/types"; +import groupMessageCalls from "./group-message-calls"; + +function call(overrides: Partial): MessageDetailsCall { + return { + id: 1, + created_at: 0, + harness: "text", + provider_id: 1, + provider_name: "OpenRouter", + provider_kind: "openrouter", + model_slug: "deepseek/deepseek-v4", + model_name: "DeepSeek V4", + prompt_tokens: 100, + completion_tokens: 10, + cost: null, + settings: null, + ...overrides, + }; +} + +describe("groupMessageCalls", () => { + test("sums calls of the same provider, model and harness", () => { + const groups = groupMessageCalls({ + calls: [call({ id: 1, cost: 0.01 }), call({ id: 2, prompt_tokens: 200, cost: 0.02 })], + }); + + expect(groups).toHaveLength(1); + expect(groups[0]).toMatchObject({ prompt_tokens: 300, completion_tokens: 20, total_tokens: 320, calls: 2 }); + expect(groups[0]!.cost).toBeCloseTo(0.03); + }); + + test("keeps harnesses and models apart, busiest first", () => { + const groups = groupMessageCalls({ + calls: [ + call({ id: 1, harness: "vision", model_slug: "small", prompt_tokens: 5, completion_tokens: 0 }), + call({ id: 2 }), + call({ id: 3, model_slug: "other" }), + call({ id: 4 }), + ], + }); + + expect(groups.map((g) => [g.harness, g.model_slug, g.calls])).toEqual([ + ["text", "deepseek/deepseek-v4", 2], + ["text", "other", 1], + ["vision", "small", 1], + ]); + }); + + test("leaves the cost out when no call reported one", () => { + const [group] = groupMessageCalls({ calls: [call({})] }); + + expect(group!.cost).toBeUndefined(); + }); +}); diff --git a/src/utils/group-message-calls.ts b/src/utils/group-message-calls.ts new file mode 100644 index 0000000..041b5a2 --- /dev/null +++ b/src/utils/group-message-calls.ts @@ -0,0 +1,44 @@ +import type { MessageDetailsCall, MessageDetailsModel } from "@/src/types"; + +type Params = { + calls: MessageDetailsCall[]; +}; + +/** + * A message's calls summed per provider, model and harness, busiest + * first: the "Models" list of its details modal. A call whose model or + * provider was never recorded is grouped with the others like it. + */ +export default function groupMessageCalls({ calls }: Params): MessageDetailsModel[] { + const groups = new Map(); + + for (const call of calls) { + const key = [call.harness, call.provider_id ?? call.provider_name ?? "", call.model_slug ?? ""].join("\u0000"); + const group = groups.get(key) || { + harness: call.harness, + provider_name: call.provider_name, + provider_kind: call.provider_kind, + model_slug: call.model_slug, + model_name: call.model_name, + settings: call.settings, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + calls: 0, + }; + + group.prompt_tokens += call.prompt_tokens; + group.completion_tokens += call.completion_tokens; + group.total_tokens = group.prompt_tokens + group.completion_tokens; + group.calls += 1; + group.settings = call.settings || group.settings; + + if (typeof call.cost == "number" && call.cost > 0) { + group.cost = (group.cost || 0) + call.cost; + } + + groups.set(key, group); + } + + return [...groups.values()].sort((a, b) => b.total_tokens - a.total_tokens || b.calls - a.calls); +} diff --git a/src/utils/parse-call-settings.test.ts b/src/utils/parse-call-settings.test.ts new file mode 100644 index 0000000..440b530 --- /dev/null +++ b/src/utils/parse-call-settings.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "bun:test"; +import parseCallSettings from "./parse-call-settings"; + +describe("parseCallSettings", () => { + test("reads what was stored", () => { + expect( + parseCallSettings({ + json: JSON.stringify({ + model_source: "pinned", + settings_source: "bot", + params: { temperature: 0.3, reasoning_effort: "high", bad: { x: 1 } }, + max_tokens: 800, + fell_back_from: ["2:a/b", 3], + }), + }), + ).toEqual({ + model_source: "pinned", + settings_source: "bot", + params: { temperature: 0.3, reasoning_effort: "high" }, + max_tokens: 800, + fell_back_from: ["2:a/b"], + }); + }); + + test("is null for an older call or a broken value", () => { + expect(parseCallSettings({ json: null })).toBeNull(); + expect(parseCallSettings({ json: "{nope" })).toBeNull(); + expect(parseCallSettings({ json: "3" })).toBeNull(); + }); +}); diff --git a/src/utils/parse-call-settings.ts b/src/utils/parse-call-settings.ts new file mode 100644 index 0000000..ca6bbc7 --- /dev/null +++ b/src/utils/parse-call-settings.ts @@ -0,0 +1,50 @@ +import type { HarnessCallSettings } from "@/src/harness/usage/types"; + +type SettingsSource = NonNullable; + +const SettingsSources: SettingsSource[] = ["default", "bot", "workspace"]; + +type Params = { + /** + * `token_usages.settings`, as stored. + */ + json?: string | null; +}; + +/** + * A call's stored settings (`HarnessCallSettings`), or null for a call + * recorded before they were, or one that doesn't parse. Only the known + * fields of the right type are kept. + */ +export default function parseCallSettings({ json }: Params): HarnessCallSettings | null { + if (!json) return null; + + let raw: unknown; + + try { + raw = JSON.parse(json); + } catch { + return null; + } + + if (!raw || typeof raw != "object") return null; + + const value = raw as Record; + const params = value.params && typeof value.params == "object" ? (value.params as Record) : null; + const kept_params = Object.fromEntries( + Object.entries(params || {}).filter(([, v]) => typeof v == "number" || typeof v == "string"), + ) as Record; + + return { + model_source: value.model_source == "pinned" ? "pinned" : "auto", + ...(SettingsSources.includes(value.settings_source as SettingsSource) + ? { settings_source: value.settings_source as SettingsSource } + : {}), + ...(Object.keys(kept_params).length ? { params: kept_params } : {}), + ...(typeof value.max_tokens == "number" ? { max_tokens: value.max_tokens } : {}), + ...(typeof value.voice == "string" && value.voice ? { voice: value.voice } : {}), + ...(Array.isArray(value.fell_back_from) + ? { fell_back_from: value.fell_back_from.filter((k): k is string => typeof k == "string") } + : {}), + }; +}