Show how each bot message was generated

Every bot message gets an info button that opens its details: the
provider and model behind it, tokens and cost, the job that ran it, each
model used and every call in order. Each model call now records how it
was made in token_usages.settings (generation settings sent, whose
settings they were, pinned or automatic model, fallbacks), shown per
model in the modal.

Live replies now carry their token usage, from the job's push and the
live relay alike, so it shows without a reload.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/harness/audio/speak-text.ts

@@ -41,6 +41,12 @@ export default async function speakText({
         grabHarnessSetting({ harness: "speech", bot_id, workspace_id }),
     ]);
 
+    const voiceFor = (candidate: HarnessCandidate) =>
+        pickVoice({
+            candidate,
+            voice: typeof setting.options.voice == "string" ? setting.options.voice : undefined,
+        });
+
     const { result } = await withCandidates<HarnessAudioStream>({
         harness: "speech",
         candidates,
@@ -49,25 +55,24 @@ export default async function speakText({
          * completion count to read: the call is recorded as it is, with
          * the text's length standing in for what was spoken.
          */
-        on_success: async ({ candidate }) =>
+        on_success: async ({ candidate, failed }) =>
             await recordHarnessUsage({
                 call: buildUsageCall({
                     harness: "speech",
                     candidate,
                     usage: { prompt_tokens: text.length, completion_tokens: 0 },
+                    failed,
+                    settings: {
+                        settings_source: setting.source,
+                        voice: voiceFor(candidate),
+                    },
                 }),
             }),
         run: async (candidate) =>
             await grabHarnessAdapter({ kind: candidate.provider.kind }).speak({
                 candidate,
                 text,
-                voice: pickVoice({
-                    candidate,
-                    voice:
-                        typeof setting.options.voice == "string"
-                            ? setting.options.voice
-                            : undefined,
-                }),
+                voice: voiceFor(candidate),
                 format,
                 signal: signal || new AbortController().signal,
             }),

src/harness/audio/transcribe-audio.ts

@@ -51,11 +51,12 @@ export default async function transcribeAudio({
     const { result } = await withCandidates<{ text: string; usage?: HarnessUsage }>({
         harness: "transcription",
         candidates,
-        on_success: async ({ candidate, result: transcript }): Promise<void> =>
+        on_success: async ({ candidate, result: transcript, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
                     harness: "transcription",
                     candidate,
+                    failed,
                     usage: transcript.usage,
                 }),
             }),

src/harness/embeddings/embed-text.ts

@@ -58,11 +58,12 @@ export default async function embedText({
     const { result } = await withCandidates<HarnessEmbedResult>({
         harness: "embedding",
         candidates,
-        on_success: async ({ candidate, result: embedded }): Promise<void> =>
+        on_success: async ({ candidate, result: embedded, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
                     harness: "embedding",
                     candidate,
+                    failed,
                     usage: embedded.usage,
                 }),
             }),

src/harness/image/generate-image.ts

@@ -41,11 +41,12 @@ export default async function generateImage({
     const { result } = await withCandidates<{ images: HarnessImageOutput[]; usage?: HarnessUsage }>({
         harness: "image",
         candidates,
-        on_success: async ({ candidate, result: made }): Promise<void> =>
+        on_success: async ({ candidate, result: made, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
                     harness: "image",
                     candidate,
+                    failed,
                     usage: made.usage,
                 }),
             }),

src/harness/media/read-image.ts

@@ -5,6 +5,7 @@ import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
 import withCandidates from "../resolve/with-candidates";
 import type { HarnessUsage } from "../types";
 import grabChatParams from "../text/grab-chat-params";
+import flattenSentParams from "../usage/flatten-sent-params";
 import buildUsageCall from "../usage/build-usage-call";
 import recordHarnessUsage from "../usage/record-harness-usage";
 
@@ -45,18 +46,35 @@ export default async function readImage({ url, focus, signal, bot_id, workspace_
         workspace_id,
     });
 
-    const { result } = await withCandidates<{ text: string; usage?: HarnessUsage }>({
+    const { result } = await withCandidates<{
+        text: string;
+        usage?: HarnessUsage;
+        sent_params: Record<string, number | string | { effort: string }>;
+    }>({
         harness: "vision",
         candidates,
-        on_success: async ({ candidate, result: answer }): Promise<void> =>
+        on_success: async ({ candidate, result: answer, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
                     harness: "vision",
                     candidate,
                     usage: answer.usage,
+                    failed,
+                    settings: {
+                        settings_source: saved_params.source,
+                        params: flattenSentParams({ params: answer.sent_params }),
+                    },
                 }),
             }),
         run: async (candidate) => {
+            /**
+             * Reading is a lookup, so it stays literal unless a
+             * temperature was set for the vision harness.
+             */
+            const sent_params = {
+                temperature: 0,
+                ...saved_params.paramsFor(candidate.model),
+            };
             const answer = await grabHarnessAdapter({
                 kind: candidate.provider.kind,
             }).chat({
@@ -75,14 +93,7 @@ export default async function readImage({ url, focus, signal, bot_id, workspace_
                         ],
                     },
                 ],
-                /**
-                 * Reading is a lookup, so it stays literal unless a
-                 * temperature was set for the vision harness.
-                 */
-                params: {
-                    temperature: 0,
-                    ...saved_params(candidate.model),
-                },
+                params: sent_params,
                 signal: signal || new AbortController().signal,
             });
 
@@ -94,7 +105,7 @@ export default async function readImage({ url, focus, signal, bot_id, workspace_
                 });
             }
 
-            return { text: answer.text.trim(), usage: answer.usage };
+            return { text: answer.text.trim(), usage: answer.usage, sent_params };
         },
     });
 

src/harness/resolve/with-candidates.ts

@@ -22,6 +22,10 @@ type Params<T> = {
     on_success?: (params: {
         candidate: HarnessCandidate;
         result: T;
+        /**
+         * The candidates that failed before this one answered, by key.
+         */
+        failed: string[];
     }) => Promise<void> | void;
     cooldowns?: HarnessCooldowns;
     now?: () => number;
@@ -105,7 +109,11 @@ export default async function withCandidates<T>({
         try {
             const result = await run(candidate);
 
-            await on_success?.({ candidate, result });
+            await on_success?.({
+                candidate,
+                result,
+                failed: errors.flatMap((e) => (e.candidate_key ? [e.candidate_key] : [])),
+            });
 
             return { result, candidate };
         } catch (error: unknown) {

src/harness/text/generate-text.ts

@@ -11,6 +11,7 @@ import type {
 import buildUsageCall from "../usage/build-usage-call";
 import recordHarnessUsage from "../usage/record-harness-usage";
 import grabChatParams from "./grab-chat-params";
+import flattenSentParams from "../usage/flatten-sent-params";
 import prepareMessagesForModel from "./prepare-messages-for-model";
 import stripThinkTags from "./strip-think-tags";
 
@@ -55,6 +56,10 @@ export type GenerateTextResult = HarnessChatResult & {
  */
 type ChatAnswer = Omit<HarnessChatResult, "text"> & {
     text: string | null;
+    /**
+     * The generation settings sent to this candidate, for its usage row.
+     */
+    sent_params?: Record<string, number | string | { effort: string }>;
 };
 
 /**
@@ -85,15 +90,22 @@ export default async function generateText({
         harness: "text",
         candidates: resolved,
         prefer_key,
-        on_success: async ({ candidate: answered, result: answer }): Promise<void> =>
+        on_success: async ({ candidate: answered, result: answer, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
                     harness: "text",
                     candidate: answered,
                     usage: answer.usage,
+                    failed,
+                    settings: {
+                        ...(saved_params ? { settings_source: saved_params.source } : {}),
+                        params: flattenSentParams({ params: answer.sent_params }),
+                        ...(max_tokens ? { max_tokens } : {}),
+                    },
                 }),
             }),
         run: async (candidate) => {
+            const sent_params = params || saved_params?.paramsFor(candidate.model);
             const answer = await grabHarnessAdapter({
                 kind: candidate.provider.kind,
             }).chat({
@@ -108,7 +120,7 @@ export default async function generateText({
                 }),
                 tools,
                 max_tokens,
-                params: params || saved_params?.(candidate.model),
+                params: sent_params,
                 signal,
             });
 
@@ -122,7 +134,7 @@ export default async function generateText({
                 });
             }
 
-            return { ...answer, text: text || null };
+            return { ...answer, text: text || null, sent_params };
         },
     });
 

src/harness/text/grab-chat-params.ts

@@ -18,22 +18,24 @@ type Params = {
  * harness's setting (the workspace's, the bot's, else the default), else
  * `HarnessParameterDefaults` so a fresh install is ready to go. Keyed by
  * the field sent on the wire, keeping only what the model says it takes
- * (a default only where the model explicitly advertises it).
+ * (a default only where the model explicitly advertises it). `source`
+ * says whose setting it was, for the call's usage row.
  */
 export default async function grabChatParams({
     harness,
     bot_id,
     workspace_id,
-}: Params): Promise<
-    (model: HarnessModelInfo) => Record<string, number | string | { effort: string }>
-> {
+}: Params): Promise<{
+    paramsFor: (model: HarnessModelInfo) => Record<string, number | string | { effort: string }>;
+    source: "default" | "bot" | "workspace";
+}> {
     const setting = await grabHarnessSetting({
         harness,
         bot_id,
         workspace_id,
     });
 
-    return (model) =>
+    const paramsFor = (model: HarnessModelInfo) =>
         Object.fromEntries(
             HarnessParameters.flatMap((parameter) => {
                 const { value, saved } = harnessParameterValue({
@@ -67,4 +69,6 @@ export default async function grabChatParams({
                 ];
             }),
         );
+
+    return { paramsFor, source: setting.source };
 }

src/harness/usage/build-usage-call.test.ts

@@ -44,6 +44,24 @@ describe("buildUsageCall", () => {
             prompt_tokens: 1200,
             completion_tokens: 340,
             cost: 0.0042,
+            settings: { model_source: "auto" },
+        });
+    });
+
+    test("keeps the settings sent and the models that failed first", () => {
+        const call = buildUsageCall({
+            harness: "text",
+            candidate: candidate({ source: "pinned" }),
+            settings: { settings_source: "bot", params: { temperature: 0.3 }, max_tokens: 800 },
+            failed: ["2:other/model"],
+        });
+
+        expect(call.settings).toEqual({
+            model_source: "pinned",
+            settings_source: "bot",
+            params: { temperature: 0.3 },
+            max_tokens: 800,
+            fell_back_from: ["2:other/model"],
         });
     });
 

src/harness/usage/build-usage-call.ts

@@ -1,18 +1,29 @@
 import type { HarnessCandidate, HarnessUsage } from "../types";
-import type { HarnessUsageCall } from "./types";
+import type { HarnessCallSettings, HarnessUsageCall } from "./types";
 
 type Params = {
     harness: HarnessUsageCall["harness"];
     candidate: HarnessCandidate;
     usage?: HarnessUsage;
+    /**
+     * How the call was made beyond the model: the settings it sent and
+     * whose they were. The model's source and the fallback are filled in
+     * here.
+     */
+    settings?: Omit<HarnessCallSettings, "model_source" | "fell_back_from">;
+    /**
+     * The candidates that failed before this one (`withCandidates`).
+     */
+    failed?: string[];
 };
 
 /**
  * The candidate that really answered a call, and what it reported, as the
  * row `recordHarnessUsage` writes. Providers that report no usage at all
- * still get a row (no tokens), so a call is never invisible.
+ * still get a row (no tokens), so a call is never invisible. Its settings
+ * say how it was made, for a message's details.
  */
-export default function buildUsageCall({ harness, candidate, usage }: Params): HarnessUsageCall {
+export default function buildUsageCall({ harness, candidate, usage, settings, failed }: Params): HarnessUsageCall {
     return {
         harness,
         provider_id: candidate.provider.id,
@@ -22,6 +33,11 @@ export default function buildUsageCall({ harness, candidate, usage }: Params): H
         prompt_tokens: Number(usage?.prompt_tokens) || 0,
         completion_tokens: Number(usage?.completion_tokens) || 0,
         cost: typeof usage?.cost == "number" && Number.isFinite(usage.cost) ? usage.cost : undefined,
+        settings: {
+            model_source: candidate.source,
+            ...settings,
+            ...(failed?.length ? { fell_back_from: failed } : {}),
+        },
     };
 }
 

src/harness/usage/flatten-sent-params.test.ts

@@ -0,0 +1,20 @@
+import { describe, expect, test } from "bun:test";
+import flattenSentParams from "./flatten-sent-params";
+
+describe("flattenSentParams", () => {
+    test("reads the nested reasoning effort back by its setting key", () => {
+        expect(flattenSentParams({ params: { temperature: 0.3, reasoning: { effort: "high" } } })).toEqual({
+            temperature: 0.3,
+            reasoning_effort: "high",
+        });
+    });
+
+    test("keeps a zero and drops empty fields", () => {
+        expect(flattenSentParams({ params: { temperature: 0, seed: "" } })).toEqual({ temperature: 0 });
+    });
+
+    test("is undefined when nothing was sent", () => {
+        expect(flattenSentParams({ params: {} })).toBeUndefined();
+        expect(flattenSentParams({ params: null })).toBeUndefined();
+    });
+});

src/harness/usage/flatten-sent-params.ts

@@ -0,0 +1,26 @@
+type Params = {
+    /**
+     * Keyed by the field sent on the wire (`grabChatParams`).
+     */
+    params?: Record<string, number | string | { effort: string }> | null;
+};
+
+/**
+ * The generation settings a call sent, keyed by their setting
+ * (`HarnessParameters`) rather than the wire: OpenRouter's nested
+ * `reasoning: { effort }` reads back as `reasoning_effort`. Empty fields
+ * are left out; undefined when nothing was sent.
+ */
+export default function flattenSentParams({ params }: Params) {
+    const flat: Record<string, number | string> = {};
+
+    for (const [key, value] of Object.entries(params || {})) {
+        if (value && typeof value == "object") {
+            if (value.effort) flat[key == "reasoning" ? "reasoning_effort" : key] = String(value.effort);
+        } else if (typeof value == "number" || (typeof value == "string" && value)) {
+            flat[key] = value;
+        }
+    }
+
+    return Object.keys(flat).length ? flat : undefined;
+}

src/harness/usage/record-harness-usage.ts

@@ -52,6 +52,7 @@ export default async function recordHarnessUsage({ scope, call }: Params) {
         prompt_tokens: Math.max(0, Math.round(call.prompt_tokens || 0)),
         completion_tokens: Math.max(0, Math.round(call.completion_tokens || 0)),
         cost: typeof call.cost == "number" && Number.isFinite(call.cost) ? call.cost : undefined,
+        settings: call.settings ? JSON.stringify(call.settings) : undefined,
     };
 
     try {

src/harness/usage/types.ts

@@ -30,6 +30,33 @@ export type HarnessUsageScope = {
  * asked, and which candidate answered. Both are filled in by the call's
  * own entry point, never by the code that runs it.
  */
+export type HarnessCallSettings = {
+    /**
+     * `pinned`: the model picked in the setting. `auto`: an automatic pick.
+     */
+    model_source: "pinned" | "auto";
+    /**
+     * Whose setting the call used: the workspace's, the bot's own or the
+     * install's default. Left out when the call overrode it.
+     */
+    settings_source?: "default" | "bot" | "workspace";
+    /**
+     * The generation settings sent, by their setting key
+     * (`HarnessParameters`: `temperature`, `reasoning_effort`, ...).
+     */
+    params?: Record<string, number | string>;
+    max_tokens?: number;
+    /**
+     * The voice asked for (speech).
+     */
+    voice?: string;
+    /**
+     * The candidates (`<provider id>:<slug>`) that failed before this one
+     * answered, in the order they were tried.
+     */
+    fell_back_from?: string[];
+};
+
 export type HarnessUsageCall = {
     harness: "text" | "vision" | "transcription" | "speech" | "image" | "embedding";
     provider_id?: number;
@@ -39,6 +66,7 @@ export type HarnessUsageCall = {
     prompt_tokens: number;
     completion_tokens: number;
     cost?: number;
+    settings?: HarnessCallSettings;
 };
 
 /**