Show how each bot message was generated

Every bot message gets an info button that opens its details: the provider and model behind it, tokens and cost, the job that ran it, each model used and every call in order. Each model call now records how it was made in token_usages.settings (generation settings sent, whose settings they were, pinned or automatic model, fallbacks), shown per model in the modal. Live replies now carry their token usage, from the job's push and the live relay alike, so it shows without a reload. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-25 04:33:47 +01:00
parent feec69adca
commit c13b9ea826
48 changed files with 1307 additions and 57 deletions
@@ -4,7 +4,10 @@ import type { ReactNode } from "react";
 type Props = {
    title: string;
    aria_label: string;
-    onClick: () => void;
+    /**
+     * Left out when a wrapper (a modal's target) handles the click.
+     */
+    onClick?: () => void;
    children: ReactNode;
 };

@@ -5,6 +5,7 @@ import type { BotMessageWithAttachments } from "@/src/types";
 import { BotMessageActionsContext } from "../(data)/message-actions-context";
 import CopyMessageButton from "./copy-message-button";
 import MessageActionButton from "./message-action-button";
+import MessageDetailsModal from "./message-details-modal";
 import MessageReactionPicker from "./message-reaction-picker";

 type Props = {
@@ -12,7 +13,7 @@ type Props = {
 };

 /**
- * A message's copy, reply and reaction actions, shown under it. A message
+ * A message's copy, reply, reaction and details actions, shown under it. A message
 * still on its way to the server has no id yet, so it can only be copied.
 */
 export default function MessageActions({ message }: Props) {
@@ -36,6 +37,7 @@ export default function MessageActions({ message }: Props) {
                        message_id={message.id}
                        onReact={onReact}
                    />
+                    {message.author == "bot" ? <MessageDetailsModal message_id={Number(message.id)} /> : null}
                </>
            ) : null}
        </Row>
@@ -0,0 +1,26 @@
+import Row from "@moduletrace/twui/layout/Row";
+import Span from "@moduletrace/twui/layout/Span";
+
+type Props = {
+    label: string;
+    value: string;
+    /**
+     * Shown on hover, for the exact figure behind a rounded one.
+     */
+    title?: string;
+};
+
+/**
+ * One fact about how a message was generated: a label on the left, its
+ * value on the right, hairline-separated like a settings list.
+ */
+export default function MessageDetailRow({ label, value, title }: Props) {
+    return (
+        <Row className="w-full py-2 gap-3! flex-nowrap items-baseline border-0 border-b border-solid border-hairline last:border-b-0">
+            <Span className="w-32 shrink-0 text-[12px] leading-4 text-ink-secondary">{label}</Span>
+            <Span title={title} className="grow min-w-0 text-[13px] leading-5 text-ink break-words tabular-nums">
+                {value}
+            </Span>
+        </Row>
+    );
+}
@@ -0,0 +1,100 @@
+import Span from "@moduletrace/twui/layout/Span";
+import Stack from "@moduletrace/twui/layout/Stack";
+import { BotJobKinds } from "@/src/dict/bot-queue-dict";
+import type { MessageDetails } from "@/src/types";
+import formatDuration from "@/src/utils/format-duration";
+import formatTokens from "@/src/utils/format-tokens";
+import relativeTime from "@/src/utils/relative-time";
+import costLabel from "../(utils)/cost-label";
+import providerLabel from "../(utils)/provider-label";
+import MessageDetailRow from "./message-detail-row";
+import MessageDetailsCallRow from "./message-details-call-row";
+import MessageDetailsHeading from "./message-details-heading";
+import MessageDetailsModelRow from "./message-details-model-row";
+
+type Props = {
+    details: MessageDetails;
+};
+
+/**
+ * How a message was generated: the provider and model that answered and
+ * what it spent, then the job that ran it, every model it used and every
+ * call in order.
+ */
+export default function MessageDetailsBody({ details }: Props) {
+    const { totals, models, calls, job } = details;
+    const providers = [
+        ...new Set(models.map((m) => providerLabel({ name: m.provider_name, kind: m.provider_kind }))),
+    ];
+    const model_names = [...new Set(models.map((m) => m.model_name || m.model_slug || "Unknown model"))];
+    const first_at = calls[0]?.created_at || 0;
+    const job_kind = BotJobKinds.find((k) => k.value == job?.kind)?.title || job?.kind;
+    const job_took =
+        job?.started_at && job.finished_at ? formatDuration({ ms: job.finished_at - job.started_at }) : "";
+
+    return (
+        <Stack className="w-full gap-0! items-stretch">
+            {totals.calls ? null : (
+                <Span className="mb-2 text-[13px] leading-[18px] text-ink-secondary">
+                    No model calls were recorded for this message. It was written by the app itself (a stop, a
+                    failure notice), or before usage was recorded.
+                </Span>
+            )}
+
+            <MessageDetailRow label={providers.length > 1 ? "Providers" : "Provider"} value={providers.join(", ") || "-"} />
+            <MessageDetailRow label={model_names.length > 1 ? "Models" : "Model"} value={model_names.join(", ") || "-"} />
+            <MessageDetailRow
+                label="Tokens"
+                value={`${formatTokens({ tokens: totals.total_tokens })} (${totals.prompt_tokens.toLocaleString()} in, ${totals.completion_tokens.toLocaleString()} out)`}
+                title={`${totals.total_tokens.toLocaleString()} tokens`}
+            />
+            <MessageDetailRow label="Cost" value={costLabel({ cost: totals.cost })} />
+            <MessageDetailRow label="Model calls" value={String(totals.calls)} />
+            {details.created_at ? (
+                <MessageDetailRow
+                    label="Written"
+                    value={relativeTime({ at: details.created_at })}
+                    title={new Date(details.created_at).toLocaleString()}
+                />
+            ) : null}
+
+            {job ? (
+                <>
+                    <MessageDetailsHeading>Run</MessageDetailsHeading>
+                    <MessageDetailRow label="Kind" value={job_kind || "-"} />
+                    {job.title ? <MessageDetailRow label="Title" value={job.title} /> : null}
+                    {job.status ? <MessageDetailRow label="Status" value={job.status} /> : null}
+                    {job_took ? <MessageDetailRow label="Took" value={job_took} /> : null}
+                    <MessageDetailRow label="Attempts" value={String(job.attempts || 1)} />
+                    <MessageDetailRow label="Steps" value={String(job.steps)} />
+                </>
+            ) : null}
+
+            {models.length ? (
+                <>
+                    <MessageDetailsHeading>{models.length > 1 ? "Models used" : "Model used"}</MessageDetailsHeading>
+                    {models.map((model) => (
+                        <MessageDetailsModelRow
+                            key={`${model.harness}:${model.provider_name}:${model.model_slug}`}
+                            model={model}
+                        />
+                    ))}
+                </>
+            ) : null}
+
+            {calls.length > 1 ? (
+                <>
+                    <MessageDetailsHeading>Calls in order (tokens in / out)</MessageDetailsHeading>
+                    {calls.map((call, index) => (
+                        <MessageDetailsCallRow key={call.id} call={call} index={index} first_at={first_at} />
+                    ))}
+                    {totals.calls > calls.length ? (
+                        <Span className="mt-1 text-[12px] leading-4 text-ink-secondary">
+                            {`Showing the first ${calls.length} of ${totals.calls} calls. The totals count every one.`}
+                        </Span>
+                    ) : null}
+                </>
+            ) : null}
+        </Stack>
+    );
+}
@@ -0,0 +1,58 @@
+import Row from "@moduletrace/twui/layout/Row";
+import Span from "@moduletrace/twui/layout/Span";
+import type { MessageDetailsCall } from "@/src/types";
+import describeCallSettings from "@/src/utils/describe-call-settings";
+import formatDuration from "@/src/utils/format-duration";
+import formatTokens from "@/src/utils/format-tokens";
+import costLabel from "../(utils)/cost-label";
+import harnessLabel from "../(utils)/harness-label";
+
+type Props = {
+    call: MessageDetailsCall;
+    index: number;
+    /**
+     * When the first call was made, so each one reads as an offset from it.
+     */
+    first_at: number;
+};
+
+/**
+ * One model call in the order it was made: its number, when it came after
+ * the first, which model and what for, and its tokens and cost. A model
+ * that answered after others failed is marked; the hover lists the
+ * call's settings.
+ */
+export default function MessageDetailsCallRow({ call, index, first_at }: Props) {
+    const offset = index ? `+${formatDuration({ ms: call.created_at - first_at })}` : "start";
+    const model = call.model_name || call.model_slug || "Unknown model";
+    const settings = call.settings
+        ? describeCallSettings({ settings: call.settings, harness: call.harness })
+              .map(({ label, value }) => `${label}: ${value}`)
+              .join("\n")
+        : "";
+    const fell_back = Boolean(call.settings?.fell_back_from?.length);
+
+    return (
+        <Row className="w-full py-1.5 gap-3! flex-nowrap items-baseline border-0 border-b border-solid border-hairline last:border-b-0">
+            <Span className="w-8 shrink-0 text-[11px] leading-4 text-ink-secondary tabular-nums">{`#${index + 1}`}</Span>
+            <Span className="w-14 shrink-0 text-[11px] leading-4 text-ink-secondary tabular-nums">{offset}</Span>
+            <Span
+                className="grow min-w-0 text-[12px] leading-4 text-ink truncate"
+                title={[
+                    `${model}${call.model_slug ? ` (${call.model_slug})` : ""}, ${harnessLabel({ harness: call.harness })}`,
+                    settings,
+                ]
+                    .filter(Boolean)
+                    .join("\n")}
+            >
+                {fell_back ? `${model} (fallback)` : model}
+            </Span>
+            <Span
+                className="shrink-0 text-[12px] leading-4 text-ink-secondary tabular-nums"
+                title={`${call.prompt_tokens.toLocaleString()} in, ${call.completion_tokens.toLocaleString()} out · ${costLabel({ cost: call.cost })}`}
+            >
+                {`${formatTokens({ tokens: call.prompt_tokens })} / ${formatTokens({ tokens: call.completion_tokens })}`}
+            </Span>
+        </Row>
+    );
+}
@@ -0,0 +1,16 @@
+import Span from "@moduletrace/twui/layout/Span";
+
+type Props = {
+    children: string;
+};
+
+/**
+ * The small label over each part of a message's details.
+ */
+export default function MessageDetailsHeading({ children }: Props) {
+    return (
+        <Span className="mt-4 mb-1 text-[11px] leading-4 font-bold uppercase tracking-wide text-ink-secondary">
+            {children}
+        </Span>
+    );
+}
@@ -0,0 +1,54 @@
+import { useState } from "react";
+import Modal from "@moduletrace/twui/elements/Modal";
+import H2 from "@moduletrace/twui/layout/H2";
+import Span from "@moduletrace/twui/layout/Span";
+import Stack from "@moduletrace/twui/layout/Stack";
+import { Info } from "lucide-react";
+import useMessageDetails from "../(hooks)/use-message-details";
+import MessageActionButton from "./message-action-button";
+import MessageDetailsBody from "./message-details-body";
+
+type Props = {
+    message_id: number;
+};
+
+/**
+ * The info button under a bot message, and the modal it opens: which
+ * provider and model generated it, the run behind it, and every model
+ * call with its tokens and cost. Loaded when it opens.
+ */
+export default function MessageDetailsModal({ message_id }: Props) {
+    const [open, set_open] = useState(false);
+    const { details, loading, error } = useMessageDetails({ message_id, enabled: open });
+
+    return (
+        <Modal
+            target={
+                <MessageActionButton title="Details" aria_label="How this message was generated">
+                    <Info size={14} aria-hidden />
+                </MessageActionButton>
+            }
+            targetWrapperProps={{ className: "shrink-0" }}
+            open={open}
+            setOpen={set_open}
+            className="max-w-xl bg-paper p-6"
+        >
+            <Stack className="w-full gap-1! items-stretch">
+                <H2 className="text-lg! leading-6! tracking-normal! font-bold text-ink">Message details</H2>
+                <Span className="text-[13px] leading-[18px] text-ink-secondary">
+                    How this answer was generated: the provider and model behind it, the settings they were called with, and what it spent.
+                </Span>
+            </Stack>
+
+            <Stack className="mt-4 max-h-[60vh] w-full gap-0! items-stretch overflow-y-auto">
+                {details ? (
+                    <MessageDetailsBody details={details} />
+                ) : (
+                    <Span className="text-[13px] leading-[18px] text-ink-secondary">
+                        {loading || !error ? "Loading details..." : error}
+                    </Span>
+                )}
+            </Stack>
+        </Modal>
+    );
+}
@@ -0,0 +1,60 @@
+import Row from "@moduletrace/twui/layout/Row";
+import Span from "@moduletrace/twui/layout/Span";
+import Stack from "@moduletrace/twui/layout/Stack";
+import type { MessageDetailsModel } from "@/src/types";
+import describeCallSettings from "@/src/utils/describe-call-settings";
+import formatTokens from "@/src/utils/format-tokens";
+import costLabel from "../(utils)/cost-label";
+import harnessLabel from "../(utils)/harness-label";
+import providerLabel from "../(utils)/provider-label";
+import MessageDetailsSetting from "./message-details-setting";
+
+type Props = {
+    model: MessageDetailsModel;
+};
+
+/**
+ * One model a message used, summed over its calls: the model by name with
+ * its exact id under it, the provider that served it and what it was used
+ * for, then its calls, tokens and cost, and the settings it was called
+ * with (temperature, reasoning effort, whose settings they were, any
+ * fallback). Calls recorded before settings were show none.
+ */
+export default function MessageDetailsModelRow({ model }: Props) {
+    const name = model.model_name || model.model_slug || "Unknown model";
+    const settings = model.settings ? describeCallSettings({ settings: model.settings, harness: model.harness }) : [];
+
+    return (
+        <Stack className="w-full py-2.5 gap-1! items-stretch border-0 border-b border-solid border-hairline last:border-b-0">
+            <Row className="w-full gap-3! flex-nowrap items-baseline">
+                <Span className="grow min-w-0 text-[13px] leading-5 font-bold text-ink truncate" title={name}>
+                    {name}
+                </Span>
+                <Span className="shrink-0 text-[13px] leading-5 text-ink tabular-nums">
+                    {`${formatTokens({ tokens: model.total_tokens })} tokens`}
+                </Span>
+            </Row>
+            {model.model_slug && model.model_slug != name ? (
+                <Span className="text-[12px] leading-4 text-ink-secondary font-mono break-all">
+                    {model.model_slug}
+                </Span>
+            ) : null}
+            <Span className="text-[12px] leading-4 text-ink-secondary">
+                {[
+                    providerLabel({ name: model.provider_name, kind: model.provider_kind }),
+                    harnessLabel({ harness: model.harness }),
+                    `${model.calls} ${model.calls == 1 ? "call" : "calls"}`,
+                    `${model.prompt_tokens.toLocaleString()} in, ${model.completion_tokens.toLocaleString()} out`,
+                    costLabel({ cost: model.cost }),
+                ].join(" · ")}
+            </Span>
+            {settings.length ? (
+                <Row className="w-full mt-0.5 gap-x-3! gap-y-1! flex-wrap items-baseline">
+                    {settings.map((setting) => (
+                        <MessageDetailsSetting key={setting.label} {...setting} />
+                    ))}
+                </Row>
+            ) : null}
+        </Stack>
+    );
+}
@@ -0,0 +1,19 @@
+import Span from "@moduletrace/twui/layout/Span";
+
+type Props = {
+    label: string;
+    value: string;
+};
+
+/**
+ * One setting a model was called with, as a small label and value pair:
+ * "Temperature 0.3", "Reasoning effort High".
+ */
+export default function MessageDetailsSetting({ label, value }: Props) {
+    return (
+        <Span className="text-[12px] leading-4 whitespace-nowrap">
+            <Span className="text-ink-secondary">{`${label} `}</Span>
+            <Span className="text-ink tabular-nums">{value}</Span>
+        </Span>
+    );
+}