Let admins hear a voice sample before picking it

The Speech section of every model settings form now has "Hear a sample", which streams a short sentence (AppData.VoicePreviewText) in the form's model and voice before they're saved, through GET api/admin/harness/preview-voice (super admins and admins, that model only via grabModelCandidate). pickVoice is shared with speakText, and explainAudioError reads why a streamed player failed for both read-aloud and the sample. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-19 21:32:49 +01:00
parent 148d02dcfe
commit bc499a7466
10 changed files with 278 additions and 29 deletions
@@ -295,6 +295,10 @@ export const AppData = {
     * Most of a bot message read aloud; the rest is cut.
     */
    SpeechMaxChars: 4000,
+    /**
+     * What a speech model reads when a voice is tried in the model settings.
+     */
+    VoicePreviewText: "Hi there. This is how I'll sound when I read my replies aloud.",
    /**
     * Largest picture a bot or a workspace can have.
     */
@@ -0,0 +1,63 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_HARNESS_MODELS } from "@/db/types/db";
+import type { HarnessKind } from "@/src/dict/harness-dict";
+import modelServesHarness from "@/src/harness/catalog/model-serves-harness";
+import type { HarnessCandidate } from "@/src/harness/types";
+import AppError from "@/src/utils/app-error";
+import grabHarnessProviders from "./grab-harness-providers";
+import harnessModelRowToCatalog from "./harness-model-row-to-catalog";
+import resolveProviderConfig from "./resolve-provider-config";
+
+type Params = {
+    model_id: number;
+    harness: HarnessKind;
+};
+
+/**
+ * One catalog model as a candidate to call directly, whatever is saved
+ * (e.g. to try it before picking it). Throws when it's gone, can't do the
+ * harness's job, or its provider is off or has no usable key.
+ */
+export default async function grabModelCandidate({
+    model_id,
+    harness,
+}: Params): Promise<HarnessCandidate> {
+    const row = (
+        await BunSQLite.sql<BUN_SQLITE_COBBLE_HARNESS_MODELS>({
+            sql: `SELECT * FROM harness_models WHERE id = ? AND available = 1`,
+            values: [model_id],
+        })
+    ).payload?.[0];
+
+    if (!row?.provider_id) {
+        throw new AppError(`That model isn't available any more.`);
+    }
+
+    const model = harnessModelRowToCatalog({ row });
+
+    if (!modelServesHarness({ model, harness })) {
+        throw new AppError(`That model can't do this.`);
+    }
+
+    const [provider] = await grabHarnessProviders({
+        provider_id: row.provider_id,
+        enabled_only: true,
+    });
+
+    if (!provider) {
+        throw new AppError(`That model's provider is turned off.`);
+    }
+
+    const { config, usable } = await resolveProviderConfig({ provider });
+
+    if (!usable) {
+        throw new AppError(`That model's provider has no usable key.`);
+    }
+
+    return {
+        key: `${row.provider_id}:${model.slug}`,
+        provider: config,
+        model,
+        source: "pinned",
+    };
+}
@@ -0,0 +1,34 @@
+type Params = {
+    /**
+     * The streaming audio route the player failed on.
+     */
+    src: string;
+    fallback?: string;
+};
+
+/**
+ * An `<audio>` only says it failed, so ask its route again for the
+ * reason: a failure answers in JSON, and audio coming back instead is
+ * dropped at once.
+ */
+export default async function explainAudioError({
+    src,
+    fallback = "Couldn't play it. Try again.",
+}: Params) {
+    const controller = new AbortController();
+
+    try {
+        const res = await fetch(src, { signal: controller.signal, cache: "no-store" });
+
+        if (!res.headers.get("content-type")?.includes("json")) {
+            controller.abort();
+            return fallback;
+        }
+
+        const json = await res.json();
+
+        return json?.msg || fallback;
+    } catch {
+        return fallback;
+    }
+}
@@ -61,7 +61,11 @@ export default async function speakText({
    return result;
 }

-function pickVoice({
+/**
+ * The voice to ask for: the picked one when the model has it (or the
+ * model was picked by hand), else the model's first, else OpenAI's.
+ */
+export function pickVoice({
    candidate,
    voice,
 }: {
@@ -1,4 +1,5 @@
 import { useEffect, useRef, useState } from "react";
+import explainAudioError from "@/src/functions/frontend/explain-audio-error";

 export type SpeakMessageState = "idle" | "loading" | "playing";

@@ -78,7 +79,7 @@ export default function useSpeakMessage({ enabled }: { enabled: boolean }): Spea
            if (!isCurrent()) return;

            reset();
-            explainError({ src }).then((text) => {
+            explainAudioError({ src, fallback: "Couldn't read it aloud. Try again." }).then((text) => {
                setError({ message_id, text });
            });
        };
@@ -97,27 +98,3 @@ export default function useSpeakMessage({ enabled }: { enabled: boolean }): Spea

    return { enabled, message_id, state, error, toggle };
 }
-
-/**
- * The player only says it failed, so ask again for the reason: a failure
- * answers in JSON, and audio coming back instead is dropped at once.
- */
-async function explainError({ src }: { src: string }) {
-    const fallback = "Couldn't read it aloud. Try again.";
-    const controller = new AbortController();
-
-    try {
-        const res = await fetch(src, { signal: controller.signal, cache: "no-store" });
-
-        if (!res.headers.get("content-type")?.includes("json")) {
-            controller.abort();
-            return fallback;
-        }
-
-        const json = await res.json();
-
-        return json?.msg || fallback;
-    } catch {
-        return fallback;
-    }
-}
@@ -4,6 +4,7 @@ import SettingsGroup from "../../(partials)/settings-group";
 import HarnessCurrentModel from "./harness-current-model";
 import HarnessModelSelect from "./harness-model-select";
 import HarnessProviderSelect from "./harness-provider-select";
+import HarnessVoicePreview from "./harness-voice-preview";
 import HarnessVoiceSelect from "./harness-voice-select";

 type Props = {
@@ -15,8 +16,8 @@ type Props = {
 /**
 * One harness: its provider, then its model (narrowed to that provider),
 * whether it falls back to the automatic picks (for a harness that has
- * them), whether it's on, and its
- * voice for speech. On a bot's page the provider select can also use the
+ * them), whether it's on, and its voice for speech, with a sample to hear
+ * it before saving. On a bot's page the provider select can also use the
 * default, which hides the rest.
 */
 export default function HarnessSettingGroup({ harness, value, onChange }: Props) {
@@ -86,6 +87,9 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
                    onChange={(voice) => onChange({ ...value, voice: voice || undefined })}
                />
            ) : null}
+            {!value.inherit && harness.harness == "speech" && picked ? (
+                <HarnessVoicePreview model_id={picked.id} voice={value.voice} />
+            ) : null}
            {value.inherit || harness.required ? null : (
                <Checkbox
                    title={`Turn ${harness.title.toLowerCase()} on`}
@@ -0,0 +1,91 @@
+import { useEffect, useRef, useState } from "react";
+import Button from "@moduletrace/twui/layout/Button";
+import Row from "@moduletrace/twui/layout/Row";
+import Span from "@moduletrace/twui/layout/Span";
+import { Square, Volume2 } from "lucide-react";
+import explainAudioError from "@/src/functions/frontend/explain-audio-error";
+
+type Props = {
+    /**
+     * The speech model picked in the form, saved or not.
+     */
+    model_id: number;
+    voice?: string;
+};
+
+/**
+ * Plays a short sample in the form's speech model and voice, before
+ * they're saved (`api/admin/harness/preview-voice`, streamed). Stops when
+ * clicked again or when the pick changes.
+ */
+export default function HarnessVoicePreview({ model_id, voice }: Props) {
+    const [state, setState] = useState<"idle" | "loading" | "playing">("idle");
+    const [error, setError] = useState<string>();
+    const audio_ref = useRef<HTMLAudioElement | null>(null);
+
+    function stop() {
+        const audio = audio_ref.current;
+
+        if (audio) {
+            audio.onerror = null;
+            audio.pause();
+            audio.removeAttribute("src");
+            audio.load();
+        }
+
+        setState("idle");
+    }
+
+    useEffect(() => {
+        setError(undefined);
+        return stop;
+    }, [model_id, voice]);
+
+    function play() {
+        stop();
+        setError(undefined);
+        setState("loading");
+
+        const src = `/api/admin/harness/preview-voice?${new URLSearchParams({
+            model_id: String(model_id),
+            ...(voice ? { voice } : {}),
+        })}`;
+        const audio = audio_ref.current || new Audio();
+
+        audio_ref.current = audio;
+        audio.onplaying = () => setState("playing");
+        audio.onended = () => setState("idle");
+        audio.onerror = () => {
+            setState("idle");
+            explainAudioError({ src, fallback: "Couldn't play the sample. Try again." }).then(setError);
+        };
+        audio.src = src;
+        audio.play().catch(() => {});
+    }
+
+    const playing = state == "playing";
+
+    return (
+        <Row className="gap-2! flex-nowrap items-center">
+            <Button
+                title={playing ? "Stop the sample" : "Hear a short sample in this voice"}
+                type="button"
+                variant="ghost"
+                color="gray"
+                size="small"
+                loading={state == "loading"}
+                beforeIcon={
+                    playing ? <Square size={13} fill="currentColor" /> : <Volume2 size={15} />
+                }
+                onClick={playing ? stop : play}
+            >
+                {playing ? "Stop" : "Hear a sample"}
+            </Button>
+            {error ? (
+                <Span role="status" className="text-[13px] leading-[18px] text-ink-secondary">
+                    {error}
+                </Span>
+            ) : null}
+        </Row>
+    );
+}
@@ -0,0 +1,65 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabModelCandidate from "@/src/functions/backend/db/harness/grab-model-candidate";
+import { AppData } from "@/src/data/app-data";
+import { pickVoice } from "@/src/harness/audio/speak-text";
+import HarnessError from "@/src/harness/harness-error";
+import grabHarnessAdapter from "@/src/harness/providers/grab-harness-adapter";
+import { PreviewVoiceRequestSchema } from "@/src/schemas/harness-schema";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import canManageCrew from "@/src/utils/rbac/can-manage-crew";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * `GET ?model_id=&voice=`: a short sample (`AppData.VoicePreviewText`) in
+ * a speech model and voice, picked or not, streamed for an `<audio>`, so
+ * a voice can be heard before it's saved. For super admins and admins,
+ * who set speech models. No fallback: it's that model or an error.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({ req, query }) => {
+    try {
+        if (req.method != "GET") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        if (!canManageCrew({ user_types })) {
+            throw new AppError(`Only admins can try voices.`);
+        }
+
+        const { model_id, voice } = parseWithSchema({
+            schema: PreviewVoiceRequestSchema,
+            data: query,
+        });
+
+        const candidate = await grabModelCandidate({ model_id, harness: "speech" });
+
+        const audio = await grabHarnessAdapter({ kind: candidate.provider.kind }).speak({
+            candidate,
+            text: AppData.VoicePreviewText,
+            voice: pickVoice({ candidate, voice }),
+            signal: req.signal,
+        });
+
+        return new Response(audio.stream, {
+            headers: {
+                "Content-Type": audio.media_type,
+                "Cache-Control": "private, max-age=3600",
+            },
+        });
+    } catch (error: unknown) {
+        return {
+            success: false,
+            msg: error instanceof HarnessError ? error.message : safeErrorMessage({ error }),
+        };
+    }
+};
@@ -90,3 +90,10 @@ export type SaveHarnessSettingsRequest = z.infer<
 export type SaveBotHarnessSettingsRequest = z.infer<
    typeof SaveBotHarnessSettingsRequestSchema
 >;
+
+export const PreviewVoiceRequestSchema = z.object({
+    model_id: z.coerce.number().int().positive("Pick a speech model"),
+    voice: z.string().trim().max(200).optional(),
+});
+
+export type PreviewVoiceRequest = z.infer<typeof PreviewVoiceRequestSchema>;