Add dictation and streamed read-aloud to the chat

- A mic in the composer records, transcribes with the bot's
  transcription model (api/admin/bots/transcribe) and writes the text
  into the input at the cursor to edit before sending. Escape cancels;
  recordings aren't stored.
- Every bot message has a read-aloud button that plays
  GET api/admin/bots/speak-message, streamed from the provider as the
  audio is made, so it starts in a couple of seconds. One message plays
  at a time; only the user's own sessions can be read.
- Both buttons stay visible but disabled while the bot has no model for
  them (bot_voice page prop).
- Transcription and speech no longer pick free models automatically:
  they're not available until a model is picked, and Settings says so.
  Transcription lists speech-to-text models only, so the chat-model
  transcript path and its missing-audio check are gone.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-19 21:21:04 +01:00
co-authored by Claude Opus 5.5
parent 727cbf53c8
commit 163d9a434c
37 changed files with 936 additions and 197 deletions
@@ -30,7 +30,7 @@ describe("modelServesHarness", () => {
).toBe(true);
});
test("transcription takes speech-to-text or audio-hearing chat models", () => {
test("transcription takes speech-to-text models only", () => {
expect(
modelServesHarness({
model: {
@@ -46,7 +46,7 @@ describe("modelServesHarness", () => {
model: { ...chat, input_modalities: ["text", "audio"] },
harness: "transcription",
}),
).toBe(true);
).toBe(false);
expect(
modelServesHarness({ model: chat, harness: "transcription" }),
).toBe(false);
+3 -5
View File
@@ -13,7 +13,8 @@ type Params = {
* Whether a model can do a harness's job:
* - text: reads and writes text, and calls tools (bots work through tools)
* - vision: reads images, writes text
* - transcription: a speech-to-text model, or a chat model that hears audio
* - transcription: a speech-to-text model (never a chat model that hears
* audio: free ones often drop it)
* - speech: a text-to-speech model
* - image: makes images
*/
@@ -31,10 +32,7 @@ export default function modelServesHarness({ model, harness }: Params) {
return input.has("image") && output.has("text");
case "transcription":
return (
output.has("transcription") ||
(input.has("audio") && output.has("text"))
);
return output.has("transcription");
case "speech":
return output.has("speech");
+15 -19
View File
@@ -106,28 +106,24 @@ describe("rankAutoModels", () => {
"google/gemma-4-31b-it:free",
"openrouter/free",
]);
expect(
slugs(rankAutoModels({ models: live, harness: "transcription" })),
).toEqual(["thinkingmachines/inkling-small:free"]);
expect(rankAutoModels({ models: live, harness: "speech" })).toEqual(
[],
);
});
test("puts speech-to-text models ahead of chat models that hear audio", () => {
const ranked = rankAutoModels({
models: [
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
model("acme/scribe", {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
}),
],
harness: "transcription",
});
test("never picks for transcription or speech", () => {
const models = [
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
model("acme/scribe", {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
}),
model("acme/voice", {
output_modalities: ["speech"],
supports_tools: false,
}),
];
expect(slugs(ranked)).toEqual(["acme/scribe", "deepseek/omni"]);
expect(rankAutoModels({ models, harness: "transcription" })).toEqual([]);
expect(rankAutoModels({ models, harness: "speech" })).toEqual([]);
});
test("drops retired and unscored small models from other vendors", () => {
+5 -12
View File
@@ -2,6 +2,7 @@ import { AppData } from "@/src/data/app-data";
import {
AutoModelRouterSlugs,
AutoModelVendorPreference,
Harnesses,
SmallModelPattern,
SpecialistModelPattern,
type HarnessKind,
@@ -26,13 +27,16 @@ type Params<T extends HarnessCatalogModel> = {
* 3. routers such as `openrouter/free`, last
*
* Within a group, benchmarked models come first by intelligence index,
* then the rest: full-size before small variants, newest first.
* then the rest: full-size before small variants, newest first. None for
* a harness that isn't `automatic` (transcription, speech).
*/
export default function rankAutoModels<T extends HarnessCatalogModel>({
models,
harness,
now = Date.now(),
}: Params<T>): T[] {
if (!Harnesses.find((h) => h.value == harness)?.automatic) return [];
const ranked = models
.filter(
(model) =>
@@ -50,23 +54,12 @@ export default function rankAutoModels<T extends HarnessCatalogModel>({
return ranked
.sort(
(a, b) =>
(harness == "transcription"
? Number(!isSpeechToText(a.model)) -
Number(!isSpeechToText(b.model))
: 0) ||
a.tier - b.tier ||
compareInTier(a.model, b.model),
)
.map((entry) => entry.model);
}
/**
* Dedicated speech-to-text models transcribe before chat models that hear
* audio: some free chat endpoints list audio input but drop it.
*/
function isSpeechToText(model: HarnessCatalogModel) {
return model.output_modalities.includes("transcription");
}
const OtherTier = AutoModelVendorPreference.length;
const RouterTier = OtherTier + 1;