Add dictation and streamed read-aloud to the chat
- A mic in the composer records, transcribes with the bot's transcription model (api/admin/bots/transcribe) and writes the text into the input at the cursor to edit before sending. Escape cancels; recordings aren't stored. - Every bot message has a read-aloud button that plays GET api/admin/bots/speak-message, streamed from the provider as the audio is made, so it starts in a couple of seconds. One message plays at a time; only the user's own sessions can be read. - Both buttons stay visible but disabled while the bot has no model for them (bot_voice page prop). - Transcription and speech no longer pick free models automatically: they're not available until a model is picked, and Settings says so. Transcription lists speech-to-text models only, so the chat-model transcript path and its missing-audio check are gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
@@ -30,7 +30,7 @@ describe("modelServesHarness", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("transcription takes speech-to-text or audio-hearing chat models", () => {
|
||||
test("transcription takes speech-to-text models only", () => {
|
||||
expect(
|
||||
modelServesHarness({
|
||||
model: {
|
||||
@@ -46,7 +46,7 @@ describe("modelServesHarness", () => {
|
||||
model: { ...chat, input_modalities: ["text", "audio"] },
|
||||
harness: "transcription",
|
||||
}),
|
||||
).toBe(true);
|
||||
).toBe(false);
|
||||
expect(
|
||||
modelServesHarness({ model: chat, harness: "transcription" }),
|
||||
).toBe(false);
|
||||
|
||||
@@ -13,7 +13,8 @@ type Params = {
|
||||
* Whether a model can do a harness's job:
|
||||
* - text: reads and writes text, and calls tools (bots work through tools)
|
||||
* - vision: reads images, writes text
|
||||
* - transcription: a speech-to-text model, or a chat model that hears audio
|
||||
* - transcription: a speech-to-text model (never a chat model that hears
|
||||
* audio: free ones often drop it)
|
||||
* - speech: a text-to-speech model
|
||||
* - image: makes images
|
||||
*/
|
||||
@@ -31,10 +32,7 @@ export default function modelServesHarness({ model, harness }: Params) {
|
||||
return input.has("image") && output.has("text");
|
||||
|
||||
case "transcription":
|
||||
return (
|
||||
output.has("transcription") ||
|
||||
(input.has("audio") && output.has("text"))
|
||||
);
|
||||
return output.has("transcription");
|
||||
|
||||
case "speech":
|
||||
return output.has("speech");
|
||||
|
||||
@@ -106,28 +106,24 @@ describe("rankAutoModels", () => {
|
||||
"google/gemma-4-31b-it:free",
|
||||
"openrouter/free",
|
||||
]);
|
||||
expect(
|
||||
slugs(rankAutoModels({ models: live, harness: "transcription" })),
|
||||
).toEqual(["thinkingmachines/inkling-small:free"]);
|
||||
expect(rankAutoModels({ models: live, harness: "speech" })).toEqual(
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("puts speech-to-text models ahead of chat models that hear audio", () => {
|
||||
const ranked = rankAutoModels({
|
||||
models: [
|
||||
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
|
||||
model("acme/scribe", {
|
||||
input_modalities: ["audio"],
|
||||
output_modalities: ["transcription"],
|
||||
supports_tools: false,
|
||||
}),
|
||||
],
|
||||
harness: "transcription",
|
||||
});
|
||||
test("never picks for transcription or speech", () => {
|
||||
const models = [
|
||||
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
|
||||
model("acme/scribe", {
|
||||
input_modalities: ["audio"],
|
||||
output_modalities: ["transcription"],
|
||||
supports_tools: false,
|
||||
}),
|
||||
model("acme/voice", {
|
||||
output_modalities: ["speech"],
|
||||
supports_tools: false,
|
||||
}),
|
||||
];
|
||||
|
||||
expect(slugs(ranked)).toEqual(["acme/scribe", "deepseek/omni"]);
|
||||
expect(rankAutoModels({ models, harness: "transcription" })).toEqual([]);
|
||||
expect(rankAutoModels({ models, harness: "speech" })).toEqual([]);
|
||||
});
|
||||
|
||||
test("drops retired and unscored small models from other vendors", () => {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { AppData } from "@/src/data/app-data";
|
||||
import {
|
||||
AutoModelRouterSlugs,
|
||||
AutoModelVendorPreference,
|
||||
Harnesses,
|
||||
SmallModelPattern,
|
||||
SpecialistModelPattern,
|
||||
type HarnessKind,
|
||||
@@ -26,13 +27,16 @@ type Params<T extends HarnessCatalogModel> = {
|
||||
* 3. routers such as `openrouter/free`, last
|
||||
*
|
||||
* Within a group, benchmarked models come first by intelligence index,
|
||||
* then the rest: full-size before small variants, newest first.
|
||||
* then the rest: full-size before small variants, newest first. None for
|
||||
* a harness that isn't `automatic` (transcription, speech).
|
||||
*/
|
||||
export default function rankAutoModels<T extends HarnessCatalogModel>({
|
||||
models,
|
||||
harness,
|
||||
now = Date.now(),
|
||||
}: Params<T>): T[] {
|
||||
if (!Harnesses.find((h) => h.value == harness)?.automatic) return [];
|
||||
|
||||
const ranked = models
|
||||
.filter(
|
||||
(model) =>
|
||||
@@ -50,23 +54,12 @@ export default function rankAutoModels<T extends HarnessCatalogModel>({
|
||||
return ranked
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(harness == "transcription"
|
||||
? Number(!isSpeechToText(a.model)) -
|
||||
Number(!isSpeechToText(b.model))
|
||||
: 0) ||
|
||||
a.tier - b.tier ||
|
||||
compareInTier(a.model, b.model),
|
||||
)
|
||||
.map((entry) => entry.model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dedicated speech-to-text models transcribe before chat models that hear
|
||||
* audio: some free chat endpoints list audio input but drop it.
|
||||
*/
|
||||
function isSpeechToText(model: HarnessCatalogModel) {
|
||||
return model.output_modalities.includes("transcription");
|
||||
}
|
||||
|
||||
const OtherTier = AutoModelVendorPreference.length;
|
||||
const RouterTier = OtherTier + 1;
|
||||
|
||||
Reference in New Issue
Block a user