Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 14 Wiki Activity

Add dictation and streamed read-aloud to the chat

Browse Source 
- A mic in the composer records, transcribes with the bot's
  transcription model (api/admin/bots/transcribe) and writes the text
  into the input at the cursor to edit before sending. Escape cancels;
  recordings aren't stored.
- Every bot message has a read-aloud button that plays
  GET api/admin/bots/speak-message, streamed from the provider as the
  audio is made, so it starts in a couple of seconds. One message plays
  at a time; only the user's own sessions can be read.
- Both buttons stay visible but disabled while the bot has no model for
  them (bot_voice page prop).
- Transcription and speech no longer pick free models automatically:
  they're not available until a model is picked, and Settings says so.
  Transcription lists speech-to-text models only, so the chat-model
  transcript path and its missing-audio check are gone.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-19 21:21:04 +01:00
co-authored by Claude Opus 5.5
parent 727cbf53c8
commit 163d9a434c
37 changed files with 936 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/harness/audio/looks-like-missing-audio.test.ts
-27
View File
@@ -1,27 +0,0 @@
import { describe, expect, test } from "bun:test";
import looksLikeMissingAudio from "./looks-like-missing-audio";
describe("looksLikeMissingAudio", () => {
test("spots a model asking for the audio it never got", () => {
for (const text of [
"Please provide the audio file you’d like transcribed.",
"I’m sorry, but I can’t transcribe without the audio.",
"I cannot hear any audio in your message.",
"There is no audio attached.",
"I didn't receive any audio.",
]) {
expect(looksLikeMissingAudio({ text })).toBe(true);
}
});
test("leaves real transcripts alone", () => {
expect(looksLikeMissingAudio({ text: "Hello from cobble." })).toBe(
false,
);
expect(
looksLikeMissingAudio({
text: `Can you send the audio report to finance? ${"And then more. ".repeat(30)}`,
}),
).toBe(false);
});
});
src/harness/audio/looks-like-missing-audio.ts
-14
View File
@@ -1,14 +0,0 @@
type Params = {
text: string;
};
const MissingAudio =
/(provide|upload|share|attach|send)\b.{0,40}\baudio|can(?:'|’)?t (?:transcribe|hear|access|listen)|cannot (?:transcribe|hear|access|listen)|no audio|without (?:the |an |any )?audio|didn(?:'|’)?t (?:receive|get) (?:the |an |any )?audio/i;
/**
* Whether a chat model's "transcript" is really it saying it got no audio.
* Some providers list audio input for a model but drop the audio.
*/
export default function looksLikeMissingAudio({ text }: Params) {
return text.length < 300 && MissingAudio.test(text);
}
src/harness/audio/speak-text.ts
+4 -2
View File
@@ -22,8 +22,10 @@ type Params = {
};
/**
* Reads text aloud with the speech harness. Uses the voice picked in the
* speech setting when the model has it, else the model's first voice.
* Reads text aloud with the speech harness, as a stream that can play
* while the rest is still being made. Falls back to the next model only
* until one answers. Uses the voice picked in the speech setting when the
* model has it, else the model's first voice.
*/
export default async function speakText({
text,
src/harness/audio/transcribe-audio.ts
+8 -53
View File
@@ -1,17 +1,10 @@
import { AppData } from "@/src/data/app-data";
import { TranscribeAudioPrompt } from "@/src/data/harness-prompts";
import AppError from "@/src/utils/app-error";
import HarnessError from "../harness-error";
import grabHarnessAdapter from "../providers/grab-harness-adapter";
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
import withCandidates from "../resolve/with-candidates";
import type { HarnessAudioInput } from "../types";
import looksLikeMissingAudio from "./looks-like-missing-audio";
/**
* A chat model that drops audio is skipped this long.
*/
const MissingAudioCooldownMs = 60 * 60 * 1000;
type Params = {
audio: HarnessAudioInput;
@@ -31,9 +24,8 @@ type Params = {
};
/**
* Turns speech into text with the transcription harness. Speech-to-text
* models get the file at `/audio/transcriptions`; chat models that hear
* audio are asked for a verbatim transcript.
* Turns speech into text with the transcription harness: its
* speech-to-text model gets the file at `/audio/transcriptions`.
*/
export default async function transcribeAudio({
audio,
@@ -62,40 +54,12 @@ export default async function transcribeAudio({
kind: candidate.provider.kind,
});
const is_speech_to_text =
candidate.model.output_modalities.includes("transcription");
const text = is_speech_to_text
? (
await adapter.transcribe({
candidate,
audio,
language,
signal: final_signal,
})
).text
: (
await adapter.chat({
candidate,
messages: [
{
role: "user",
content: [
{ type: "text", text: TranscribeAudioPrompt },
{
type: "audio",
data: Buffer.from(audio.data).toString(
"base64",
),
format: audio.format,
},
],
},
],
temperature: 0,
signal: final_signal,
})
).text;
const { text } = await adapter.transcribe({
candidate,
audio,
language,
signal: final_signal,
});
if (text === null || text === undefined) {
throw new HarnessError({
@@ -105,15 +69,6 @@ export default async function transcribeAudio({
});
}
if (!is_speech_to_text && looksLikeMissingAudio({ text })) {
throw new HarnessError({
message: `${candidate.model.name || candidate.model.slug} didn't get the audio.`,
retryable: true,
retry_at: Date.now() + MissingAudioCooldownMs,
candidate_key: candidate.key,
});
}
return text.trim();
},
});
src/harness/catalog/model-serves-harness.test.ts
+2 -2
View File
@@ -30,7 +30,7 @@ describe("modelServesHarness", () => {
).toBe(true);
});
test("transcription takes speech-to-text or audio-hearing chat models", () => {
test("transcription takes speech-to-text models only", () => {
expect(
modelServesHarness({
model: {
@@ -46,7 +46,7 @@ describe("modelServesHarness", () => {
model: { ...chat, input_modalities: ["text", "audio"] },
harness: "transcription",
}),
).toBe(true);
).toBe(false);
expect(
modelServesHarness({ model: chat, harness: "transcription" }),
).toBe(false);
src/harness/catalog/model-serves-harness.ts
+3 -5
View File
@@ -13,7 +13,8 @@ type Params = {
* Whether a model can do a harness's job:
* - text: reads and writes text, and calls tools (bots work through tools)
* - vision: reads images, writes text
* - transcription: a speech-to-text model, or a chat model that hears audio
* - transcription: a speech-to-text model (never a chat model that hears
* audio: free ones often drop it)
* - speech: a text-to-speech model
* - image: makes images
*/
@@ -31,10 +32,7 @@ export default function modelServesHarness({ model, harness }: Params) {
return input.has("image") && output.has("text");
case "transcription":
return (
output.has("transcription") ||
(input.has("audio") && output.has("text"))
);
return output.has("transcription");
case "speech":
return output.has("speech");
src/harness/catalog/rank-auto-models.test.ts
+15 -19
View File
@@ -106,28 +106,24 @@ describe("rankAutoModels", () => {
"google/gemma-4-31b-it:free",
"openrouter/free",
]);
expect(
slugs(rankAutoModels({ models: live, harness: "transcription" })),
).toEqual(["thinkingmachines/inkling-small:free"]);
expect(rankAutoModels({ models: live, harness: "speech" })).toEqual(
[],
);
});
test("puts speech-to-text models ahead of chat models that hear audio", () => {
const ranked = rankAutoModels({
models: [
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
model("acme/scribe", {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
}),
],
harness: "transcription",
});
test("never picks for transcription or speech", () => {
const models = [
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
model("acme/scribe", {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
}),
model("acme/voice", {
output_modalities: ["speech"],
supports_tools: false,
}),
];
expect(slugs(ranked)).toEqual(["acme/scribe", "deepseek/omni"]);
expect(rankAutoModels({ models, harness: "transcription" })).toEqual([]);
expect(rankAutoModels({ models, harness: "speech" })).toEqual([]);
});
test("drops retired and unscored small models from other vendors", () => {
src/harness/catalog/rank-auto-models.ts
+5 -12
View File
@@ -2,6 +2,7 @@ import { AppData } from "@/src/data/app-data";
import {
AutoModelRouterSlugs,
AutoModelVendorPreference,
Harnesses,
SmallModelPattern,
SpecialistModelPattern,
type HarnessKind,
@@ -26,13 +27,16 @@ type Params<T extends HarnessCatalogModel> = {
* 3. routers such as `openrouter/free`, last
*
* Within a group, benchmarked models come first by intelligence index,
* then the rest: full-size before small variants, newest first.
* then the rest: full-size before small variants, newest first. None for
* a harness that isn't `automatic` (transcription, speech).
*/
export default function rankAutoModels<T extends HarnessCatalogModel>({
models,
harness,
now = Date.now(),
}: Params<T>): T[] {
if (!Harnesses.find((h) => h.value == harness)?.automatic) return [];
const ranked = models
.filter(
(model) =>
@@ -50,23 +54,12 @@ export default function rankAutoModels<T extends HarnessCatalogModel>({
return ranked
.sort(
(a, b) =>
(harness == "transcription"
? Number(!isSpeechToText(a.model)) -
Number(!isSpeechToText(b.model))
: 0) ||
a.tier - b.tier ||
compareInTier(a.model, b.model),
)
.map((entry) => entry.model);
}
/**
* Dedicated speech-to-text models transcribe before chat models that hear
* audio: some free chat endpoints list audio input but drop it.
*/
function isSpeechToText(model: HarnessCatalogModel) {
return model.output_modalities.includes("transcription");
}
const OtherTier = AutoModelVendorPreference.length;
const RouterTier = OtherTier + 1;
src/harness/providers/openai-compatible/speak.ts
+12 -2
View File
@@ -1,8 +1,10 @@
import type { HarnessAdapter } from "../../types";
import HarnessError from "../../harness-error";
import providerRequest from "./provider-request";
/**
* `POST /audio/speech`. Returns the audio bytes.
* `POST /audio/speech`. Resolves once the provider answers, with its
* audio as a stream (providers send it in chunks as it's made).
*/
const speak: HarnessAdapter["speak"] = async ({
candidate,
@@ -24,8 +26,16 @@ const speak: HarnessAdapter["speak"] = async ({
candidate_key: candidate.key,
});
if (!res.body) {
throw new HarnessError({
message: `The model returned no audio.`,
retryable: true,
candidate_key: candidate.key,
});
}
return {
data: new Uint8Array(await res.arrayBuffer()),
stream: res.body,
media_type:
res.headers.get("content-type")?.split(";")[0] ||
`audio/${format == "mp3" ? "mpeg" : format}`,
src/harness/types.ts
+7 -3
View File
@@ -174,8 +174,12 @@ export type HarnessSpeakParams = {
signal: AbortSignal;
};
export type HarnessAudioOutput = {
data: Uint8Array;
/**
* Speech as the provider sends it: the stream starts once the provider
* answers, so it can play while the rest is still being made.
*/
export type HarnessAudioStream = {
stream: ReadableStream<Uint8Array>;
media_type: string;
};
@@ -206,7 +210,7 @@ export type HarnessImageOutput = {
export type HarnessAdapter = {
chat: (params: HarnessChatParams) => Promise<HarnessChatResult>;
transcribe: (params: HarnessTranscribeParams) => Promise<{ text: string }>;
speak: (params: HarnessSpeakParams) => Promise<HarnessAudioOutput>;
speak: (params: HarnessSpeakParams) => Promise<HarnessAudioStream>;
generateImage: (
params: HarnessImageParams,
) => Promise<{ images: HarnessImageOutput[] }>;