Add dictation and streamed read-aloud to the chat
- A mic in the composer records, transcribes with the bot's transcription model (api/admin/bots/transcribe) and writes the text into the input at the cursor to edit before sending. Escape cancels; recordings aren't stored. - Every bot message has a read-aloud button that plays GET api/admin/bots/speak-message, streamed from the provider as the audio is made, so it starts in a couple of seconds. One message plays at a time; only the user's own sessions can be read. - Both buttons stay visible but disabled while the bot has no model for them (bot_voice page prop). - Transcription and speech no longer pick free models automatically: they're not available until a model is picked, and Settings says so. Transcription lists speech-to-text models only, so the chat-model transcript path and its missing-audio check are gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -1,27 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import looksLikeMissingAudio from "./looks-like-missing-audio";
|
||||
|
||||
describe("looksLikeMissingAudio", () => {
|
||||
test("spots a model asking for the audio it never got", () => {
|
||||
for (const text of [
|
||||
"Please provide the audio file you’d like transcribed.",
|
||||
"I’m sorry, but I can’t transcribe without the audio.",
|
||||
"I cannot hear any audio in your message.",
|
||||
"There is no audio attached.",
|
||||
"I didn't receive any audio.",
|
||||
]) {
|
||||
expect(looksLikeMissingAudio({ text })).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test("leaves real transcripts alone", () => {
|
||||
expect(looksLikeMissingAudio({ text: "Hello from cobble." })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
looksLikeMissingAudio({
|
||||
text: `Can you send the audio report to finance? ${"And then more. ".repeat(30)}`,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
type Params = {
|
||||
text: string;
|
||||
};
|
||||
|
||||
const MissingAudio =
|
||||
/(provide|upload|share|attach|send)\b.{0,40}\baudio|can(?:'|’)?t (?:transcribe|hear|access|listen)|cannot (?:transcribe|hear|access|listen)|no audio|without (?:the |an |any )?audio|didn(?:'|’)?t (?:receive|get) (?:the |an |any )?audio/i;
|
||||
|
||||
/**
|
||||
* Whether a chat model's "transcript" is really it saying it got no audio.
|
||||
* Some providers list audio input for a model but drop the audio.
|
||||
*/
|
||||
export default function looksLikeMissingAudio({ text }: Params) {
|
||||
return text.length < 300 && MissingAudio.test(text);
|
||||
}
|
||||
@@ -22,8 +22,10 @@ type Params = {
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads text aloud with the speech harness. Uses the voice picked in the
|
||||
* speech setting when the model has it, else the model's first voice.
|
||||
* Reads text aloud with the speech harness, as a stream that can play
|
||||
* while the rest is still being made. Falls back to the next model only
|
||||
* until one answers. Uses the voice picked in the speech setting when the
|
||||
* model has it, else the model's first voice.
|
||||
*/
|
||||
export default async function speakText({
|
||||
text,
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import { TranscribeAudioPrompt } from "@/src/data/harness-prompts";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import HarnessError from "../harness-error";
|
||||
import grabHarnessAdapter from "../providers/grab-harness-adapter";
|
||||
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
|
||||
import withCandidates from "../resolve/with-candidates";
|
||||
import type { HarnessAudioInput } from "../types";
|
||||
import looksLikeMissingAudio from "./looks-like-missing-audio";
|
||||
|
||||
/**
|
||||
* A chat model that drops audio is skipped this long.
|
||||
*/
|
||||
const MissingAudioCooldownMs = 60 * 60 * 1000;
|
||||
|
||||
type Params = {
|
||||
audio: HarnessAudioInput;
|
||||
@@ -31,9 +24,8 @@ type Params = {
|
||||
};
|
||||
|
||||
/**
|
||||
* Turns speech into text with the transcription harness. Speech-to-text
|
||||
* models get the file at `/audio/transcriptions`; chat models that hear
|
||||
* audio are asked for a verbatim transcript.
|
||||
* Turns speech into text with the transcription harness: its
|
||||
* speech-to-text model gets the file at `/audio/transcriptions`.
|
||||
*/
|
||||
export default async function transcribeAudio({
|
||||
audio,
|
||||
@@ -62,40 +54,12 @@ export default async function transcribeAudio({
|
||||
kind: candidate.provider.kind,
|
||||
});
|
||||
|
||||
const is_speech_to_text =
|
||||
candidate.model.output_modalities.includes("transcription");
|
||||
|
||||
const text = is_speech_to_text
|
||||
? (
|
||||
await adapter.transcribe({
|
||||
candidate,
|
||||
audio,
|
||||
language,
|
||||
signal: final_signal,
|
||||
})
|
||||
).text
|
||||
: (
|
||||
await adapter.chat({
|
||||
candidate,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: TranscribeAudioPrompt },
|
||||
{
|
||||
type: "audio",
|
||||
data: Buffer.from(audio.data).toString(
|
||||
"base64",
|
||||
),
|
||||
format: audio.format,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0,
|
||||
signal: final_signal,
|
||||
})
|
||||
).text;
|
||||
const { text } = await adapter.transcribe({
|
||||
candidate,
|
||||
audio,
|
||||
language,
|
||||
signal: final_signal,
|
||||
});
|
||||
|
||||
if (text === null || text === undefined) {
|
||||
throw new HarnessError({
|
||||
@@ -105,15 +69,6 @@ export default async function transcribeAudio({
|
||||
});
|
||||
}
|
||||
|
||||
if (!is_speech_to_text && looksLikeMissingAudio({ text })) {
|
||||
throw new HarnessError({
|
||||
message: `${candidate.model.name || candidate.model.slug} didn't get the audio.`,
|
||||
retryable: true,
|
||||
retry_at: Date.now() + MissingAudioCooldownMs,
|
||||
candidate_key: candidate.key,
|
||||
});
|
||||
}
|
||||
|
||||
return text.trim();
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user