Add dictation and streamed read-aloud to the chat
- A mic in the composer records, transcribes with the bot's transcription model (api/admin/bots/transcribe) and writes the text into the input at the cursor to edit before sending. Escape cancels; recordings aren't stored. - Every bot message has a read-aloud button that plays GET api/admin/bots/speak-message, streamed from the provider as the audio is made, so it starts in a couple of seconds. One message plays at a time; only the user's own sessions can be read. - Both buttons stay visible but disabled while the bot has no model for them (bot_voice page prop). - Transcription and speech no longer pick free models automatically: they're not available until a model is picked, and Settings says so. Transcription lists speech-to-text models only, so the chat-model transcript path and its missing-audio check are gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,123 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
export type SpeakMessageState = "idle" | "loading" | "playing";
|
||||
|
||||
export type SpeakMessage = {
|
||||
/**
|
||||
* The bot has a speech model here; the play buttons are disabled
|
||||
* without one.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* The message being fetched or played, if any.
|
||||
*/
|
||||
message_id: number | null;
|
||||
state: SpeakMessageState;
|
||||
error?: { message_id: number; text: string };
|
||||
/**
|
||||
* Plays the message, or stops it when it's the one playing. Starting
|
||||
* one stops any other.
|
||||
*/
|
||||
toggle: (params: { message_id: number }) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The conversation's read-aloud: one audio element for every message,
|
||||
* pointed at `api/admin/bots/speak-message`, which streams, so playing
|
||||
* starts before the whole message is spoken. Stopping drops the stream.
|
||||
*/
|
||||
export default function useSpeakMessage({ enabled }: { enabled: boolean }): SpeakMessage {
|
||||
const [message_id, setMessageId] = useState<number | null>(null);
|
||||
const [state, setState] = useState<SpeakMessageState>("idle");
|
||||
const [error, setError] = useState<SpeakMessage["error"]>();
|
||||
const audio_ref = useRef<HTMLAudioElement | null>(null);
|
||||
const current_ref = useRef<number | null>(null);
|
||||
|
||||
useEffect(() => () => release(), []);
|
||||
|
||||
function release() {
|
||||
const audio = audio_ref.current;
|
||||
|
||||
if (!audio) return;
|
||||
|
||||
audio.pause();
|
||||
audio.removeAttribute("src");
|
||||
audio.load();
|
||||
}
|
||||
|
||||
function reset() {
|
||||
current_ref.current = null;
|
||||
setMessageId(null);
|
||||
setState("idle");
|
||||
}
|
||||
|
||||
function stop() {
|
||||
release();
|
||||
reset();
|
||||
}
|
||||
|
||||
function play({ message_id }: { message_id: number }) {
|
||||
stop();
|
||||
setError(undefined);
|
||||
current_ref.current = message_id;
|
||||
setMessageId(message_id);
|
||||
setState("loading");
|
||||
|
||||
const src = `/api/admin/bots/speak-message?message_id=${message_id}`;
|
||||
const audio = audio_ref.current || new Audio();
|
||||
const isCurrent = () => current_ref.current == message_id;
|
||||
|
||||
audio_ref.current = audio;
|
||||
audio.onplaying = () => {
|
||||
if (isCurrent()) setState("playing");
|
||||
};
|
||||
audio.onended = () => {
|
||||
if (isCurrent()) reset();
|
||||
};
|
||||
audio.onerror = () => {
|
||||
if (!isCurrent()) return;
|
||||
|
||||
reset();
|
||||
explainError({ src }).then((text) => {
|
||||
setError({ message_id, text });
|
||||
});
|
||||
};
|
||||
audio.src = src;
|
||||
audio.play().catch(() => {});
|
||||
}
|
||||
|
||||
function toggle({ message_id: id }: { message_id: number }) {
|
||||
if (current_ref.current == id) {
|
||||
stop();
|
||||
return;
|
||||
}
|
||||
|
||||
play({ message_id: id });
|
||||
}
|
||||
|
||||
return { enabled, message_id, state, error, toggle };
|
||||
}
|
||||
|
||||
/**
|
||||
* The player only says it failed, so ask again for the reason: a failure
|
||||
* answers in JSON, and audio coming back instead is dropped at once.
|
||||
*/
|
||||
async function explainError({ src }: { src: string }) {
|
||||
const fallback = "Couldn't read it aloud. Try again.";
|
||||
const controller = new AbortController();
|
||||
|
||||
try {
|
||||
const res = await fetch(src, { signal: controller.signal, cache: "no-store" });
|
||||
|
||||
if (!res.headers.get("content-type")?.includes("json")) {
|
||||
controller.abort();
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const json = await res.json();
|
||||
|
||||
return json?.msg || fallback;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
|
||||
export type VoiceRecorderState = "idle" | "recording" | "transcribing";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* Gets the finished recording. The hook stays `transcribing` until it
|
||||
* resolves.
|
||||
*/
|
||||
onRecorded: (params: { audio: Blob }) => Promise<void>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Types the browser may record in, best first. Safari only has mp4.
|
||||
*/
|
||||
const RecordingTypes = ["audio/webm;codecs=opus", "audio/webm", "audio/mp4", "audio/ogg"];
|
||||
|
||||
/**
|
||||
* Records the microphone for the composer's dictation. Stops on its own
|
||||
* after `AppData.ComposerVoiceMaxMs`; Escape cancels. The mic is released whenever
|
||||
* recording ends, and on unmount. `supported` is false without a secure
|
||||
* context or `MediaRecorder`.
|
||||
*/
|
||||
export default function useVoiceRecorder({ onRecorded }: Params) {
|
||||
const [state, setState] = useState<VoiceRecorderState>("idle");
|
||||
const [elapsed_ms, setElapsedMs] = useState(0);
|
||||
const [error, setError] = useState<string>();
|
||||
const [supported, setSupported] = useState(false);
|
||||
const recorder_ref = useRef<MediaRecorder | null>(null);
|
||||
const cancelled_ref = useRef(false);
|
||||
const on_recorded_ref = useRef(onRecorded);
|
||||
|
||||
on_recorded_ref.current = onRecorded;
|
||||
|
||||
useEffect(() => {
|
||||
setSupported(
|
||||
typeof MediaRecorder != "undefined" &&
|
||||
Boolean(navigator.mediaDevices?.getUserMedia),
|
||||
);
|
||||
|
||||
return () => {
|
||||
cancelled_ref.current = true;
|
||||
recorder_ref.current?.stop();
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (state != "recording") return;
|
||||
|
||||
const started_at = Date.now();
|
||||
const timer = setInterval(() => {
|
||||
const ms = Date.now() - started_at;
|
||||
|
||||
setElapsedMs(ms);
|
||||
|
||||
if (ms >= AppData.ComposerVoiceMaxMs) stop();
|
||||
}, 250);
|
||||
|
||||
const onKeyDown = (e: KeyboardEvent) => {
|
||||
if (e.key != "Escape") return;
|
||||
|
||||
e.preventDefault();
|
||||
cancel();
|
||||
};
|
||||
|
||||
window.addEventListener("keydown", onKeyDown);
|
||||
|
||||
return () => {
|
||||
clearInterval(timer);
|
||||
window.removeEventListener("keydown", onKeyDown);
|
||||
};
|
||||
}, [state]);
|
||||
|
||||
async function start() {
|
||||
if (state != "idle") return;
|
||||
|
||||
setError(undefined);
|
||||
setElapsedMs(0);
|
||||
|
||||
let stream: MediaStream;
|
||||
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
} catch {
|
||||
setError("Microphone blocked. Allow it in the browser's site settings.");
|
||||
return;
|
||||
}
|
||||
|
||||
const type = RecordingTypes.find((t) => MediaRecorder.isTypeSupported(t));
|
||||
const recorder = new MediaRecorder(stream, type ? { mimeType: type } : undefined);
|
||||
const chunks: Blob[] = [];
|
||||
|
||||
cancelled_ref.current = false;
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size) chunks.push(e.data);
|
||||
};
|
||||
recorder.onstop = async () => {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
recorder_ref.current = null;
|
||||
|
||||
if (cancelled_ref.current || !chunks.length) {
|
||||
setState("idle");
|
||||
return;
|
||||
}
|
||||
|
||||
setState("transcribing");
|
||||
|
||||
/**
|
||||
* The bare type, without `;codecs=`, so it reads as a plain
|
||||
* data URL.
|
||||
*/
|
||||
const audio = new Blob(chunks, {
|
||||
type: (recorder.mimeType || type || "audio/webm").split(";")[0],
|
||||
});
|
||||
|
||||
await on_recorded_ref.current({ audio }).catch(() => {});
|
||||
setState("idle");
|
||||
};
|
||||
|
||||
recorder_ref.current = recorder;
|
||||
recorder.start();
|
||||
setState("recording");
|
||||
}
|
||||
|
||||
function stop() {
|
||||
recorder_ref.current?.stop();
|
||||
}
|
||||
|
||||
function cancel() {
|
||||
cancelled_ref.current = true;
|
||||
recorder_ref.current?.stop();
|
||||
}
|
||||
|
||||
return { state, elapsed_ms, error, setError, supported, start, stop, cancel };
|
||||
}
|
||||
Reference in New Issue
Block a user