Add dictation and streamed read-aloud to the chat
- A mic in the composer records, transcribes with the bot's transcription model (api/admin/bots/transcribe) and writes the text into the input at the cursor to edit before sending. Escape cancels; recordings aren't stored. - Every bot message has a read-aloud button that plays GET api/admin/bots/speak-message, streamed from the provider as the audio is made, so it starts in a couple of seconds. One message plays at a time; only the user's own sessions can be read. - Both buttons stay visible but disabled while the bot has no model for them (bot_voice page prop). - Transcription and speech no longer pick free models automatically: they're not available until a model is picked, and Settings says so. Transcription lists speech-to-text models only, so the chat-model transcript path and its missing-audio check are gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
@@ -6,10 +6,12 @@ import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi";
|
||||
import useLiveBots from "@/src/hooks/use-live-bots";
|
||||
import useServerEvent from "@/src/hooks/use-server-event";
|
||||
import { BotComputersContext } from "../(data)/bot-computers-context";
|
||||
import { SpeakMessageContext } from "../(data)/speak-message-context";
|
||||
import useBotComputers from "../(hooks)/use-bot-computers";
|
||||
import useTaskProgress from "../(hooks)/use-task-progress";
|
||||
import useReplyProgress from "../(hooks)/use-reply-progress";
|
||||
import useReplyCatchUp from "../(hooks)/use-reply-catch-up";
|
||||
import useSpeakMessage from "../(hooks)/use-speak-message";
|
||||
import ConversationHeader from "../(partials)/conversation-header";
|
||||
import MessageThread from "../(partials)/message-thread";
|
||||
import Composer from "../(partials)/composer";
|
||||
@@ -21,7 +23,8 @@ import ComputerViewerModal from "../(partials)/computer-viewer-modal";
|
||||
* bot's conversation, on its own or in a workspace (`pageProps.workspace`).
|
||||
* The bot's computers are shared with the header's modal and the computer
|
||||
* buttons in the thread, so a switch from either shows in both; either
|
||||
* can open one in the viewer.
|
||||
* can open one in the viewer. Bot messages can be read aloud, one at a
|
||||
* time.
|
||||
*/
|
||||
export default function Conversation() {
|
||||
const { pageProps } = useContext(AppContext);
|
||||
@@ -47,6 +50,8 @@ export default function Conversation() {
|
||||
computers: pageProps?.bot_computers || [],
|
||||
});
|
||||
|
||||
const speak = useSpeakMessage({ enabled: Boolean(pageProps?.bot_voice?.speech) });
|
||||
|
||||
const progress = useTaskProgress({
|
||||
bot_id: bot?.id || 0,
|
||||
session_id,
|
||||
@@ -112,40 +117,43 @@ export default function Conversation() {
|
||||
|
||||
return (
|
||||
<BotComputersContext.Provider value={bot_computers}>
|
||||
<Stack className="grow min-w-0 h-full gap-0! items-stretch">
|
||||
<ConversationHeader
|
||||
bot={live_bot}
|
||||
workspace={workspace}
|
||||
has_other_unread={has_other_unread}
|
||||
harnesses={pageProps?.bot_harness_overview?.harnesses}
|
||||
/>
|
||||
<MessageThread
|
||||
bot={live_bot}
|
||||
messages={messages}
|
||||
progress={progress.text}
|
||||
reply_progress={reply_progress}
|
||||
has_task={Boolean(progress.task_id)}
|
||||
/>
|
||||
<Composer
|
||||
bot={bot}
|
||||
workspace_id={workspace_id}
|
||||
extensions={pageProps?.bot_extensions}
|
||||
onSent={({ message }) => {
|
||||
if (startsNewSession(message) && message.session_id != session_id) {
|
||||
setSessionId(Number(message.session_id));
|
||||
setMessages([message]);
|
||||
return;
|
||||
}
|
||||
|
||||
setMessages((prev) =>
|
||||
prev.some((m) => m.id == message.id)
|
||||
? prev
|
||||
: [...prev, message],
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Stack>
|
||||
<ComputerViewerModal bot={live_bot} />
|
||||
<SpeakMessageContext.Provider value={speak}>
|
||||
<Stack className="grow min-w-0 h-full gap-0! items-stretch">
|
||||
<ConversationHeader
|
||||
bot={live_bot}
|
||||
workspace={workspace}
|
||||
has_other_unread={has_other_unread}
|
||||
harnesses={pageProps?.bot_harness_overview?.harnesses}
|
||||
/>
|
||||
<MessageThread
|
||||
bot={live_bot}
|
||||
messages={messages}
|
||||
progress={progress.text}
|
||||
reply_progress={reply_progress}
|
||||
has_task={Boolean(progress.task_id)}
|
||||
/>
|
||||
<Composer
|
||||
bot={bot}
|
||||
workspace_id={workspace_id}
|
||||
extensions={pageProps?.bot_extensions}
|
||||
voice={pageProps?.bot_voice}
|
||||
onSent={({ message }) => {
|
||||
if (startsNewSession(message) && message.session_id != session_id) {
|
||||
setSessionId(Number(message.session_id));
|
||||
setMessages([message]);
|
||||
return;
|
||||
}
|
||||
|
||||
setMessages((prev) =>
|
||||
prev.some((m) => m.id == message.id)
|
||||
? prev
|
||||
: [...prev, message],
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Stack>
|
||||
<ComputerViewerModal bot={live_bot} />
|
||||
</SpeakMessageContext.Provider>
|
||||
</BotComputersContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user