Add dictation and streamed read-aloud to the chat
- A mic in the composer records, transcribes with the bot's transcription model (api/admin/bots/transcribe) and writes the text into the input at the cursor to edit before sending. Escape cancels; recordings aren't stored. - Every bot message has a read-aloud button that plays GET api/admin/bots/speak-message, streamed from the provider as the audio is made, so it starts in a couple of seconds. One message plays at a time; only the user's own sessions can be read. - Both buttons stay visible but disabled while the bot has no model for them (bot_voice page prop). - Transcription and speech no longer pick free models automatically: they're not available until a model is picked, and Settings says so. Transcription lists speech-to-text models only, so the chat-model transcript path and its missing-audio check are gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -29,6 +29,7 @@ export default function BotMessageGroup({ bot, group }: Props) {
|
||||
{group.messages.map((message) => (
|
||||
<BotMessageLine
|
||||
key={message.id}
|
||||
message_id={message.id || null}
|
||||
text={message.text || ""}
|
||||
needs_you={Boolean(message.needs_you)}
|
||||
/>
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import Markdown from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import Row from "@moduletrace/twui/layout/Row";
|
||||
import Stack from "@moduletrace/twui/layout/Stack";
|
||||
import linkComputerRefs from "@/src/utils/link-computer-refs";
|
||||
import { MessageMarkdownComponents } from "../(data)/message-markdown-components";
|
||||
import NeedsYouChip from "./needs-you-chip";
|
||||
import SpeakMessageButton from "./speak-message-button";
|
||||
|
||||
type Props = {
|
||||
message_id?: number | null;
|
||||
text: string;
|
||||
needs_you?: boolean;
|
||||
};
|
||||
@@ -13,17 +16,30 @@ type Props = {
|
||||
/**
|
||||
* One bot message, rendered from markdown (GFM; raw HTML is never
|
||||
* rendered). Computers it names (`computerRef`) show as buttons that
|
||||
* switch the bot to them.
|
||||
* switch the bot to them. A play button under it reads it aloud,
|
||||
* disabled while the bot has no speech model.
|
||||
*/
|
||||
export default function BotMessageLine({ text, needs_you }: Props) {
|
||||
export default function BotMessageLine({ message_id, text, needs_you }: Props) {
|
||||
const can_speak = Boolean(message_id && text.trim());
|
||||
|
||||
return (
|
||||
<Stack className="gap-1.5! items-start max-w-full">
|
||||
<Stack className="w-full gap-2.5! items-start break-words">
|
||||
<Markdown remarkPlugins={[remarkGfm]} components={MessageMarkdownComponents}>
|
||||
<Markdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={MessageMarkdownComponents}
|
||||
>
|
||||
{linkComputerRefs({ text })}
|
||||
</Markdown>
|
||||
</Stack>
|
||||
{needs_you ? <NeedsYouChip /> : null}
|
||||
{needs_you || can_speak ? (
|
||||
<Row className="gap-2! items-center">
|
||||
{needs_you ? <NeedsYouChip /> : null}
|
||||
{can_speak ? (
|
||||
<SpeakMessageButton message_id={message_id!} />
|
||||
) : null}
|
||||
</Row>
|
||||
) : null}
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import Button from "@moduletrace/twui/layout/Button";
|
||||
import Row from "@moduletrace/twui/layout/Row";
|
||||
import Span from "@moduletrace/twui/layout/Span";
|
||||
import { Mic, Square } from "lucide-react";
|
||||
import type { VoiceRecorderState } from "../(hooks)/use-voice-recorder";
|
||||
|
||||
const UnavailableTitle =
|
||||
"Dictation needs a transcription model. A super admin can pick one in Settings, under Models.";
|
||||
|
||||
type Props = {
|
||||
state: VoiceRecorderState;
|
||||
elapsed_ms: number;
|
||||
error?: string;
|
||||
/**
|
||||
* No transcription model for the bot here: shown disabled, saying so.
|
||||
*/
|
||||
unavailable?: boolean;
|
||||
disabled?: boolean;
|
||||
onStart: () => void;
|
||||
onStop: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The composer's dictation button: a mic, a stop square with the time
|
||||
* while recording, a spinner while the recording is turned into text.
|
||||
* Its title says why it last failed, or that it needs a model.
|
||||
*/
|
||||
export default function ComposerVoiceButton({
|
||||
state,
|
||||
elapsed_ms,
|
||||
error,
|
||||
unavailable,
|
||||
disabled,
|
||||
onStart,
|
||||
onStop,
|
||||
}: Props) {
|
||||
const recording = state == "recording";
|
||||
const seconds = Math.floor(elapsed_ms / 1000);
|
||||
|
||||
return (
|
||||
<Row className="shrink-0 gap-1.5! flex-nowrap items-center">
|
||||
{recording ? (
|
||||
<Span className="text-xs tabular-nums text-ink-secondary" aria-live="off">
|
||||
{Math.floor(seconds / 60)}:{String(seconds % 60).padStart(2, "0")}
|
||||
</Span>
|
||||
) : null}
|
||||
<Button
|
||||
title={
|
||||
unavailable
|
||||
? UnavailableTitle
|
||||
: recording
|
||||
? "Stop and write it out (Esc cancels)"
|
||||
: error || "Dictate"
|
||||
}
|
||||
aria-label={recording ? "Stop dictating" : "Dictate"}
|
||||
aria-invalid={error ? true : undefined}
|
||||
aria-pressed={recording}
|
||||
type="button"
|
||||
variant="ghost"
|
||||
color="text"
|
||||
disabled={disabled || unavailable}
|
||||
loading={state == "transcribing"}
|
||||
loadingIconSize="smaller"
|
||||
onClick={recording ? onStop : onStart}
|
||||
className="shrink-0 p-1! -mx-1 my-1 text-ink dark:text-ink rounded-default-xs focus-visible:ring-2 focus-visible:ring-ink"
|
||||
>
|
||||
{recording ? (
|
||||
<Square size={14} aria-hidden fill="currentColor" />
|
||||
) : (
|
||||
<Mic size={16} aria-hidden />
|
||||
)}
|
||||
</Button>
|
||||
</Row>
|
||||
);
|
||||
}
|
||||
@@ -2,20 +2,25 @@ import { useRef, useState, type KeyboardEvent } from "react";
|
||||
import Row from "@moduletrace/twui/layout/Row";
|
||||
import Stack from "@moduletrace/twui/layout/Stack";
|
||||
import Textarea from "@moduletrace/twui/form/Textarea";
|
||||
import P from "@moduletrace/twui/layout/P";
|
||||
import type {
|
||||
BotExtensionsPayload,
|
||||
BotMessageWithAttachments,
|
||||
BotVoicePayload,
|
||||
BotWithLastMessage,
|
||||
} from "@/src/types";
|
||||
import sendBotMessage from "../(functions)/send-bot-message";
|
||||
import transcribeVoice from "../(functions)/transcribe-voice";
|
||||
import useComposerAttachments from "../(hooks)/use-composer-attachments";
|
||||
import useComposerDraft from "../(hooks)/use-composer-draft";
|
||||
import useAutoGrowTextarea from "../(hooks)/use-auto-grow-textarea";
|
||||
import useVoiceRecorder from "../(hooks)/use-voice-recorder";
|
||||
import useOnlineStatus from "@/src/hooks/use-online-status";
|
||||
import ComposerAttachButton from "./composer-attach-button";
|
||||
import ComposerAttachments from "./composer-attachments";
|
||||
import ComposerSendButton from "./composer-send-button";
|
||||
import ComposerTools from "./composer-tools";
|
||||
import ComposerVoiceButton from "./composer-voice-button";
|
||||
|
||||
type Props = {
|
||||
bot: BotWithLastMessage;
|
||||
@@ -27,6 +32,10 @@ type Props = {
|
||||
* Its MCP servers and tools, listed under the input.
|
||||
*/
|
||||
extensions?: BotExtensionsPayload | null;
|
||||
/**
|
||||
* The dictation mic is disabled while transcription has no model.
|
||||
*/
|
||||
voice?: BotVoicePayload | null;
|
||||
onSent: (params: { message: BotMessageWithAttachments }) => void;
|
||||
};
|
||||
|
||||
@@ -36,9 +45,10 @@ type Props = {
|
||||
* just above it. A click anywhere on the row focuses the input. Files can
|
||||
* also be pasted or dropped in. Unsent text is kept per conversation
|
||||
* across reloads, with its uploaded files. The bot's MCP servers and tools
|
||||
* are listed under the input.
|
||||
* are listed under the input. The mic dictates: what's said is written
|
||||
* into the input at the cursor, to edit before sending.
|
||||
*/
|
||||
export default function Composer({ bot, workspace_id, extensions, onSent }: Props) {
|
||||
export default function Composer({ bot, workspace_id, extensions, voice, onSent }: Props) {
|
||||
const { text, setText } = useComposerDraft({ bot_id: bot.id, workspace_id });
|
||||
const [sending, setSending] = useState(false);
|
||||
const [error, setError] = useState<string>();
|
||||
@@ -48,12 +58,52 @@ export default function Composer({ bot, workspace_id, extensions, onSent }: Prop
|
||||
|
||||
useAutoGrowTextarea({ ref: input_ref, value: text });
|
||||
|
||||
const recorder = useVoiceRecorder({
|
||||
onRecorded: async ({ audio }) => {
|
||||
if (!bot.id) return;
|
||||
|
||||
const res = await transcribeVoice({ bot_id: bot.id, workspace_id, audio });
|
||||
|
||||
if (!res.success || !res.singleRes?.text) {
|
||||
recorder.setError(res.msg || "Couldn't write that out");
|
||||
return;
|
||||
}
|
||||
|
||||
insertText({ insert: res.singleRes.text });
|
||||
},
|
||||
});
|
||||
|
||||
const can_send =
|
||||
Boolean(text.trim() || files.media_ids.length) &&
|
||||
!sending &&
|
||||
!files.uploading &&
|
||||
recorder.state == "idle" &&
|
||||
online;
|
||||
|
||||
/**
|
||||
* Puts dictated text at the cursor (or the end), spaced from what's
|
||||
* around it, and leaves the cursor after it.
|
||||
*/
|
||||
function insertText({ insert }: { insert: string }) {
|
||||
const el = input_ref.current;
|
||||
const start = el ? el.selectionStart : text.length;
|
||||
const end = el ? el.selectionEnd : text.length;
|
||||
const before = text.slice(0, start);
|
||||
const after = text.slice(end);
|
||||
const piece =
|
||||
(before && !/\s$/.test(before) ? " " : "") +
|
||||
insert +
|
||||
(after && !/^\s/.test(after) ? " " : "");
|
||||
const cursor = before.length + piece.length;
|
||||
|
||||
setText(before + piece + after);
|
||||
|
||||
requestAnimationFrame(() => {
|
||||
input_ref.current?.focus();
|
||||
input_ref.current?.setSelectionRange(cursor, cursor);
|
||||
});
|
||||
}
|
||||
|
||||
async function send() {
|
||||
if (!can_send || !bot.id) return;
|
||||
|
||||
@@ -149,12 +199,28 @@ export default function Composer({ bot, workspace_id, extensions, onSent }: Prop
|
||||
}}
|
||||
autoFocus
|
||||
/>
|
||||
{recorder.supported ? (
|
||||
<ComposerVoiceButton
|
||||
state={recorder.state}
|
||||
elapsed_ms={recorder.elapsed_ms}
|
||||
error={recorder.error}
|
||||
unavailable={!voice?.transcription}
|
||||
disabled={!online}
|
||||
onStart={recorder.start}
|
||||
onStop={recorder.stop}
|
||||
/>
|
||||
) : null}
|
||||
<ComposerSendButton
|
||||
disabled={!can_send}
|
||||
loading={sending || files.uploading}
|
||||
onClick={send}
|
||||
/>
|
||||
</Row>
|
||||
{recorder.error ? (
|
||||
<P role="alert" className="w-full max-w-container mx-auto px-3 md:px-6 -mt-5 pb-3 text-xs text-ink-secondary">
|
||||
{recorder.error}
|
||||
</P>
|
||||
) : null}
|
||||
<ComposerTools bot={bot} workspace_id={workspace_id} initial={extensions} />
|
||||
</Stack>
|
||||
</Stack>
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { useContext } from "react";
|
||||
import Button from "@moduletrace/twui/layout/Button";
|
||||
import { Square, Volume2 } from "lucide-react";
|
||||
import { SpeakMessageContext } from "../(data)/speak-message-context";
|
||||
|
||||
const UnavailableTitle =
|
||||
"Reading aloud needs a speech model. A super admin can pick one in Settings, under Models.";
|
||||
|
||||
type Props = {
|
||||
message_id: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads a bot message aloud with its speech model, or stops it. Its title
|
||||
* says why it last failed. Disabled, saying so, while speech has no model.
|
||||
*/
|
||||
export default function SpeakMessageButton({ message_id }: Props) {
|
||||
const speak = useContext(SpeakMessageContext);
|
||||
const active = speak.message_id == message_id;
|
||||
const playing = active && speak.state == "playing";
|
||||
const error = speak.error?.message_id == message_id ? speak.error.text : undefined;
|
||||
|
||||
return (
|
||||
<Button
|
||||
title={
|
||||
!speak.enabled
|
||||
? UnavailableTitle
|
||||
: playing
|
||||
? "Stop"
|
||||
: error || "Read aloud"
|
||||
}
|
||||
aria-label={playing ? "Stop reading" : "Read aloud"}
|
||||
aria-invalid={error ? true : undefined}
|
||||
type="button"
|
||||
disabled={!speak.enabled}
|
||||
variant="ghost"
|
||||
color="text"
|
||||
loading={active && speak.state == "loading"}
|
||||
loadingIconSize="smaller"
|
||||
onClick={() => speak.toggle({ message_id })}
|
||||
className="shrink-0 p-1! -mx-1 text-ink-secondary dark:text-ink-secondary hover:text-ink rounded-default-xs focus-visible:ring-2 focus-visible:ring-ink"
|
||||
>
|
||||
{playing ? (
|
||||
<Square size={12} aria-hidden fill="currentColor" />
|
||||
) : (
|
||||
<Volume2 size={14} aria-hidden />
|
||||
)}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user