Add dictation and streamed read-aloud to the chat
- A mic in the composer records, transcribes with the bot's transcription model (api/admin/bots/transcribe) and writes the text into the input at the cursor to edit before sending. Escape cancels; recordings aren't stored. - Every bot message has a read-aloud button that plays GET api/admin/bots/speak-message, streamed from the provider as the audio is made, so it starts in a couple of seconds. One message plays at a time; only the user's own sessions can be read. - Both buttons stay visible but disabled while the bot has no model for them (bot_voice page prop). - Transcription and speech no longer pick free models automatically: they're not available until a model is picked, and Settings says so. Transcription lists speech-to-text models only, so the chat-model transcript path and its missing-audio check are gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
@@ -6,6 +6,7 @@ import grabBotExtensionsOverview from "../extensions/grab-bot-extensions-overvie
|
||||
import grabBotComputers from "../computers/grab-bot-computers";
|
||||
import grabBotHarnessOverview from "../harness/grab-bot-harness-overview";
|
||||
import grabHarnessOverview from "../harness/grab-harness-overview";
|
||||
import resolveHarnessCandidates from "@/src/harness/resolve/resolve-harness-candidates";
|
||||
import grabUserWorkspaces from "../workspaces/grab-user-workspaces";
|
||||
import ensureBotSession from "./ensure-bot-session";
|
||||
import grabUserConversation from "./grab-user-conversation";
|
||||
@@ -25,7 +26,8 @@ type Params = {
|
||||
* `/admin/workspaces/<id>/bots/<id>`): the rail's crew (the workspace's
|
||||
* in a workspace) and workspaces, the bot, its active session (marked
|
||||
* read) and its messages, its models, computers, tools and MCP servers
|
||||
* and active task there.
|
||||
* and active task there, and whether its mic and read-aloud have a model
|
||||
* to run on.
|
||||
* Null when the user can't chat with the bot there.
|
||||
*/
|
||||
export default async function grabConversationProps({
|
||||
@@ -64,7 +66,7 @@ export default async function grabConversationProps({
|
||||
* workspace the workspace's own models come first, so what runs now
|
||||
* shows its pick.
|
||||
*/
|
||||
const [bot_messages, bot_harness_overview, bot_computers, bot_task, bot_extensions] = await Promise.all([
|
||||
const [bot_messages, bot_harness_overview, bot_computers, bot_task, bot_extensions, transcription, speech] = await Promise.all([
|
||||
grabBotMessages({ session_id: session.id! }),
|
||||
(bot.kind == "coordinator" && !scope
|
||||
? grabHarnessOverview().then(({ harnesses }) => ({ harnesses }))
|
||||
@@ -76,6 +78,8 @@ export default async function grabConversationProps({
|
||||
grabBotComputers({ bot_id: bot.id, workspace_id: scope }),
|
||||
grabActiveBotTask({ bot_id: bot.id, session_id: session.id }),
|
||||
grabBotExtensionsOverview({ user_id, bot_id: bot.id, workspace_id: scope }),
|
||||
hasHarnessModel({ harness: "transcription", bot_id: bot.id, workspace_id: scope }),
|
||||
hasHarnessModel({ harness: "speech", bot_id: bot.id, workspace_id: scope }),
|
||||
]);
|
||||
|
||||
return {
|
||||
@@ -89,5 +93,15 @@ export default async function grabConversationProps({
|
||||
bot_computers,
|
||||
bot_task,
|
||||
bot_extensions,
|
||||
bot_voice: { transcription, speech },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the harness is on and has a model to run on for the bot there.
|
||||
*/
|
||||
async function hasHarnessModel(params: Parameters<typeof resolveHarnessCandidates>[0]) {
|
||||
return await resolveHarnessCandidates(params)
|
||||
.then((candidates) => candidates.length > 0)
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_BOT_MESSAGES } from "@/db/types/db";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabUserConversation from "../bot-sessions/grab-user-conversation";
|
||||
|
||||
type Params = {
|
||||
user_id: number;
|
||||
message_id: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* A bot message in one of the user's own sessions, for reading aloud,
|
||||
* with the conversation it's in (so a lost grant blocks it). Throws when
|
||||
* it isn't theirs.
|
||||
*/
|
||||
export default async function grabSpeakableMessage({ user_id, message_id }: Params) {
|
||||
const message = (
|
||||
await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_MESSAGES>({
|
||||
sql: `SELECT m.* FROM bot_messages m
|
||||
JOIN bot_sessions s ON s.id = m.session_id
|
||||
WHERE m.id = ? AND m.author = 'bot' AND s.user_id = ?`,
|
||||
values: [message_id, user_id],
|
||||
})
|
||||
).payload?.[0];
|
||||
|
||||
if (!message?.id || !message.bot_id) {
|
||||
throw new AppError(`Message not found`);
|
||||
}
|
||||
|
||||
const conversation = await grabUserConversation({
|
||||
user_id,
|
||||
bot_id: message.bot_id,
|
||||
workspace_id: message.workspace_id || null,
|
||||
});
|
||||
|
||||
return { message, conversation };
|
||||
}
|
||||
@@ -55,6 +55,7 @@ export default async function grabHarnessOverviewItem({
|
||||
title: meta.title,
|
||||
description: meta.description,
|
||||
required: meta.required,
|
||||
automatic: meta.automatic,
|
||||
model_id: setting.model_id,
|
||||
provider_id: setting.provider_id,
|
||||
providers,
|
||||
|
||||
Reference in New Issue
Block a user