Add dictation and streamed read-aloud to the chat

- A mic in the composer records, transcribes with the bot's
  transcription model (api/admin/bots/transcribe) and writes the text
  into the input at the cursor to edit before sending. Escape cancels;
  recordings aren't stored.
- Every bot message has a read-aloud button that plays
  GET api/admin/bots/speak-message, streamed from the provider as the
  audio is made, so it starts in a couple of seconds. One message plays
  at a time; only the user's own sessions can be read.
- Both buttons stay visible but disabled while the bot has no model for
  them (bot_voice page prop).
- Transcription and speech no longer pick free models automatically:
  they're not available until a model is picked, and Settings says so.
  Transcription lists speech-to-text models only, so the chat-model
  transcript path and its missing-audio check are gone.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-19 21:21:04 +01:00
co-authored by Claude Opus 5.5
parent 727cbf53c8
commit 163d9a434c
37 changed files with 936 additions and 197 deletions
@@ -21,7 +21,9 @@ export default function HarnessCurrentModel({ harness }: Props) {
}`
: harness.required
? "Nothing to run on yet: no provider has a usable key."
: "No free model can do this right now. Pick one to turn it on.";
: !harness.automatic
? "Not available until a model is picked."
: "No free model can do this right now. Pick one to turn it on.";
return (
<Span className="text-[13px] leading-[18px] text-ink-secondary">
@@ -12,17 +12,20 @@ type Props = {
models: HarnessModelOption[];
model_id: number | null;
provider_name?: string;
automatic: boolean;
onChange: (model_id: number | null) => void;
};
/**
* Searchable model picker: automatic, or any model that can do the job.
* Searchable model picker: automatic (or not available, for a harness
* without automatic picks), or any model that can do the job.
*/
export default function HarnessModelSelect({
id,
models,
model_id,
provider_name,
automatic,
onChange,
}: Props) {
return (
@@ -32,7 +35,7 @@ export default function HarnessModelSelect({
label="Model"
showLabel
placeholder="Search models"
options={harnessModelSelectOptions({ models, model_id, provider_name })}
options={harnessModelSelectOptions({ models, model_id, provider_name, automatic })}
defaultValues={[model_id ? String(model_id) : AutoModelValue]}
changeHandler={(value) => {
const picked = Array.isArray(value) ? value[0] : value;
@@ -14,7 +14,8 @@ type Props = {
/**
* One harness: its provider, then its model (narrowed to that provider),
* whether it falls back to the automatic picks, whether it's on, and its
* whether it falls back to the automatic picks (for a harness that has
* them), whether it's on, and its
* voice for speech. On a bot's page the provider select can also use the
* default, which hides the rest.
*/
@@ -31,7 +32,7 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
? "turned off"
: harness.default?.current
? `${harness.default.current.name} on ${harness.default.current.provider_name}`
: "nothing available";
: "not available";
return (
<SettingsGroup title={harness.title} description={harness.description}>
@@ -60,6 +61,7 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
models={models}
model_id={value.model_id}
provider_name={provider?.name}
automatic={harness.automatic}
onChange={(model_id) => onChange({ ...value, model_id, voice: undefined })}
/>
)}
@@ -68,7 +70,7 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
) : (
<HarnessCurrentModel harness={harness} />
)}
{!value.inherit && value.model_id ? (
{!value.inherit && value.model_id && harness.automatic ? (
<Checkbox
title="Fall back to the automatic picks if this model fails"
label={`Fall back to the automatic picks${provider ? ` on ${provider.name}` : ""} if this model fails`}
@@ -13,24 +13,33 @@ type Params = {
* The picked provider's name; undefined for any provider.
*/
provider_name?: string;
/**
* False: there's no automatic pick, so the empty choice reads "Not
* available".
*/
automatic: boolean;
};
/**
* The model picker's options: automatic (the best free model, on the
* picked provider if there is one), then every model that can do the job
* (free ones first, as the server sorts them).
* (free ones first, as the server sorts them). A harness without
* automatic picks starts with "Not available" instead.
*/
export default function harnessModelSelectOptions({
models,
model_id,
provider_name,
automatic,
}: Params): TWUISelectOptionObject[] {
const has_free = models.some((model) => model.free);
return [
{
value: AutoModelValue,
title: provider_name
title: !automatic
? "Not available (pick a model to turn it on)"
: provider_name
? has_free
? `Automatic (best free model on ${provider_name})`
: `Automatic (no free models on ${provider_name}: pick one)`