Add the harness, envs, per-bot models and chat attachments
- Harness: providers, model catalog synced hourly, automatic free picks, cooldowns, tool loop with context trimming, MCP computer tools - Bots queue runs replies and tasks through the harness; jobs wait (attempt refunded) while every model is resting - Envs: Settings > Environment, scoped general/workspace/computer, written to computers and redacted from tool output - Models: Settings > Models, per-harness provider select defaulting to OpenRouter, and a models modal on each bot's chat (the coordinator's edits the defaults) - Chat: thinking row while a reply runs, file attachments (upload, paste, drop) that replies read as images, audio and document text - Fix media uploads failing after a user's first file, and private files being served from the public media path Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import looksLikeMissingAudio from "./looks-like-missing-audio";
|
||||
|
||||
describe("looksLikeMissingAudio", () => {
|
||||
test("spots a model asking for the audio it never got", () => {
|
||||
for (const text of [
|
||||
"Please provide the audio file you’d like transcribed.",
|
||||
"I’m sorry, but I can’t transcribe without the audio.",
|
||||
"I cannot hear any audio in your message.",
|
||||
"There is no audio attached.",
|
||||
"I didn't receive any audio.",
|
||||
]) {
|
||||
expect(looksLikeMissingAudio({ text })).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test("leaves real transcripts alone", () => {
|
||||
expect(looksLikeMissingAudio({ text: "Hello from cobble." })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
looksLikeMissingAudio({
|
||||
text: `Can you send the audio report to finance? ${"And then more. ".repeat(30)}`,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
type Params = {
|
||||
text: string;
|
||||
};
|
||||
|
||||
const MissingAudio =
|
||||
/(provide|upload|share|attach|send)\b.{0,40}\baudio|can(?:'|’)?t (?:transcribe|hear|access|listen)|cannot (?:transcribe|hear|access|listen)|no audio|without (?:the |an |any )?audio|didn(?:'|’)?t (?:receive|get) (?:the |an |any )?audio/i;
|
||||
|
||||
/**
|
||||
* Whether a chat model's "transcript" is really it saying it got no audio.
|
||||
* Some providers list audio input for a model but drop the audio.
|
||||
*/
|
||||
export default function looksLikeMissingAudio({ text }: Params) {
|
||||
return text.length < 300 && MissingAudio.test(text);
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
|
||||
import grabHarnessAdapter from "../providers/grab-harness-adapter";
|
||||
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
|
||||
import withCandidates from "../resolve/with-candidates";
|
||||
import type { HarnessCandidate } from "../types";
|
||||
|
||||
type Params = {
|
||||
text: string;
|
||||
/**
|
||||
* `mp3` unless set.
|
||||
*/
|
||||
format?: string;
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Uses this bot's speech model and voice when it has its own.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads text aloud with the speech harness. Uses the voice picked in the
|
||||
* speech setting when the model has it, else the model's first voice.
|
||||
*/
|
||||
export default async function speakText({
|
||||
text,
|
||||
format,
|
||||
signal,
|
||||
bot_id,
|
||||
}: Params) {
|
||||
const [candidates, setting] = await Promise.all([
|
||||
resolveHarnessCandidates({ harness: "speech", bot_id }),
|
||||
grabHarnessSetting({ harness: "speech", bot_id }),
|
||||
]);
|
||||
|
||||
const { result } = await withCandidates({
|
||||
harness: "speech",
|
||||
candidates,
|
||||
run: async (candidate) =>
|
||||
await grabHarnessAdapter({ kind: candidate.provider.kind }).speak({
|
||||
candidate,
|
||||
text,
|
||||
voice: pickVoice({
|
||||
candidate,
|
||||
voice:
|
||||
typeof setting.options.voice == "string"
|
||||
? setting.options.voice
|
||||
: undefined,
|
||||
}),
|
||||
format,
|
||||
signal: signal || new AbortController().signal,
|
||||
}),
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function pickVoice({
|
||||
candidate,
|
||||
voice,
|
||||
}: {
|
||||
candidate: HarnessCandidate;
|
||||
voice?: string;
|
||||
}) {
|
||||
const voices = candidate.model.voices;
|
||||
|
||||
if (voice && (candidate.source == "pinned" || voices?.includes(voice))) {
|
||||
return voice;
|
||||
}
|
||||
|
||||
if (voices?.[0]) return voices[0];
|
||||
|
||||
return candidate.provider.kind == "openai" ? "alloy" : undefined;
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import { TranscribeAudioPrompt } from "@/src/data/harness-prompts";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import HarnessError from "../harness-error";
|
||||
import grabHarnessAdapter from "../providers/grab-harness-adapter";
|
||||
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
|
||||
import withCandidates from "../resolve/with-candidates";
|
||||
import type { HarnessAudioInput } from "../types";
|
||||
import looksLikeMissingAudio from "./looks-like-missing-audio";
|
||||
|
||||
/**
|
||||
* A chat model that drops audio is skipped this long.
|
||||
*/
|
||||
const MissingAudioCooldownMs = 60 * 60 * 1000;
|
||||
|
||||
type Params = {
|
||||
audio: HarnessAudioInput;
|
||||
/**
|
||||
* ISO 639-1 code, when known. Helps speech-to-text models.
|
||||
*/
|
||||
language?: string;
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Uses this bot's transcription model when it has its own.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Turns speech into text with the transcription harness. Speech-to-text
|
||||
* models get the file at `/audio/transcriptions`; chat models that hear
|
||||
* audio are asked for a verbatim transcript.
|
||||
*/
|
||||
export default async function transcribeAudio({
|
||||
audio,
|
||||
language,
|
||||
signal,
|
||||
bot_id,
|
||||
}: Params) {
|
||||
if (audio.data.byteLength > AppData.HarnessMaxAudioBytes) {
|
||||
throw new AppError(`That recording is too long to transcribe.`);
|
||||
}
|
||||
|
||||
const candidates = await resolveHarnessCandidates({
|
||||
harness: "transcription",
|
||||
bot_id,
|
||||
});
|
||||
|
||||
const final_signal = signal || new AbortController().signal;
|
||||
|
||||
const { result } = await withCandidates({
|
||||
harness: "transcription",
|
||||
candidates,
|
||||
run: async (candidate) => {
|
||||
const adapter = grabHarnessAdapter({
|
||||
kind: candidate.provider.kind,
|
||||
});
|
||||
|
||||
const is_speech_to_text =
|
||||
candidate.model.output_modalities.includes("transcription");
|
||||
|
||||
const text = is_speech_to_text
|
||||
? (
|
||||
await adapter.transcribe({
|
||||
candidate,
|
||||
audio,
|
||||
language,
|
||||
signal: final_signal,
|
||||
})
|
||||
).text
|
||||
: (
|
||||
await adapter.chat({
|
||||
candidate,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: TranscribeAudioPrompt },
|
||||
{
|
||||
type: "audio",
|
||||
data: Buffer.from(audio.data).toString(
|
||||
"base64",
|
||||
),
|
||||
format: audio.format,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0,
|
||||
signal: final_signal,
|
||||
})
|
||||
).text;
|
||||
|
||||
if (text === null || text === undefined) {
|
||||
throw new HarnessError({
|
||||
message: `The model returned no transcript.`,
|
||||
retryable: true,
|
||||
candidate_key: candidate.key,
|
||||
});
|
||||
}
|
||||
|
||||
if (!is_speech_to_text && looksLikeMissingAudio({ text })) {
|
||||
throw new HarnessError({
|
||||
message: `${candidate.model.name || candidate.model.slug} didn't get the audio.`,
|
||||
retryable: true,
|
||||
retry_at: Date.now() + MissingAudioCooldownMs,
|
||||
candidate_key: candidate.key,
|
||||
});
|
||||
}
|
||||
|
||||
return text.trim();
|
||||
},
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
Reference in New Issue
Block a user