Let bots take in messages sent while they reply

A reply read the session once at the start, so a message sent while it
ran was never seen, and the follow-up reply queued for it then finished
silently: the message was saved before the reply, so the thread ended
with the bot. Those messages went unanswered.

The text loop now reads new user messages after every round of tools and
before an answer goes out, and the bot decides from what each one means
whether it changes the work (folded in, or the unsent answer redrafted)
or is a separate request, which it acknowledges at once with the new
send_message tool and does after the current work. Messages sent that
way are marked interim, so the thinking row stays up while the bot
works. A reply job records the newest user message it read, and a queued
reply only stays silent when a finished reply already read everything.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/harness/text/push-user-turn.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, test } from "bun:test";
+import type { HarnessMessage } from "../types";
+import pushUserTurn from "./push-user-turn";
+
+describe("pushUserTurn", () => {
+    test("adds a turn after the bot's", () => {
+        const transcript: HarnessMessage[] = [{ role: "assistant", content: "Hi." }];
+
+        pushUserTurn({ transcript, content: "Hello" });
+
+        expect(transcript).toEqual([
+            { role: "assistant", content: "Hi." },
+            { role: "user", content: "Hello" },
+        ]);
+    });
+
+    test("joins text onto a user turn, so roles never repeat", () => {
+        const transcript: HarnessMessage[] = [{ role: "user", content: "One" }];
+
+        pushUserTurn({ transcript, content: "Two" });
+
+        expect(transcript).toEqual([{ role: "user", content: "One\n\nTwo" }]);
+    });
+
+    test("joins parts once either side has files", () => {
+        const transcript: HarnessMessage[] = [{ role: "user", content: "Look" }];
+
+        pushUserTurn({ transcript, content: [{ type: "image", url: "u" }] });
+
+        expect(transcript).toEqual([
+            {
+                role: "user",
+                content: [
+                    { type: "text", text: "Look" },
+                    { type: "image", url: "u" },
+                ],
+            },
+        ]);
+    });
+});

src/harness/text/push-user-turn.ts

@@ -0,0 +1,35 @@
+import type { HarnessContentPart, HarnessMessage } from "../types";
+
+type UserContent = Extract<HarnessMessage, { role: "user" }>["content"];
+
+type Params = {
+    transcript: HarnessMessage[];
+    content: UserContent;
+};
+
+/**
+ * Adds a user turn to the transcript, joined onto the last one when that is
+ * the user's too, since some providers reject repeated roles. Text stays a
+ * string while both sides are text.
+ */
+export default function pushUserTurn({ transcript, content }: Params) {
+    const last = transcript[transcript.length - 1];
+
+    if (last?.role != "user") {
+        transcript.push({ role: "user", content });
+        return;
+    }
+
+    if (typeof last.content == "string" && typeof content == "string") {
+        last.content = [last.content, content].filter(Boolean).join("\n\n");
+        return;
+    }
+
+    last.content = [...toParts({ content: last.content }), ...toParts({ content })];
+}
+
+function toParts({ content }: { content: UserContent }): HarnessContentPart[] {
+    if (typeof content != "string") return content;
+
+    return content ? [{ type: "text", text: content }] : [];
+}

src/harness/text/run-text-loop.test.ts

@@ -3,7 +3,7 @@ import { z } from "zod";
 import HarnessError from "../harness-error";
 import type { HarnessCandidate, HarnessTool } from "../types";
 import type { GenerateTextParams, GenerateTextResult } from "./generate-text";
-import runTextLoop from "./run-text-loop";
+import runTextLoop, { IncomingMessagesNote, RedraftNote } from "./run-text-loop";
 
 const candidate: HarnessCandidate = {
     key: "1:m",
@@ -152,4 +152,140 @@ describe("runTextLoop", () => {
         expect(error).toBeInstanceOf(HarnessError);
         expect(error.code).toBe("aborted");
     });
+
+    test("takes in what the user wrote while tools ran, before the next turn", async () => {
+        memory.length = 0;
+        const { generate, seen } = scripted([
+            { tool_calls: [{ id: "c1", name: "remember", arguments: `{"note":"tea"}` }] },
+            { text: "Coffee it is." },
+        ]);
+        const arrivals = [[{ role: "user" as const, content: "Actually, coffee." }]];
+
+        const result = await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "I like tea." }],
+            tools: [remember],
+            generate,
+            resolveCandidates,
+            incoming: async () => arrivals.shift() || [],
+        });
+
+        expect(result.text).toBe("Coffee it is.");
+        expect(seen).toHaveLength(2);
+        expect(seen[1]!.messages.slice(-2)).toEqual([
+            { role: "tool", tool_call_id: "c1", content: "Saved." },
+            { role: "user", content: `${IncomingMessagesNote}\n\nActually, coffee.` },
+        ]);
+    });
+
+    test("writes an answer again when the user wrote past it, keeping the draft and tool turns", async () => {
+        const { generate, seen } = scripted([
+            { text: "Here's the summary in English." },
+            { text: "Here's the summary in French." },
+        ]);
+        const arrivals = [
+            [
+                { role: "user" as const, content: "In French, please." },
+                { role: "user" as const, content: "And short." },
+            ],
+        ];
+
+        const result = await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "Summarise it." }],
+            generate,
+            resolveCandidates,
+            incoming: async () => arrivals.shift() || [],
+        });
+
+        expect(result.text).toBe("Here's the summary in French.");
+        expect(result.steps).toBe(2);
+        expect(seen[1]!.messages.slice(-2)).toEqual([
+            { role: "assistant", content: "Here's the summary in English." },
+            {
+                role: "user",
+                content: `${RedraftNote}\n\nIn French, please.\n\nAnd short.`,
+            },
+        ]);
+    });
+
+    test("joins files from new messages onto the same user turn", async () => {
+        const { generate, seen } = scripted([{ text: "Draft." }, { text: "With the photo." }]);
+        const arrivals = [
+            [
+                {
+                    role: "user" as const,
+                    content: [
+                        { type: "text" as const, text: "Use this one." },
+                        { type: "image" as const, url: "data:image/png;base64,x" },
+                    ],
+                },
+            ],
+        ];
+
+        await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "Write the caption." }],
+            generate,
+            resolveCandidates,
+            incoming: async () => arrivals.shift() || [],
+        });
+
+        expect(seen[1]!.messages.at(-1)).toEqual({
+            role: "user",
+            content: [
+                { type: "text", text: RedraftNote },
+                { type: "text", text: "Use this one." },
+                { type: "image", url: "data:image/png;base64,x" },
+            ],
+        });
+    });
+
+    test("returns at once when nothing new came in", async () => {
+        const { generate, seen } = scripted([{ text: "Hi." }]);
+        let looks = 0;
+
+        const result = await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "Hello" }],
+            generate,
+            resolveCandidates,
+            incoming: async () => {
+                looks++;
+                return [];
+            },
+        });
+
+        expect(result.text).toBe("Hi.");
+        expect(seen).toHaveLength(1);
+        expect(looks).toBe(1);
+    });
+
+    test("doesn't redraft the wrap up: a later reply picks the new messages up", async () => {
+        const loop_call = {
+            tool_calls: [{ id: "c", name: "remember", arguments: `{"note":"x"}` }],
+        };
+        const { generate, seen } = scripted([loop_call, { text: "Out of steps." }]);
+        let looks = 0;
+
+        const result = await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "go" }],
+            tools: [remember],
+            max_steps: 1,
+            generate,
+            resolveCandidates,
+            incoming: async () => {
+                looks++;
+                return looks == 1 ? [{ role: "user", content: "Also this." }] : [];
+            },
+        });
+
+        expect(result.text).toBe("Out of steps.");
+        expect(seen).toHaveLength(2);
+        expect(looks).toBe(1);
+        expect(typeof seen[1]!.messages.at(-1)!.content).toBe("string");
+        expect(seen[1]!.messages.at(-1)!.content).toContain("Also this.");
+        expect(seen[1]!.messages.at(-1)!.content).toContain("You've used every step");
+    });
 });

src/harness/text/run-text-loop.ts

@@ -12,6 +12,7 @@ import type {
 } from "../types";
 import fitTranscriptToContext from "./fit-transcript-to-context";
 import generateText from "./generate-text";
+import pushUserTurn from "./push-user-turn";
 
 export type TextLoopStep = {
     index: number;
@@ -37,6 +38,14 @@ type Params = {
      * progress and checkpoints.
      */
     onStep?: (step: TextLoopStep) => Promise<void> | void;
+    /**
+     * The user's turns that arrived since the run started (and since the
+     * last call), read after each round of tools and before an answer is
+     * returned. They join the transcript, so a message sent while the bot
+     * works can change what it does, and an answer they arrive in time for
+     * is written again with them in mind.
+     */
+    incoming?: () => Promise<HarnessMessage[]>;
     /**
      * The bot the run is for: its own models are used where it has them.
      */
@@ -65,6 +74,25 @@ export type TextLoopResult = {
 
 const WrapUpPrompt = `You've used every step this run allows. Don't call any more tools. Report what you did, what is left, and anything you need from the user.`;
 
+/**
+ * Told with what the user wrote while the run worked. Whether a message
+ * belongs to the work is the model's call, from what it says: a correction
+ * changes the work, a separate request is acknowledged (`send_message`,
+ * where the run has it) and done after.
+ */
+export const IncomingMessagesNote = `The user wrote again while you were working (below). Decide what it is from what it means, not how it's worded:
+- If it changes, corrects or adds to what you're doing, take it into account and carry on with the updated request.
+- If it's a separate request, don't drop what you're doing: if you can send_message, acknowledge it in a line now (say you'll get to it once this is done), then finish the current work, deal with the new request, and cover both in your answer.
+- If it only needs a quick answer, give it with send_message if you can and carry on.`;
+
+/**
+ * Told when the user wrote past an answer that hadn't gone out yet.
+ */
+export const RedraftNote = `Your answer above was not sent: the user wrote again before it went out (below). Decide what it is from what it means, not how it's worded:
+- If it changes or adds to what you were answering, write the answer again with it in mind.
+- If it's separate, keep your answer as it is and deal with the new message too: in the same answer, or send the first with send_message and then answer the new one.
+Don't repeat tool calls you have already made.`;
+
 /**
  * The text harness at work: the model answers or calls tools, tool results
  * go back to it, and so on until it answers without calling a tool. Tools
@@ -78,6 +106,7 @@ export default async function runTextLoop({
     max_steps = AppData.HarnessMaxToolSteps,
     signal = new AbortController().signal,
     onStep,
+    incoming,
     bot_id,
     workspace_id,
     generate = generateText,
@@ -102,7 +131,7 @@ export default async function runTextLoop({
         const wrap_up = index == max_steps;
 
         if (wrap_up) {
-            transcript.push({ role: "user", content: WrapUpPrompt });
+            pushUserTurn({ transcript, content: WrapUpPrompt });
         }
 
         /**
@@ -148,6 +177,14 @@ export default async function runTextLoop({
             candidate: answer.candidate,
         });
 
+        /**
+         * An answer the user wrote past is drafted again. Not on the wrap
+         * up: the run is out of steps, and a later reply picks them up.
+         */
+        if (!tool_calls.length && !wrap_up && (await takeIncoming({ note: RedraftNote }))) {
+            continue;
+        }
+
         if (!tool_calls.length) {
             return {
                 text:
@@ -167,7 +204,28 @@ export default async function runTextLoop({
                 content: await runToolCall({ call, tools, signal }),
             });
         }
+
+        await takeIncoming({ note: IncomingMessagesNote });
     }
 
     throw new Error(`Unreachable: the text loop ended without an answer.`);
+
+    /**
+     * Adds what the user wrote since the last look, after `note`. True when
+     * there was anything.
+     */
+    async function takeIncoming({ note }: { note: string }) {
+        const turns = (await incoming?.()) || [];
+        const user_turns = turns.filter((turn) => turn.role == "user");
+
+        if (!user_turns.length) return false;
+
+        pushUserTurn({ transcript, content: note });
+
+        for (const turn of user_turns) {
+            pushUserTurn({ transcript, content: turn.content });
+        }
+
+        return true;
+    }
 }

src/harness/tools/bot/send-message-tool.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, test } from "bun:test";
+import { AppData } from "@/src/data/app-data";
+import sendMessageTool from "./send-message-tool";
+
+const signal = new AbortController().signal;
+
+function tool() {
+    const posted: string[] = [];
+    const sent: number[] = [];
+    const send = sendMessageTool({
+        bot_id: 2,
+        session_id: 100,
+        post: async ({ text }) => {
+            posted.push(text);
+            return { id: posted.length };
+        },
+        onSend: ({ message_id }) => sent.push(message_id),
+    });
+
+    return { send, posted, sent };
+}
+
+describe("sendMessageTool", () => {
+    test("posts the message now and tells the bot to carry on", async () => {
+        const { send, posted, sent } = tool();
+
+        const result = await send.run({ args: { text: "Noted, I'll do that once this is done." }, signal });
+
+        expect(posted).toEqual(["Noted, I'll do that once this is done."]);
+        expect(sent).toEqual([1]);
+        expect(result).toContain("Carry on");
+    });
+
+    test("stops at the cap, so a reply can't flood the chat", async () => {
+        const { send, posted } = tool();
+
+        for (let i = 0; i < AppData.BotReplyInterimMessagesMax; i++) {
+            await send.run({ args: { text: `Heads-up ${i}` }, signal });
+        }
+
+        const error = await Promise.resolve(send.run({ args: { text: "One more" }, signal })).catch(
+            (e) => e,
+        );
+
+        expect(error).toBeInstanceOf(Error);
+        expect(posted).toHaveLength(AppData.BotReplyInterimMessagesMax);
+    });
+
+    test("refuses an empty message", () => {
+        expect(tool().send.schema?.safeParse({ text: "   " }).success).toBe(false);
+    });
+});

src/harness/tools/bot/send-message-tool.ts

@@ -0,0 +1,71 @@
+import { z } from "zod";
+import pushBotNotice from "@/src/bots-queue/push-bot-notice";
+import { AppData } from "@/src/data/app-data";
+import insertBotMessage from "@/src/functions/backend/db/bots/insert-bot-message";
+import type { HarnessTool } from "../../types";
+
+type Params = {
+    bot_id: number;
+    /**
+     * The session being answered: the message goes in it.
+     */
+    session_id: number;
+    workspace_id?: number | null;
+    /**
+     * Called with each message sent, so the reply knows it said something.
+     */
+    onSend?: (params: { message_id: number }) => void;
+    /**
+     * Saves and pushes the message (`insertBotMessage`, `pushBotNotice`).
+     */
+    post?: (params: { text: string }) => Promise<{ id?: number | "" }>;
+};
+
+type Args = {
+    text: string;
+};
+
+/**
+ * Lets a reply post a message in the conversation now, without ending the
+ * run: the bot acknowledges something the user wrote while it works ("Noted,
+ * I'll do that once this is done") and carries on. The message is marked
+ * `interim`, so the conversation keeps showing the bot at work, and is
+ * pushed at once. At most `AppData.BotReplyInterimMessagesMax` per run.
+ */
+export default function sendMessageTool({
+    bot_id,
+    session_id,
+    workspace_id,
+    onSend,
+    post = async ({ text }) => {
+        const message = await insertBotMessage({ bot_id, session_id, workspace_id, text, interim: true });
+
+        await pushBotNotice({ bot_id, message });
+
+        return message;
+    },
+}: Params): HarnessTool<Args> {
+    let sent = 0;
+
+    return {
+        name: "send_message",
+        description: `Send the user a message now and keep working: your reply still comes when you finish. Use it to acknowledge something they wrote while you were working that is separate from what you're doing (say you'll get to it, then carry on), or for a short heads-up on long work. Not for your answer, and not for something that changes what you're doing: take that into account instead. At most ${AppData.BotReplyInterimMessagesMax} per reply.`,
+        schema: z.object({
+            text: z.string().trim().min(1).max(2000).describe(`The message (markdown), short`),
+        }),
+        run: async ({ args }) => {
+            if (sent >= AppData.BotReplyInterimMessagesMax) {
+                throw new Error(
+                    `You've sent ${sent} messages in this reply already. Carry on, and put the rest in your answer.`,
+                );
+            }
+
+            const message = await post({ text: args.text });
+
+            sent++;
+            onSend?.({ message_id: Number(message.id) });
+
+            return `Sent. Carry on with what you were doing.`;
+        },
+    };
+}