Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/bots-queue/run-bot-reply.ts

@@ -28,6 +28,8 @@ import grabBotFileTools from "@/src/harness/tools/storage/grab-bot-file-tools";
 import grabRunScopes from "@/src/functions/backend/db/bot-files/grab-run-scopes";
 import openBotExtensions from "@/src/harness/tools/extensions/open-bot-extensions";
 import openBotSkills from "@/src/harness/tools/skills/open-bot-skills";
+import openBotKnowledgebases from "@/src/harness/tools/knowledgebases/open-bot-knowledgebases";
+import grabKnowledgebaseContext from "@/src/harness/tools/knowledgebases/grab-knowledgebase-context";
 import botMessagesToHarness from "./bot-messages-to-harness";
 import composeComputersNote from "./compose-computers-note";
 import grabSessionCarryOver from "./grab-session-carry-over";
@@ -262,6 +264,22 @@ export default async function runBotReply({
         taken: [...tools, ...extensions.tools].map((tool) => tool.name),
     });
 
+    /**
+     * The run's knowledgebases: a search tool and a reader, and the
+     * passages found for what the user just asked, so a plain question is
+     * answered from the documents.
+     */
+    const knowledge = openBotKnowledgebases({
+        knowledgebases: extensions.knowledgebases,
+        taken: [...tools, ...extensions.tools, ...skills.tools].map((tool) => tool.name),
+    });
+    const knowledge_context = await grabKnowledgebaseContext({
+        knowledgebases: extensions.knowledgebases,
+        ids: knowledge.ids,
+        query: unansweredText({ messages: history, answered_id }),
+        signal,
+    });
+
     /**
      * delegate_task is only there while the coordinator, or a workspace's
      * admin, has bots to hand work to. Anyone else in a workspace with an
@@ -298,7 +316,9 @@ export default async function runBotReply({
                 }),
                 crew_note,
                 skills.note,
+                knowledge.note,
                 extensions.note,
+                knowledge_context,
                 `Now: ${new Date().toUTCString()}.`,
             ]
                 .filter(Boolean)
@@ -332,6 +352,7 @@ export default async function runBotReply({
                 ...file_tools,
                 ...extensions.tools,
                 ...skills.tools,
+                ...knowledge.tools,
             ],
             max_steps: await grabReplyMaxToolSteps(),
             bot_id: bot.id,
@@ -365,3 +386,22 @@ function newestUserMessageId({ messages }: { messages: Pick<BotMessageWithAttach
         ...messages.filter((message) => message.author == "user").map((message) => Number(message.id)),
     );
 }
+
+/**
+ * The text of the user messages the reply answers (those after the last
+ * answered one), else the newest user message: what the knowledgebases are
+ * searched for ahead of the reply.
+ */
+function unansweredText({
+    messages,
+    answered_id,
+}: {
+    messages: Pick<BotMessageWithAttachments, "id" | "author" | "text">[];
+    answered_id?: number | string | null;
+}) {
+    const from_user = messages.filter((message) => message.author == "user");
+    const unanswered = from_user.filter((message) => Number(message.id) > Number(answered_id || 0));
+    const picked = unanswered.length ? unanswered : from_user.slice(-1);
+
+    return picked.map((message) => message.text || "").join("\n");
+}

src/bots-queue/run-bot-task.ts

@@ -22,6 +22,7 @@ import runTextLoop from "@/src/harness/text/run-text-loop";
 import openComputerTools from "@/src/harness/tools/computer/open-computer-tools";
 import openBotExtensions, { type OpenBotExtensions } from "@/src/harness/tools/extensions/open-bot-extensions";
 import openBotSkills, { type OpenBotSkills } from "@/src/harness/tools/skills/open-bot-skills";
+import openBotKnowledgebases from "@/src/harness/tools/knowledgebases/open-bot-knowledgebases";
 import grabBotTools from "@/src/harness/tools/bot/grab-bot-tools";
 import grabBotSecretTools from "@/src/harness/tools/bot/secrets/grab-bot-secret-tools";
 import createWorkspaceTool from "@/src/harness/tools/bot/create-workspace-tool";
@@ -242,6 +243,11 @@ export default async function runBotTask({
             taken: [...tools, ...extensions.tools].map((tool) => tool.name),
         });
 
+        const knowledge = openBotKnowledgebases({
+            knowledgebases: extensions.knowledgebases,
+            taken: [...tools, ...extensions.tools, ...skills.tools].map((tool) => tool.name),
+        });
+
         const result = await runTextLoop({
             system: [
                 prompt,
@@ -277,6 +283,7 @@ export default async function runBotTask({
                           .join("\n")}`
                     : "",
                 skills.note,
+                knowledge.note,
                 extensions.note,
             ]
                 .filter(Boolean)
@@ -291,7 +298,7 @@ export default async function runBotTask({
                     }),
                 },
             ],
-            tools: [...tools, ...extensions.tools, ...skills.tools],
+            tools: [...tools, ...extensions.tools, ...skills.tools, ...knowledge.tools],
             max_steps: await grabHarnessMaxToolSteps(),
             bot_id: bot.id,
             workspace_id,