Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/pages/admin/bots/[bot_id]/(data)/composer-extensions-copy.ts

@@ -0,0 +1,38 @@
+import { Library, Wrench } from "lucide-react";
+import type { ComposerExtensionsCopy } from "../(partials)/composer-tools-modal";
+
+/**
+ * The composer's tools button: MCP servers, tools and skills.
+ */
+export function composerToolsCopy({ bot_name }: { bot_name: string }): ComposerExtensionsCopy {
+    return {
+        id: "composer-add-extension",
+        Icon: Wrench,
+        one: "tool",
+        many: "tools",
+        empty_label: "Add tools",
+        title: `Tools ${bot_name} can use here`,
+        some: "MCP servers, tools and skills it was given, plus this workspace's. Remove one to take it away, or add another below.",
+        none: "It has none yet. Give it an MCP server, tool or skill below, or make one under Settings.",
+        add_hint: `Give ${bot_name} another one. It can use it everywhere it works, for everyone it works for.`,
+        add_noun: "tool, MCP server or skill",
+    };
+}
+
+/**
+ * The composer's knowledgebases button, beside the tools.
+ */
+export function composerKnowledgebasesCopy({ bot_name }: { bot_name: string }): ComposerExtensionsCopy {
+    return {
+        id: "composer-add-knowledgebase",
+        Icon: Library,
+        one: "knowledgebase",
+        many: "knowledgebases",
+        empty_label: "Add knowledge",
+        title: `Knowledgebases ${bot_name} searches here`,
+        some: "Before it answers, it searches these for passages that fit, and it can search them again while it works. Remove one to take it away, or add another below.",
+        none: "It has none yet. Give it a knowledgebase below, or make one under Settings, under Knowledgebases.",
+        add_hint: `Give ${bot_name} another one. It searches it everywhere it works, for everyone it works for.`,
+        add_noun: "knowledgebase",
+    };
+}

src/pages/admin/bots/[bot_id]/(partials)/composer-tool-row.tsx

@@ -5,6 +5,7 @@ import Stack from "@moduletrace/twui/layout/Stack";
 import { X } from "lucide-react";
 import extensionIcon from "@/src/components/general/extension-icon";
 import type { BotExtension } from "@/src/types";
+import extensionStateNote from "@/src/utils/extension-state-note";
 
 type Props = {
     item: BotExtension;
@@ -17,24 +18,16 @@ type Props = {
 };
 
 /**
- * One MCP server, tool or skill the bot can use here, in the composer's
- * tools modal. One that isn't working says so, one that's off is struck
- * out. A skill has nothing to connect or run, so it never reports one.
+ * One MCP server, tool, skill or knowledgebase the bot can use here, in
+ * the composer's modals. One that isn't working says so
+ * (`extensionStateNote`), one that's off is struck out.
  */
 export default function ComposerToolRow({ item, busy, onRemove }: Props) {
     const Icon = extensionIcon({ kind: item.kind });
     const notes = [
         item.detail,
         item.source == "workspace" ? "From this workspace" : "",
-        !item.enabled
-            ? "Off for every bot"
-            : item.kind == "skill"
-              ? ""
-              : item.state == "error"
-                ? "Not working"
-                : item.state == "untested"
-                  ? "Not checked yet"
-                  : "",
+        extensionStateNote({ item }),
     ].filter(Boolean);
 
     return (

src/pages/admin/bots/[bot_id]/(partials)/composer-tools-modal.tsx

@@ -4,13 +4,43 @@ import Button from "@moduletrace/twui/layout/Button";
 import H2 from "@moduletrace/twui/layout/H2";
 import Span from "@moduletrace/twui/layout/Span";
 import Stack from "@moduletrace/twui/layout/Stack";
-import { CircleAlert, Wrench } from "lucide-react";
+import { CircleAlert, type LucideIcon } from "lucide-react";
 import ExtensionAddSelect from "@/src/components/general/extension-add-select";
-import type { BotExtension, BotExtensionsPayload } from "@/src/types";
+import type { BotExtension, BotExtensionKind, BotExtensionsPayload } from "@/src/types";
 import ComposerToolRow from "./composer-tool-row";
 
+/**
+ * What one of the composer's buttons is about, in copy: its tools, or its
+ * knowledgebases.
+ */
+export type ComposerExtensionsCopy = {
+    id: string;
+    Icon: LucideIcon;
+    /**
+     * "tool" and "tools", for the button's count.
+     */
+    one: string;
+    many: string;
+    /**
+     * The button's label with nothing yet, e.g. "Add tools".
+     */
+    empty_label: string;
+    /**
+     * The modal's heading, with the bot's name.
+     */
+    title: string;
+    some: string;
+    none: string;
+    /**
+     * Above the picker, and in its placeholder.
+     */
+    add_hint: string;
+    add_noun: string;
+};
+
 type Props = {
     bot_name: string;
+    copy: ComposerExtensionsCopy;
     attached: BotExtension[];
     addable: BotExtensionsPayload["addable"];
     can_manage: boolean;
@@ -18,22 +48,19 @@ type Props = {
      * The extension being changed, as `kind:id`.
      */
     busy?: string;
-    onAdd: (params: { kind: "mcp" | "tool" | "skill"; id: number }) => Promise<boolean>;
-    onRemove: (params: {
-        kind: "mcp" | "tool" | "skill";
-        id: number;
-        name: string;
-    }) => Promise<boolean>;
+    onAdd: (params: { kind: BotExtensionKind; id: number }) => Promise<boolean>;
+    onRemove: (params: { kind: BotExtensionKind; id: number; name: string }) => Promise<boolean>;
 };
 
 /**
- * The composer's tools button: how many MCP servers, tools and skills the
+ * One of the composer's buttons: how many tools (or knowledgebases) the
  * bot can use here, opening them in a modal rather than filling the
  * composer with chips. Super admins and admins remove any it was given
  * (even one every bot started with) and add more; a workspace's own stay.
  */
 export default function ComposerToolsModal({
     bot_name,
+    copy,
     attached,
     addable,
     can_manage,
@@ -46,14 +73,14 @@ export default function ComposerToolsModal({
     const has_problem = attached.some(
         (item) => !item.enabled || item.state == "error",
     );
-    const TriggerIcon = has_problem ? CircleAlert : Wrench;
-    const label = count ? `${count} ${count == 1 ? "tool" : "tools"}` : "Add tools";
+    const TriggerIcon = has_problem ? CircleAlert : copy.Icon;
+    const label = count ? `${count} ${count == 1 ? copy.one : copy.many}` : copy.empty_label;
 
     return (
         <Modal
             target={
                 <Button
-                    title={`The tools ${bot_name} can use here`}
+                    title={copy.title}
                     type="button"
                     variant="ghost"
                     color="gray"
@@ -71,18 +98,16 @@ export default function ComposerToolsModal({
         >
             <Stack className="w-full gap-1! items-stretch">
                 <H2 className="text-lg! leading-6! tracking-normal! font-bold text-ink">
-                    {`Tools ${bot_name} can use here`}
+                    {copy.title}
                 </H2>
                 <Span className="text-[13px] leading-[18px] text-ink-secondary">
-                    {count
-                        ? "MCP servers, tools and skills it was given, plus this workspace's. Remove one to take it away, or add another below."
-                        : "It has none yet. Give it an MCP server, tool or skill below, or make one under Settings."}
+                    {count ? copy.some : copy.none}
                 </Span>
             </Stack>
 
             {count ? (
                 <Stack
-                    aria-label={`Tools ${bot_name} can use here`}
+                    aria-label={copy.title}
                     className="w-full mt-4 py-1 gap-0! items-stretch rounded-default border border-solid border-hairline bg-surface"
                 >
                     {attached.map((item) => (
@@ -108,10 +133,11 @@ export default function ComposerToolsModal({
             {can_manage && addable.length ? (
                 <Stack className="w-full mt-4 gap-2! items-stretch">
                     <Span className="text-[13px] leading-[18px] text-ink-secondary">
-                        {`Give ${bot_name} another one. It can use it everywhere it works, for everyone it works for.`}
+                        {copy.add_hint}
                     </Span>
                     <ExtensionAddSelect
-                        id="composer-add-extension"
+                        id={copy.id}
+                        noun={copy.add_noun}
                         addable={addable}
                         loading={Boolean(busy)}
                         onAdd={onAdd}

src/pages/admin/bots/[bot_id]/(partials)/composer-tools.tsx

@@ -4,6 +4,7 @@ import Span from "@moduletrace/twui/layout/Span";
 import useExtensions from "@/src/hooks/use-extensions";
 import type { BotExtensionsPayload, BotWithLastMessage } from "@/src/types";
 import { BotComputersContext } from "../(data)/bot-computers-context";
+import { composerKnowledgebasesCopy, composerToolsCopy } from "../(data)/composer-extensions-copy";
 import ComposerComputerButton from "./composer-computer-button";
 import ComposerToolsModal from "./composer-tools-modal";
 import { Dot } from "lucide-react";
@@ -15,33 +16,44 @@ type Props = {
 };
 
 /**
- * Under the composer: one button counting the MCP servers, tools and skills the
- * bot can use here (they open in a modal, so ten of them don't fill the
+ * Under the composer: one button counting the MCP servers, tools and
+ * skills the bot can use here, one beside it counting the knowledgebases it
+ * searches (each opens in a modal, so ten of them don't fill the
  * composer), then the computer it's on, opening it in the viewer.
  */
 export default function ComposerTools({ bot, workspace_id, initial }: Props) {
+    const bot_name = bot.name || "this bot";
     const extensions = useExtensions({
         owner: { bot_id: bot.id || 0, workspace_id },
-        owner_name: bot.name || "this bot",
+        owner_name: bot_name,
         initial,
     });
 
     const { computers } = useContext(BotComputersContext);
 
-    const has_tools = Boolean(
-        extensions.attached.length ||
-        (extensions.can_manage && extensions.addable.length),
-    );
+    const is_knowledge = (item: { kind: string }) => item.kind == "knowledgebase";
+    const tools = {
+        attached: extensions.attached.filter((item) => !is_knowledge(item)),
+        addable: extensions.addable.filter((item) => !is_knowledge(item)),
+    };
+    const knowledge = {
+        attached: extensions.attached.filter(is_knowledge),
+        addable: extensions.addable.filter(is_knowledge),
+    };
 
-    if (!has_tools && !computers.length) return null;
+    const has_tools = Boolean(tools.attached.length || (extensions.can_manage && tools.addable.length));
+    const has_knowledge = Boolean(knowledge.attached.length || (extensions.can_manage && knowledge.addable.length));
+
+    if (!has_tools && !has_knowledge && !computers.length) return null;
 
     return (
         <Row className="w-full max-w-container mx-auto gap-1.5! items-center px-3 md:px-6 pb-3 -mt-3">
             {has_tools ? (
                 <ComposerToolsModal
-                    bot_name={bot.name || "this bot"}
-                    attached={extensions.attached}
-                    addable={extensions.addable}
+                    bot_name={bot_name}
+                    copy={composerToolsCopy({ bot_name })}
+                    attached={tools.attached}
+                    addable={tools.addable}
                     can_manage={extensions.can_manage}
                     busy={extensions.busy}
                     onAdd={extensions.add}
@@ -49,7 +61,20 @@ export default function ComposerTools({ bot, workspace_id, initial }: Props) {
                 />
             ) : null}
 
-            {has_tools && computers.length ? (
+            {has_knowledge ? (
+                <ComposerToolsModal
+                    bot_name={bot_name}
+                    copy={composerKnowledgebasesCopy({ bot_name })}
+                    attached={knowledge.attached}
+                    addable={knowledge.addable}
+                    can_manage={extensions.can_manage}
+                    busy={extensions.busy}
+                    onAdd={extensions.add}
+                    onRemove={extensions.remove}
+                />
+            ) : null}
+
+            {(has_tools || has_knowledge) && computers.length ? (
                 <Span className="-mr-2 opacity-40">
                     <Dot size={20} />
                 </Span>