Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/pages/admin/bots/[bot_id]/(data)/composer-extensions-copy.ts

@@ -0,0 +1,38 @@
+import { Library, Wrench } from "lucide-react";
+import type { ComposerExtensionsCopy } from "../(partials)/composer-tools-modal";
+
+/**
+ * The composer's tools button: MCP servers, tools and skills.
+ */
+export function composerToolsCopy({ bot_name }: { bot_name: string }): ComposerExtensionsCopy {
+    return {
+        id: "composer-add-extension",
+        Icon: Wrench,
+        one: "tool",
+        many: "tools",
+        empty_label: "Add tools",
+        title: `Tools ${bot_name} can use here`,
+        some: "MCP servers, tools and skills it was given, plus this workspace's. Remove one to take it away, or add another below.",
+        none: "It has none yet. Give it an MCP server, tool or skill below, or make one under Settings.",
+        add_hint: `Give ${bot_name} another one. It can use it everywhere it works, for everyone it works for.`,
+        add_noun: "tool, MCP server or skill",
+    };
+}
+
+/**
+ * The composer's knowledgebases button, beside the tools.
+ */
+export function composerKnowledgebasesCopy({ bot_name }: { bot_name: string }): ComposerExtensionsCopy {
+    return {
+        id: "composer-add-knowledgebase",
+        Icon: Library,
+        one: "knowledgebase",
+        many: "knowledgebases",
+        empty_label: "Add knowledge",
+        title: `Knowledgebases ${bot_name} searches here`,
+        some: "Before it answers, it searches these for passages that fit, and it can search them again while it works. Remove one to take it away, or add another below.",
+        none: "It has none yet. Give it a knowledgebase below, or make one under Settings, under Knowledgebases.",
+        add_hint: `Give ${bot_name} another one. It searches it everywhere it works, for everyone it works for.`,
+        add_noun: "knowledgebase",
+    };
+}

src/pages/admin/bots/[bot_id]/(partials)/composer-tool-row.tsx

@@ -5,6 +5,7 @@ import Stack from "@moduletrace/twui/layout/Stack";
 import { X } from "lucide-react";
 import extensionIcon from "@/src/components/general/extension-icon";
 import type { BotExtension } from "@/src/types";
+import extensionStateNote from "@/src/utils/extension-state-note";
 
 type Props = {
     item: BotExtension;
@@ -17,24 +18,16 @@ type Props = {
 };
 
 /**
- * One MCP server, tool or skill the bot can use here, in the composer's
- * tools modal. One that isn't working says so, one that's off is struck
- * out. A skill has nothing to connect or run, so it never reports one.
+ * One MCP server, tool, skill or knowledgebase the bot can use here, in
+ * the composer's modals. One that isn't working says so
+ * (`extensionStateNote`), one that's off is struck out.
  */
 export default function ComposerToolRow({ item, busy, onRemove }: Props) {
     const Icon = extensionIcon({ kind: item.kind });
     const notes = [
         item.detail,
         item.source == "workspace" ? "From this workspace" : "",
-        !item.enabled
-            ? "Off for every bot"
-            : item.kind == "skill"
-              ? ""
-              : item.state == "error"
-                ? "Not working"
-                : item.state == "untested"
-                  ? "Not checked yet"
-                  : "",
+        extensionStateNote({ item }),
     ].filter(Boolean);
 
     return (

src/pages/admin/bots/[bot_id]/(partials)/composer-tools-modal.tsx

@@ -4,13 +4,43 @@ import Button from "@moduletrace/twui/layout/Button";
 import H2 from "@moduletrace/twui/layout/H2";
 import Span from "@moduletrace/twui/layout/Span";
 import Stack from "@moduletrace/twui/layout/Stack";
-import { CircleAlert, Wrench } from "lucide-react";
+import { CircleAlert, type LucideIcon } from "lucide-react";
 import ExtensionAddSelect from "@/src/components/general/extension-add-select";
-import type { BotExtension, BotExtensionsPayload } from "@/src/types";
+import type { BotExtension, BotExtensionKind, BotExtensionsPayload } from "@/src/types";
 import ComposerToolRow from "./composer-tool-row";
 
+/**
+ * What one of the composer's buttons is about, in copy: its tools, or its
+ * knowledgebases.
+ */
+export type ComposerExtensionsCopy = {
+    id: string;
+    Icon: LucideIcon;
+    /**
+     * "tool" and "tools", for the button's count.
+     */
+    one: string;
+    many: string;
+    /**
+     * The button's label with nothing yet, e.g. "Add tools".
+     */
+    empty_label: string;
+    /**
+     * The modal's heading, with the bot's name.
+     */
+    title: string;
+    some: string;
+    none: string;
+    /**
+     * Above the picker, and in its placeholder.
+     */
+    add_hint: string;
+    add_noun: string;
+};
+
 type Props = {
     bot_name: string;
+    copy: ComposerExtensionsCopy;
     attached: BotExtension[];
     addable: BotExtensionsPayload["addable"];
     can_manage: boolean;
@@ -18,22 +48,19 @@ type Props = {
      * The extension being changed, as `kind:id`.
      */
     busy?: string;
-    onAdd: (params: { kind: "mcp" | "tool" | "skill"; id: number }) => Promise<boolean>;
-    onRemove: (params: {
-        kind: "mcp" | "tool" | "skill";
-        id: number;
-        name: string;
-    }) => Promise<boolean>;
+    onAdd: (params: { kind: BotExtensionKind; id: number }) => Promise<boolean>;
+    onRemove: (params: { kind: BotExtensionKind; id: number; name: string }) => Promise<boolean>;
 };
 
 /**
- * The composer's tools button: how many MCP servers, tools and skills the
+ * One of the composer's buttons: how many tools (or knowledgebases) the
  * bot can use here, opening them in a modal rather than filling the
  * composer with chips. Super admins and admins remove any it was given
  * (even one every bot started with) and add more; a workspace's own stay.
  */
 export default function ComposerToolsModal({
     bot_name,
+    copy,
     attached,
     addable,
     can_manage,
@@ -46,14 +73,14 @@ export default function ComposerToolsModal({
     const has_problem = attached.some(
         (item) => !item.enabled || item.state == "error",
     );
-    const TriggerIcon = has_problem ? CircleAlert : Wrench;
-    const label = count ? `${count} ${count == 1 ? "tool" : "tools"}` : "Add tools";
+    const TriggerIcon = has_problem ? CircleAlert : copy.Icon;
+    const label = count ? `${count} ${count == 1 ? copy.one : copy.many}` : copy.empty_label;
 
     return (
         <Modal
             target={
                 <Button
-                    title={`The tools ${bot_name} can use here`}
+                    title={copy.title}
                     type="button"
                     variant="ghost"
                     color="gray"
@@ -71,18 +98,16 @@ export default function ComposerToolsModal({
         >
             <Stack className="w-full gap-1! items-stretch">
                 <H2 className="text-lg! leading-6! tracking-normal! font-bold text-ink">
-                    {`Tools ${bot_name} can use here`}
+                    {copy.title}
                 </H2>
                 <Span className="text-[13px] leading-[18px] text-ink-secondary">
-                    {count
-                        ? "MCP servers, tools and skills it was given, plus this workspace's. Remove one to take it away, or add another below."
-                        : "It has none yet. Give it an MCP server, tool or skill below, or make one under Settings."}
+                    {count ? copy.some : copy.none}
                 </Span>
             </Stack>
 
             {count ? (
                 <Stack
-                    aria-label={`Tools ${bot_name} can use here`}
+                    aria-label={copy.title}
                     className="w-full mt-4 py-1 gap-0! items-stretch rounded-default border border-solid border-hairline bg-surface"
                 >
                     {attached.map((item) => (
@@ -108,10 +133,11 @@ export default function ComposerToolsModal({
             {can_manage && addable.length ? (
                 <Stack className="w-full mt-4 gap-2! items-stretch">
                     <Span className="text-[13px] leading-[18px] text-ink-secondary">
-                        {`Give ${bot_name} another one. It can use it everywhere it works, for everyone it works for.`}
+                        {copy.add_hint}
                     </Span>
                     <ExtensionAddSelect
-                        id="composer-add-extension"
+                        id={copy.id}
+                        noun={copy.add_noun}
                         addable={addable}
                         loading={Boolean(busy)}
                         onAdd={onAdd}

src/pages/admin/bots/[bot_id]/(partials)/composer-tools.tsx

@@ -4,6 +4,7 @@ import Span from "@moduletrace/twui/layout/Span";
 import useExtensions from "@/src/hooks/use-extensions";
 import type { BotExtensionsPayload, BotWithLastMessage } from "@/src/types";
 import { BotComputersContext } from "../(data)/bot-computers-context";
+import { composerKnowledgebasesCopy, composerToolsCopy } from "../(data)/composer-extensions-copy";
 import ComposerComputerButton from "./composer-computer-button";
 import ComposerToolsModal from "./composer-tools-modal";
 import { Dot } from "lucide-react";
@@ -15,33 +16,44 @@ type Props = {
 };
 
 /**
- * Under the composer: one button counting the MCP servers, tools and skills the
- * bot can use here (they open in a modal, so ten of them don't fill the
+ * Under the composer: one button counting the MCP servers, tools and
+ * skills the bot can use here, one beside it counting the knowledgebases it
+ * searches (each opens in a modal, so ten of them don't fill the
  * composer), then the computer it's on, opening it in the viewer.
  */
 export default function ComposerTools({ bot, workspace_id, initial }: Props) {
+    const bot_name = bot.name || "this bot";
     const extensions = useExtensions({
         owner: { bot_id: bot.id || 0, workspace_id },
-        owner_name: bot.name || "this bot",
+        owner_name: bot_name,
         initial,
     });
 
     const { computers } = useContext(BotComputersContext);
 
-    const has_tools = Boolean(
-        extensions.attached.length ||
-        (extensions.can_manage && extensions.addable.length),
-    );
+    const is_knowledge = (item: { kind: string }) => item.kind == "knowledgebase";
+    const tools = {
+        attached: extensions.attached.filter((item) => !is_knowledge(item)),
+        addable: extensions.addable.filter((item) => !is_knowledge(item)),
+    };
+    const knowledge = {
+        attached: extensions.attached.filter(is_knowledge),
+        addable: extensions.addable.filter(is_knowledge),
+    };
 
-    if (!has_tools && !computers.length) return null;
+    const has_tools = Boolean(tools.attached.length || (extensions.can_manage && tools.addable.length));
+    const has_knowledge = Boolean(knowledge.attached.length || (extensions.can_manage && knowledge.addable.length));
+
+    if (!has_tools && !has_knowledge && !computers.length) return null;
 
     return (
         <Row className="w-full max-w-container mx-auto gap-1.5! items-center px-3 md:px-6 pb-3 -mt-3">
             {has_tools ? (
                 <ComposerToolsModal
-                    bot_name={bot.name || "this bot"}
-                    attached={extensions.attached}
-                    addable={extensions.addable}
+                    bot_name={bot_name}
+                    copy={composerToolsCopy({ bot_name })}
+                    attached={tools.attached}
+                    addable={tools.addable}
                     can_manage={extensions.can_manage}
                     busy={extensions.busy}
                     onAdd={extensions.add}
@@ -49,7 +61,20 @@ export default function ComposerTools({ bot, workspace_id, initial }: Props) {
                 />
             ) : null}
 
-            {has_tools && computers.length ? (
+            {has_knowledge ? (
+                <ComposerToolsModal
+                    bot_name={bot_name}
+                    copy={composerKnowledgebasesCopy({ bot_name })}
+                    attached={knowledge.attached}
+                    addable={knowledge.addable}
+                    can_manage={extensions.can_manage}
+                    busy={extensions.busy}
+                    onAdd={extensions.add}
+                    onRemove={extensions.remove}
+                />
+            ) : null}
+
+            {(has_tools || has_knowledge) && computers.length ? (
                 <Span className="-mr-2 opacity-40">
                     <Dot size={20} />
                 </Span>

src/pages/admin/help/(data)/help-topics.ts

@@ -119,4 +119,16 @@ A bot's own models (the model icon in its chat header) win over the defaults, an
 
 Tell the bot your time zone once ("I'm in Lagos", "I'm on New York time") and it remembers it and converts for you. To fix one yourself, open **Routines**, click **Edit**, and change the schedule to the UTC hour you want. Routines don't follow daylight saving changes, so ones near a clock change may need a nudge twice a year.`,
     },
+    {
+        id: "knowledgebase-not-found",
+        title: "A bot doesn't use a knowledgebase",
+        summary: "It answers from memory, or says it found nothing, though the document is there.",
+        body: `A super admin makes knowledgebases in **Settings, Knowledgebases**, and a bot searches one only once it's given it: from the knowledgebases button under its composer, its settings page, or a workspace it works in.
+
+If it's given and still misses:
+
+- Open the knowledgebase's documents. A document still **being read** isn't searchable yet; one that **couldn't be read** names why (a scan with no text, a file too big). Write the text in instead, or upload a version with text in it.
+- Bots search by meaning only with an embedding model, picked once for the whole install in **Settings, Models** under Embeddings. Without one they match words, which misses questions worded differently from the document. Picking or changing the model reads every document again on its own; a document shows **needs re-indexing** until it's done.
+- If a document still says it needs re-indexing, use the refresh icon on its knowledgebase's row to read them all again.`,
+    },
 ];

src/pages/admin/settings/(partials)/extension-access-modal.tsx

@@ -18,13 +18,18 @@ type Props<T> = {
     /**
      * Under `/api/admin/`.
      */
-    path: "mcps/set-mcp-access" | "tools/set-tool-access" | "skills/set-skill-access";
+    path:
+        | "mcps/set-mcp-access"
+        | "tools/set-tool-access"
+        | "skills/set-skill-access"
+        | "knowledgebases/set-knowledgebase-access";
     onSaved: (overview: T, msg: string) => void;
     onCancel: () => void;
 };
 
 /**
- * Who gets an MCP server, tool or skill: the bots and workspaces picked here.
+ * Who gets an MCP server, tool, skill or knowledgebase: the bots and
+ * workspaces picked here.
  * "Every bot" (super admins only) gives it to every bot now and every new
  * one; each can still drop it. A workspace gives it to every bot in
  * it, there.

src/pages/admin/settings/(partials)/extension-row.tsx

@@ -1,6 +1,7 @@
 import IconTile from "@/src/components/general/icon-tile";
 import extensionIcon from "@/src/components/general/extension-icon";
 import type { BotExtension } from "@/src/types";
+import extensionStateNote from "@/src/utils/extension-state-note";
 import SettingsListRow from "./settings-list-row";
 import SettingsRowAction from "./settings-row-action";
 
@@ -17,23 +18,15 @@ const ExtensionHrefs = {
     mcp: "/admin/settings/mcps",
     tool: "/admin/settings/tools",
     skill: "/admin/settings/skills",
+    knowledgebase: "/admin/settings/knowledgebases",
 } as const;
 
 /**
- * One MCP server, tool or skill a bot or workspace has, and whether it's
- * working. A skill has nothing to connect or run, so it never reports a
- * failure.
+ * One MCP server, tool, skill or knowledgebase a bot or workspace has,
+ * and whether it's working (`extensionStateNote`).
  */
 export default function ExtensionRow({ item, busy, onRemove }: Props) {
-    const state = !item.enabled
-        ? "Off for every bot"
-        : item.kind == "skill"
-          ? ""
-          : item.state == "error"
-            ? "Not working"
-            : item.state == "untested"
-              ? "Not checked yet"
-              : "";
+    const state = extensionStateNote({ item });
 
     return (
         <SettingsListRow

src/pages/admin/settings/(partials)/extensions-group.tsx

@@ -1,6 +1,6 @@
 import Button from "@moduletrace/twui/layout/Button";
 import Row from "@moduletrace/twui/layout/Row";
-import { BookOpen, Plug, Wrench } from "lucide-react";
+import { BookOpen, Library, Plug, Wrench } from "lucide-react";
 import ExtensionAddSelect from "@/src/components/general/extension-add-select";
 import useExtensions from "@/src/hooks/use-extensions";
 import type { BotExtensionsPayload } from "@/src/types";
@@ -14,20 +14,36 @@ type Props = {
     owner_name: string;
     description: string;
     initial?: BotExtensionsPayload | null;
+    /**
+     * `tools`: MCP servers, tools and skills. `knowledgebases`: only those.
+     */
+    variant?: "tools" | "knowledgebases";
 };
 
 /**
- * The MCP servers, tools and skills a bot or a workspace has, removable,
- * and a picker to add more. On the bot and workspace settings pages.
+ * The MCP servers, tools and skills (or the knowledgebases) a bot or a
+ * workspace has, removable, and a picker to add more. On the bot and
+ * workspace settings pages.
  */
-export default function ExtensionsGroup({ owner, owner_name, description, initial }: Props) {
+export default function ExtensionsGroup({ owner, owner_name, description, initial, variant = "tools" }: Props) {
     const extensions = useExtensions({ owner, owner_name, initial });
     const id = "bot_id" in owner ? `bot-${owner.bot_id}` : `workspace-${owner.workspace_id}`;
+    const is_knowledge = variant == "knowledgebases";
+    const fits = (item: { kind: string }) => (item.kind == "knowledgebase") == is_knowledge;
+    const attached = extensions.attached.filter(fits);
+    const addable = extensions.addable.filter(fits);
 
     return (
-        <SettingsGroup id="tools" title="Tools, MCP servers and skills" description={description}>
-            <SettingsList is_empty={!extensions.attached.length} empty="No tools, MCP servers or skills yet.">
-                {extensions.attached.map((item) => (
+        <SettingsGroup
+            id={is_knowledge ? "knowledgebases" : "tools"}
+            title={is_knowledge ? "Knowledgebases" : "Tools, MCP servers and skills"}
+            description={description}
+        >
+            <SettingsList
+                is_empty={!attached.length}
+                empty={is_knowledge ? "No knowledgebases yet." : "No tools, MCP servers or skills yet."}
+            >
+                {attached.map((item) => (
                     <ExtensionRow
                         key={`${item.kind}:${item.id}`}
                         item={item}
@@ -42,12 +58,27 @@ export default function ExtensionsGroup({ owner, owner_name, description, initia
             </SettingsList>
             {extensions.can_manage ? (
                 <ExtensionAddSelect
-                    id={`${id}-add-extension`}
-                    addable={extensions.addable}
+                    id={`${id}-add-${is_knowledge ? "knowledgebase" : "extension"}`}
+                    noun={is_knowledge ? "knowledgebase" : undefined}
+                    addable={addable}
                     loading={Boolean(extensions.busy)}
                     onAdd={extensions.add}
                 />
             ) : null}
+            {is_knowledge ? (
+                <Button
+                    title="Manage knowledgebases"
+                    type="button"
+                    variant="ghost"
+                    color="gray"
+                    size="small"
+                    href="/admin/settings/knowledgebases"
+                    beforeIcon={<Library size={14} aria-hidden />}
+                    className="self-start gap-1!"
+                >
+                    Knowledgebases
+                </Button>
+            ) : (
             <Row className="gap-1!">
                 <Button
                     title="Manage MCP servers"
@@ -86,6 +117,7 @@ export default function ExtensionsGroup({ owner, owner_name, description, initia
                     Skills
                 </Button>
             </Row>
+            )}
             <SettingsStatus status={extensions.status} />
         </SettingsGroup>
     );

src/pages/admin/settings/bots/[bot_id]/(sections)/bot-settings-section.tsx

@@ -77,6 +77,13 @@ export default function BotSettingsSection() {
                 description="What it can use besides its own tools, everywhere it works: MCP servers, tools and skills. A workspace's own come on top while it works there."
                 initial={pageProps?.bot_extensions}
             />
+            <ExtensionsGroup
+                variant="knowledgebases"
+                owner={{ bot_id: bot.id }}
+                owner_name={name}
+                description="The documents it searches before it answers, everywhere it works. A workspace's own come on top while it works there."
+                initial={pageProps?.bot_extensions}
+            />
             <ScopedEnvsGroup initial={pageProps?.bot_envs} />
             {is_coordinator ? null : <BotDeleteGroup bot={bot} />}
         </SettingsShell>

src/pages/admin/settings/knowledgebases/(data)/index-state-marks.ts

@@ -0,0 +1,33 @@
+import { CircleAlert, CircleCheck, LoaderCircle, TriangleAlert, type LucideIcon } from "lucide-react";
+import type { KnowledgebaseIndexState } from "@/src/utils/knowledgebase-index-state";
+
+/**
+ * The mark beside a knowledgebase or document for each index state, and
+ * what it says on hover. Green only when bots search all of it by meaning.
+ */
+export const IndexStateMarks: Record<
+    Exclude<KnowledgebaseIndexState, "empty">,
+    { Icon: LucideIcon; label: string; className: string[] }
+> = {
+    indexing: { Icon: LoaderCircle, label: "Being read", className: ["animate-spin", "text-ink-secondary"] },
+    failed: {
+        Icon: TriangleAlert,
+        label: "Something couldn't be read",
+        className: ["text-warning", "dark:text-warning-dark"],
+    },
+    words: {
+        Icon: CircleCheck,
+        label: "Read, found by words only: no embedding model is set",
+        className: ["text-ink-disabled"],
+    },
+    stale: {
+        Icon: CircleAlert,
+        label: "Read with another embedding model: needs re-indexing",
+        className: ["text-ink-disabled"],
+    },
+    indexed: {
+        Icon: CircleCheck,
+        label: "Indexed: bots search all of it by meaning",
+        className: ["text-success", "dark:text-success-dark"],
+    },
+};

src/pages/admin/settings/knowledgebases/(functions)/knowledgebase-document-summary.ts

@@ -0,0 +1,39 @@
+import type { KnowledgebaseDocument } from "@/src/types";
+import formatBytes from "@/src/utils/format-bytes";
+import checkedAgo from "../../(utils)/checked-ago";
+
+type Params = {
+    document: KnowledgebaseDocument;
+    /**
+     * The install's embedding model now, to tell a document embedded by
+     * another one.
+     */
+    embedding_model: string | null;
+};
+
+/**
+ * One document's line: what it is, its size, how much of it is searchable
+ * and when it was added. "Text · 3.1 KB · 4 passages · Added 5m ago".
+ */
+export default function knowledgebaseDocumentSummary({ document, embedding_model }: Params) {
+    const kind = document.source == "text" ? "Text" : (document.name.split(".").pop() || "File").toUpperCase();
+
+    const state = document.indexing
+        ? "Being read"
+        : document.ingest_error
+          ? document.ingest_error
+          : !document.embedding_model
+            ? `${document.chunk_count} ${document.chunk_count == 1 ? "passage" : "passages"}, by words only`
+            : document.embedding_model != embedding_model
+              ? `${document.chunk_count} ${document.chunk_count == 1 ? "passage" : "passages"}, needs re-indexing`
+              : `${document.chunk_count} ${document.chunk_count == 1 ? "passage" : "passages"}`;
+
+    return [
+        kind,
+        formatBytes({ bytes: document.size }),
+        state,
+        document.created_at ? `Added ${checkedAgo({ at: document.created_at })}` : "",
+    ]
+        .filter(Boolean)
+        .join(" · ");
+}

src/pages/admin/settings/knowledgebases/(functions)/knowledgebase-summary.test.ts

@@ -0,0 +1,21 @@
+import { describe, expect, test } from "bun:test";
+import knowledgebaseSummary from "./knowledgebase-summary";
+
+const empty = { document_count: 0, pending_count: 0, error_count: 0, stale_count: 0, size: 0 };
+
+describe("knowledgebaseSummary", () => {
+    test("an empty knowledgebase says so", () => {
+        expect(knowledgebaseSummary({ knowledgebase: empty })).toBe("No documents yet");
+    });
+
+    test("counts documents and names only the problems there are", () => {
+        expect(knowledgebaseSummary({ knowledgebase: { ...empty, document_count: 1, size: 2048 } })).toBe(
+            "1 document · 2.0 KB",
+        );
+        expect(
+            knowledgebaseSummary({
+                knowledgebase: { document_count: 3, pending_count: 1, error_count: 1, stale_count: 2, size: 2048 },
+            }),
+        ).toBe("3 documents · 2.0 KB · 1 being read · 1 couldn't be read · 2 need re-indexing");
+    });
+});

src/pages/admin/settings/knowledgebases/(functions)/knowledgebase-summary.ts

@@ -0,0 +1,29 @@
+import type { PublicKnowledgebase } from "@/src/types";
+import formatBytes from "@/src/utils/format-bytes";
+
+type Params = {
+    knowledgebase: Pick<
+        PublicKnowledgebase,
+        "document_count" | "pending_count" | "error_count" | "stale_count" | "size"
+    >;
+};
+
+/**
+ * What a knowledgebase holds and how much of it bots can search, in a few
+ * words: "3 documents · 1.2 MB · 1 being read · 1 needs re-indexing".
+ */
+export default function knowledgebaseSummary({ knowledgebase }: Params) {
+    const { document_count, pending_count, error_count, stale_count, size } = knowledgebase;
+
+    if (!document_count) return "No documents yet";
+
+    return [
+        `${document_count} ${document_count == 1 ? "document" : "documents"}`,
+        formatBytes({ bytes: size }),
+        pending_count ? `${pending_count} being read` : "",
+        error_count ? `${error_count} couldn't be read` : "",
+        stale_count ? `${stale_count} ${stale_count == 1 ? "needs" : "need"} re-indexing` : "",
+    ]
+        .filter(Boolean)
+        .join(" · ");
+}

src/pages/admin/settings/knowledgebases/(functions)/read-file-data-url.ts

@@ -0,0 +1,16 @@
+type Params = {
+    file: File;
+};
+
+/**
+ * A picked file as a data URL, for an upload. Empty when the browser can't
+ * read it.
+ */
+export default async function readFileDataUrl({ file }: Params) {
+    return await new Promise<string>((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(String(reader.result || ""));
+        reader.onerror = () => reject(reader.error);
+        reader.readAsDataURL(file);
+    }).catch(() => "");
+}

src/pages/admin/settings/knowledgebases/(hooks)/use-knowledgebase-documents.ts

@@ -0,0 +1,111 @@
+import { useEffect, useState } from "react";
+import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
+import type { KnowledgebaseDocument, KnowledgebaseDocumentsPayload } from "@/src/types";
+import readFileDataUrl from "../(functions)/read-file-data-url";
+
+type Params = {
+    knowledgebase_id: number;
+};
+
+type Status = { error?: boolean; success?: boolean; msg?: string };
+
+/**
+ * How often the list is read again while a document is still being read.
+ */
+const PollMs = 4000;
+
+/**
+ * A knowledgebase's documents, loaded when they're opened and read again
+ * every few seconds while one is still waiting to be read, with uploading,
+ * writing and deleting them (each asks first).
+ */
+export default function useKnowledgebaseDocuments({ knowledgebase_id }: Params) {
+    const [documents, setDocuments] = useState<KnowledgebaseDocument[]>();
+    const [busy, setBusy] = useState(false);
+    const [status, setStatus] = useState<Status>();
+
+    const load = async () => {
+        const res = await postAdminApi<KnowledgebaseDocumentsPayload>({
+            path: "knowledgebases/documents",
+            body: { knowledgebase_id },
+        });
+
+        if (res.success && res.singleRes) setDocuments(res.singleRes.documents);
+        else setStatus({ error: true, msg: res.msg || "Couldn't load the documents." });
+    };
+
+    useEffect(() => {
+        load();
+    }, [knowledgebase_id]);
+
+    const waiting = Boolean(documents?.some((document) => document.indexing));
+
+    useEffect(() => {
+        if (!waiting) return;
+
+        const timer = setInterval(load, PollMs);
+
+        return () => clearInterval(timer);
+    }, [waiting, knowledgebase_id]);
+
+    const post = async ({ path, body }: { path: string; body: Record<string, unknown> }) => {
+        setBusy(true);
+        setStatus(undefined);
+
+        const res = await postAdminApi<KnowledgebaseDocumentsPayload>({ path, body });
+
+        setBusy(false);
+
+        if (!res.success || !res.singleRes) {
+            setStatus({ error: true, msg: res.msg || "That didn't work." });
+            return false;
+        }
+
+        setDocuments(res.singleRes.documents);
+        setStatus({ success: true, msg: res.msg || "Saved." });
+        return true;
+    };
+
+    const upload = async ({ files }: { files: File[] }) => {
+        if (!files.length || busy) return;
+
+        const names = files.map((file) => file.name).join(", ");
+
+        if (!window.confirm(`Add ${names} to this knowledgebase?`)) return;
+
+        for (const file of files) {
+            const data_url = await readFileDataUrl({ file });
+
+            if (!data_url) {
+                setStatus({ error: true, msg: `${file.name} couldn't be read.` });
+                return;
+            }
+
+            const done = await post({
+                path: "knowledgebases/upload-document",
+                body: { knowledgebase_id, name: file.name, data_url },
+            });
+
+            if (!done) return;
+        }
+    };
+
+    const saveText = async ({ document_id, name, content }: { document_id?: number; name: string; content: string }) => {
+        if (busy || !window.confirm(document_id ? `Save ${name || "this document"}?` : `Add ${name || "this document"}?`)) {
+            return false;
+        }
+
+        return await post({
+            path: "knowledgebases/save-text-document",
+            body: { knowledgebase_id, document_id, name, content },
+        });
+    };
+
+    const remove = async ({ document }: { document: KnowledgebaseDocument }) => {
+        if (busy || !window.confirm(`Delete ${document.name}? Bots stop finding it at once.`)) return;
+
+        await post({ path: "knowledgebases/delete-document", body: { document_id: document.id } });
+    };
+
+    return { documents, busy, status, upload, saveText, remove };
+}

src/pages/admin/settings/knowledgebases/(partials)/embedding-model-note.tsx

@@ -0,0 +1,31 @@
+import Link from "@moduletrace/twui/layout/Link";
+import Span from "@moduletrace/twui/layout/Span";
+
+type Props = {
+    /**
+     * The install's embedding model, null when none is set.
+     */
+    model: string | null;
+};
+
+/**
+ * Which model turns every knowledgebase into vectors: one for the whole
+ * install, picked under Settings > Models. Without one, bots still find
+ * passages by their words. Picking or changing it reads every document
+ * again (`reindexAfterEmbeddingChange`).
+ */
+export default function EmbeddingModelNote({ model }: Props) {
+    return (
+        <Span className="text-[13px] leading-[18px] text-ink-secondary">
+            {model
+                ? `Searched by meaning with ${model}, the install's embedding model. Change it under `
+                : "No embedding model is set, so bots find passages by their words only, which misses questions worded differently. Pick one under "}
+            <Link href="/admin/settings/models" title="Models settings" className="text-ink underline">
+                Settings, Models
+            </Link>
+            {model
+                ? "; every document is then read again with the new one."
+                : "; every document is then read again with it."}
+        </Span>
+    );
+}

src/pages/admin/settings/knowledgebases/(partials)/index-state-mark.tsx

@@ -0,0 +1,28 @@
+import { twMerge } from "tailwind-merge";
+import type { KnowledgebaseIndexState } from "@/src/utils/knowledgebase-index-state";
+import { IndexStateMarks } from "../(data)/index-state-marks";
+
+type Props = {
+    state: KnowledgebaseIndexState;
+    /**
+     * Said on hover instead of the state's own words.
+     */
+    label?: string;
+};
+
+/**
+ * A spinner, check or warning beside a knowledgebase or document's name:
+ * how far it's indexed (`knowledgebaseIndexState`).
+ */
+export default function IndexStateMark({ state, label }: Props) {
+    if (state == "empty") return null;
+
+    const { Icon, label: own, className } = IndexStateMarks[state];
+    const text = label || own;
+
+    return (
+        <Icon size={15} role="img" aria-label={text} className={twMerge("shrink-0 inline-block align-middle", ...className)}>
+            <title>{text}</title>
+        </Icon>
+    );
+}

src/pages/admin/settings/knowledgebases/(partials)/knowledgebase-document-row.tsx

@@ -0,0 +1,54 @@
+import { FileText, NotebookPen, Pencil, Trash2 } from "lucide-react";
+import Row from "@moduletrace/twui/layout/Row";
+import IconTile from "@/src/components/general/icon-tile";
+import RowIconButton from "@/src/components/general/row-icon-button";
+import type { KnowledgebaseDocument } from "@/src/types";
+import SettingsListRow from "../../(partials)/settings-list-row";
+import knowledgebaseIndexState from "@/src/utils/knowledgebase-index-state";
+import knowledgebaseDocumentSummary from "../(functions)/knowledgebase-document-summary";
+import IndexStateMark from "./index-state-mark";
+
+type Props = {
+    document: KnowledgebaseDocument;
+    embedding_model: string | null;
+    busy?: boolean;
+    /**
+     * Opens a text document to edit; not given for a file or when the user
+     * can't edit.
+     */
+    onEdit?: () => void;
+    onDelete?: () => void;
+};
+
+/**
+ * One document in a knowledgebase: what it is and how much of it bots can
+ * search, or why it couldn't be read.
+ */
+export default function KnowledgebaseDocumentRow({ document, embedding_model, busy, onEdit, onDelete }: Props) {
+    return (
+        <SettingsListRow
+            avatar={<IconTile Icon={document.source == "text" ? NotebookPen : FileText} />}
+            title={document.name}
+            title_icon={
+                <IndexStateMark
+                    state={knowledgebaseIndexState({
+                        total: 1,
+                        indexing: document.indexing ? 1 : 0,
+                        failed: document.ingest_error ? 1 : 0,
+                        stale: document.embedding_model && document.embedding_model == embedding_model ? 0 : 1,
+                        embedding_model,
+                    })}
+                    label={document.ingest_error && !document.indexing ? document.ingest_error : undefined}
+                />
+            }
+            subtitle={knowledgebaseDocumentSummary({ document, embedding_model })}
+        >
+            <Row className="gap-0! flex-nowrap">
+                {onEdit ? <RowIconButton icon={Pencil} title={`Edit ${document.name}`} disabled={busy} onClick={onEdit} /> : null}
+                {onDelete ? (
+                    <RowIconButton icon={Trash2} title={`Delete ${document.name}`} disabled={busy} onClick={onDelete} />
+                ) : null}
+            </Row>
+        </SettingsListRow>
+    );
+}

src/pages/admin/settings/knowledgebases/(partials)/knowledgebase-documents-modal.tsx

@@ -0,0 +1,133 @@
+import { useRef, useState } from "react";
+import Button from "@moduletrace/twui/layout/Button";
+import Loading from "@moduletrace/twui/elements/Loading";
+import Row from "@moduletrace/twui/layout/Row";
+import { NotebookPen, Upload } from "lucide-react";
+import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
+import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
+import type { PublicKnowledgebase } from "@/src/types";
+import SettingsList from "../../(partials)/settings-list";
+import SettingsStatus from "../../(partials)/settings-status";
+import useKnowledgebaseDocuments from "../(hooks)/use-knowledgebase-documents";
+import KnowledgebaseDocumentRow from "./knowledgebase-document-row";
+import KnowledgebaseTextForm from "./knowledgebase-text-form";
+
+type Props = {
+    knowledgebase: PublicKnowledgebase;
+    /**
+     * A super admin: adds, writes and deletes documents. Admins only read
+     * the list.
+     */
+    can_edit: boolean;
+    /**
+     * The install's embedding model, to tell a document that needs
+     * re-indexing.
+     */
+    embedding_model: string | null;
+    onClose: () => void;
+};
+
+/**
+ * A knowledgebase's documents: each with how much of it is searchable,
+ * files to upload, and text to write straight in. A document is read
+ * (chunked and embedded) in the background; the list follows until it's
+ * done.
+ */
+export default function KnowledgebaseDocumentsModal({ knowledgebase, can_edit, embedding_model, onClose }: Props) {
+    const { documents, busy, status, upload, saveText, remove } = useKnowledgebaseDocuments({
+        knowledgebase_id: knowledgebase.id,
+    });
+    /**
+     * `new` writes a text document, a number edits that one.
+     */
+    const [writing, setWriting] = useState<"new" | number>();
+    const file_input = useRef<HTMLInputElement>(null);
+
+    if (writing) {
+        return (
+            <AddModalShell
+                open
+                setOpen={(next) => !next && setWriting(undefined)}
+                title={writing == "new" ? `Write a document in ${knowledgebase.name}` : "Edit the document"}
+                description="Bots search it once it's read, a few moments after it's saved."
+            >
+                <KnowledgebaseTextForm
+                    document_id={writing == "new" ? undefined : writing}
+                    loading={busy}
+                    onSave={saveText}
+                    onCancel={() => setWriting(undefined)}
+                />
+                <SettingsStatus status={status?.error ? status : undefined} />
+            </AddModalShell>
+        );
+    }
+
+    return (
+        <AddModalShell
+            open
+            setOpen={(next) => !next && onClose()}
+            title={`Documents in ${knowledgebase.name}`}
+            description={`Upload ${BotTextFileExtensions.join(", ")} files, or write text straight in. Each is split into passages bots search.`}
+        >
+            {documents ? (
+                <SettingsList is_empty={!documents.length} empty="No documents yet.">
+                    {documents.map((document) => (
+                        <KnowledgebaseDocumentRow
+                            key={document.id}
+                            document={document}
+                            embedding_model={embedding_model}
+                            busy={busy}
+                            onEdit={can_edit && document.source == "text" ? () => setWriting(document.id) : undefined}
+                            onDelete={can_edit ? () => remove({ document }) : undefined}
+                        />
+                    ))}
+                </SettingsList>
+            ) : (
+                <Loading />
+            )}
+            {can_edit ? (
+                <Row className="gap-1!">
+                    <input
+                        ref={file_input}
+                        type="file"
+                        multiple
+                        accept={BotTextFileExtensions.map((ext) => `.${ext}`).join(",")}
+                        className="hidden"
+                        onChange={(event) => {
+                            const files = Array.from(event.target.files || []);
+                            upload({ files }).catch(() => {});
+                            event.target.value = "";
+                        }}
+                    />
+                    <Button
+                        title="Upload files"
+                        type="button"
+                        variant="ghost"
+                        color="gray"
+                        size="small"
+                        loading={busy}
+                        onClick={() => file_input.current?.click()}
+                        beforeIcon={<Upload size={14} aria-hidden />}
+                        className="gap-1!"
+                    >
+                        Upload files
+                    </Button>
+                    <Button
+                        title="Write a document"
+                        type="button"
+                        variant="ghost"
+                        color="gray"
+                        size="small"
+                        disabled={busy}
+                        onClick={() => setWriting("new")}
+                        beforeIcon={<NotebookPen size={14} aria-hidden />}
+                        className="gap-1!"
+                    >
+                        Write a document
+                    </Button>
+                </Row>
+            ) : null}
+            <SettingsStatus status={status} />
+        </AddModalShell>
+    );
+}

src/pages/admin/settings/knowledgebases/(partials)/knowledgebase-modal.tsx

@@ -0,0 +1,107 @@
+import { useState } from "react";
+import AddFormActions from "@/src/layouts/bots/(partials)/add-form-actions";
+import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
+import Checkbox from "@moduletrace/twui/form/Checkbox";
+import Form from "@moduletrace/twui/form/Form";
+import Input from "@moduletrace/twui/form/Input";
+import Textarea from "@moduletrace/twui/form/Textarea";
+import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
+import type { KnowledgebasesOverviewPayload, PublicKnowledgebase } from "@/src/types";
+
+type Props = {
+    /**
+     * Edits this knowledgebase; adds one when not given.
+     */
+    knowledgebase?: PublicKnowledgebase;
+    onSaved: (overview: KnowledgebasesOverviewPayload, msg: string) => void;
+    onCancel: () => void;
+};
+
+/**
+ * Adds or edits a knowledgebase: its name, the one line that tells a bot
+ * what it holds, whether it's on, and whether every bot gets it. Its
+ * documents are added from its row.
+ */
+export default function KnowledgebaseModal({ knowledgebase, onSaved, onCancel }: Props) {
+    const [name, setName] = useState(knowledgebase?.name || "");
+    const [description, setDescription] = useState(knowledgebase?.description || "");
+    const [enabled, setEnabled] = useState(knowledgebase?.enabled ?? true);
+    const [all_bots, setAllBots] = useState(knowledgebase?.access.all_bots ?? false);
+    const [loading, setLoading] = useState(false);
+    const [error, setError] = useState<string>();
+
+    const id = `knowledgebase-${knowledgebase?.id || "new"}`;
+
+    const submit = async () => {
+        if (
+            loading ||
+            !window.confirm(knowledgebase ? `Save ${knowledgebase.name}?` : `Add ${name || "this knowledgebase"}?`)
+        ) {
+            return;
+        }
+
+        setLoading(true);
+        setError(undefined);
+
+        const res = await postAdminApi<KnowledgebasesOverviewPayload>({
+            path: "knowledgebases/save-knowledgebase",
+            body: { knowledgebase_id: knowledgebase?.id, name, description, enabled, all_bots },
+        });
+
+        setLoading(false);
+
+        if (!res.success || !res.singleRes) {
+            setError(res.msg || "Couldn't save the knowledgebase.");
+            return;
+        }
+
+        onSaved(res.singleRes, res.msg || "Saved.");
+    };
+
+    return (
+        <AddModalShell
+            open
+            setOpen={(next) => !next && onCancel()}
+            title={knowledgebase ? `Edit ${knowledgebase.name}` : "Add a knowledgebase"}
+            description="Bots search it once it's attached to them. Add its documents from its row."
+        >
+            <Form className="w-full gap-4! items-stretch pt-4" onSubmit={submit}>
+                <Input
+                    id={`${id}-name`}
+                    title="Name"
+                    label="Name"
+                    showLabel
+                    placeholder="Product handbook"
+                    defaultValue={name}
+                    autoComplete="off"
+                    info="What the settings page and the composer call it."
+                    onChange={(e) => setName(e.target.value)}
+                />
+                <Textarea
+                    id={`${id}-description`}
+                    title="What it holds"
+                    label="What it holds"
+                    showLabel
+                    rows={2}
+                    placeholder="Pricing, plans and refund rules, as the sales team uses them."
+                    defaultValue={description}
+                    info="The one line a bot reads to know when to search it. Say what's in it."
+                    onChange={(e) => setDescription(e.target.value)}
+                />
+                <Checkbox title="On" label="On: bots search it" checked={enabled} changeHandler={setEnabled} />
+                <Checkbox
+                    title="Every bot"
+                    label="Every bot by default: every bot now and every new one, the coordinator included. Each can still drop it."
+                    checked={all_bots}
+                    changeHandler={setAllBots}
+                />
+                <AddFormActions
+                    submit_label={knowledgebase ? "Save" : "Add knowledgebase"}
+                    loading={loading}
+                    error={error}
+                    onCancel={onCancel}
+                />
+            </Form>
+        </AddModalShell>
+    );
+}

src/pages/admin/settings/knowledgebases/(partials)/knowledgebase-row.tsx

@@ -0,0 +1,169 @@
+import { useState } from "react";
+import Row from "@moduletrace/twui/layout/Row";
+import Stack from "@moduletrace/twui/layout/Stack";
+import { FileText, Library, Pencil, RefreshCw, Trash2, Users } from "lucide-react";
+import IconTile from "@/src/components/general/icon-tile";
+import RowIconButton from "@/src/components/general/row-icon-button";
+import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
+import type { ExtensionTargets, KnowledgebasesOverviewPayload, PublicKnowledgebase } from "@/src/types";
+import ExtensionAccessModal from "../../(partials)/extension-access-modal";
+import SettingsListRow from "../../(partials)/settings-list-row";
+import extensionAccessLabel from "../../(utils)/extension-access-label";
+import updatedAgo from "../../(utils)/updated-ago";
+import knowledgebaseIndexState from "@/src/utils/knowledgebase-index-state";
+import knowledgebaseSummary from "../(functions)/knowledgebase-summary";
+import IndexStateMark from "./index-state-mark";
+import KnowledgebaseDocumentsModal from "./knowledgebase-documents-modal";
+import KnowledgebaseModal from "./knowledgebase-modal";
+
+type Props = {
+    knowledgebase: PublicKnowledgebase;
+    targets: ExtensionTargets;
+    /**
+     * The install's embedding model, for the documents' lines.
+     */
+    embedding_model: string | null;
+    onChanged: (overview: KnowledgebasesOverviewPayload, msg: string) => void;
+    onError: (msg: string) => void;
+};
+
+/**
+ * One knowledgebase: how many documents it holds and how much of them is
+ * searchable, and who gets it. Documents, Edit and Access open in a modal;
+ * Re-index reads every document again with the install's embedding model.
+ */
+export default function KnowledgebaseRow({ knowledgebase, targets, embedding_model, onChanged, onError }: Props) {
+    const [open, setOpen] = useState<"documents" | "edit" | "access">();
+    const [busy, setBusy] = useState(false);
+
+    const saved = (overview: KnowledgebasesOverviewPayload, msg: string) => {
+        setOpen(undefined);
+        onChanged(overview, msg);
+    };
+
+    const post = async ({ path, confirm }: { path: string; confirm: string }) => {
+        if (busy || !window.confirm(confirm)) return;
+
+        setBusy(true);
+        const res = await postAdminApi<KnowledgebasesOverviewPayload>({
+            path,
+            body: { knowledgebase_id: knowledgebase.id },
+        });
+        setBusy(false);
+
+        if (res.success && res.singleRes) onChanged(res.singleRes, res.msg || "Done.");
+        else onError(res.msg || "That didn't work.");
+    };
+
+    /**
+     * The documents changed while open: their counts on the row are
+     * read again.
+     */
+    const closeDocuments = async () => {
+        setOpen(undefined);
+
+        const res = await postAdminApi<KnowledgebasesOverviewPayload>({ path: "knowledgebases/overview", body: {} });
+
+        if (res.success && res.singleRes) onChanged(res.singleRes, "");
+    };
+
+    return (
+        <Stack className="w-full gap-0! items-stretch border-0 border-b border-solid border-hairline last:border-b-0">
+            <SettingsListRow
+                avatar={<IconTile Icon={Library} />}
+                title={knowledgebase.name}
+                badge={knowledgebase.enabled ? undefined : "Off"}
+                title_icon={
+                    <IndexStateMark
+                        state={knowledgebaseIndexState({
+                            total: knowledgebase.document_count,
+                            indexing: knowledgebase.pending_count,
+                            failed: knowledgebase.warning_count,
+                            stale: knowledgebase.stale_count,
+                            embedding_model,
+                        })}
+                    />
+                }
+                subtitle={[
+                    knowledgebaseSummary({ knowledgebase }),
+                    extensionAccessLabel({ access: knowledgebase.access }),
+                    updatedAgo({ at: knowledgebase.updated_at }),
+                    knowledgebase.description,
+                ]
+                    .filter(Boolean)
+                    .join(" · ")}
+            >
+                <Row className="gap-0! flex-nowrap">
+                    <RowIconButton
+                        icon={FileText}
+                        title={`Open the documents in ${knowledgebase.name}`}
+                        disabled={busy}
+                        onClick={() => setOpen("documents")}
+                    />
+                    {targets.can_edit ? (
+                        <RowIconButton
+                            icon={Pencil}
+                            title={`Edit ${knowledgebase.name}`}
+                            disabled={busy}
+                            onClick={() => setOpen("edit")}
+                        />
+                    ) : null}
+                    <RowIconButton
+                        icon={Users}
+                        title={`Choose who gets ${knowledgebase.name}`}
+                        disabled={busy}
+                        onClick={() => setOpen("access")}
+                    />
+                    {targets.can_edit && knowledgebase.document_count ? (
+                        <RowIconButton
+                            icon={RefreshCw}
+                            title={`Re-index ${knowledgebase.name}: read every document again`}
+                            disabled={busy}
+                            onClick={() =>
+                                post({
+                                    path: "knowledgebases/reindex-knowledgebase",
+                                    confirm: `Read every document in ${knowledgebase.name} again? Bots keep finding the old passages until the new ones are in.`,
+                                })
+                            }
+                        />
+                    ) : null}
+                    {targets.can_edit ? (
+                        <RowIconButton
+                            icon={Trash2}
+                            title={`Delete ${knowledgebase.name}`}
+                            disabled={busy}
+                            onClick={() =>
+                                post({
+                                    path: "knowledgebases/delete-knowledgebase",
+                                    confirm: `Delete ${knowledgebase.name} and every document in it? Bots lose it at once.`,
+                                })
+                            }
+                        />
+                    ) : null}
+                </Row>
+            </SettingsListRow>
+            {open == "documents" ? (
+                <KnowledgebaseDocumentsModal
+                    knowledgebase={knowledgebase}
+                    can_edit={targets.can_edit}
+                    embedding_model={embedding_model}
+                    onClose={closeDocuments}
+                />
+            ) : null}
+            {open == "edit" ? (
+                <KnowledgebaseModal knowledgebase={knowledgebase} onSaved={saved} onCancel={() => setOpen(undefined)} />
+            ) : null}
+            {open == "access" ? (
+                <ExtensionAccessModal<KnowledgebasesOverviewPayload>
+                    id={knowledgebase.id}
+                    name={knowledgebase.name}
+                    access={knowledgebase.access}
+                    targets={targets}
+                    path="knowledgebases/set-knowledgebase-access"
+                    onSaved={saved}
+                    onCancel={() => setOpen(undefined)}
+                />
+            ) : null}
+        </Stack>
+    );
+}

src/pages/admin/settings/knowledgebases/(partials)/knowledgebase-text-form.tsx

@@ -0,0 +1,81 @@
+import { useEffect, useState } from "react";
+import Form from "@moduletrace/twui/form/Form";
+import Input from "@moduletrace/twui/form/Input";
+import Loading from "@moduletrace/twui/elements/Loading";
+import CodeEditor from "@/src/components/general/code-editor";
+import { AppData } from "@/src/data/app-data";
+import AddFormActions from "@/src/layouts/bots/(partials)/add-form-actions";
+import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
+
+type Props = {
+    /**
+     * Edits this text document (its content loads first); writes a new one
+     * when not given.
+     */
+    document_id?: number;
+    loading?: boolean;
+    onSave: (params: { document_id?: number; name: string; content: string }) => Promise<boolean>;
+    onCancel: () => void;
+};
+
+/**
+ * Writes a document straight into a knowledgebase, in markdown: a title
+ * and the text bots search.
+ */
+export default function KnowledgebaseTextForm({ document_id, loading, onSave, onCancel }: Props) {
+    const [name, setName] = useState("");
+    const [content, setContent] = useState("");
+    const [ready, setReady] = useState(!document_id);
+    const [error, setError] = useState<string>();
+
+    useEffect(() => {
+        if (!document_id) return;
+
+        postAdminApi<{ document_id: number; name: string; content: string }>({
+            path: "knowledgebases/document-text",
+            body: { document_id },
+        }).then((res) => {
+            if (res.success && res.singleRes) {
+                setName(res.singleRes.name);
+                setContent(res.singleRes.content);
+                setReady(true);
+            } else {
+                setError(res.msg || "Couldn't open the document.");
+            }
+        });
+    }, [document_id]);
+
+    if (!ready) {
+        return error ? <AddFormActions submit_label="Save" error={error} onCancel={onCancel} /> : <Loading />;
+    }
+
+    return (
+        <Form
+            className="w-full gap-4! items-stretch"
+            onSubmit={async () => {
+                if (await onSave({ document_id, name, content })) onCancel();
+            }}
+        >
+            <Input
+                id={`knowledgebase-text-${document_id || "new"}-name`}
+                title="Title"
+                label="Title"
+                showLabel
+                placeholder="Refund rules"
+                defaultValue={name}
+                autoComplete="off"
+                info="What bots see as the document's name when they find a passage in it."
+                onChange={(e) => setName(e.target.value)}
+            />
+            <CodeEditor
+                mode="markdown"
+                className="h-80"
+                title="Text"
+                content={content}
+                info={`Markdown, up to ${AppData.KnowledgebaseTextMaxChars} characters. Headings help: a passage carries the heading above it.`}
+                onChange={(value) => setContent(value)}
+            />
+            <AddFormActions submit_label={document_id ? "Save" : "Add document"} loading={loading} onCancel={onCancel} />
+        </Form>
+    );
+}

src/pages/admin/settings/knowledgebases/(sections)/knowledgebases-settings-section.tsx

@@ -0,0 +1,107 @@
+import { useContext, useEffect, useState } from "react";
+import Button from "@moduletrace/twui/layout/Button";
+import { Plus } from "lucide-react";
+import { AppContext } from "@/src/pages/__root";
+import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import SettingsGroup from "../../(partials)/settings-group";
+import SettingsList from "../../(partials)/settings-list";
+import SettingsStatus from "../../(partials)/settings-status";
+import EmbeddingModelNote from "../(partials)/embedding-model-note";
+import KnowledgebaseModal from "../(partials)/knowledgebase-modal";
+import KnowledgebaseRow from "../(partials)/knowledgebase-row";
+
+type Status = { error?: boolean; success?: boolean; msg?: string };
+
+/**
+ * How often the list is read again while a document is being read.
+ */
+const PollMs = 4000;
+
+/**
+ * Every knowledgebase with who gets it. Super admins add, fill, edit and
+ * delete them; admins choose which bots and workspaces get each one.
+ */
+export default function KnowledgebasesSettingsSection() {
+    const { pageProps } = useContext(AppContext);
+    const [overview, setOverview] = useState(pageProps?.knowledgebases_overview);
+    const [adding, setAdding] = useState(false);
+    const [status, setStatus] = useState<Status>();
+    /**
+     * Remounts the rows when the data changes under them.
+     */
+    const [version, setVersion] = useState(0);
+
+    const reading = Boolean(overview?.knowledgebases.some((knowledgebase) => knowledgebase.pending_count));
+
+    /**
+     * Kept current while documents are read, so the marks beside them
+     * turn from spinners into checks. The rows aren't remounted: a modal
+     * open on one stays.
+     */
+    useEffect(() => {
+        if (!reading) return;
+
+        const timer = setInterval(async () => {
+            const res = await postAdminApi<KnowledgebasesOverviewPayload>({ path: "knowledgebases/overview", body: {} });
+
+            if (res.success && res.singleRes) setOverview(res.singleRes);
+        }, PollMs);
+
+        return () => clearInterval(timer);
+    }, [reading]);
+
+    if (!overview) return null;
+
+    const changed = (next: KnowledgebasesOverviewPayload, msg: string) => {
+        setOverview(next);
+        setStatus(msg ? { success: true, msg } : undefined);
+        setVersion((prev) => prev + 1);
+    };
+    const failed = (msg: string) => setStatus({ error: true, msg });
+
+    return (
+        <SettingsGroup
+            title="Knowledgebases"
+            description="A set of documents (handbooks, policies, product notes) a bot searches for the passages that answer a question. Before every reply it's handed the passages that fit the newest message, and it can search again while it works. Give one to a bot on its settings page or in its chat, or to a workspace to reach every bot there."
+        >
+            <EmbeddingModelNote model={overview.embedding_model} />
+            {adding ? (
+                <KnowledgebaseModal
+                    onSaved={(next, msg) => {
+                        setAdding(false);
+                        changed(next, msg);
+                    }}
+                    onCancel={() => setAdding(false)}
+                />
+            ) : null}
+            <SettingsList is_empty={!overview.knowledgebases.length} empty="No knowledgebases yet.">
+                {overview.knowledgebases.map((knowledgebase) => (
+                    <KnowledgebaseRow
+                        key={`${knowledgebase.id}-${version}`}
+                        knowledgebase={knowledgebase}
+                        targets={overview}
+                        embedding_model={overview.embedding_model}
+                        onChanged={changed}
+                        onError={failed}
+                    />
+                ))}
+            </SettingsList>
+            {overview.can_edit && !adding ? (
+                <Button
+                    title="Add a knowledgebase"
+                    type="button"
+                    variant="ghost"
+                    color="gray"
+                    size="small"
+                    onClick={() => setAdding(true)}
+                    className="self-start gap-1!"
+                >
+                    <Plus size={14} aria-hidden />
+                    New knowledgebase
+                </Button>
+            ) : null}
+            <SettingsStatus status={status} />
+        </SettingsGroup>
+    );
+}

src/pages/admin/settings/knowledgebases/index.server.ts

@@ -0,0 +1,21 @@
+import type { PagePropsType } from "@/src/types";
+import type { BunextPageServerFn } from "@moduletrace/bunext/types";
+import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
+import grabCrewSettingsServerProps from "../(functions)/grab-crew-settings-server-props";
+
+/**
+ * Super admins and admins: admins only choose who gets each knowledgebase.
+ * Documents load when a knowledgebase's documents are opened.
+ */
+export const server: BunextPageServerFn<PagePropsType> = async (params) => {
+    const settings = await grabCrewSettingsServerProps(params);
+
+    if (!settings.props?.person?.id) return settings;
+
+    return {
+        props: {
+            ...settings.props,
+            knowledgebases_overview: await grabKnowledgebasesOverview({ user_id: settings.props.person.id }),
+        },
+    };
+};

src/pages/admin/settings/knowledgebases/index.tsx

@@ -0,0 +1,13 @@
+import KnowledgebasesSettingsSection from "./(sections)/knowledgebases-settings-section";
+import SettingsShell from "../(partials)/settings-shell";
+import settingsPageMeta from "../(functions)/settings-page-meta";
+
+export default function AdminSettingsKnowledgebasesPage() {
+    return (
+        <SettingsShell title="Knowledgebases" description="Documents your bots search before they answer.">
+            <KnowledgebasesSettingsSection />
+        </SettingsShell>
+    );
+}
+
+export const meta = settingsPageMeta({ label: "Knowledgebases" });

src/pages/admin/settings/workspaces/[workspace_id]/(sections)/workspace-overview.tsx

@@ -70,6 +70,13 @@ export default function WorkspaceOverview() {
                 description="Every bot working here gets these, the coordinator too, on top of its own, only while it works here: MCP servers, tools and skills."
                 initial={pageProps?.workspace_extensions}
             />
+            <ExtensionsGroup
+                variant="knowledgebases"
+                owner={{ workspace_id }}
+                owner_name={name}
+                description="Every bot working here searches these before it answers, the coordinator too, on top of its own, only while it works here."
+                initial={pageProps?.workspace_extensions}
+            />
             <ScopedEnvsGroup initial={pageProps?.workspace_envs} />
             {harnesses?.length ? (
                 <>

src/pages/api/admin/harness/delete-provider.ts

@@ -1,3 +1,5 @@
+import grabInstallEmbeddingModel from "@/src/functions/backend/db/knowledgebases/grab-install-embedding-model";
+import reindexAfterEmbeddingChange from "@/src/functions/backend/db/knowledgebases/reindex-after-embedding-change";
 import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
 import deleteHarnessProvider from "@/src/functions/backend/db/harness/delete-harness-provider";
 import grabHarnessOverview from "@/src/functions/backend/db/harness/grab-harness-overview";
@@ -35,7 +37,10 @@ export const handler: BunextAPIRouteHandler<
             data: body,
         });
 
+        const embedding_before = await grabInstallEmbeddingModel();
+
         await deleteHarnessProvider({ provider_id });
+        await reindexAfterEmbeddingChange({ before: embedding_before });
 
         await logEvent({
             title: "Model provider removed",

src/pages/api/admin/harness/save-provider.ts

@@ -1,3 +1,5 @@
+import grabInstallEmbeddingModel from "@/src/functions/backend/db/knowledgebases/grab-install-embedding-model";
+import reindexAfterEmbeddingChange from "@/src/functions/backend/db/knowledgebases/reindex-after-embedding-change";
 import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
 import grabHarnessOverview from "@/src/functions/backend/db/harness/grab-harness-overview";
 import saveHarnessProvider from "@/src/functions/backend/db/harness/save-harness-provider";
@@ -36,6 +38,7 @@ export const handler: BunextAPIRouteHandler<
             data: body,
         });
 
+        const embedding_before = await grabInstallEmbeddingModel();
         const provider = await saveHarnessProvider(request);
 
         await logEvent({
@@ -63,6 +66,12 @@ export const handler: BunextAPIRouteHandler<
             }
         }
 
+        const reindexing = await reindexAfterEmbeddingChange({ before: embedding_before });
+
+        if (reindexing) {
+            msg = `${msg} Reading ${reindexing} knowledgebase ${reindexing == 1 ? "document" : "documents"} again with the new embedding model.`;
+        }
+
         return {
             success: true,
             msg,

src/pages/api/admin/harness/save-settings.ts

@@ -1,3 +1,5 @@
+import grabInstallEmbeddingModel from "@/src/functions/backend/db/knowledgebases/grab-install-embedding-model";
+import reindexAfterEmbeddingChange from "@/src/functions/backend/db/knowledgebases/reindex-after-embedding-change";
 import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
 import grabHarnessOverview from "@/src/functions/backend/db/harness/grab-harness-overview";
 import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
@@ -36,6 +38,8 @@ export const handler: BunextAPIRouteHandler<
             data: body,
         });
 
+        const embedding_before = await grabInstallEmbeddingModel();
+
         for (const setting of settings) {
             const existing = await grabHarnessSetting({ harness: setting.harness });
             const { voice, ...options } = existing.options;
@@ -65,9 +69,13 @@ export const handler: BunextAPIRouteHandler<
             ip_address: grabRequestIP(req),
         });
 
+        const reindexing = await reindexAfterEmbeddingChange({ before: embedding_before });
+
         return {
             success: true,
-            msg: `Saved.`,
+            msg: reindexing
+                ? `Saved. Reading ${reindexing} knowledgebase ${reindexing == 1 ? "document" : "documents"} again with the new embedding model.`
+                : `Saved.`,
             singleRes: await grabHarnessOverview(),
         };
     } catch (error: unknown) {

src/pages/api/admin/knowledgebases/delete-document.ts

@@ -0,0 +1,57 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import deleteKnowledgebaseDocument from "@/src/functions/backend/db/knowledgebases/delete-knowledgebase-document";
+import listKnowledgebaseDocuments from "@/src/functions/backend/db/knowledgebases/list-knowledgebase-documents";
+import { touchKnowledgebase } from "@/src/functions/backend/db/knowledgebases/record-knowledgebase-document";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { KnowledgebaseDocumentIdRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebaseDocumentsPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: deletes one document from a knowledgebase. Bots stop
+ * finding it at once.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebaseDocumentsPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) {
+            throw new AppError(`Only a super admin can do that.`);
+        }
+
+        const { document_id } = parseWithSchema({ schema: KnowledgebaseDocumentIdRequestSchema, data: body });
+        const document = await deleteKnowledgebaseDocument({ document_id });
+        const knowledgebase_id = Number(document.knowledgebase_id);
+
+        await touchKnowledgebase({ knowledgebase_id });
+
+        await logEvent({
+            title: "Knowledgebase document deleted",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} deleted ${document.name} (#${document.id}) from the knowledgebase #${knowledgebase_id}.`,
+            metadata: { knowledgebase_id, document_id },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: `Deleted ${document.name}.`,
+            singleRes: await listKnowledgebaseDocuments({ knowledgebase_id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/delete-knowledgebase.ts

@@ -0,0 +1,53 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import deleteKnowledgebase from "@/src/functions/backend/db/knowledgebases/delete-knowledgebase";
+import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { KnowledgebaseIdRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: deletes a knowledgebase and every document in it.
+ * Bots lose it at once.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebasesOverviewPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) {
+            throw new AppError(`Only a super admin can do that.`);
+        }
+
+        const { knowledgebase_id } = parseWithSchema({ schema: KnowledgebaseIdRequestSchema, data: body });
+        const knowledgebase = await deleteKnowledgebase({ knowledgebase_id });
+
+        await logEvent({
+            title: "Knowledgebase deleted",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} deleted the knowledgebase ${knowledgebase.name} (#${knowledgebase.id}).`,
+            metadata: { knowledgebase_id },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: `Deleted ${knowledgebase.name}.`,
+            singleRes: await grabKnowledgebasesOverview({ user_id: user.id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/document-text.ts

@@ -0,0 +1,49 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabKnowledgebaseDocumentRow from "@/src/functions/backend/db/knowledgebases/grab-knowledgebase-document-row";
+import { KnowledgebaseDocumentIdRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import canManageCrew from "@/src/utils/rbac/can-manage-crew";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admins and admins: a text document's title and content, loaded
+ * when it's opened to read or edit.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<{ document_id: number; name: string; content: string }>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        if (!canManageCrew({ user_types })) {
+            throw new AppError(`Only an admin can do that.`);
+        }
+
+        const { document_id } = parseWithSchema({ schema: KnowledgebaseDocumentIdRequestSchema, data: body });
+        const document = await grabKnowledgebaseDocumentRow({ document_id });
+
+        if (document.source != "text") {
+            throw new AppError(`Only a text document can be opened here.`);
+        }
+
+        return {
+            success: true,
+            singleRes: { document_id: document.id, name: document.name || "", content: document.content || "" },
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/documents.ts

@@ -0,0 +1,42 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import listKnowledgebaseDocuments from "@/src/functions/backend/db/knowledgebases/list-knowledgebase-documents";
+import { KnowledgebaseIdRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebaseDocumentsPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import canManageCrew from "@/src/utils/rbac/can-manage-crew";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admins and admins: a knowledgebase's documents, loaded when its
+ * documents are opened on the settings page.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebaseDocumentsPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        if (!canManageCrew({ user_types })) {
+            throw new AppError(`Only an admin can do that.`);
+        }
+
+        const { knowledgebase_id } = parseWithSchema({ schema: KnowledgebaseIdRequestSchema, data: body });
+
+        return { success: true, singleRes: await listKnowledgebaseDocuments({ knowledgebase_id }) };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/overview.ts

@@ -0,0 +1,34 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import canManageCrew from "@/src/utils/rbac/can-manage-crew";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admins and admins: the Knowledgebases page's list again, after its
+ * documents changed (their counts and how much of each is read).
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebasesOverviewPayload>
+> = async ({ req }) => {
+    try {
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        if (!canManageCrew({ user_types })) {
+            throw new AppError(`Only an admin can do that.`);
+        }
+
+        return { success: true, singleRes: await grabKnowledgebasesOverview({ user_id: user.id }) };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/reindex-knowledgebase.ts

@@ -0,0 +1,53 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
+import reindexKnowledgebase from "@/src/functions/backend/db/knowledgebases/reindex-knowledgebase";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { KnowledgebaseIdRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: reads every document of a knowledgebase again, with the
+ * install's embedding model as it is now.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebasesOverviewPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) {
+            throw new AppError(`Only a super admin can do that.`);
+        }
+
+        const { knowledgebase_id } = parseWithSchema({ schema: KnowledgebaseIdRequestSchema, data: body });
+        const queued = await reindexKnowledgebase({ knowledgebase_id });
+
+        await logEvent({
+            title: "Knowledgebase re-indexed",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} re-indexed the knowledgebase #${knowledgebase_id} (${queued} documents).`,
+            metadata: { knowledgebase_id, queued },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: queued ? `Reading ${queued} ${queued == 1 ? "document" : "documents"} again.` : "It has no documents yet.",
+            singleRes: await grabKnowledgebasesOverview({ user_id: user.id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/save-knowledgebase.ts

@@ -0,0 +1,52 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
+import saveKnowledgebase from "@/src/functions/backend/db/knowledgebases/save-knowledgebase";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { SaveKnowledgebaseRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: adds or edits a knowledgebase.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebasesOverviewPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) {
+            throw new AppError(`Only a super admin can do that.`);
+        }
+
+        const data = parseWithSchema({ schema: SaveKnowledgebaseRequestSchema, data: body });
+        const knowledgebase_id = await saveKnowledgebase({ user_id: user.id, data });
+
+        await logEvent({
+            title: data.knowledgebase_id ? "Knowledgebase changed" : "Knowledgebase added",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} ${data.knowledgebase_id ? "changed" : "added"} the knowledgebase ${data.name} (#${knowledgebase_id}).`,
+            metadata: { knowledgebase_id },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: `Saved ${data.name}.`,
+            singleRes: await grabKnowledgebasesOverview({ user_id: user.id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/save-text-document.ts

@@ -0,0 +1,53 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import listKnowledgebaseDocuments from "@/src/functions/backend/db/knowledgebases/list-knowledgebase-documents";
+import saveKnowledgebaseText from "@/src/functions/backend/db/knowledgebases/save-knowledgebase-text";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { SaveKnowledgebaseTextRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebaseDocumentsPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: writes a text document into a knowledgebase, or
+ * changes one.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebaseDocumentsPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) {
+            throw new AppError(`Only a super admin can do that.`);
+        }
+
+        const data = parseWithSchema({ schema: SaveKnowledgebaseTextRequestSchema, data: body });
+        const document_id = await saveKnowledgebaseText({ user_id: user.id, data });
+
+        await logEvent({
+            title: data.document_id ? "Knowledgebase document changed" : "Knowledgebase document added",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} ${data.document_id ? "changed" : "wrote"} ${data.name} (#${document_id}) in the knowledgebase #${data.knowledgebase_id}.`,
+            metadata: { knowledgebase_id: data.knowledgebase_id, document_id },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: `Saved ${data.name}. It's searchable once it's read.`,
+            singleRes: await listKnowledgebaseDocuments({ knowledgebase_id: data.knowledgebase_id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/set-knowledgebase-access.ts

@@ -0,0 +1,59 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import setExtensionAccess from "@/src/functions/backend/db/extensions/set-extension-access";
+import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { SetExtensionAccessRequestSchema } from "@/src/schemas/extension-schema";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admins and admins: attaches a knowledgebase to exactly these bots
+ * and workspaces. Only a super admin sets "all bots" (`setExtensionAccess`).
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebasesOverviewPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const { user, user_types } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        if (isReadOnlyUser({ user_types: user_types || [] })) {
+            throw new AppError(`This user can only read records.`);
+        }
+
+        const access = parseWithSchema({ schema: SetExtensionAccessRequestSchema, data: body });
+
+        await setExtensionAccess({ user_id: user.id, kind: "knowledgebase", ...access });
+
+        await logEvent({
+            title: "Knowledgebase access changed",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} changed who gets the knowledgebase #${access.id}.`,
+            metadata: access,
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: `Saved who gets it.`,
+            singleRes: await grabKnowledgebasesOverview({ user_id: user.id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/knowledgebases/upload-document.ts

@@ -0,0 +1,111 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import listKnowledgebaseDocuments from "@/src/functions/backend/db/knowledgebases/list-knowledgebase-documents";
+import recordKnowledgebaseDocument from "@/src/functions/backend/db/knowledgebases/record-knowledgebase-document";
+import uploadAndRecordMedia from "@/src/functions/backend/db/users/media/upload-and-record-media";
+import logEvent, { grabRequestIP, logError } from "@/src/functions/backend/logger";
+import { AppData } from "@/src/data/app-data";
+import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
+import { UploadKnowledgebaseDocumentRequestSchema } from "@/src/schemas/knowledgebase-schema";
+import type { KnowledgebaseDocumentsPayload } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import attachmentFileKind from "@/src/utils/attachment-file-kind";
+import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
+import canExtractText from "@/src/utils/can-extract-text";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: adds one file to a knowledgebase. Only documents whose
+ * text can be read are taken (PDF, DOCX, markdown, plain text and the
+ * like); the bytes are kept in private media and the document is queued to
+ * be chunked and embedded.
+ */
+export const handler: BunextAPIRouteHandler<
+    APIResponseObject<KnowledgebaseDocumentsPayload>
+> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") {
+            throw new AppError(`Method not allowed`);
+        }
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) {
+            throw new AppError(`Only a super admin can do that.`);
+        }
+
+        const { knowledgebase_id, name, data_url } = parseWithSchema({
+            schema: UploadKnowledgebaseDocumentRequestSchema,
+            data: body,
+        });
+
+        const mime = data_url.match(/^data:([^;,]*)/)?.[1] || "";
+
+        if (!canExtractText({ name, mime_type: mime })) {
+            throw new AppError(
+                `${name} has no text to read. A knowledgebase takes ${BotTextFileExtensions.join(", ")} files.`,
+            );
+        }
+
+        const { media_type, ext } = attachmentFileKind({ mime, name });
+        const { buffer } = await bufferFromBase64(data_url);
+
+        if (!buffer.length) {
+            throw new AppError(`${name} is empty.`);
+        }
+
+        if (buffer.length > AppData.KnowledgebaseDocumentMaxBytes) {
+            throw new AppError(
+                `${name} is too big. Documents can be up to ${Math.round(AppData.KnowledgebaseDocumentMaxBytes / 1024 / 1024)} MB.`,
+            );
+        }
+
+        const media = await uploadAndRecordMedia({
+            data: buffer,
+            media_type,
+            media_mime_type: ext,
+            media_paradigms: ["knowledgebase-document"],
+            media: { title: name },
+            extract_media_text_content: true,
+            user: { id: Number(user.id) },
+        }).catch(async (error) => {
+            await logError({ title: `Knowledgebase document ${name} couldn't be saved`, error }).catch(() => {});
+            throw new AppError(`Couldn't save ${name}.`);
+        });
+
+        if (!media.singleRes?.id) {
+            throw new AppError(`Couldn't save ${name}.`);
+        }
+
+        const document_id = await recordKnowledgebaseDocument({
+            user_id: user.id,
+            knowledgebase_id,
+            name,
+            source: "file",
+            media_id: media.singleRes.id,
+            mime_type: mime,
+            size: buffer.length,
+        });
+
+        await logEvent({
+            title: "Knowledgebase document added",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} added ${name} (#${document_id}) to the knowledgebase #${knowledgebase_id}.`,
+            metadata: { knowledgebase_id, document_id },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+            msg: `Added ${name}. It's searchable once it's read.`,
+            singleRes: await listKnowledgebaseDocuments({ knowledgebase_id }),
+        };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};