Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-26 16:41:01 +01:00
co-authored by Claude Opus 5.5
parent 2259b6895f
commit e7b23b7454
111 changed files with 4573 additions and 148 deletions
@@ -18,13 +18,18 @@ type Props<T> = {
/**
* Under `/api/admin/`.
*/
path: "mcps/set-mcp-access" | "tools/set-tool-access" | "skills/set-skill-access";
path:
| "mcps/set-mcp-access"
| "tools/set-tool-access"
| "skills/set-skill-access"
| "knowledgebases/set-knowledgebase-access";
onSaved: (overview: T, msg: string) => void;
onCancel: () => void;
};
/**
* Who gets an MCP server, tool or skill: the bots and workspaces picked here.
* Who gets an MCP server, tool, skill or knowledgebase: the bots and
* workspaces picked here.
* "Every bot" (super admins only) gives it to every bot now and every new
* one; each can still drop it. A workspace gives it to every bot in
* it, there.
@@ -1,6 +1,7 @@
import IconTile from "@/src/components/general/icon-tile";
import extensionIcon from "@/src/components/general/extension-icon";
import type { BotExtension } from "@/src/types";
import extensionStateNote from "@/src/utils/extension-state-note";
import SettingsListRow from "./settings-list-row";
import SettingsRowAction from "./settings-row-action";
@@ -17,23 +18,15 @@ const ExtensionHrefs = {
mcp: "/admin/settings/mcps",
tool: "/admin/settings/tools",
skill: "/admin/settings/skills",
knowledgebase: "/admin/settings/knowledgebases",
} as const;
/**
* One MCP server, tool or skill a bot or workspace has, and whether it's
* working. A skill has nothing to connect or run, so it never reports a
* failure.
* One MCP server, tool, skill or knowledgebase a bot or workspace has,
* and whether it's working (`extensionStateNote`).
*/
export default function ExtensionRow({ item, busy, onRemove }: Props) {
const state = !item.enabled
? "Off for every bot"
: item.kind == "skill"
? ""
: item.state == "error"
? "Not working"
: item.state == "untested"
? "Not checked yet"
: "";
const state = extensionStateNote({ item });
return (
<SettingsListRow
@@ -1,6 +1,6 @@
import Button from "@moduletrace/twui/layout/Button";
import Row from "@moduletrace/twui/layout/Row";
import { BookOpen, Plug, Wrench } from "lucide-react";
import { BookOpen, Library, Plug, Wrench } from "lucide-react";
import ExtensionAddSelect from "@/src/components/general/extension-add-select";
import useExtensions from "@/src/hooks/use-extensions";
import type { BotExtensionsPayload } from "@/src/types";
@@ -14,20 +14,36 @@ type Props = {
owner_name: string;
description: string;
initial?: BotExtensionsPayload | null;
/**
* `tools`: MCP servers, tools and skills. `knowledgebases`: only those.
*/
variant?: "tools" | "knowledgebases";
};
/**
* The MCP servers, tools and skills a bot or a workspace has, removable,
* and a picker to add more. On the bot and workspace settings pages.
* The MCP servers, tools and skills (or the knowledgebases) a bot or a
* workspace has, removable, and a picker to add more. On the bot and
* workspace settings pages.
*/
export default function ExtensionsGroup({ owner, owner_name, description, initial }: Props) {
export default function ExtensionsGroup({ owner, owner_name, description, initial, variant = "tools" }: Props) {
const extensions = useExtensions({ owner, owner_name, initial });
const id = "bot_id" in owner ? `bot-${owner.bot_id}` : `workspace-${owner.workspace_id}`;
const is_knowledge = variant == "knowledgebases";
const fits = (item: { kind: string }) => (item.kind == "knowledgebase") == is_knowledge;
const attached = extensions.attached.filter(fits);
const addable = extensions.addable.filter(fits);
return (
<SettingsGroup id="tools" title="Tools, MCP servers and skills" description={description}>
<SettingsList is_empty={!extensions.attached.length} empty="No tools, MCP servers or skills yet.">
{extensions.attached.map((item) => (
<SettingsGroup
id={is_knowledge ? "knowledgebases" : "tools"}
title={is_knowledge ? "Knowledgebases" : "Tools, MCP servers and skills"}
description={description}
>
<SettingsList
is_empty={!attached.length}
empty={is_knowledge ? "No knowledgebases yet." : "No tools, MCP servers or skills yet."}
>
{attached.map((item) => (
<ExtensionRow
key={`${item.kind}:${item.id}`}
item={item}
@@ -42,12 +58,27 @@ export default function ExtensionsGroup({ owner, owner_name, description, initia
</SettingsList>
{extensions.can_manage ? (
<ExtensionAddSelect
id={`${id}-add-extension`}
addable={extensions.addable}
id={`${id}-add-${is_knowledge ? "knowledgebase" : "extension"}`}
noun={is_knowledge ? "knowledgebase" : undefined}
addable={addable}
loading={Boolean(extensions.busy)}
onAdd={extensions.add}
/>
) : null}
{is_knowledge ? (
<Button
title="Manage knowledgebases"
type="button"
variant="ghost"
color="gray"
size="small"
href="/admin/settings/knowledgebases"
beforeIcon={<Library size={14} aria-hidden />}
className="self-start gap-1!"
>
Knowledgebases
</Button>
) : (
<Row className="gap-1!">
<Button
title="Manage MCP servers"
@@ -86,6 +117,7 @@ export default function ExtensionsGroup({ owner, owner_name, description, initia
Skills
</Button>
</Row>
)}
<SettingsStatus status={extensions.status} />
</SettingsGroup>
);