Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-26 16:41:01 +01:00
co-authored by Claude Opus 5.5
parent 2259b6895f
commit e7b23b7454
111 changed files with 4573 additions and 148 deletions
@@ -0,0 +1,31 @@
import Link from "@moduletrace/twui/layout/Link";
import Span from "@moduletrace/twui/layout/Span";
type Props = {
/**
* The install's embedding model, null when none is set.
*/
model: string | null;
};
/**
* Which model turns every knowledgebase into vectors: one for the whole
* install, picked under Settings > Models. Without one, bots still find
* passages by their words. Picking or changing it reads every document
* again (`reindexAfterEmbeddingChange`).
*/
export default function EmbeddingModelNote({ model }: Props) {
return (
<Span className="text-[13px] leading-[18px] text-ink-secondary">
{model
? `Searched by meaning with ${model}, the install's embedding model. Change it under `
: "No embedding model is set, so bots find passages by their words only, which misses questions worded differently. Pick one under "}
<Link href="/admin/settings/models" title="Models settings" className="text-ink underline">
Settings, Models
</Link>
{model
? "; every document is then read again with the new one."
: "; every document is then read again with it."}
</Span>
);
}
@@ -0,0 +1,28 @@
import { twMerge } from "tailwind-merge";
import type { KnowledgebaseIndexState } from "@/src/utils/knowledgebase-index-state";
import { IndexStateMarks } from "../(data)/index-state-marks";
type Props = {
state: KnowledgebaseIndexState;
/**
* Said on hover instead of the state's own words.
*/
label?: string;
};
/**
* A spinner, check or warning beside a knowledgebase or document's name:
* how far it's indexed (`knowledgebaseIndexState`).
*/
export default function IndexStateMark({ state, label }: Props) {
if (state == "empty") return null;
const { Icon, label: own, className } = IndexStateMarks[state];
const text = label || own;
return (
<Icon size={15} role="img" aria-label={text} className={twMerge("shrink-0 inline-block align-middle", ...className)}>
<title>{text}</title>
</Icon>
);
}
@@ -0,0 +1,54 @@
import { FileText, NotebookPen, Pencil, Trash2 } from "lucide-react";
import Row from "@moduletrace/twui/layout/Row";
import IconTile from "@/src/components/general/icon-tile";
import RowIconButton from "@/src/components/general/row-icon-button";
import type { KnowledgebaseDocument } from "@/src/types";
import SettingsListRow from "../../(partials)/settings-list-row";
import knowledgebaseIndexState from "@/src/utils/knowledgebase-index-state";
import knowledgebaseDocumentSummary from "../(functions)/knowledgebase-document-summary";
import IndexStateMark from "./index-state-mark";
type Props = {
document: KnowledgebaseDocument;
embedding_model: string | null;
busy?: boolean;
/**
* Opens a text document to edit; not given for a file or when the user
* can't edit.
*/
onEdit?: () => void;
onDelete?: () => void;
};
/**
* One document in a knowledgebase: what it is and how much of it bots can
* search, or why it couldn't be read.
*/
export default function KnowledgebaseDocumentRow({ document, embedding_model, busy, onEdit, onDelete }: Props) {
return (
<SettingsListRow
avatar={<IconTile Icon={document.source == "text" ? NotebookPen : FileText} />}
title={document.name}
title_icon={
<IndexStateMark
state={knowledgebaseIndexState({
total: 1,
indexing: document.indexing ? 1 : 0,
failed: document.ingest_error ? 1 : 0,
stale: document.embedding_model && document.embedding_model == embedding_model ? 0 : 1,
embedding_model,
})}
label={document.ingest_error && !document.indexing ? document.ingest_error : undefined}
/>
}
subtitle={knowledgebaseDocumentSummary({ document, embedding_model })}
>
<Row className="gap-0! flex-nowrap">
{onEdit ? <RowIconButton icon={Pencil} title={`Edit ${document.name}`} disabled={busy} onClick={onEdit} /> : null}
{onDelete ? (
<RowIconButton icon={Trash2} title={`Delete ${document.name}`} disabled={busy} onClick={onDelete} />
) : null}
</Row>
</SettingsListRow>
);
}
@@ -0,0 +1,133 @@
import { useRef, useState } from "react";
import Button from "@moduletrace/twui/layout/Button";
import Loading from "@moduletrace/twui/elements/Loading";
import Row from "@moduletrace/twui/layout/Row";
import { NotebookPen, Upload } from "lucide-react";
import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
import type { PublicKnowledgebase } from "@/src/types";
import SettingsList from "../../(partials)/settings-list";
import SettingsStatus from "../../(partials)/settings-status";
import useKnowledgebaseDocuments from "../(hooks)/use-knowledgebase-documents";
import KnowledgebaseDocumentRow from "./knowledgebase-document-row";
import KnowledgebaseTextForm from "./knowledgebase-text-form";
type Props = {
knowledgebase: PublicKnowledgebase;
/**
* A super admin: adds, writes and deletes documents. Admins only read
* the list.
*/
can_edit: boolean;
/**
* The install's embedding model, to tell a document that needs
* re-indexing.
*/
embedding_model: string | null;
onClose: () => void;
};
/**
* A knowledgebase's documents: each with how much of it is searchable,
* files to upload, and text to write straight in. A document is read
* (chunked and embedded) in the background; the list follows until it's
* done.
*/
export default function KnowledgebaseDocumentsModal({ knowledgebase, can_edit, embedding_model, onClose }: Props) {
const { documents, busy, status, upload, saveText, remove } = useKnowledgebaseDocuments({
knowledgebase_id: knowledgebase.id,
});
/**
* `new` writes a text document, a number edits that one.
*/
const [writing, setWriting] = useState<"new" | number>();
const file_input = useRef<HTMLInputElement>(null);
if (writing) {
return (
<AddModalShell
open
setOpen={(next) => !next && setWriting(undefined)}
title={writing == "new" ? `Write a document in ${knowledgebase.name}` : "Edit the document"}
description="Bots search it once it's read, a few moments after it's saved."
>
<KnowledgebaseTextForm
document_id={writing == "new" ? undefined : writing}
loading={busy}
onSave={saveText}
onCancel={() => setWriting(undefined)}
/>
<SettingsStatus status={status?.error ? status : undefined} />
</AddModalShell>
);
}
return (
<AddModalShell
open
setOpen={(next) => !next && onClose()}
title={`Documents in ${knowledgebase.name}`}
description={`Upload ${BotTextFileExtensions.join(", ")} files, or write text straight in. Each is split into passages bots search.`}
>
{documents ? (
<SettingsList is_empty={!documents.length} empty="No documents yet.">
{documents.map((document) => (
<KnowledgebaseDocumentRow
key={document.id}
document={document}
embedding_model={embedding_model}
busy={busy}
onEdit={can_edit && document.source == "text" ? () => setWriting(document.id) : undefined}
onDelete={can_edit ? () => remove({ document }) : undefined}
/>
))}
</SettingsList>
) : (
<Loading />
)}
{can_edit ? (
<Row className="gap-1!">
<input
ref={file_input}
type="file"
multiple
accept={BotTextFileExtensions.map((ext) => `.${ext}`).join(",")}
className="hidden"
onChange={(event) => {
const files = Array.from(event.target.files || []);
upload({ files }).catch(() => {});
event.target.value = "";
}}
/>
<Button
title="Upload files"
type="button"
variant="ghost"
color="gray"
size="small"
loading={busy}
onClick={() => file_input.current?.click()}
beforeIcon={<Upload size={14} aria-hidden />}
className="gap-1!"
>
Upload files
</Button>
<Button
title="Write a document"
type="button"
variant="ghost"
color="gray"
size="small"
disabled={busy}
onClick={() => setWriting("new")}
beforeIcon={<NotebookPen size={14} aria-hidden />}
className="gap-1!"
>
Write a document
</Button>
</Row>
) : null}
<SettingsStatus status={status} />
</AddModalShell>
);
}
@@ -0,0 +1,107 @@
import { useState } from "react";
import AddFormActions from "@/src/layouts/bots/(partials)/add-form-actions";
import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
import Checkbox from "@moduletrace/twui/form/Checkbox";
import Form from "@moduletrace/twui/form/Form";
import Input from "@moduletrace/twui/form/Input";
import Textarea from "@moduletrace/twui/form/Textarea";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
import type { KnowledgebasesOverviewPayload, PublicKnowledgebase } from "@/src/types";
type Props = {
/**
* Edits this knowledgebase; adds one when not given.
*/
knowledgebase?: PublicKnowledgebase;
onSaved: (overview: KnowledgebasesOverviewPayload, msg: string) => void;
onCancel: () => void;
};
/**
* Adds or edits a knowledgebase: its name, the one line that tells a bot
* what it holds, whether it's on, and whether every bot gets it. Its
* documents are added from its row.
*/
export default function KnowledgebaseModal({ knowledgebase, onSaved, onCancel }: Props) {
const [name, setName] = useState(knowledgebase?.name || "");
const [description, setDescription] = useState(knowledgebase?.description || "");
const [enabled, setEnabled] = useState(knowledgebase?.enabled ?? true);
const [all_bots, setAllBots] = useState(knowledgebase?.access.all_bots ?? false);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string>();
const id = `knowledgebase-${knowledgebase?.id || "new"}`;
const submit = async () => {
if (
loading ||
!window.confirm(knowledgebase ? `Save ${knowledgebase.name}?` : `Add ${name || "this knowledgebase"}?`)
) {
return;
}
setLoading(true);
setError(undefined);
const res = await postAdminApi<KnowledgebasesOverviewPayload>({
path: "knowledgebases/save-knowledgebase",
body: { knowledgebase_id: knowledgebase?.id, name, description, enabled, all_bots },
});
setLoading(false);
if (!res.success || !res.singleRes) {
setError(res.msg || "Couldn't save the knowledgebase.");
return;
}
onSaved(res.singleRes, res.msg || "Saved.");
};
return (
<AddModalShell
open
setOpen={(next) => !next && onCancel()}
title={knowledgebase ? `Edit ${knowledgebase.name}` : "Add a knowledgebase"}
description="Bots search it once it's attached to them. Add its documents from its row."
>
<Form className="w-full gap-4! items-stretch pt-4" onSubmit={submit}>
<Input
id={`${id}-name`}
title="Name"
label="Name"
showLabel
placeholder="Product handbook"
defaultValue={name}
autoComplete="off"
info="What the settings page and the composer call it."
onChange={(e) => setName(e.target.value)}
/>
<Textarea
id={`${id}-description`}
title="What it holds"
label="What it holds"
showLabel
rows={2}
placeholder="Pricing, plans and refund rules, as the sales team uses them."
defaultValue={description}
info="The one line a bot reads to know when to search it. Say what's in it."
onChange={(e) => setDescription(e.target.value)}
/>
<Checkbox title="On" label="On: bots search it" checked={enabled} changeHandler={setEnabled} />
<Checkbox
title="Every bot"
label="Every bot by default: every bot now and every new one, the coordinator included. Each can still drop it."
checked={all_bots}
changeHandler={setAllBots}
/>
<AddFormActions
submit_label={knowledgebase ? "Save" : "Add knowledgebase"}
loading={loading}
error={error}
onCancel={onCancel}
/>
</Form>
</AddModalShell>
);
}
@@ -0,0 +1,169 @@
import { useState } from "react";
import Row from "@moduletrace/twui/layout/Row";
import Stack from "@moduletrace/twui/layout/Stack";
import { FileText, Library, Pencil, RefreshCw, Trash2, Users } from "lucide-react";
import IconTile from "@/src/components/general/icon-tile";
import RowIconButton from "@/src/components/general/row-icon-button";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
import type { ExtensionTargets, KnowledgebasesOverviewPayload, PublicKnowledgebase } from "@/src/types";
import ExtensionAccessModal from "../../(partials)/extension-access-modal";
import SettingsListRow from "../../(partials)/settings-list-row";
import extensionAccessLabel from "../../(utils)/extension-access-label";
import updatedAgo from "../../(utils)/updated-ago";
import knowledgebaseIndexState from "@/src/utils/knowledgebase-index-state";
import knowledgebaseSummary from "../(functions)/knowledgebase-summary";
import IndexStateMark from "./index-state-mark";
import KnowledgebaseDocumentsModal from "./knowledgebase-documents-modal";
import KnowledgebaseModal from "./knowledgebase-modal";
type Props = {
knowledgebase: PublicKnowledgebase;
targets: ExtensionTargets;
/**
* The install's embedding model, for the documents' lines.
*/
embedding_model: string | null;
onChanged: (overview: KnowledgebasesOverviewPayload, msg: string) => void;
onError: (msg: string) => void;
};
/**
* One knowledgebase: how many documents it holds and how much of them is
* searchable, and who gets it. Documents, Edit and Access open in a modal;
* Re-index reads every document again with the install's embedding model.
*/
export default function KnowledgebaseRow({ knowledgebase, targets, embedding_model, onChanged, onError }: Props) {
const [open, setOpen] = useState<"documents" | "edit" | "access">();
const [busy, setBusy] = useState(false);
const saved = (overview: KnowledgebasesOverviewPayload, msg: string) => {
setOpen(undefined);
onChanged(overview, msg);
};
const post = async ({ path, confirm }: { path: string; confirm: string }) => {
if (busy || !window.confirm(confirm)) return;
setBusy(true);
const res = await postAdminApi<KnowledgebasesOverviewPayload>({
path,
body: { knowledgebase_id: knowledgebase.id },
});
setBusy(false);
if (res.success && res.singleRes) onChanged(res.singleRes, res.msg || "Done.");
else onError(res.msg || "That didn't work.");
};
/**
* The documents changed while open: their counts on the row are
* read again.
*/
const closeDocuments = async () => {
setOpen(undefined);
const res = await postAdminApi<KnowledgebasesOverviewPayload>({ path: "knowledgebases/overview", body: {} });
if (res.success && res.singleRes) onChanged(res.singleRes, "");
};
return (
<Stack className="w-full gap-0! items-stretch border-0 border-b border-solid border-hairline last:border-b-0">
<SettingsListRow
avatar={<IconTile Icon={Library} />}
title={knowledgebase.name}
badge={knowledgebase.enabled ? undefined : "Off"}
title_icon={
<IndexStateMark
state={knowledgebaseIndexState({
total: knowledgebase.document_count,
indexing: knowledgebase.pending_count,
failed: knowledgebase.warning_count,
stale: knowledgebase.stale_count,
embedding_model,
})}
/>
}
subtitle={[
knowledgebaseSummary({ knowledgebase }),
extensionAccessLabel({ access: knowledgebase.access }),
updatedAgo({ at: knowledgebase.updated_at }),
knowledgebase.description,
]
.filter(Boolean)
.join(" · ")}
>
<Row className="gap-0! flex-nowrap">
<RowIconButton
icon={FileText}
title={`Open the documents in ${knowledgebase.name}`}
disabled={busy}
onClick={() => setOpen("documents")}
/>
{targets.can_edit ? (
<RowIconButton
icon={Pencil}
title={`Edit ${knowledgebase.name}`}
disabled={busy}
onClick={() => setOpen("edit")}
/>
) : null}
<RowIconButton
icon={Users}
title={`Choose who gets ${knowledgebase.name}`}
disabled={busy}
onClick={() => setOpen("access")}
/>
{targets.can_edit && knowledgebase.document_count ? (
<RowIconButton
icon={RefreshCw}
title={`Re-index ${knowledgebase.name}: read every document again`}
disabled={busy}
onClick={() =>
post({
path: "knowledgebases/reindex-knowledgebase",
confirm: `Read every document in ${knowledgebase.name} again? Bots keep finding the old passages until the new ones are in.`,
})
}
/>
) : null}
{targets.can_edit ? (
<RowIconButton
icon={Trash2}
title={`Delete ${knowledgebase.name}`}
disabled={busy}
onClick={() =>
post({
path: "knowledgebases/delete-knowledgebase",
confirm: `Delete ${knowledgebase.name} and every document in it? Bots lose it at once.`,
})
}
/>
) : null}
</Row>
</SettingsListRow>
{open == "documents" ? (
<KnowledgebaseDocumentsModal
knowledgebase={knowledgebase}
can_edit={targets.can_edit}
embedding_model={embedding_model}
onClose={closeDocuments}
/>
) : null}
{open == "edit" ? (
<KnowledgebaseModal knowledgebase={knowledgebase} onSaved={saved} onCancel={() => setOpen(undefined)} />
) : null}
{open == "access" ? (
<ExtensionAccessModal<KnowledgebasesOverviewPayload>
id={knowledgebase.id}
name={knowledgebase.name}
access={knowledgebase.access}
targets={targets}
path="knowledgebases/set-knowledgebase-access"
onSaved={saved}
onCancel={() => setOpen(undefined)}
/>
) : null}
</Stack>
);
}
@@ -0,0 +1,81 @@
import { useEffect, useState } from "react";
import Form from "@moduletrace/twui/form/Form";
import Input from "@moduletrace/twui/form/Input";
import Loading from "@moduletrace/twui/elements/Loading";
import CodeEditor from "@/src/components/general/code-editor";
import { AppData } from "@/src/data/app-data";
import AddFormActions from "@/src/layouts/bots/(partials)/add-form-actions";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
type Props = {
/**
* Edits this text document (its content loads first); writes a new one
* when not given.
*/
document_id?: number;
loading?: boolean;
onSave: (params: { document_id?: number; name: string; content: string }) => Promise<boolean>;
onCancel: () => void;
};
/**
* Writes a document straight into a knowledgebase, in markdown: a title
* and the text bots search.
*/
export default function KnowledgebaseTextForm({ document_id, loading, onSave, onCancel }: Props) {
const [name, setName] = useState("");
const [content, setContent] = useState("");
const [ready, setReady] = useState(!document_id);
const [error, setError] = useState<string>();
useEffect(() => {
if (!document_id) return;
postAdminApi<{ document_id: number; name: string; content: string }>({
path: "knowledgebases/document-text",
body: { document_id },
}).then((res) => {
if (res.success && res.singleRes) {
setName(res.singleRes.name);
setContent(res.singleRes.content);
setReady(true);
} else {
setError(res.msg || "Couldn't open the document.");
}
});
}, [document_id]);
if (!ready) {
return error ? <AddFormActions submit_label="Save" error={error} onCancel={onCancel} /> : <Loading />;
}
return (
<Form
className="w-full gap-4! items-stretch"
onSubmit={async () => {
if (await onSave({ document_id, name, content })) onCancel();
}}
>
<Input
id={`knowledgebase-text-${document_id || "new"}-name`}
title="Title"
label="Title"
showLabel
placeholder="Refund rules"
defaultValue={name}
autoComplete="off"
info="What bots see as the document's name when they find a passage in it."
onChange={(e) => setName(e.target.value)}
/>
<CodeEditor
mode="markdown"
className="h-80"
title="Text"
content={content}
info={`Markdown, up to ${AppData.KnowledgebaseTextMaxChars} characters. Headings help: a passage carries the heading above it.`}
onChange={(value) => setContent(value)}
/>
<AddFormActions submit_label={document_id ? "Save" : "Add document"} loading={loading} onCancel={onCancel} />
</Form>
);
}