Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-26 16:41:01 +01:00
co-authored by Claude Opus 5.5
parent 2259b6895f
commit e7b23b7454
111 changed files with 4573 additions and 148 deletions
@@ -18,13 +18,18 @@ type Props<T> = {
/**
* Under `/api/admin/`.
*/
path: "mcps/set-mcp-access" | "tools/set-tool-access" | "skills/set-skill-access";
path:
| "mcps/set-mcp-access"
| "tools/set-tool-access"
| "skills/set-skill-access"
| "knowledgebases/set-knowledgebase-access";
onSaved: (overview: T, msg: string) => void;
onCancel: () => void;
};
/**
* Who gets an MCP server, tool or skill: the bots and workspaces picked here.
* Who gets an MCP server, tool, skill or knowledgebase: the bots and
* workspaces picked here.
* "Every bot" (super admins only) gives it to every bot now and every new
* one; each can still drop it. A workspace gives it to every bot in
* it, there.
@@ -1,6 +1,7 @@
import IconTile from "@/src/components/general/icon-tile";
import extensionIcon from "@/src/components/general/extension-icon";
import type { BotExtension } from "@/src/types";
import extensionStateNote from "@/src/utils/extension-state-note";
import SettingsListRow from "./settings-list-row";
import SettingsRowAction from "./settings-row-action";
@@ -17,23 +18,15 @@ const ExtensionHrefs = {
mcp: "/admin/settings/mcps",
tool: "/admin/settings/tools",
skill: "/admin/settings/skills",
knowledgebase: "/admin/settings/knowledgebases",
} as const;
/**
* One MCP server, tool or skill a bot or workspace has, and whether it's
* working. A skill has nothing to connect or run, so it never reports a
* failure.
* One MCP server, tool, skill or knowledgebase a bot or workspace has,
* and whether it's working (`extensionStateNote`).
*/
export default function ExtensionRow({ item, busy, onRemove }: Props) {
const state = !item.enabled
? "Off for every bot"
: item.kind == "skill"
? ""
: item.state == "error"
? "Not working"
: item.state == "untested"
? "Not checked yet"
: "";
const state = extensionStateNote({ item });
return (
<SettingsListRow
@@ -1,6 +1,6 @@
import Button from "@moduletrace/twui/layout/Button";
import Row from "@moduletrace/twui/layout/Row";
import { BookOpen, Plug, Wrench } from "lucide-react";
import { BookOpen, Library, Plug, Wrench } from "lucide-react";
import ExtensionAddSelect from "@/src/components/general/extension-add-select";
import useExtensions from "@/src/hooks/use-extensions";
import type { BotExtensionsPayload } from "@/src/types";
@@ -14,20 +14,36 @@ type Props = {
owner_name: string;
description: string;
initial?: BotExtensionsPayload | null;
/**
* `tools`: MCP servers, tools and skills. `knowledgebases`: only those.
*/
variant?: "tools" | "knowledgebases";
};
/**
* The MCP servers, tools and skills a bot or a workspace has, removable,
* and a picker to add more. On the bot and workspace settings pages.
* The MCP servers, tools and skills (or the knowledgebases) a bot or a
* workspace has, removable, and a picker to add more. On the bot and
* workspace settings pages.
*/
export default function ExtensionsGroup({ owner, owner_name, description, initial }: Props) {
export default function ExtensionsGroup({ owner, owner_name, description, initial, variant = "tools" }: Props) {
const extensions = useExtensions({ owner, owner_name, initial });
const id = "bot_id" in owner ? `bot-${owner.bot_id}` : `workspace-${owner.workspace_id}`;
const is_knowledge = variant == "knowledgebases";
const fits = (item: { kind: string }) => (item.kind == "knowledgebase") == is_knowledge;
const attached = extensions.attached.filter(fits);
const addable = extensions.addable.filter(fits);
return (
<SettingsGroup id="tools" title="Tools, MCP servers and skills" description={description}>
<SettingsList is_empty={!extensions.attached.length} empty="No tools, MCP servers or skills yet.">
{extensions.attached.map((item) => (
<SettingsGroup
id={is_knowledge ? "knowledgebases" : "tools"}
title={is_knowledge ? "Knowledgebases" : "Tools, MCP servers and skills"}
description={description}
>
<SettingsList
is_empty={!attached.length}
empty={is_knowledge ? "No knowledgebases yet." : "No tools, MCP servers or skills yet."}
>
{attached.map((item) => (
<ExtensionRow
key={`${item.kind}:${item.id}`}
item={item}
@@ -42,12 +58,27 @@ export default function ExtensionsGroup({ owner, owner_name, description, initia
</SettingsList>
{extensions.can_manage ? (
<ExtensionAddSelect
id={`${id}-add-extension`}
addable={extensions.addable}
id={`${id}-add-${is_knowledge ? "knowledgebase" : "extension"}`}
noun={is_knowledge ? "knowledgebase" : undefined}
addable={addable}
loading={Boolean(extensions.busy)}
onAdd={extensions.add}
/>
) : null}
{is_knowledge ? (
<Button
title="Manage knowledgebases"
type="button"
variant="ghost"
color="gray"
size="small"
href="/admin/settings/knowledgebases"
beforeIcon={<Library size={14} aria-hidden />}
className="self-start gap-1!"
>
Knowledgebases
</Button>
) : (
<Row className="gap-1!">
<Button
title="Manage MCP servers"
@@ -86,6 +117,7 @@ export default function ExtensionsGroup({ owner, owner_name, description, initia
Skills
</Button>
</Row>
)}
<SettingsStatus status={extensions.status} />
</SettingsGroup>
);
@@ -77,6 +77,13 @@ export default function BotSettingsSection() {
description="What it can use besides its own tools, everywhere it works: MCP servers, tools and skills. A workspace's own come on top while it works there."
initial={pageProps?.bot_extensions}
/>
<ExtensionsGroup
variant="knowledgebases"
owner={{ bot_id: bot.id }}
owner_name={name}
description="The documents it searches before it answers, everywhere it works. A workspace's own come on top while it works there."
initial={pageProps?.bot_extensions}
/>
<ScopedEnvsGroup initial={pageProps?.bot_envs} />
{is_coordinator ? null : <BotDeleteGroup bot={bot} />}
</SettingsShell>
@@ -0,0 +1,33 @@
import { CircleAlert, CircleCheck, LoaderCircle, TriangleAlert, type LucideIcon } from "lucide-react";
import type { KnowledgebaseIndexState } from "@/src/utils/knowledgebase-index-state";
/**
* The mark beside a knowledgebase or document for each index state, and
* what it says on hover. Green only when bots search all of it by meaning.
*/
export const IndexStateMarks: Record<
Exclude<KnowledgebaseIndexState, "empty">,
{ Icon: LucideIcon; label: string; className: string[] }
> = {
indexing: { Icon: LoaderCircle, label: "Being read", className: ["animate-spin", "text-ink-secondary"] },
failed: {
Icon: TriangleAlert,
label: "Something couldn't be read",
className: ["text-warning", "dark:text-warning-dark"],
},
words: {
Icon: CircleCheck,
label: "Read, found by words only: no embedding model is set",
className: ["text-ink-disabled"],
},
stale: {
Icon: CircleAlert,
label: "Read with another embedding model: needs re-indexing",
className: ["text-ink-disabled"],
},
indexed: {
Icon: CircleCheck,
label: "Indexed: bots search all of it by meaning",
className: ["text-success", "dark:text-success-dark"],
},
};
@@ -0,0 +1,39 @@
import type { KnowledgebaseDocument } from "@/src/types";
import formatBytes from "@/src/utils/format-bytes";
import checkedAgo from "../../(utils)/checked-ago";
type Params = {
document: KnowledgebaseDocument;
/**
* The install's embedding model now, to tell a document embedded by
* another one.
*/
embedding_model: string | null;
};
/**
* One document's line: what it is, its size, how much of it is searchable
* and when it was added. "Text · 3.1 KB · 4 passages · Added 5m ago".
*/
export default function knowledgebaseDocumentSummary({ document, embedding_model }: Params) {
const kind = document.source == "text" ? "Text" : (document.name.split(".").pop() || "File").toUpperCase();
const state = document.indexing
? "Being read"
: document.ingest_error
? document.ingest_error
: !document.embedding_model
? `${document.chunk_count} ${document.chunk_count == 1 ? "passage" : "passages"}, by words only`
: document.embedding_model != embedding_model
? `${document.chunk_count} ${document.chunk_count == 1 ? "passage" : "passages"}, needs re-indexing`
: `${document.chunk_count} ${document.chunk_count == 1 ? "passage" : "passages"}`;
return [
kind,
formatBytes({ bytes: document.size }),
state,
document.created_at ? `Added ${checkedAgo({ at: document.created_at })}` : "",
]
.filter(Boolean)
.join(" · ");
}
@@ -0,0 +1,21 @@
import { describe, expect, test } from "bun:test";
import knowledgebaseSummary from "./knowledgebase-summary";
const empty = { document_count: 0, pending_count: 0, error_count: 0, stale_count: 0, size: 0 };
describe("knowledgebaseSummary", () => {
test("an empty knowledgebase says so", () => {
expect(knowledgebaseSummary({ knowledgebase: empty })).toBe("No documents yet");
});
test("counts documents and names only the problems there are", () => {
expect(knowledgebaseSummary({ knowledgebase: { ...empty, document_count: 1, size: 2048 } })).toBe(
"1 document · 2.0 KB",
);
expect(
knowledgebaseSummary({
knowledgebase: { document_count: 3, pending_count: 1, error_count: 1, stale_count: 2, size: 2048 },
}),
).toBe("3 documents · 2.0 KB · 1 being read · 1 couldn't be read · 2 need re-indexing");
});
});
@@ -0,0 +1,29 @@
import type { PublicKnowledgebase } from "@/src/types";
import formatBytes from "@/src/utils/format-bytes";
type Params = {
knowledgebase: Pick<
PublicKnowledgebase,
"document_count" | "pending_count" | "error_count" | "stale_count" | "size"
>;
};
/**
* What a knowledgebase holds and how much of it bots can search, in a few
* words: "3 documents · 1.2 MB · 1 being read · 1 needs re-indexing".
*/
export default function knowledgebaseSummary({ knowledgebase }: Params) {
const { document_count, pending_count, error_count, stale_count, size } = knowledgebase;
if (!document_count) return "No documents yet";
return [
`${document_count} ${document_count == 1 ? "document" : "documents"}`,
formatBytes({ bytes: size }),
pending_count ? `${pending_count} being read` : "",
error_count ? `${error_count} couldn't be read` : "",
stale_count ? `${stale_count} ${stale_count == 1 ? "needs" : "need"} re-indexing` : "",
]
.filter(Boolean)
.join(" · ");
}
@@ -0,0 +1,16 @@
type Params = {
file: File;
};
/**
* A picked file as a data URL, for an upload. Empty when the browser can't
* read it.
*/
export default async function readFileDataUrl({ file }: Params) {
return await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result || ""));
reader.onerror = () => reject(reader.error);
reader.readAsDataURL(file);
}).catch(() => "");
}
@@ -0,0 +1,111 @@
import { useEffect, useState } from "react";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
import type { KnowledgebaseDocument, KnowledgebaseDocumentsPayload } from "@/src/types";
import readFileDataUrl from "../(functions)/read-file-data-url";
type Params = {
knowledgebase_id: number;
};
type Status = { error?: boolean; success?: boolean; msg?: string };
/**
* How often the list is read again while a document is still being read.
*/
const PollMs = 4000;
/**
* A knowledgebase's documents, loaded when they're opened and read again
* every few seconds while one is still waiting to be read, with uploading,
* writing and deleting them (each asks first).
*/
export default function useKnowledgebaseDocuments({ knowledgebase_id }: Params) {
const [documents, setDocuments] = useState<KnowledgebaseDocument[]>();
const [busy, setBusy] = useState(false);
const [status, setStatus] = useState<Status>();
const load = async () => {
const res = await postAdminApi<KnowledgebaseDocumentsPayload>({
path: "knowledgebases/documents",
body: { knowledgebase_id },
});
if (res.success && res.singleRes) setDocuments(res.singleRes.documents);
else setStatus({ error: true, msg: res.msg || "Couldn't load the documents." });
};
useEffect(() => {
load();
}, [knowledgebase_id]);
const waiting = Boolean(documents?.some((document) => document.indexing));
useEffect(() => {
if (!waiting) return;
const timer = setInterval(load, PollMs);
return () => clearInterval(timer);
}, [waiting, knowledgebase_id]);
const post = async ({ path, body }: { path: string; body: Record<string, unknown> }) => {
setBusy(true);
setStatus(undefined);
const res = await postAdminApi<KnowledgebaseDocumentsPayload>({ path, body });
setBusy(false);
if (!res.success || !res.singleRes) {
setStatus({ error: true, msg: res.msg || "That didn't work." });
return false;
}
setDocuments(res.singleRes.documents);
setStatus({ success: true, msg: res.msg || "Saved." });
return true;
};
const upload = async ({ files }: { files: File[] }) => {
if (!files.length || busy) return;
const names = files.map((file) => file.name).join(", ");
if (!window.confirm(`Add ${names} to this knowledgebase?`)) return;
for (const file of files) {
const data_url = await readFileDataUrl({ file });
if (!data_url) {
setStatus({ error: true, msg: `${file.name} couldn't be read.` });
return;
}
const done = await post({
path: "knowledgebases/upload-document",
body: { knowledgebase_id, name: file.name, data_url },
});
if (!done) return;
}
};
const saveText = async ({ document_id, name, content }: { document_id?: number; name: string; content: string }) => {
if (busy || !window.confirm(document_id ? `Save ${name || "this document"}?` : `Add ${name || "this document"}?`)) {
return false;
}
return await post({
path: "knowledgebases/save-text-document",
body: { knowledgebase_id, document_id, name, content },
});
};
const remove = async ({ document }: { document: KnowledgebaseDocument }) => {
if (busy || !window.confirm(`Delete ${document.name}? Bots stop finding it at once.`)) return;
await post({ path: "knowledgebases/delete-document", body: { document_id: document.id } });
};
return { documents, busy, status, upload, saveText, remove };
}
@@ -0,0 +1,31 @@
import Link from "@moduletrace/twui/layout/Link";
import Span from "@moduletrace/twui/layout/Span";
type Props = {
/**
* The install's embedding model, null when none is set.
*/
model: string | null;
};
/**
* Which model turns every knowledgebase into vectors: one for the whole
* install, picked under Settings > Models. Without one, bots still find
* passages by their words. Picking or changing it reads every document
* again (`reindexAfterEmbeddingChange`).
*/
export default function EmbeddingModelNote({ model }: Props) {
return (
<Span className="text-[13px] leading-[18px] text-ink-secondary">
{model
? `Searched by meaning with ${model}, the install's embedding model. Change it under `
: "No embedding model is set, so bots find passages by their words only, which misses questions worded differently. Pick one under "}
<Link href="/admin/settings/models" title="Models settings" className="text-ink underline">
Settings, Models
</Link>
{model
? "; every document is then read again with the new one."
: "; every document is then read again with it."}
</Span>
);
}
@@ -0,0 +1,28 @@
import { twMerge } from "tailwind-merge";
import type { KnowledgebaseIndexState } from "@/src/utils/knowledgebase-index-state";
import { IndexStateMarks } from "../(data)/index-state-marks";
type Props = {
state: KnowledgebaseIndexState;
/**
* Said on hover instead of the state's own words.
*/
label?: string;
};
/**
* A spinner, check or warning beside a knowledgebase or document's name:
* how far it's indexed (`knowledgebaseIndexState`).
*/
export default function IndexStateMark({ state, label }: Props) {
if (state == "empty") return null;
const { Icon, label: own, className } = IndexStateMarks[state];
const text = label || own;
return (
<Icon size={15} role="img" aria-label={text} className={twMerge("shrink-0 inline-block align-middle", ...className)}>
<title>{text}</title>
</Icon>
);
}
@@ -0,0 +1,54 @@
import { FileText, NotebookPen, Pencil, Trash2 } from "lucide-react";
import Row from "@moduletrace/twui/layout/Row";
import IconTile from "@/src/components/general/icon-tile";
import RowIconButton from "@/src/components/general/row-icon-button";
import type { KnowledgebaseDocument } from "@/src/types";
import SettingsListRow from "../../(partials)/settings-list-row";
import knowledgebaseIndexState from "@/src/utils/knowledgebase-index-state";
import knowledgebaseDocumentSummary from "../(functions)/knowledgebase-document-summary";
import IndexStateMark from "./index-state-mark";
type Props = {
document: KnowledgebaseDocument;
embedding_model: string | null;
busy?: boolean;
/**
* Opens a text document to edit; not given for a file or when the user
* can't edit.
*/
onEdit?: () => void;
onDelete?: () => void;
};
/**
* One document in a knowledgebase: what it is and how much of it bots can
* search, or why it couldn't be read.
*/
export default function KnowledgebaseDocumentRow({ document, embedding_model, busy, onEdit, onDelete }: Props) {
return (
<SettingsListRow
avatar={<IconTile Icon={document.source == "text" ? NotebookPen : FileText} />}
title={document.name}
title_icon={
<IndexStateMark
state={knowledgebaseIndexState({
total: 1,
indexing: document.indexing ? 1 : 0,
failed: document.ingest_error ? 1 : 0,
stale: document.embedding_model && document.embedding_model == embedding_model ? 0 : 1,
embedding_model,
})}
label={document.ingest_error && !document.indexing ? document.ingest_error : undefined}
/>
}
subtitle={knowledgebaseDocumentSummary({ document, embedding_model })}
>
<Row className="gap-0! flex-nowrap">
{onEdit ? <RowIconButton icon={Pencil} title={`Edit ${document.name}`} disabled={busy} onClick={onEdit} /> : null}
{onDelete ? (
<RowIconButton icon={Trash2} title={`Delete ${document.name}`} disabled={busy} onClick={onDelete} />
) : null}
</Row>
</SettingsListRow>
);
}
@@ -0,0 +1,133 @@
import { useRef, useState } from "react";
import Button from "@moduletrace/twui/layout/Button";
import Loading from "@moduletrace/twui/elements/Loading";
import Row from "@moduletrace/twui/layout/Row";
import { NotebookPen, Upload } from "lucide-react";
import { BotTextFileExtensions } from "@/src/dict/bot-file-dict";
import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
import type { PublicKnowledgebase } from "@/src/types";
import SettingsList from "../../(partials)/settings-list";
import SettingsStatus from "../../(partials)/settings-status";
import useKnowledgebaseDocuments from "../(hooks)/use-knowledgebase-documents";
import KnowledgebaseDocumentRow from "./knowledgebase-document-row";
import KnowledgebaseTextForm from "./knowledgebase-text-form";
type Props = {
knowledgebase: PublicKnowledgebase;
/**
* A super admin: adds, writes and deletes documents. Admins only read
* the list.
*/
can_edit: boolean;
/**
* The install's embedding model, to tell a document that needs
* re-indexing.
*/
embedding_model: string | null;
onClose: () => void;
};
/**
* A knowledgebase's documents: each with how much of it is searchable,
* files to upload, and text to write straight in. A document is read
* (chunked and embedded) in the background; the list follows until it's
* done.
*/
export default function KnowledgebaseDocumentsModal({ knowledgebase, can_edit, embedding_model, onClose }: Props) {
const { documents, busy, status, upload, saveText, remove } = useKnowledgebaseDocuments({
knowledgebase_id: knowledgebase.id,
});
/**
* `new` writes a text document, a number edits that one.
*/
const [writing, setWriting] = useState<"new" | number>();
const file_input = useRef<HTMLInputElement>(null);
if (writing) {
return (
<AddModalShell
open
setOpen={(next) => !next && setWriting(undefined)}
title={writing == "new" ? `Write a document in ${knowledgebase.name}` : "Edit the document"}
description="Bots search it once it's read, a few moments after it's saved."
>
<KnowledgebaseTextForm
document_id={writing == "new" ? undefined : writing}
loading={busy}
onSave={saveText}
onCancel={() => setWriting(undefined)}
/>
<SettingsStatus status={status?.error ? status : undefined} />
</AddModalShell>
);
}
return (
<AddModalShell
open
setOpen={(next) => !next && onClose()}
title={`Documents in ${knowledgebase.name}`}
description={`Upload ${BotTextFileExtensions.join(", ")} files, or write text straight in. Each is split into passages bots search.`}
>
{documents ? (
<SettingsList is_empty={!documents.length} empty="No documents yet.">
{documents.map((document) => (
<KnowledgebaseDocumentRow
key={document.id}
document={document}
embedding_model={embedding_model}
busy={busy}
onEdit={can_edit && document.source == "text" ? () => setWriting(document.id) : undefined}
onDelete={can_edit ? () => remove({ document }) : undefined}
/>
))}
</SettingsList>
) : (
<Loading />
)}
{can_edit ? (
<Row className="gap-1!">
<input
ref={file_input}
type="file"
multiple
accept={BotTextFileExtensions.map((ext) => `.${ext}`).join(",")}
className="hidden"
onChange={(event) => {
const files = Array.from(event.target.files || []);
upload({ files }).catch(() => {});
event.target.value = "";
}}
/>
<Button
title="Upload files"
type="button"
variant="ghost"
color="gray"
size="small"
loading={busy}
onClick={() => file_input.current?.click()}
beforeIcon={<Upload size={14} aria-hidden />}
className="gap-1!"
>
Upload files
</Button>
<Button
title="Write a document"
type="button"
variant="ghost"
color="gray"
size="small"
disabled={busy}
onClick={() => setWriting("new")}
beforeIcon={<NotebookPen size={14} aria-hidden />}
className="gap-1!"
>
Write a document
</Button>
</Row>
) : null}
<SettingsStatus status={status} />
</AddModalShell>
);
}
@@ -0,0 +1,107 @@
import { useState } from "react";
import AddFormActions from "@/src/layouts/bots/(partials)/add-form-actions";
import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
import Checkbox from "@moduletrace/twui/form/Checkbox";
import Form from "@moduletrace/twui/form/Form";
import Input from "@moduletrace/twui/form/Input";
import Textarea from "@moduletrace/twui/form/Textarea";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
import type { KnowledgebasesOverviewPayload, PublicKnowledgebase } from "@/src/types";
type Props = {
/**
* Edits this knowledgebase; adds one when not given.
*/
knowledgebase?: PublicKnowledgebase;
onSaved: (overview: KnowledgebasesOverviewPayload, msg: string) => void;
onCancel: () => void;
};
/**
* Adds or edits a knowledgebase: its name, the one line that tells a bot
* what it holds, whether it's on, and whether every bot gets it. Its
* documents are added from its row.
*/
export default function KnowledgebaseModal({ knowledgebase, onSaved, onCancel }: Props) {
const [name, setName] = useState(knowledgebase?.name || "");
const [description, setDescription] = useState(knowledgebase?.description || "");
const [enabled, setEnabled] = useState(knowledgebase?.enabled ?? true);
const [all_bots, setAllBots] = useState(knowledgebase?.access.all_bots ?? false);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string>();
const id = `knowledgebase-${knowledgebase?.id || "new"}`;
const submit = async () => {
if (
loading ||
!window.confirm(knowledgebase ? `Save ${knowledgebase.name}?` : `Add ${name || "this knowledgebase"}?`)
) {
return;
}
setLoading(true);
setError(undefined);
const res = await postAdminApi<KnowledgebasesOverviewPayload>({
path: "knowledgebases/save-knowledgebase",
body: { knowledgebase_id: knowledgebase?.id, name, description, enabled, all_bots },
});
setLoading(false);
if (!res.success || !res.singleRes) {
setError(res.msg || "Couldn't save the knowledgebase.");
return;
}
onSaved(res.singleRes, res.msg || "Saved.");
};
return (
<AddModalShell
open
setOpen={(next) => !next && onCancel()}
title={knowledgebase ? `Edit ${knowledgebase.name}` : "Add a knowledgebase"}
description="Bots search it once it's attached to them. Add its documents from its row."
>
<Form className="w-full gap-4! items-stretch pt-4" onSubmit={submit}>
<Input
id={`${id}-name`}
title="Name"
label="Name"
showLabel
placeholder="Product handbook"
defaultValue={name}
autoComplete="off"
info="What the settings page and the composer call it."
onChange={(e) => setName(e.target.value)}
/>
<Textarea
id={`${id}-description`}
title="What it holds"
label="What it holds"
showLabel
rows={2}
placeholder="Pricing, plans and refund rules, as the sales team uses them."
defaultValue={description}
info="The one line a bot reads to know when to search it. Say what's in it."
onChange={(e) => setDescription(e.target.value)}
/>
<Checkbox title="On" label="On: bots search it" checked={enabled} changeHandler={setEnabled} />
<Checkbox
title="Every bot"
label="Every bot by default: every bot now and every new one, the coordinator included. Each can still drop it."
checked={all_bots}
changeHandler={setAllBots}
/>
<AddFormActions
submit_label={knowledgebase ? "Save" : "Add knowledgebase"}
loading={loading}
error={error}
onCancel={onCancel}
/>
</Form>
</AddModalShell>
);
}
@@ -0,0 +1,169 @@
import { useState } from "react";
import Row from "@moduletrace/twui/layout/Row";
import Stack from "@moduletrace/twui/layout/Stack";
import { FileText, Library, Pencil, RefreshCw, Trash2, Users } from "lucide-react";
import IconTile from "@/src/components/general/icon-tile";
import RowIconButton from "@/src/components/general/row-icon-button";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
import type { ExtensionTargets, KnowledgebasesOverviewPayload, PublicKnowledgebase } from "@/src/types";
import ExtensionAccessModal from "../../(partials)/extension-access-modal";
import SettingsListRow from "../../(partials)/settings-list-row";
import extensionAccessLabel from "../../(utils)/extension-access-label";
import updatedAgo from "../../(utils)/updated-ago";
import knowledgebaseIndexState from "@/src/utils/knowledgebase-index-state";
import knowledgebaseSummary from "../(functions)/knowledgebase-summary";
import IndexStateMark from "./index-state-mark";
import KnowledgebaseDocumentsModal from "./knowledgebase-documents-modal";
import KnowledgebaseModal from "./knowledgebase-modal";
type Props = {
knowledgebase: PublicKnowledgebase;
targets: ExtensionTargets;
/**
* The install's embedding model, for the documents' lines.
*/
embedding_model: string | null;
onChanged: (overview: KnowledgebasesOverviewPayload, msg: string) => void;
onError: (msg: string) => void;
};
/**
* One knowledgebase: how many documents it holds and how much of them is
* searchable, and who gets it. Documents, Edit and Access open in a modal;
* Re-index reads every document again with the install's embedding model.
*/
export default function KnowledgebaseRow({ knowledgebase, targets, embedding_model, onChanged, onError }: Props) {
const [open, setOpen] = useState<"documents" | "edit" | "access">();
const [busy, setBusy] = useState(false);
const saved = (overview: KnowledgebasesOverviewPayload, msg: string) => {
setOpen(undefined);
onChanged(overview, msg);
};
const post = async ({ path, confirm }: { path: string; confirm: string }) => {
if (busy || !window.confirm(confirm)) return;
setBusy(true);
const res = await postAdminApi<KnowledgebasesOverviewPayload>({
path,
body: { knowledgebase_id: knowledgebase.id },
});
setBusy(false);
if (res.success && res.singleRes) onChanged(res.singleRes, res.msg || "Done.");
else onError(res.msg || "That didn't work.");
};
/**
* The documents changed while open: their counts on the row are
* read again.
*/
const closeDocuments = async () => {
setOpen(undefined);
const res = await postAdminApi<KnowledgebasesOverviewPayload>({ path: "knowledgebases/overview", body: {} });
if (res.success && res.singleRes) onChanged(res.singleRes, "");
};
return (
<Stack className="w-full gap-0! items-stretch border-0 border-b border-solid border-hairline last:border-b-0">
<SettingsListRow
avatar={<IconTile Icon={Library} />}
title={knowledgebase.name}
badge={knowledgebase.enabled ? undefined : "Off"}
title_icon={
<IndexStateMark
state={knowledgebaseIndexState({
total: knowledgebase.document_count,
indexing: knowledgebase.pending_count,
failed: knowledgebase.warning_count,
stale: knowledgebase.stale_count,
embedding_model,
})}
/>
}
subtitle={[
knowledgebaseSummary({ knowledgebase }),
extensionAccessLabel({ access: knowledgebase.access }),
updatedAgo({ at: knowledgebase.updated_at }),
knowledgebase.description,
]
.filter(Boolean)
.join(" · ")}
>
<Row className="gap-0! flex-nowrap">
<RowIconButton
icon={FileText}
title={`Open the documents in ${knowledgebase.name}`}
disabled={busy}
onClick={() => setOpen("documents")}
/>
{targets.can_edit ? (
<RowIconButton
icon={Pencil}
title={`Edit ${knowledgebase.name}`}
disabled={busy}
onClick={() => setOpen("edit")}
/>
) : null}
<RowIconButton
icon={Users}
title={`Choose who gets ${knowledgebase.name}`}
disabled={busy}
onClick={() => setOpen("access")}
/>
{targets.can_edit && knowledgebase.document_count ? (
<RowIconButton
icon={RefreshCw}
title={`Re-index ${knowledgebase.name}: read every document again`}
disabled={busy}
onClick={() =>
post({
path: "knowledgebases/reindex-knowledgebase",
confirm: `Read every document in ${knowledgebase.name} again? Bots keep finding the old passages until the new ones are in.`,
})
}
/>
) : null}
{targets.can_edit ? (
<RowIconButton
icon={Trash2}
title={`Delete ${knowledgebase.name}`}
disabled={busy}
onClick={() =>
post({
path: "knowledgebases/delete-knowledgebase",
confirm: `Delete ${knowledgebase.name} and every document in it? Bots lose it at once.`,
})
}
/>
) : null}
</Row>
</SettingsListRow>
{open == "documents" ? (
<KnowledgebaseDocumentsModal
knowledgebase={knowledgebase}
can_edit={targets.can_edit}
embedding_model={embedding_model}
onClose={closeDocuments}
/>
) : null}
{open == "edit" ? (
<KnowledgebaseModal knowledgebase={knowledgebase} onSaved={saved} onCancel={() => setOpen(undefined)} />
) : null}
{open == "access" ? (
<ExtensionAccessModal<KnowledgebasesOverviewPayload>
id={knowledgebase.id}
name={knowledgebase.name}
access={knowledgebase.access}
targets={targets}
path="knowledgebases/set-knowledgebase-access"
onSaved={saved}
onCancel={() => setOpen(undefined)}
/>
) : null}
</Stack>
);
}
@@ -0,0 +1,81 @@
import { useEffect, useState } from "react";
import Form from "@moduletrace/twui/form/Form";
import Input from "@moduletrace/twui/form/Input";
import Loading from "@moduletrace/twui/elements/Loading";
import CodeEditor from "@/src/components/general/code-editor";
import { AppData } from "@/src/data/app-data";
import AddFormActions from "@/src/layouts/bots/(partials)/add-form-actions";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
type Props = {
/**
* Edits this text document (its content loads first); writes a new one
* when not given.
*/
document_id?: number;
loading?: boolean;
onSave: (params: { document_id?: number; name: string; content: string }) => Promise<boolean>;
onCancel: () => void;
};
/**
* Writes a document straight into a knowledgebase, in markdown: a title
* and the text bots search.
*/
export default function KnowledgebaseTextForm({ document_id, loading, onSave, onCancel }: Props) {
const [name, setName] = useState("");
const [content, setContent] = useState("");
const [ready, setReady] = useState(!document_id);
const [error, setError] = useState<string>();
useEffect(() => {
if (!document_id) return;
postAdminApi<{ document_id: number; name: string; content: string }>({
path: "knowledgebases/document-text",
body: { document_id },
}).then((res) => {
if (res.success && res.singleRes) {
setName(res.singleRes.name);
setContent(res.singleRes.content);
setReady(true);
} else {
setError(res.msg || "Couldn't open the document.");
}
});
}, [document_id]);
if (!ready) {
return error ? <AddFormActions submit_label="Save" error={error} onCancel={onCancel} /> : <Loading />;
}
return (
<Form
className="w-full gap-4! items-stretch"
onSubmit={async () => {
if (await onSave({ document_id, name, content })) onCancel();
}}
>
<Input
id={`knowledgebase-text-${document_id || "new"}-name`}
title="Title"
label="Title"
showLabel
placeholder="Refund rules"
defaultValue={name}
autoComplete="off"
info="What bots see as the document's name when they find a passage in it."
onChange={(e) => setName(e.target.value)}
/>
<CodeEditor
mode="markdown"
className="h-80"
title="Text"
content={content}
info={`Markdown, up to ${AppData.KnowledgebaseTextMaxChars} characters. Headings help: a passage carries the heading above it.`}
onChange={(value) => setContent(value)}
/>
<AddFormActions submit_label={document_id ? "Save" : "Add document"} loading={loading} onCancel={onCancel} />
</Form>
);
}
@@ -0,0 +1,107 @@
import { useContext, useEffect, useState } from "react";
import Button from "@moduletrace/twui/layout/Button";
import { Plus } from "lucide-react";
import { AppContext } from "@/src/pages/__root";
import postAdminApi from "@/src/layouts/bots/(functions)/post-admin-api";
import type { KnowledgebasesOverviewPayload } from "@/src/types";
import SettingsGroup from "../../(partials)/settings-group";
import SettingsList from "../../(partials)/settings-list";
import SettingsStatus from "../../(partials)/settings-status";
import EmbeddingModelNote from "../(partials)/embedding-model-note";
import KnowledgebaseModal from "../(partials)/knowledgebase-modal";
import KnowledgebaseRow from "../(partials)/knowledgebase-row";
type Status = { error?: boolean; success?: boolean; msg?: string };
/**
* How often the list is read again while a document is being read.
*/
const PollMs = 4000;
/**
* Every knowledgebase with who gets it. Super admins add, fill, edit and
* delete them; admins choose which bots and workspaces get each one.
*/
export default function KnowledgebasesSettingsSection() {
const { pageProps } = useContext(AppContext);
const [overview, setOverview] = useState(pageProps?.knowledgebases_overview);
const [adding, setAdding] = useState(false);
const [status, setStatus] = useState<Status>();
/**
* Remounts the rows when the data changes under them.
*/
const [version, setVersion] = useState(0);
const reading = Boolean(overview?.knowledgebases.some((knowledgebase) => knowledgebase.pending_count));
/**
* Kept current while documents are read, so the marks beside them
* turn from spinners into checks. The rows aren't remounted: a modal
* open on one stays.
*/
useEffect(() => {
if (!reading) return;
const timer = setInterval(async () => {
const res = await postAdminApi<KnowledgebasesOverviewPayload>({ path: "knowledgebases/overview", body: {} });
if (res.success && res.singleRes) setOverview(res.singleRes);
}, PollMs);
return () => clearInterval(timer);
}, [reading]);
if (!overview) return null;
const changed = (next: KnowledgebasesOverviewPayload, msg: string) => {
setOverview(next);
setStatus(msg ? { success: true, msg } : undefined);
setVersion((prev) => prev + 1);
};
const failed = (msg: string) => setStatus({ error: true, msg });
return (
<SettingsGroup
title="Knowledgebases"
description="A set of documents (handbooks, policies, product notes) a bot searches for the passages that answer a question. Before every reply it's handed the passages that fit the newest message, and it can search again while it works. Give one to a bot on its settings page or in its chat, or to a workspace to reach every bot there."
>
<EmbeddingModelNote model={overview.embedding_model} />
{adding ? (
<KnowledgebaseModal
onSaved={(next, msg) => {
setAdding(false);
changed(next, msg);
}}
onCancel={() => setAdding(false)}
/>
) : null}
<SettingsList is_empty={!overview.knowledgebases.length} empty="No knowledgebases yet.">
{overview.knowledgebases.map((knowledgebase) => (
<KnowledgebaseRow
key={`${knowledgebase.id}-${version}`}
knowledgebase={knowledgebase}
targets={overview}
embedding_model={overview.embedding_model}
onChanged={changed}
onError={failed}
/>
))}
</SettingsList>
{overview.can_edit && !adding ? (
<Button
title="Add a knowledgebase"
type="button"
variant="ghost"
color="gray"
size="small"
onClick={() => setAdding(true)}
className="self-start gap-1!"
>
<Plus size={14} aria-hidden />
New knowledgebase
</Button>
) : null}
<SettingsStatus status={status} />
</SettingsGroup>
);
}
@@ -0,0 +1,21 @@
import type { PagePropsType } from "@/src/types";
import type { BunextPageServerFn } from "@moduletrace/bunext/types";
import grabKnowledgebasesOverview from "@/src/functions/backend/db/knowledgebases/grab-knowledgebases-overview";
import grabCrewSettingsServerProps from "../(functions)/grab-crew-settings-server-props";
/**
* Super admins and admins: admins only choose who gets each knowledgebase.
* Documents load when a knowledgebase's documents are opened.
*/
export const server: BunextPageServerFn<PagePropsType> = async (params) => {
const settings = await grabCrewSettingsServerProps(params);
if (!settings.props?.person?.id) return settings;
return {
props: {
...settings.props,
knowledgebases_overview: await grabKnowledgebasesOverview({ user_id: settings.props.person.id }),
},
};
};
@@ -0,0 +1,13 @@
import KnowledgebasesSettingsSection from "./(sections)/knowledgebases-settings-section";
import SettingsShell from "../(partials)/settings-shell";
import settingsPageMeta from "../(functions)/settings-page-meta";
export default function AdminSettingsKnowledgebasesPage() {
return (
<SettingsShell title="Knowledgebases" description="Documents your bots search before they answer.">
<KnowledgebasesSettingsSection />
</SettingsShell>
);
}
export const meta = settingsPageMeta({ label: "Knowledgebases" });
@@ -70,6 +70,13 @@ export default function WorkspaceOverview() {
description="Every bot working here gets these, the coordinator too, on top of its own, only while it works here: MCP servers, tools and skills."
initial={pageProps?.workspace_extensions}
/>
<ExtensionsGroup
variant="knowledgebases"
owner={{ workspace_id }}
owner_name={name}
description="Every bot working here searches these before it answers, the coordinator too, on top of its own, only while it works here."
initial={pageProps?.workspace_extensions}
/>
<ScopedEnvsGroup initial={pageProps?.workspace_envs} />
{harnesses?.length ? (
<>