Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-26 16:41:01 +01:00
co-authored by Claude Opus 5.5
parent 2259b6895f
commit e7b23b7454
111 changed files with 4573 additions and 148 deletions
@@ -0,0 +1,33 @@
import { CircleAlert, CircleCheck, LoaderCircle, TriangleAlert, type LucideIcon } from "lucide-react";
import type { KnowledgebaseIndexState } from "@/src/utils/knowledgebase-index-state";
/**
* The mark beside a knowledgebase or document for each index state, and
* what it says on hover. Green only when bots search all of it by meaning.
*/
export const IndexStateMarks: Record<
Exclude<KnowledgebaseIndexState, "empty">,
{ Icon: LucideIcon; label: string; className: string[] }
> = {
indexing: { Icon: LoaderCircle, label: "Being read", className: ["animate-spin", "text-ink-secondary"] },
failed: {
Icon: TriangleAlert,
label: "Something couldn't be read",
className: ["text-warning", "dark:text-warning-dark"],
},
words: {
Icon: CircleCheck,
label: "Read, found by words only: no embedding model is set",
className: ["text-ink-disabled"],
},
stale: {
Icon: CircleAlert,
label: "Read with another embedding model: needs re-indexing",
className: ["text-ink-disabled"],
},
indexed: {
Icon: CircleCheck,
label: "Indexed: bots search all of it by meaning",
className: ["text-success", "dark:text-success-dark"],
},
};