Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/functions/backend/db/harness/grab-bot-harness-overview.ts

@@ -1,4 +1,10 @@
 import { Harnesses } from "@/src/dict/harness-dict";
+
+/**
+ * Only what a bot or workspace may set: install-wide harnesses are the
+ * defaults' alone.
+ */
+const OwnHarnesses = Harnesses.filter((harness) => !harness.install_wide);
 import type { BotHarnessOverviewPayload } from "@/src/types";
 import grabHarnessModelEntries from "./grab-harness-model-entries";
 import grabHarnessOverviewItem from "./grab-harness-overview-item";
@@ -20,7 +26,8 @@ type Params = {
 /**
  * A worker bot's (or a workspace's) model settings: per harness, its own
  * setting (or what it inherits), the models it can pick, what it runs on
- * now, and what it would run on without its own setting.
+ * now, and what it would run on without its own setting. Install-wide
+ * harnesses (embeddings) aren't listed.
  */
 export default async function grabBotHarnessOverview({
     bot_id,
@@ -34,7 +41,7 @@ export default async function grabBotHarnessOverview({
 
     const [own, defaults] = await Promise.all([
         Promise.all(
-            Harnesses.map((harness) =>
+            OwnHarnesses.map((harness) =>
                 grabHarnessSetting({
                     harness: harness.value,
                     bot_id: workspace_id ? undefined : bot_id,
@@ -43,7 +50,7 @@ export default async function grabBotHarnessOverview({
             ),
         ),
         Promise.all(
-            Harnesses.map((harness) =>
+            OwnHarnesses.map((harness) =>
                 grabHarnessSetting({ harness: harness.value, bot_id: inherited_bot_id }),
             ),
         ),

src/functions/backend/db/harness/grab-harness-setting.ts

@@ -4,7 +4,7 @@ import type {
     BUN_SQLITE_COBBLE_HARNESS_SETTINGS,
     BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS,
 } from "@/db/types/db";
-import type { HarnessKind } from "@/src/dict/harness-dict";
+import { isInstallWideHarness, type HarnessKind } from "@/src/dict/harness-dict";
 import parseJobJson from "@/src/bots-queue/parse-job-json";
 import grabDefaultHarnessProviderId from "./grab-default-harness-provider-id";
 
@@ -51,14 +51,18 @@ type SettingRow = Pick<
 /**
  * A harness's setting for a call: the workspace's, else the bot's, else
  * the default. With no row it is automatic on the default provider
- * (OpenRouter), on, with fallback. Text is always on.
+ * (OpenRouter), on, with fallback. Text is always on. An install-wide
+ * harness (embeddings) always takes the default: rows a bot or workspace
+ * saved before it was one are ignored.
  */
 export default async function grabHarnessSetting({
     harness,
     bot_id,
     workspace_id,
 }: Params): Promise<HarnessSetting> {
-    const workspace_row = workspace_id
+    const install_wide = isInstallWideHarness({ harness });
+
+    const workspace_row = workspace_id && !install_wide
         ? await grabWorkspaceOverride({ harness, workspace_id })
         : undefined;
 
@@ -66,7 +70,7 @@ export default async function grabHarnessSetting({
         return toSetting({ harness, row: workspace_row, source: "workspace" });
     }
 
-    const bot_row = bot_id
+    const bot_row = bot_id && !install_wide
         ? await grabBotOverride({ harness, bot_id })
         : undefined;
 

src/functions/backend/db/harness/save-bot-harness-setting.ts

@@ -1,5 +1,5 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
-import type { HarnessKind } from "@/src/dict/harness-dict";
+import { Harnesses, isInstallWideHarness, type HarnessKind } from "@/src/dict/harness-dict";
 import AppError from "@/src/utils/app-error";
 import assertHarnessModel from "./assert-harness-model";
 import assertHarnessProvider from "./assert-harness-provider";
@@ -47,6 +47,12 @@ export default async function saveBotHarnessSetting({
         return;
     }
 
+    if (isInstallWideHarness({ harness })) {
+        const title = Harnesses.find((h) => h.value == harness)?.title || harness;
+
+        throw new AppError(`${title} is set for the whole install, in Settings, under Models.`);
+    }
+
     if (harness == "text" && !enabled) {
         throw new AppError(`The text model can't be turned off.`);
     }

src/functions/backend/db/harness/save-workspace-harness-setting.ts

@@ -1,5 +1,5 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
-import type { HarnessKind } from "@/src/dict/harness-dict";
+import { Harnesses, isInstallWideHarness, type HarnessKind } from "@/src/dict/harness-dict";
 import AppError from "@/src/utils/app-error";
 import assertHarnessModel from "./assert-harness-model";
 import assertHarnessProvider from "./assert-harness-provider";
@@ -48,6 +48,12 @@ export default async function saveWorkspaceHarnessSetting({
         return;
     }
 
+    if (isInstallWideHarness({ harness })) {
+        const title = Harnesses.find((h) => h.value == harness)?.title || harness;
+
+        throw new AppError(`${title} is set for the whole install, in Settings, under Models.`);
+    }
+
     if (harness == "text" && !enabled) {
         throw new AppError(`The text model can't be turned off.`);
     }