Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/functions/backend/db/bot-files/ingest-bot-file.ts

@@ -166,6 +166,12 @@ export default async function ingestBotFile({
 
         return { status: "ingested", chunks: embedded };
     } catch (error) {
+        /**
+         * Every model resting means nothing was tried: the job waits for
+         * them, and the file keeps whatever it said before.
+         */
+        if (error instanceof HarnessError && error.resting) throw error;
+
         const message = error instanceof Error ? error.message : String(error);
 
         await BunSQLite.sql({

src/functions/backend/db/extensions/attach-default-extensions.ts

@@ -5,7 +5,7 @@ type Params = {
 };
 
 /**
- * Gives a new bot every MCP server, tool and skill set to "every bot".
+ * Gives a new bot every MCP server, tool, skill and knowledgebase set to "every bot".
  * Called when a bot is created; it can drop any of them afterwards.
  */
 export default async function attachDefaultExtensions({ bot_id }: Params) {
@@ -28,4 +28,10 @@ export default async function attachDefaultExtensions({ bot_id }: Params) {
             SELECT ?, ?, ?, id FROM skills WHERE all_bots = 1`,
         values: [now, now, Number(bot_id)],
     });
+
+    await BunSQLite.sql({
+        sql: `INSERT OR IGNORE INTO bot_knowledgebases (created_at, updated_at, bot_id, knowledgebase_id)
+            SELECT ?, ?, ?, id FROM knowledgebases WHERE all_bots = 1`,
+        values: [now, now, Number(bot_id)],
+    });
 }

src/functions/backend/db/extensions/extension-tables.ts

@@ -24,6 +24,13 @@ export const ExtensionTables = {
         column: "skill_id",
         label: "skill",
     },
+    knowledgebase: {
+        table: "knowledgebases",
+        bot_link: "bot_knowledgebases",
+        workspace_link: "workspace_knowledgebases",
+        column: "knowledgebase_id",
+        label: "knowledgebase",
+    },
 } as const;
 
 export type ExtensionKind = keyof typeof ExtensionTables;

src/functions/backend/db/extensions/grab-bot-extensions.ts

@@ -1,10 +1,12 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
 import type {
     BUN_SQLITE_COBBLE_CUSTOM_TOOLS,
+    BUN_SQLITE_COBBLE_KNOWLEDGEBASES,
     BUN_SQLITE_COBBLE_MCP_SERVERS,
     BUN_SQLITE_COBBLE_SKILLS,
 } from "@/db/types/db";
 import botExtensionsSql from "@/src/utils/bot-extensions-sql";
+import parseKnowledgebaseRow from "../knowledgebases/parse-knowledgebase-row";
 import parseCustomToolRow from "./parse-custom-tool-row";
 import parseMcpServerRow from "./parse-mcp-server-row";
 import parseSkillRow from "./parse-skill-row";
@@ -15,19 +17,23 @@ type Params = {
 };
 
 /**
- * The MCP servers, custom tools and skills a bot gets on one run
+ * The MCP servers, custom tools, skills and knowledgebases a bot gets on one run
  * (`botExtensionsSql`): every bot's, its own, and its workspace's there.
  */
 export default async function grabBotExtensions({ bot_id, workspace_id }: Params) {
-    const [servers, tools, skills] = await Promise.all([
+    const [servers, tools, skills, knowledgebases] = await Promise.all([
         BunSQLite.sql<BUN_SQLITE_COBBLE_MCP_SERVERS>(botExtensionsSql({ kind: "mcp", bot_id, workspace_id })),
         BunSQLite.sql<BUN_SQLITE_COBBLE_CUSTOM_TOOLS>(botExtensionsSql({ kind: "tool", bot_id, workspace_id })),
         BunSQLite.sql<BUN_SQLITE_COBBLE_SKILLS>(botExtensionsSql({ kind: "skill", bot_id, workspace_id })),
+        BunSQLite.sql<BUN_SQLITE_COBBLE_KNOWLEDGEBASES>(
+            botExtensionsSql({ kind: "knowledgebase", bot_id, workspace_id }),
+        ),
     ]);
 
     return {
         servers: (servers.payload || []).map((row) => parseMcpServerRow({ row })),
         tools: (tools.payload || []).map((row) => parseCustomToolRow({ row })),
         skills: (skills.payload || []).map((row) => parseSkillRow({ row })),
+        knowledgebases: (knowledgebases.payload || []).map((row) => parseKnowledgebaseRow({ row })),
     };
 }

src/functions/backend/db/extensions/grab-extensions-overview.ts

@@ -1,12 +1,14 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
 import type {
     BUN_SQLITE_COBBLE_CUSTOM_TOOLS,
+    BUN_SQLITE_COBBLE_KNOWLEDGEBASES,
     BUN_SQLITE_COBBLE_MCP_SERVERS,
     BUN_SQLITE_COBBLE_SKILLS,
 } from "@/db/types/db";
 import { CustomToolKinds } from "@/src/dict/custom-tool-dict";
 import { McpTransports } from "@/src/dict/mcp-dict";
 import type { BotExtension, BotExtensionsPayload } from "@/src/types";
+import parseKnowledgebaseRow from "../knowledgebases/parse-knowledgebase-row";
 import parseCustomToolRow from "./parse-custom-tool-row";
 import parseMcpServerRow from "./parse-mcp-server-row";
 import parseSkillRow from "./parse-skill-row";
@@ -28,7 +30,7 @@ type Params = {
 type Linked<T> = T & { source: "bot" | "workspace" };
 
 /**
- * The MCP servers, tools and skills a bot or a workspace has, and the rest of
+ * The MCP servers, tools, skills and knowledgebases a bot or a workspace has, and the rest of
  * them, which can be added (`grabBotExtensionsOverview`,
  * `grabWorkspaceExtensionsOverview`).
  */
@@ -51,8 +53,10 @@ export default async function grabExtensionsOverview({
     const mcp_query = linked("mcp_server_id", "bot_mcp_servers", "workspace_mcp_servers");
     const tool_query = linked("custom_tool_id", "bot_custom_tools", "workspace_custom_tools");
     const skill_query = linked("skill_id", "bot_skills", "workspace_skills");
+    const knowledgebase_query = linked("knowledgebase_id", "bot_knowledgebases", "workspace_knowledgebases");
 
-    const [servers, tools, skills, all_servers, all_tools, all_skills] = await Promise.all([
+    const [servers, tools, skills, knowledgebases, documents, all_servers, all_tools, all_skills, all_knowledgebases] =
+        await Promise.all([
         BunSQLite.sql<Linked<BUN_SQLITE_COBBLE_MCP_SERVERS>>({
             sql: mcp_query.sql.replace("%TABLE%", "mcp_servers"),
             values: mcp_query.values,
@@ -65,6 +69,15 @@ export default async function grabExtensionsOverview({
             sql: skill_query.sql.replace("%TABLE%", "skills"),
             values: skill_query.values,
         }),
+        BunSQLite.sql<Linked<BUN_SQLITE_COBBLE_KNOWLEDGEBASES>>({
+            sql: knowledgebase_query.sql.replace("%TABLE%", "knowledgebases"),
+            values: knowledgebase_query.values,
+        }),
+        BunSQLite.sql<{ knowledgebase_id: number; documents: number; failed: number }>({
+            sql: `SELECT knowledgebase_id, COUNT(*) AS documents,
+                    SUM(CASE WHEN ingest_error IS NOT NULL AND COALESCE(chunk_count, 0) = 0 THEN 1 ELSE 0 END) AS failed
+                FROM knowledgebase_documents GROUP BY knowledgebase_id`,
+        }),
         can_manage
             ? BunSQLite.sql<{ id: number; name: string }>({ sql: `SELECT id, name FROM mcp_servers ORDER BY name COLLATE NOCASE` })
             : null,
@@ -76,8 +89,17 @@ export default async function grabExtensionsOverview({
                   sql: `SELECT id, name FROM skills ORDER BY name COLLATE NOCASE`,
               })
             : null,
+        can_manage
+            ? BunSQLite.sql<{ id: number; name: string }>({
+                  sql: `SELECT id, name FROM knowledgebases ORDER BY name COLLATE NOCASE`,
+              })
+            : null,
     ]);
 
+    const document_counts = new Map(
+        (documents.payload || []).map((row) => [Number(row.knowledgebase_id), row]),
+    );
+
     const attached: BotExtension[] = [
         ...(servers.payload || []).map((row) => {
             const server = parseMcpServerRow({ row });
@@ -123,6 +145,22 @@ export default async function grabExtensionsOverview({
                 source: row.source,
             };
         }),
+        ...(knowledgebases.payload || []).map((row) => {
+            const knowledgebase = parseKnowledgebaseRow({ row });
+            const count = document_counts.get(knowledgebase.id);
+            const total = Number(count?.documents) || 0;
+
+            return {
+                kind: "knowledgebase" as const,
+                id: knowledgebase.id,
+                name: knowledgebase.name,
+                description: knowledgebase.description,
+                detail: `Knowledgebase · ${total} ${total == 1 ? "document" : "documents"}`,
+                state: Number(count?.failed) ? ("error" as const) : ("ok" as const),
+                enabled: knowledgebase.enabled,
+                source: row.source,
+            };
+        }),
     ];
 
     /**
@@ -144,6 +182,9 @@ export default async function grabExtensionsOverview({
             ...(all_skills?.payload || [])
                 .filter((row) => !has("skill", Number(row.id)))
                 .map((row) => ({ kind: "skill" as const, id: Number(row.id), name: row.name })),
+            ...(all_knowledgebases?.payload || [])
+                .filter((row) => !has("knowledgebase", Number(row.id)))
+                .map((row) => ({ kind: "knowledgebase" as const, id: Number(row.id), name: row.name })),
         ],
         can_manage,
     };

src/functions/backend/db/harness/grab-bot-harness-overview.ts

@@ -1,4 +1,10 @@
 import { Harnesses } from "@/src/dict/harness-dict";
+
+/**
+ * Only what a bot or workspace may set: install-wide harnesses are the
+ * defaults' alone.
+ */
+const OwnHarnesses = Harnesses.filter((harness) => !harness.install_wide);
 import type { BotHarnessOverviewPayload } from "@/src/types";
 import grabHarnessModelEntries from "./grab-harness-model-entries";
 import grabHarnessOverviewItem from "./grab-harness-overview-item";
@@ -20,7 +26,8 @@ type Params = {
 /**
  * A worker bot's (or a workspace's) model settings: per harness, its own
  * setting (or what it inherits), the models it can pick, what it runs on
- * now, and what it would run on without its own setting.
+ * now, and what it would run on without its own setting. Install-wide
+ * harnesses (embeddings) aren't listed.
  */
 export default async function grabBotHarnessOverview({
     bot_id,
@@ -34,7 +41,7 @@ export default async function grabBotHarnessOverview({
 
     const [own, defaults] = await Promise.all([
         Promise.all(
-            Harnesses.map((harness) =>
+            OwnHarnesses.map((harness) =>
                 grabHarnessSetting({
                     harness: harness.value,
                     bot_id: workspace_id ? undefined : bot_id,
@@ -43,7 +50,7 @@ export default async function grabBotHarnessOverview({
             ),
         ),
         Promise.all(
-            Harnesses.map((harness) =>
+            OwnHarnesses.map((harness) =>
                 grabHarnessSetting({ harness: harness.value, bot_id: inherited_bot_id }),
             ),
         ),

src/functions/backend/db/harness/grab-harness-setting.ts

@@ -4,7 +4,7 @@ import type {
     BUN_SQLITE_COBBLE_HARNESS_SETTINGS,
     BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS,
 } from "@/db/types/db";
-import type { HarnessKind } from "@/src/dict/harness-dict";
+import { isInstallWideHarness, type HarnessKind } from "@/src/dict/harness-dict";
 import parseJobJson from "@/src/bots-queue/parse-job-json";
 import grabDefaultHarnessProviderId from "./grab-default-harness-provider-id";
 
@@ -51,14 +51,18 @@ type SettingRow = Pick<
 /**
  * A harness's setting for a call: the workspace's, else the bot's, else
  * the default. With no row it is automatic on the default provider
- * (OpenRouter), on, with fallback. Text is always on.
+ * (OpenRouter), on, with fallback. Text is always on. An install-wide
+ * harness (embeddings) always takes the default: rows a bot or workspace
+ * saved before it was one are ignored.
  */
 export default async function grabHarnessSetting({
     harness,
     bot_id,
     workspace_id,
 }: Params): Promise<HarnessSetting> {
-    const workspace_row = workspace_id
+    const install_wide = isInstallWideHarness({ harness });
+
+    const workspace_row = workspace_id && !install_wide
         ? await grabWorkspaceOverride({ harness, workspace_id })
         : undefined;
 
@@ -66,7 +70,7 @@ export default async function grabHarnessSetting({
         return toSetting({ harness, row: workspace_row, source: "workspace" });
     }
 
-    const bot_row = bot_id
+    const bot_row = bot_id && !install_wide
         ? await grabBotOverride({ harness, bot_id })
         : undefined;
 

src/functions/backend/db/harness/save-bot-harness-setting.ts

@@ -1,5 +1,5 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
-import type { HarnessKind } from "@/src/dict/harness-dict";
+import { Harnesses, isInstallWideHarness, type HarnessKind } from "@/src/dict/harness-dict";
 import AppError from "@/src/utils/app-error";
 import assertHarnessModel from "./assert-harness-model";
 import assertHarnessProvider from "./assert-harness-provider";
@@ -47,6 +47,12 @@ export default async function saveBotHarnessSetting({
         return;
     }
 
+    if (isInstallWideHarness({ harness })) {
+        const title = Harnesses.find((h) => h.value == harness)?.title || harness;
+
+        throw new AppError(`${title} is set for the whole install, in Settings, under Models.`);
+    }
+
     if (harness == "text" && !enabled) {
         throw new AppError(`The text model can't be turned off.`);
     }

src/functions/backend/db/harness/save-workspace-harness-setting.ts

@@ -1,5 +1,5 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
-import type { HarnessKind } from "@/src/dict/harness-dict";
+import { Harnesses, isInstallWideHarness, type HarnessKind } from "@/src/dict/harness-dict";
 import AppError from "@/src/utils/app-error";
 import assertHarnessModel from "./assert-harness-model";
 import assertHarnessProvider from "./assert-harness-provider";
@@ -48,6 +48,12 @@ export default async function saveWorkspaceHarnessSetting({
         return;
     }
 
+    if (isInstallWideHarness({ harness })) {
+        const title = Harnesses.find((h) => h.value == harness)?.title || harness;
+
+        throw new AppError(`${title} is set for the whole install, in Settings, under Models.`);
+    }
+
     if (harness == "text" && !enabled) {
         throw new AppError(`The text model can't be turned off.`);
     }

src/functions/backend/db/knowledgebases/delete-knowledgebase-document.ts

@@ -0,0 +1,49 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import { rmSync } from "node:fs";
+import mediaFilePath from "@/src/utils/media-file-path";
+import grabKnowledgebaseDocumentRow from "./grab-knowledgebase-document-row";
+
+type Params = {
+    document_id: number | string;
+};
+
+/**
+ * Deletes a knowledgebase document: its vectors, chunks (and their
+ * full-text index rows), pending ingest,
+ * row and (for a file) its bytes and `media` row. Children are deleted here
+ * rather than left to the schema's cascade, since foreign keys are off
+ * outside the server process.
+ */
+export default async function deleteKnowledgebaseDocument({ document_id }: Params) {
+    const document = await grabKnowledgebaseDocumentRow({ document_id });
+
+    await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_vectors WHERE document_id = ?`, values: [document.id] });
+    await BunSQLite.sql({
+        sql: `DELETE FROM knowledgebase_chunks_fts WHERE document_id = ?`,
+        values: [document.id],
+    }).catch(() => {});
+    await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_chunks WHERE document_id = ?`, values: [document.id] });
+    await BunSQLite.sql({
+        sql: `DELETE FROM file_ingest_queue WHERE document_id = ? AND status IN ('pending', 'processing')`,
+        values: [document.id],
+    }).catch(() => {});
+    await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_documents WHERE id = ?`, values: [document.id] });
+
+    if (document.media_id) {
+        const media = await BunSQLite.sql<{ media_write_path?: string; media_thumbnail_write_path?: string }>({
+            sql: `SELECT media_write_path, media_thumbnail_write_path FROM media WHERE id = ? LIMIT 1`,
+            values: [document.media_id],
+        }).then((res) => res.payload?.[0]);
+
+        for (const write_path of [media?.media_write_path, media?.media_thumbnail_write_path]) {
+            const absolute = mediaFilePath({ write_path });
+
+            if (absolute) rmSync(absolute, { force: true });
+        }
+
+        await BunSQLite.sql({ sql: `DELETE FROM media_paradigms WHERE media_id = ?`, values: [document.media_id] });
+        await BunSQLite.sql({ sql: `DELETE FROM media WHERE id = ?`, values: [document.media_id] });
+    }
+
+    return document;
+}

src/functions/backend/db/knowledgebases/delete-knowledgebase.ts

@@ -0,0 +1,38 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import deleteKnowledgebaseDocument from "./delete-knowledgebase-document";
+import grabKnowledgebase from "./grab-knowledgebase";
+
+type Params = {
+    knowledgebase_id: number | string;
+};
+
+/**
+ * Deletes a knowledgebase with everything in it: each document (its
+ * vectors, chunks and bytes), then its bot and workspace links. All
+ * deleted here rather than left to the schema's cascade, since foreign
+ * keys are off outside the server process.
+ */
+export default async function deleteKnowledgebase({ knowledgebase_id }: Params) {
+    const knowledgebase = await grabKnowledgebase({ knowledgebase_id });
+
+    const documents = await BunSQLite.sql<{ id: number }>({
+        sql: `SELECT id FROM knowledgebase_documents WHERE knowledgebase_id = ?`,
+        values: [knowledgebase.id],
+    });
+
+    for (const document of documents.payload || []) {
+        await deleteKnowledgebaseDocument({ document_id: document.id });
+    }
+
+    await BunSQLite.sql({ sql: `DELETE FROM bot_knowledgebases WHERE knowledgebase_id = ?`, values: [knowledgebase.id] });
+    await BunSQLite.sql({
+        sql: `DELETE FROM workspace_knowledgebases WHERE knowledgebase_id = ?`,
+        values: [knowledgebase.id],
+    });
+
+    const res = await BunSQLite.sql({ sql: `DELETE FROM knowledgebases WHERE id = ?`, values: [knowledgebase.id] });
+
+    if (!res.success) throw new Error(`Couldn't delete the knowledgebase: ${res.error}`);
+
+    return knowledgebase;
+}

src/functions/backend/db/knowledgebases/fts/ensure-knowledgebase-fts.ts

@@ -0,0 +1,33 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+
+let ensured: Promise<void> | undefined;
+
+/**
+ * Makes sure the full-text index of knowledgebase passages exists, once
+ * per process. An FTS5 table can't be declared in `db/schema.ts`, so it is
+ * made here; schema sync only drops tables it made, so it leaves this one
+ * alone. `unicode61 remove_diacritics 2` matches words in any script that
+ * separates them with spaces, accents or not ("conges" finds "congés").
+ */
+export default function ensureKnowledgebaseFts() {
+    ensured ||= create().catch((error) => {
+        ensured = undefined;
+        throw error;
+    });
+
+    return ensured;
+}
+
+async function create() {
+    const res = await BunSQLite.sql({
+        sql: `CREATE VIRTUAL TABLE IF NOT EXISTS knowledgebase_chunks_fts USING fts5(
+            text,
+            chunk_id UNINDEXED,
+            document_id UNINDEXED,
+            knowledgebase_id UNINDEXED,
+            tokenize = 'unicode61 remove_diacritics 2'
+        )`,
+    });
+
+    if (!res.success) throw new Error(`Couldn't make the knowledgebase search index: ${res.error || res.msg}`);
+}

src/functions/backend/db/knowledgebases/fts/sync-knowledgebase-fts.ts

@@ -0,0 +1,41 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import ensureKnowledgebaseFts from "./ensure-knowledgebase-fts";
+
+/**
+ * Brings the full-text index in line with `knowledgebase_chunks`, at boot:
+ * adds passages it doesn't have (stored before it existed, or by a process
+ * that failed halfway) and drops ones whose passage is gone (a delete from
+ * a script). Returns how many rows it added and dropped.
+ */
+export default async function syncKnowledgebaseFts() {
+    await ensureKnowledgebaseFts();
+
+    /**
+     * Counted here: FTS5 reports its own shadow-table writes as changes.
+     */
+    const count = async () =>
+        Number(
+            (await BunSQLite.sql<{ n: number }>({ sql: `SELECT COUNT(*) AS n FROM knowledgebase_chunks_fts` }))
+                .payload?.[0]?.n,
+        ) || 0;
+
+    const start = await count();
+
+    const added = await BunSQLite.sql({
+        sql: `INSERT INTO knowledgebase_chunks_fts (text, chunk_id, document_id, knowledgebase_id)
+            SELECT c.text, c.id, c.document_id, c.knowledgebase_id FROM knowledgebase_chunks c
+            WHERE c.id NOT IN (SELECT chunk_id FROM knowledgebase_chunks_fts)`,
+    });
+
+    const after_add = await count();
+
+    const dropped = await BunSQLite.sql({
+        sql: `DELETE FROM knowledgebase_chunks_fts WHERE chunk_id NOT IN (SELECT id FROM knowledgebase_chunks)`,
+    });
+
+    if (!added.success || !dropped.success) {
+        throw new Error(`Couldn't sync the knowledgebase search index: ${added.error || dropped.error}`);
+    }
+
+    return { added: after_add - start, dropped: after_add - (await count()) };
+}

src/functions/backend/db/knowledgebases/grab-install-embedding-model.ts

@@ -0,0 +1,11 @@
+import resolveHarnessCandidates from "@/src/harness/resolve/resolve-harness-candidates";
+
+/**
+ * The model the install embeds with now (the first candidate of the
+ * default embedding setting), null when none is set or it's turned off.
+ */
+export default async function grabInstallEmbeddingModel() {
+    const candidates = await resolveHarnessCandidates({ harness: "embedding" }).catch(() => []);
+
+    return candidates[0]?.model.slug || null;
+}

src/functions/backend/db/knowledgebases/grab-knowledgebase-document-row.ts

@@ -0,0 +1,24 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS } from "@/db/types/db";
+import AppError from "@/src/utils/app-error";
+
+type Params = {
+    document_id: number | string;
+};
+
+/**
+ * One knowledgebase document's row, content included, or an `AppError`
+ * when there's none.
+ */
+export default async function grabKnowledgebaseDocumentRow({ document_id }: Params) {
+    const row = (
+        await BunSQLite.sql<BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS>({
+            sql: `SELECT * FROM knowledgebase_documents WHERE id = ?`,
+            values: [document_id],
+        })
+    ).payload?.[0];
+
+    if (!row?.id) throw new AppError(`That document doesn't exist.`);
+
+    return { ...row, id: Number(row.id) };
+}

src/functions/backend/db/knowledgebases/grab-knowledgebase-document-text.ts

@@ -0,0 +1,36 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import extractBotFileText from "../bot-files/extract-bot-file-text";
+
+type Params = {
+    document: {
+        source?: string | null;
+        content?: string | null;
+        media_id?: number | null;
+        name?: string | null;
+        mime_type?: string | null;
+    };
+};
+
+/**
+ * A knowledgebase document's whole text: what was written in, or what its
+ * file holds (the text pulled out at upload, else unpacked now).
+ */
+export default async function grabKnowledgebaseDocumentText({ document }: Params) {
+    if (document.source == "text") return document.content || "";
+
+    if (!document.media_id) return "";
+
+    const media = await BunSQLite.sql<{ media_write_path?: string; text_content?: string }>({
+        sql: `SELECT media_write_path, text_content FROM media WHERE id = ? LIMIT 1`,
+        values: [document.media_id],
+    }).then((res) => res.payload?.[0]);
+
+    return (
+        (await extractBotFileText({
+            write_path: media?.media_write_path,
+            text_content: media?.text_content,
+            name: document.name,
+            mime_type: document.mime_type,
+        })) || ""
+    );
+}

src/functions/backend/db/knowledgebases/grab-knowledgebase.ts

@@ -0,0 +1,24 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASES } from "@/db/types/db";
+import AppError from "@/src/utils/app-error";
+import parseKnowledgebaseRow from "./parse-knowledgebase-row";
+
+type Params = {
+    knowledgebase_id: number | string;
+};
+
+/**
+ * One knowledgebase, or an `AppError` when there's none.
+ */
+export default async function grabKnowledgebase({ knowledgebase_id }: Params) {
+    const row = (
+        await BunSQLite.sql<BUN_SQLITE_COBBLE_KNOWLEDGEBASES>({
+            sql: `SELECT * FROM knowledgebases WHERE id = ?`,
+            values: [knowledgebase_id],
+        })
+    ).payload?.[0];
+
+    if (!row) throw new AppError(`That knowledgebase doesn't exist.`);
+
+    return parseKnowledgebaseRow({ row });
+}

src/functions/backend/db/knowledgebases/grab-knowledgebases-overview.ts

@@ -0,0 +1,79 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASES } from "@/db/types/db";
+import type { KnowledgebasesOverviewPayload } from "@/src/types";
+import grabCrewRole from "../access/grab-crew-role";
+import grabExtensionAccess from "../extensions/grab-extension-access";
+import grabExtensionTargets from "../extensions/grab-extension-targets";
+import { KnowledgebaseDocumentIndexingSql } from "./knowledgebase-document-indexing-sql";
+import grabInstallEmbeddingModel from "./grab-install-embedding-model";
+import parseKnowledgebaseRow from "./parse-knowledgebase-row";
+
+type Params = {
+    user_id: number | string;
+};
+
+type Counts = {
+    knowledgebase_id: number;
+    documents: number;
+    pending: number;
+    errors: number;
+    warnings: number;
+    stale: number;
+    size: number;
+};
+
+/**
+ * The Knowledgebases settings page: every knowledgebase with who gets it
+ * and how much of it is searchable, what it can be attached to, and the
+ * install's embedding model. Admins read it and choose who gets each;
+ * only a super admin adds, edits, fills or deletes one.
+ */
+export default async function grabKnowledgebasesOverview({
+    user_id,
+}: Params): Promise<KnowledgebasesOverviewPayload> {
+    const role = await grabCrewRole({ user_id });
+    const embedding_model = await grabInstallEmbeddingModel();
+
+    const [rows, counts, access, targets] = await Promise.all([
+        BunSQLite.sql<BUN_SQLITE_COBBLE_KNOWLEDGEBASES>({
+            sql: `SELECT * FROM knowledgebases ORDER BY name COLLATE NOCASE ASC, id ASC`,
+        }),
+        BunSQLite.sql<Counts>({
+            sql: `SELECT knowledgebase_id,
+                    COUNT(*) AS documents,
+                    SUM(CASE WHEN ${KnowledgebaseDocumentIndexingSql} THEN 1 ELSE 0 END) AS pending,
+                    SUM(CASE WHEN ingest_error IS NOT NULL AND COALESCE(chunk_count, 0) = 0 THEN 1 ELSE 0 END) AS errors,
+                    SUM(CASE WHEN ingest_error IS NOT NULL AND NOT ${KnowledgebaseDocumentIndexingSql}
+                        THEN 1 ELSE 0 END) AS warnings,
+                    SUM(CASE WHEN ingested_at IS NOT NULL AND chunk_count > 0
+                        AND (embedding_model IS NULL OR embedding_model IS NOT ?) THEN 1 ELSE 0 END) AS stale,
+                    SUM(size) AS size
+                FROM knowledgebase_documents d GROUP BY knowledgebase_id`,
+            values: [embedding_model],
+        }),
+        grabExtensionAccess({ kind: "knowledgebase" }),
+        grabExtensionTargets({ role }),
+    ]);
+
+    const by_id = new Map((counts.payload || []).map((row) => [Number(row.knowledgebase_id), row]));
+
+    return {
+        ...targets,
+        embedding_model,
+        knowledgebases: (rows.payload || []).map((row) => {
+            const { all_bots, ...knowledgebase } = parseKnowledgebaseRow({ row });
+            const count = by_id.get(knowledgebase.id);
+
+            return {
+                ...knowledgebase,
+                access: { all_bots, ...access(knowledgebase.id) },
+                document_count: Number(count?.documents) || 0,
+                pending_count: Number(count?.pending) || 0,
+                error_count: Number(count?.errors) || 0,
+                warning_count: Number(count?.warnings) || 0,
+                stale_count: Number(count?.stale) || 0,
+                size: Number(count?.size) || 0,
+            };
+        }),
+    };
+}

src/functions/backend/db/knowledgebases/ingest-knowledgebase-document.ts

@@ -0,0 +1,297 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASE_CHUNKS, BunSQLiteTables } from "@/db/types/db";
+import { AppData } from "@/src/data/app-data";
+import embedText from "@/src/harness/embeddings/embed-text";
+import HarnessError from "@/src/harness/harness-error";
+import runWithFallbackScope from "@/src/harness/usage/run-with-fallback-scope";
+import chunkText from "../bot-files/chunk-text";
+import ensureKnowledgebaseFts from "./fts/ensure-knowledgebase-fts";
+import grabKnowledgebaseDocumentText from "./grab-knowledgebase-document-text";
+
+type Params = {
+    document_id: number | string;
+    signal?: AbortSignal;
+};
+
+export type KnowledgebaseIngestResult = {
+    /**
+     * `ingested`, `empty` (no text to read), `skipped` (too large) or
+     * `words` (no embedding model: searchable by words only).
+     */
+    status: "ingested" | "empty" | "skipped" | "words" | "missing";
+    chunks: number;
+};
+
+/**
+ * Reads a knowledgebase document's text, splits it into chunks and embeds
+ * them with the install's embedding model (never a bot's: every vector in
+ * a knowledgebase must be comparable). The old passages stay searchable
+ * until the new ones are in. Without an embedding model the chunks are
+ * still kept, and indexed for ranked word matching
+ * (`knowledgebase_chunks_fts`).
+ *
+ * Failures are recorded on the document (`ingest_error`) and rethrown, so
+ * the queue can retry.
+ */
+export default async function ingestKnowledgebaseDocument({
+    document_id,
+    signal,
+}: Params): Promise<KnowledgebaseIngestResult> {
+    const document = (
+        await BunSQLite.sql<{
+            id: number;
+            knowledgebase_id: number;
+            name: string;
+            source: string;
+            media_id: number | null;
+            content: string | null;
+            mime_type: string | null;
+            size: number;
+            user_id: number | null;
+        }>({
+            sql: `SELECT id, knowledgebase_id, name, source, media_id, content, mime_type, size, user_id
+                FROM knowledgebase_documents WHERE id = ? LIMIT 1`,
+            values: [document_id],
+        })
+    ).payload?.[0];
+
+    if (!document?.id) return { status: "missing", chunks: 0 };
+
+    if (Number(document.size) > AppData.KnowledgebaseDocumentMaxBytes) {
+        await finish({
+            document_id: document.id,
+            chunks: 0,
+            model: null,
+            error: `Too large to read (over ${AppData.KnowledgebaseDocumentMaxBytes / (1024 * 1024)} MB).`,
+        });
+
+        return { status: "skipped", chunks: 0 };
+    }
+
+    /**
+     * The ingest queue has no run behind it: the embedding is charged to
+     * whoever added the document, so it shows on the usage page.
+     */
+    const scope = document.user_id ? { user_id: document.user_id } : null;
+    /**
+     * Set once the document's outcome is written, so the catch below
+     * doesn't write over it.
+     */
+    let recorded = false;
+
+    try {
+        const text = await grabKnowledgebaseDocumentText({ document });
+
+        const old_ids = await chunkIds({ document_id: document.id });
+        const chunks = text.trim() ? chunkText({ text }) : [];
+
+        if (!chunks.length) {
+            await dropChunks({ ids: old_ids });
+            await finish({ document_id: document.id, chunks: 0, model: null, error: "It has no text to read." });
+
+            return { status: "empty", chunks: 0 };
+        }
+
+        const new_ids = await insertChunks({ document, chunks });
+        let model: string | null = null;
+
+        try {
+            for (let start = 0; start < chunks.length; start += AppData.EmbeddingBatchSize) {
+                const batch = chunks.slice(start, start + AppData.EmbeddingBatchSize);
+                const embedded = await runWithFallbackScope({
+                    scope,
+                    task: () => embedText({ input: batch, signal }),
+                });
+
+                model = embedded.model || model;
+
+                for (let i = 0; i < embedded.vectors.length; i++) {
+                    await insertVector({
+                        vector: embedded.vectors[i]!,
+                        chunk_id: new_ids[start + i]!,
+                        document_id: document.id,
+                        knowledgebase_id: document.knowledgebase_id,
+                    });
+                }
+            }
+        } catch (error) {
+            if (!(error instanceof HarnessError) || error.resting) {
+                /**
+                 * Not the model's answer (a bug, a database error), or no
+                 * model was tried yet: the old passages are still whole,
+                 * the new ones go, and the job tries again.
+                 */
+                await dropChunks({ ids: new_ids });
+                throw error;
+            }
+
+            /**
+             * No embedding model, or it refused: the new passages stay,
+             * found by their words, and none of their vectors are kept
+             * (half a document embedded would rank unevenly).
+             */
+            await dropVectors({ ids: new_ids });
+
+            if (error.code != "unavailable") {
+                await dropChunks({ ids: old_ids });
+                await finish({
+                    document_id: document.id,
+                    chunks: chunks.length,
+                    model: null,
+                    error: `Found by its words only: the embedding model failed. ${error.message}`.slice(0, 500),
+                });
+                recorded = true;
+
+                throw error;
+            }
+
+            model = null;
+        }
+
+        await dropChunks({ ids: old_ids });
+        await finish({ document_id: document.id, chunks: chunks.length, model });
+
+        return { status: model ? "ingested" : "words", chunks: chunks.length };
+    } catch (error) {
+        /**
+         * Every model resting: nothing was tried, the job waits for them
+         * and the document keeps what it said before.
+         */
+        if (recorded || (error instanceof HarnessError && error.resting)) throw error;
+
+        const message = error instanceof Error ? error.message : String(error);
+
+        await BunSQLite.sql({
+            sql: `UPDATE knowledgebase_documents SET ingest_error = ?, updated_at = ? WHERE id = ?`,
+            values: [message.slice(0, 500), Date.now(), document.id],
+        }).catch(() => {});
+
+        throw error;
+    }
+}
+
+async function chunkIds({ document_id }: { document_id: number }) {
+    const res = await BunSQLite.sql<{ id: number }>({
+        sql: `SELECT id FROM knowledgebase_chunks WHERE document_id = ?`,
+        values: [document_id],
+    });
+
+    return (res.payload || []).map((row) => Number(row.id));
+}
+
+async function dropVectors({ ids }: { ids: number[] }) {
+    for (let start = 0; start < ids.length; start += 500) {
+        const batch = ids.slice(start, start + 500);
+
+        await BunSQLite.sql({
+            sql: `DELETE FROM knowledgebase_vectors WHERE chunk_id IN (${batch.map(() => "?").join(", ")})`,
+            values: batch,
+        });
+    }
+}
+
+async function dropChunks({ ids }: { ids: number[] }) {
+    for (let start = 0; start < ids.length; start += 500) {
+        const batch = ids.slice(start, start + 500);
+        const placeholders = batch.map(() => "?").join(", ");
+
+        await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_vectors WHERE chunk_id IN (${placeholders})`, values: batch });
+        await BunSQLite.sql({
+            sql: `DELETE FROM knowledgebase_chunks_fts WHERE chunk_id IN (${placeholders})`,
+            values: batch,
+        });
+        await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_chunks WHERE id IN (${placeholders})`, values: batch });
+    }
+}
+
+async function insertChunks({
+    document,
+    chunks,
+}: {
+    document: { id: number; knowledgebase_id: number };
+    chunks: string[];
+}) {
+    const ids: number[] = [];
+
+    await ensureKnowledgebaseFts();
+
+    for (let index = 0; index < chunks.length; index++) {
+        const text = chunks[index]!;
+
+        const res = await BunSQLite.insert<BUN_SQLITE_COBBLE_KNOWLEDGEBASE_CHUNKS, (typeof BunSQLiteTables)[number]>({
+            table: "knowledgebase_chunks",
+            data: [
+                {
+                    document_id: document.id,
+                    knowledgebase_id: Number(document.knowledgebase_id),
+                    chunk_index: index,
+                    text,
+                    token_estimate: Math.ceil(text.length / 4),
+                },
+            ],
+        });
+
+        const id = res.postInsertReturn?.insertId;
+
+        if (!res.success || !id) throw new Error(`Couldn't store a passage: ${res.msg || res.error}`);
+
+        ids.push(Number(id));
+
+        /**
+         * Its words go in the full-text index, which ranks a word search.
+         */
+        const indexed = await BunSQLite.sql({
+            sql: `INSERT INTO knowledgebase_chunks_fts (text, chunk_id, document_id, knowledgebase_id) VALUES (?, ?, ?, ?)`,
+            values: [text, Number(id), document.id, Number(document.knowledgebase_id)],
+        });
+
+        if (!indexed.success) throw new Error(`Couldn't index a passage: ${indexed.error || indexed.msg}`);
+    }
+
+    return ids;
+}
+
+async function insertVector({
+    vector,
+    chunk_id,
+    document_id,
+    knowledgebase_id,
+}: {
+    vector: Float32Array;
+    chunk_id: number;
+    document_id: number;
+    knowledgebase_id: number;
+}) {
+    /**
+     * vec0's metadata columns (its id and timestamps) are NOT NULL, so a
+     * raw insert fills them; the vector needs `vec_f32`.
+     */
+    const now = Date.now();
+
+    const res = await BunSQLite.sql({
+        sql: `INSERT INTO knowledgebase_vectors (created_at, updated_at, embedding, chunk_id, document_id, knowledgebase_id) VALUES (?, ?, vec_f32(?), ?, ?, ?)`,
+        values: [now, now, vector as any, chunk_id, document_id, knowledgebase_id],
+    });
+
+    if (!res.success) throw new Error(`Couldn't store an embedding: ${res.error || res.msg}`);
+}
+
+async function finish({
+    document_id,
+    chunks,
+    model,
+    error,
+}: {
+    document_id: number;
+    chunks: number;
+    model: string | null;
+    error?: string;
+}) {
+    const now = Date.now();
+
+    await BunSQLite.sql({
+        sql: `UPDATE knowledgebase_documents SET ingested_at = ?, chunk_count = ?, embedding_model = ?, ingest_error = ?,
+                updated_at = ? WHERE id = ?`,
+        values: [now, chunks, model, error || null, now, document_id],
+    });
+}

src/functions/backend/db/knowledgebases/knowledgebase-document-indexing-sql.ts

@@ -0,0 +1,7 @@
+/**
+ * Whether a `knowledgebase_documents` row aliased `d` is being read: never
+ * read yet, or with an ingest job still waiting or running (a re-index
+ * leaves `ingested_at` set while it reads the document again).
+ */
+export const KnowledgebaseDocumentIndexingSql = `((d.ingested_at IS NULL AND d.ingest_error IS NULL)
+    OR EXISTS (SELECT 1 FROM file_ingest_queue q WHERE q.document_id = d.id AND q.status IN ('pending', 'processing')))`;

src/functions/backend/db/knowledgebases/list-knowledgebase-documents.ts

@@ -0,0 +1,31 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS } from "@/db/types/db";
+import type { KnowledgebaseDocumentsPayload } from "@/src/types";
+import grabKnowledgebase from "./grab-knowledgebase";
+import { KnowledgebaseDocumentIndexingSql } from "./knowledgebase-document-indexing-sql";
+import parseKnowledgebaseDocumentRow from "./parse-knowledgebase-document-row";
+
+type Params = {
+    knowledgebase_id: number | string;
+};
+
+/**
+ * A knowledgebase's documents, newest first, without their text.
+ */
+export default async function listKnowledgebaseDocuments({
+    knowledgebase_id,
+}: Params): Promise<KnowledgebaseDocumentsPayload> {
+    const knowledgebase = await grabKnowledgebase({ knowledgebase_id });
+
+    const rows = await BunSQLite.sql<BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS & { indexing?: number }>({
+        sql: `SELECT id, knowledgebase_id, name, source, mime_type, size, chunk_count, embedding_model,
+                ingested_at, ingest_error, created_at, ${KnowledgebaseDocumentIndexingSql} AS indexing
+            FROM knowledgebase_documents d WHERE knowledgebase_id = ? ORDER BY id DESC`,
+        values: [knowledgebase.id],
+    });
+
+    return {
+        knowledgebase_id: knowledgebase.id,
+        documents: (rows.payload || []).map((row) => parseKnowledgebaseDocumentRow({ row })),
+    };
+}

src/functions/backend/db/knowledgebases/locate-passage.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, test } from "bun:test";
+import locatePassage from "./locate-passage";
+
+describe("locatePassage", () => {
+    const text = "# Leave\n\nTwenty days a year.\n\n# Refunds\n\nRefunds are issued within fourteen days.";
+
+    test("finds a passage by its last line, counted back by what comes before it", () => {
+        const at = locatePassage({ text, passage: "# Refunds\n\nRefunds are issued within fourteen days." });
+
+        expect(text.slice(at).startsWith("# Refunds")).toBe(true);
+    });
+
+    test("starts at the top when it can't be found", () => {
+        expect(locatePassage({ text, passage: "Nothing like this" })).toBe(0);
+    });
+});

src/functions/backend/db/knowledgebases/locate-passage.ts

@@ -0,0 +1,31 @@
+type Params = {
+    /**
+     * The document's whole text.
+     */
+    text: string;
+    /**
+     * One of its passages, which may start with a heading carried over and
+     * a stretch repeated from the passage before.
+     */
+    passage: string;
+};
+
+/**
+ * Where a passage sits in its document, roughly: found by its last line
+ * (the part least likely to be a carried heading or an overlap), counted
+ * back by where that line sits in the passage. 0 when it can't be found.
+ */
+export default function locatePassage({ text, passage }: Params) {
+    const lines = passage.split("\n").map((line) => line.trim()).filter(Boolean);
+
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i]!;
+        const at = text.indexOf(line);
+
+        if (at < 0) continue;
+
+        return Math.max(0, at - passage.lastIndexOf(line));
+    }
+
+    return 0;
+}

src/functions/backend/db/knowledgebases/parse-knowledgebase-document-row.ts

@@ -0,0 +1,30 @@
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS } from "@/db/types/db";
+import type { KnowledgebaseDocument } from "@/src/types";
+
+type Params = {
+    /**
+     * With `indexing` when the query asked (`KnowledgebaseDocumentIndexingSql`).
+     */
+    row: BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS & { indexing?: number | boolean };
+};
+
+/**
+ * A `knowledgebase_documents` row as the settings page shows it (a text
+ * document's content is left out: it loads when it's opened).
+ */
+export default function parseKnowledgebaseDocumentRow({ row }: Params): KnowledgebaseDocument {
+    return {
+        id: Number(row.id),
+        knowledgebase_id: Number(row.knowledgebase_id),
+        name: row.name || "",
+        source: row.source == "text" ? "text" : "file",
+        mime_type: row.mime_type || "",
+        size: Number(row.size) || 0,
+        chunk_count: Number(row.chunk_count) || 0,
+        embedding_model: row.embedding_model || null,
+        ingested_at: typeof row.ingested_at == "number" ? row.ingested_at : null,
+        ingest_error: row.ingest_error || null,
+        indexing: row.indexing == undefined ? !row.ingested_at && !row.ingest_error : Boolean(row.indexing),
+        created_at: typeof row.created_at == "number" ? row.created_at : null,
+    };
+}

src/functions/backend/db/knowledgebases/parse-knowledgebase-row.ts

@@ -0,0 +1,20 @@
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASES } from "@/db/types/db";
+import type { KnowledgebaseRecord } from "@/src/types";
+
+type Params = {
+    row: BUN_SQLITE_COBBLE_KNOWLEDGEBASES;
+};
+
+/**
+ * A `knowledgebases` row, with its flags read as booleans.
+ */
+export default function parseKnowledgebaseRow({ row }: Params): KnowledgebaseRecord {
+    return {
+        id: Number(row.id),
+        name: row.name || "",
+        description: row.description || "",
+        all_bots: row.all_bots == 1,
+        enabled: row.enabled != 0,
+        updated_at: typeof row.updated_at == "number" ? row.updated_at : null,
+    };
+}

src/functions/backend/db/knowledgebases/read-knowledgebase-document.ts

@@ -0,0 +1,79 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import AppError from "@/src/utils/app-error";
+import grabKnowledgebaseDocumentText from "./grab-knowledgebase-document-text";
+import locatePassage from "./locate-passage";
+
+type Params = {
+    /**
+     * The knowledgebases the run reaches: a document outside them is
+     * refused.
+     */
+    knowledgebase_ids: number[];
+    document_id: number;
+    /**
+     * Where to start: a passage a search returned, or a character offset
+     * the last read said to go on from. Neither starts at the top.
+     */
+    from_passage?: number;
+    from_char?: number;
+    max_chars: number;
+};
+
+/**
+ * A stretch of a knowledgebase document's own text (not its passages,
+ * which overlap), up to `max_chars`, with the offset to go on from when
+ * there's more.
+ */
+export default async function readKnowledgebaseDocument({
+    knowledgebase_ids,
+    document_id,
+    from_passage,
+    from_char,
+    max_chars,
+}: Params) {
+    const document = (
+        await BunSQLite.sql<{
+            id: number;
+            knowledgebase_id: number;
+            name: string;
+            source: string;
+            content: string | null;
+            media_id: number | null;
+            mime_type: string | null;
+        }>({
+            sql: `SELECT id, knowledgebase_id, name, source, content, media_id, mime_type
+                FROM knowledgebase_documents WHERE id = ? LIMIT 1`,
+            values: [document_id],
+        })
+    ).payload?.[0];
+
+    if (!document || !knowledgebase_ids.includes(Number(document.knowledgebase_id))) {
+        throw new AppError(`That document isn't in a knowledgebase you have.`);
+    }
+
+    const text = (await grabKnowledgebaseDocumentText({ document })).replace(/\r\n/g, "\n");
+
+    const passage =
+        from_passage != null && from_char == null
+            ? (
+                  await BunSQLite.sql<{ text: string }>({
+                      sql: `SELECT text FROM knowledgebase_chunks WHERE document_id = ? AND chunk_index = ? LIMIT 1`,
+                      values: [document.id, from_passage],
+                  })
+              ).payload?.[0]?.text
+            : undefined;
+
+    const start = Math.min(
+        text.length,
+        Math.max(0, from_char ?? (passage ? locatePassage({ text, passage }) : 0)),
+    );
+    const end = Math.min(text.length, start + max_chars);
+
+    return {
+        name: document.name,
+        start,
+        length: text.length,
+        text: text.slice(start, end),
+        next: end < text.length ? end : null,
+    };
+}

src/functions/backend/db/knowledgebases/record-knowledgebase-document.ts

@@ -0,0 +1,68 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS, BunSQLiteTables } from "@/db/types/db";
+import type { KnowledgebaseDocumentSource } from "@/src/dict/knowledgebase-dict";
+import enqueueDocumentIngest from "@/src/file-ingest/enqueue-document-ingest";
+import grabKnowledgebase from "./grab-knowledgebase";
+
+type Params = {
+    user_id: number | string;
+    knowledgebase_id: number | string;
+    name: string;
+    source: KnowledgebaseDocumentSource;
+    media_id?: number | null;
+    content?: string | null;
+    mime_type?: string;
+    size: number;
+};
+
+/**
+ * Adds a document to a knowledgebase and queues it to be chunked and
+ * embedded. Returns its id.
+ */
+export default async function recordKnowledgebaseDocument({
+    user_id,
+    knowledgebase_id,
+    name,
+    source,
+    media_id,
+    content,
+    mime_type,
+    size,
+}: Params) {
+    const knowledgebase = await grabKnowledgebase({ knowledgebase_id });
+
+    const res = await BunSQLite.insert<BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS, (typeof BunSQLiteTables)[number]>({
+        table: "knowledgebase_documents",
+        data: [
+            {
+                knowledgebase_id: knowledgebase.id,
+                name,
+                source,
+                media_id: media_id || undefined,
+                content: content || undefined,
+                mime_type: mime_type || "",
+                size,
+                user_id: Number(user_id),
+            },
+        ],
+    });
+
+    const id = res.postInsertReturn?.insertId;
+
+    if (!res.success || !id) throw new Error(`Couldn't add the document: ${res.msg || res.error}`);
+
+    await touchKnowledgebase({ knowledgebase_id: knowledgebase.id });
+    await enqueueDocumentIngest({ document_id: id });
+
+    return Number(id);
+}
+
+/**
+ * Marks a knowledgebase changed, for its "updated" line.
+ */
+export async function touchKnowledgebase({ knowledgebase_id }: { knowledgebase_id: number | string }) {
+    await BunSQLite.sql({
+        sql: `UPDATE knowledgebases SET updated_at = ? WHERE id = ?`,
+        values: [Date.now(), knowledgebase_id],
+    });
+}

src/functions/backend/db/knowledgebases/reindex-after-embedding-change.ts

@@ -0,0 +1,38 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import enqueueDocumentIngest from "@/src/file-ingest/enqueue-document-ingest";
+import grabInstallEmbeddingModel from "./grab-install-embedding-model";
+
+type Params = {
+    /**
+     * The install's embedding model before the change
+     * (`grabInstallEmbeddingModel`), null when there was none.
+     */
+    before: string | null;
+};
+
+/**
+ * After something that can change the install's embedding model (its
+ * setting, a provider, the catalog), queues every knowledgebase document
+ * that was read with another model, or by words only, to be read again:
+ * a query embedded by the new model can't be compared with vectors from
+ * the old one. Nothing happens when the model is the same, or there's
+ * none. Returns how many documents were queued.
+ */
+export default async function reindexAfterEmbeddingChange({ before }: Params) {
+    const after = await grabInstallEmbeddingModel();
+
+    if (!after || after == before) return 0;
+
+    const documents = await BunSQLite.sql<{ id: number }>({
+        sql: `SELECT id FROM knowledgebase_documents
+            WHERE ingested_at IS NOT NULL AND chunk_count > 0
+              AND (embedding_model IS NULL OR embedding_model != ?)`,
+        values: [after],
+    });
+
+    for (const document of documents.payload || []) {
+        await enqueueDocumentIngest({ document_id: document.id });
+    }
+
+    return documents.payload?.length || 0;
+}

src/functions/backend/db/knowledgebases/reindex-knowledgebase.ts

@@ -0,0 +1,32 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import enqueueDocumentIngest from "@/src/file-ingest/enqueue-document-ingest";
+import grabKnowledgebase from "./grab-knowledgebase";
+
+type Params = {
+    knowledgebase_id: number | string;
+};
+
+/**
+ * Reads every document of a knowledgebase again (after the install's
+ * embedding model changed, or one failed). Each keeps its old passages
+ * until its new ones are in. Returns how many were queued.
+ */
+export default async function reindexKnowledgebase({ knowledgebase_id }: Params) {
+    const knowledgebase = await grabKnowledgebase({ knowledgebase_id });
+
+    const documents = await BunSQLite.sql<{ id: number }>({
+        sql: `SELECT id FROM knowledgebase_documents WHERE knowledgebase_id = ?`,
+        values: [knowledgebase.id],
+    });
+
+    await BunSQLite.sql({
+        sql: `UPDATE knowledgebase_documents SET ingest_error = NULL WHERE knowledgebase_id = ?`,
+        values: [knowledgebase.id],
+    });
+
+    for (const document of documents.payload || []) {
+        await enqueueDocumentIngest({ document_id: document.id });
+    }
+
+    return documents.payload?.length || 0;
+}

src/functions/backend/db/knowledgebases/save-knowledgebase-text.ts

@@ -0,0 +1,50 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import enqueueDocumentIngest from "@/src/file-ingest/enqueue-document-ingest";
+import type { SaveKnowledgebaseTextRequest } from "@/src/schemas/knowledgebase-schema";
+import AppError from "@/src/utils/app-error";
+import grabKnowledgebaseDocumentRow from "./grab-knowledgebase-document-row";
+import recordKnowledgebaseDocument, { touchKnowledgebase } from "./record-knowledgebase-document";
+
+type Params = {
+    user_id: number | string;
+    data: SaveKnowledgebaseTextRequest;
+};
+
+/**
+ * Writes a text document into a knowledgebase, or changes one, and queues
+ * it to be read again. Returns its id.
+ */
+export default async function saveKnowledgebaseText({ user_id, data }: Params) {
+    const size = Buffer.byteLength(data.content, "utf8");
+
+    if (!data.document_id) {
+        return await recordKnowledgebaseDocument({
+            user_id,
+            knowledgebase_id: data.knowledgebase_id,
+            name: data.name,
+            source: "text",
+            content: data.content,
+            mime_type: "text/markdown",
+            size,
+        });
+    }
+
+    const document = await grabKnowledgebaseDocumentRow({ document_id: data.document_id });
+
+    if (Number(document.knowledgebase_id) != data.knowledgebase_id || document.source != "text") {
+        throw new AppError(`Only a text document can be edited here.`);
+    }
+
+    const res = await BunSQLite.sql({
+        sql: `UPDATE knowledgebase_documents SET name = ?, content = ?, size = ?, ingested_at = NULL,
+                ingest_error = NULL, updated_at = ? WHERE id = ?`,
+        values: [data.name, data.content, size, Date.now(), document.id],
+    });
+
+    if (!res.success) throw new Error(`Couldn't save the document: ${res.error}`);
+
+    await touchKnowledgebase({ knowledgebase_id: data.knowledgebase_id });
+    await enqueueDocumentIngest({ document_id: document.id! });
+
+    return Number(document.id);
+}

src/functions/backend/db/knowledgebases/save-knowledgebase.ts

@@ -0,0 +1,57 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_KNOWLEDGEBASES, BunSQLiteTables } from "@/db/types/db";
+import type { SaveKnowledgebaseRequest } from "@/src/schemas/knowledgebase-schema";
+import attachToAllBots from "../extensions/attach-to-all-bots";
+import grabKnowledgebase from "./grab-knowledgebase";
+
+type Params = {
+    /**
+     * The super admin saving it (checked by the caller).
+     */
+    user_id: number | string;
+    data: SaveKnowledgebaseRequest;
+};
+
+/**
+ * Adds or edits a knowledgebase (its name, what it holds, on or off, every
+ * bot) and returns its id. Its documents are added on their own.
+ */
+export default async function saveKnowledgebase({ user_id, data }: Params) {
+    const fields = {
+        name: data.name,
+        description: data.description,
+        enabled: data.enabled ? 1 : 0,
+        all_bots: data.all_bots ? 1 : 0,
+    } as const;
+
+    if (data.knowledgebase_id) {
+        const knowledgebase = await grabKnowledgebase({ knowledgebase_id: data.knowledgebase_id });
+
+        const res = await BunSQLite.update<BUN_SQLITE_COBBLE_KNOWLEDGEBASES, (typeof BunSQLiteTables)[number]>({
+            table: "knowledgebases",
+            targetId: knowledgebase.id,
+            data: fields,
+        });
+
+        if (!res.success) throw new Error(`Couldn't save the knowledgebase: ${res.msg}`);
+
+        if (data.all_bots && !knowledgebase.all_bots) {
+            await attachToAllBots({ kind: "knowledgebase", id: knowledgebase.id });
+        }
+
+        return knowledgebase.id;
+    }
+
+    const res = await BunSQLite.insert<BUN_SQLITE_COBBLE_KNOWLEDGEBASES, (typeof BunSQLiteTables)[number]>({
+        table: "knowledgebases",
+        data: [{ ...fields, user_id: Number(user_id) }],
+    });
+
+    const id = res.singleRes?.id || res.postInsertReturn?.insertId;
+
+    if (!res.success || !id) throw new Error(`Couldn't add the knowledgebase: ${res.msg}`);
+
+    if (data.all_bots) await attachToAllBots({ kind: "knowledgebase", id: Number(id) });
+
+    return Number(id);
+}

src/functions/backend/db/knowledgebases/search-knowledgebases.ts

@@ -0,0 +1,245 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import { AppData } from "@/src/data/app-data";
+import { embedQuery } from "@/src/harness/embeddings/embed-text";
+import HarnessError from "@/src/harness/harness-error";
+import knowledgebaseWordQuery from "@/src/utils/knowledgebase-word-query";
+import ensureKnowledgebaseFts from "./fts/ensure-knowledgebase-fts";
+
+type Params = {
+    /**
+     * The knowledgebases the run reaches: nothing outside them is returned.
+     */
+    knowledgebase_ids: number[];
+    query: string;
+    /**
+     * Only this document's passages.
+     */
+    document_id?: number | null;
+    limit?: number;
+    signal?: AbortSignal;
+};
+
+export type KnowledgebaseSearchHit = {
+    knowledgebase_id: number;
+    document_id: number;
+    chunk_id: number;
+    chunk_index: number;
+    document_name: string;
+    /**
+     * Cosine distance when the vector index answered, null for a word
+     * match.
+     */
+    distance: number | null;
+    text: string;
+};
+
+export type KnowledgebaseSearchResult = {
+    hits: KnowledgebaseSearchHit[];
+    mode: "vector" | "words" | "none";
+    /**
+     * Said plainly for the bot's own reading of what it got back.
+     */
+    note: string;
+};
+
+/**
+ * Searches knowledgebases: by meaning through the install's embedding
+ * model, else (or when nothing matches) by words over the stored passages.
+ * Never throws for an unavailable model.
+ */
+export default async function searchKnowledgebases({
+    knowledgebase_ids,
+    query,
+    document_id,
+    limit = AppData.RagTopChunks,
+    signal,
+}: Params): Promise<KnowledgebaseSearchResult> {
+    const text = query.trim();
+    const ids = [...new Set(knowledgebase_ids.map(Number).filter(Boolean))];
+
+    if (!ids.length || !text) {
+        return { hits: [], mode: "none", note: "No knowledgebase to search." };
+    }
+
+    try {
+        const vector = await embedQuery({ text, signal });
+
+        const hits = await vectorSearch({
+            ids,
+            query_vector: vector,
+            document_id,
+            limit,
+            seek: Math.min(limit * AppData.RagOverFetch, AppData.RagMaxK),
+        });
+
+        if (hits.length) {
+            return { hits, mode: "vector", note: `${hits.length} passage(s), found by meaning.` };
+        }
+    } catch (error) {
+        if (!(error instanceof HarnessError)) throw error;
+    }
+
+    const hits = await wordSearch({ ids, query: text, document_id, limit });
+
+    return {
+        hits,
+        mode: hits.length ? "words" : "none",
+        note: hits.length
+            ? `${hits.length} passage(s), found by matching words (no embedding model, or nothing matched by meaning).`
+            : `Nothing in the knowledgebase matched "${text}".`,
+    };
+}
+
+/**
+ * The nearest passages by vector. sqlite-vec refuses a filter on a vec0
+ * side-car column in the same query as the match, so the index is asked
+ * for its nearest `seek` and the run's knowledgebases are applied here.
+ */
+async function vectorSearch({
+    ids,
+    query_vector,
+    document_id,
+    limit,
+    seek,
+}: {
+    ids: number[];
+    query_vector: Float32Array;
+    document_id?: number | null;
+    limit: number;
+    seek: number;
+}): Promise<KnowledgebaseSearchHit[]> {
+    const nearest = await BunSQLite.sql<{
+        chunk_id: number;
+        document_id: number;
+        knowledgebase_id: number;
+        distance: number;
+    }>({
+        sql: `SELECT chunk_id, document_id, knowledgebase_id, distance FROM knowledgebase_vectors
+            WHERE embedding MATCH vec_f32(?) AND k = ? ORDER BY distance`,
+        values: [query_vector as any, seek],
+    });
+
+    if (!nearest.success) {
+        throw new Error(`The knowledgebase index couldn't be read: ${nearest.error || nearest.msg}`);
+    }
+
+    const picked = (nearest.payload || [])
+        .filter((row) => ids.includes(Number(row.knowledgebase_id)))
+        .filter((row) => !document_id || Number(row.document_id) == Number(document_id))
+        .slice(0, limit);
+
+    if (!picked.length) return [];
+
+    const chunk_ids = picked.map((row) => Number(row.chunk_id));
+    const chunks = await BunSQLite.sql<{ id: number; text: string; chunk_index: number; name: string }>({
+        sql: `SELECT c.id, c.text, c.chunk_index, d.name FROM knowledgebase_chunks c
+            JOIN knowledgebase_documents d ON d.id = c.document_id
+            WHERE c.id IN (${chunk_ids.map(() => "?").join(", ")})`,
+        values: chunk_ids,
+    });
+
+    const by_id = new Map((chunks.payload || []).map((row) => [Number(row.id), row]));
+
+    return picked.flatMap((row) => {
+        const chunk = by_id.get(Number(row.chunk_id));
+
+        if (!chunk) return [];
+
+        return [
+            {
+                knowledgebase_id: Number(row.knowledgebase_id),
+                document_id: Number(row.document_id),
+                chunk_id: Number(row.chunk_id),
+                chunk_index: Number(chunk.chunk_index),
+                document_name: chunk.name || "",
+                distance: Number(row.distance),
+                text: chunk.text || "",
+            },
+        ];
+    });
+}
+
+/**
+ * Passages by their words: ranked by the full-text index (BM25, so a rare
+ * word counts for more than a common one), else, when it finds nothing
+ * (text written without spaces, a word cut differently), by the most
+ * substrings they hold.
+ */
+async function wordSearch({
+    ids,
+    query,
+    document_id,
+    limit,
+}: {
+    ids: number[];
+    query: string;
+    document_id?: number | null;
+    limit: number;
+}): Promise<KnowledgebaseSearchHit[]> {
+    const { match, like_terms } = knowledgebaseWordQuery({ query });
+    const in_ids = ids.map(() => "?").join(", ");
+
+    if (match) {
+        await ensureKnowledgebaseFts();
+
+        const res = await BunSQLite.sql<HitRow>({
+            sql: `SELECT c.knowledgebase_id, c.document_id, c.id AS chunk_id, c.chunk_index, d.name, c.text
+                FROM knowledgebase_chunks_fts f
+                JOIN knowledgebase_chunks c ON c.id = f.chunk_id
+                JOIN knowledgebase_documents d ON d.id = c.document_id
+                WHERE knowledgebase_chunks_fts MATCH ?
+                  AND f.knowledgebase_id IN (${in_ids})
+                  ${document_id ? "AND f.document_id = ?" : ""}
+                ORDER BY bm25(knowledgebase_chunks_fts)
+                LIMIT ?`,
+            values: [match, ...ids, ...(document_id ? [document_id] : []), limit],
+        });
+
+        if (!res.success) {
+            throw new Error(`The knowledgebase word index couldn't be read: ${res.error || res.msg}`);
+        }
+
+        if (res.payload?.length) return res.payload.map(toHit);
+    }
+
+    if (!like_terms.length) return [];
+
+    const score = like_terms.map(() => `(CASE WHEN lower(c.text) LIKE ? THEN 1 ELSE 0 END)`).join(" + ");
+    const like_values = like_terms.map((term) => `%${term}%`);
+
+    const res = await BunSQLite.sql<HitRow>({
+        sql: `SELECT c.knowledgebase_id, c.document_id, c.id AS chunk_id, c.chunk_index, d.name, c.text,
+                (${score}) AS score
+            FROM knowledgebase_chunks c
+            JOIN knowledgebase_documents d ON d.id = c.document_id
+            WHERE c.knowledgebase_id IN (${in_ids})
+              ${document_id ? "AND c.document_id = ?" : ""}
+              AND (${like_terms.map(() => `lower(c.text) LIKE ?`).join(" OR ")})
+            ORDER BY score DESC, d.updated_at DESC, c.chunk_index
+            LIMIT ?`,
+        values: [...like_values, ...ids, ...(document_id ? [document_id] : []), ...like_values, limit],
+    });
+
+    return (res.payload || []).map(toHit);
+}
+
+type HitRow = {
+    knowledgebase_id: number;
+    document_id: number;
+    chunk_id: number;
+    chunk_index: number;
+    name: string;
+    text: string;
+};
+
+function toHit(row: HitRow): KnowledgebaseSearchHit {
+    return {
+        knowledgebase_id: Number(row.knowledgebase_id),
+        document_id: Number(row.document_id),
+        chunk_id: Number(row.chunk_id),
+        chunk_index: Number(row.chunk_index),
+        document_name: row.name || "",
+        distance: null,
+        text: row.text || "",
+    };
+}