Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/file-ingest/process-ingest-job.ts

@@ -1,18 +1,27 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
 import type { BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE } from "@/db/types/db";
 import ingestBotFile from "@/src/functions/backend/db/bot-files/ingest-bot-file";
+import ingestKnowledgebaseDocument from "@/src/functions/backend/db/knowledgebases/ingest-knowledgebase-document";
 import { logError } from "@/src/functions/backend/logger";
+import settleFailedIngestJob from "./settle-failed-ingest-job";
 
 type Params = {
     job: BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE;
 };
 
 /**
- * Ingests one queued file and completes (or fails) its job. A file that
- * was deleted meanwhile is simply done.
+ * Ingests one queued file (or knowledgebase document) and completes (or
+ * fails) its job. One that was deleted meanwhile is simply done.
  */
 export default async function processIngestJob({ job }: Params) {
-    if (!job.id || !job.file_id) return;
+    if (!job.id) return;
+
+    if (job.document_id) return await processDocumentJob({ job_id: job.id, document_id: job.document_id });
+
+    if (!job.file_id) {
+        await BunSQLite.queue.complete({ table: "file_ingest_queue", id: job.id }).catch(() => {});
+        return;
+    }
 
     try {
         const result = await ingestBotFile({ file_id: job.file_id });
@@ -26,11 +35,9 @@ export default async function processIngestJob({ job }: Params) {
             }).catch(() => {});
         }
     } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
+        const { deferred } = await settleFailedIngestJob({ job_id: job.id, error });
 
-        await BunSQLite.queue
-            .fail({ table: "file_ingest_queue", id: job.id, error: message.slice(0, 500) })
-            .catch(() => {});
+        if (deferred) return;
 
         await logError({
             title: `Bot file #${job.file_id} couldn't be ingested`,
@@ -39,3 +46,20 @@ export default async function processIngestJob({ job }: Params) {
         }).catch(() => {});
     }
 }
+
+async function processDocumentJob({ job_id, document_id }: { job_id: number; document_id: number }) {
+    try {
+        await ingestKnowledgebaseDocument({ document_id });
+        await BunSQLite.queue.complete({ table: "file_ingest_queue", id: job_id });
+    } catch (error) {
+        const { deferred } = await settleFailedIngestJob({ job_id, error });
+
+        if (deferred) return;
+
+        await logError({
+            title: `Knowledgebase document #${document_id} couldn't be ingested`,
+            error,
+            metadata: { document_id },
+        }).catch(() => {});
+    }
+}