Add knowledgebases

Knowledgebases give bots RAG over documents, scoped like extensions: to particular bots, to a workspace, or to every bot, with their own button beside Tools in the composer. Documents are chunked and embedded by the file-ingest worker with the install's embedding model, which is now an install-wide setting rather than a per-bot one. - Word search runs on an FTS5 index ranked by BM25, with a scored LIKE fallback for scripts written without spaces. - Changing the embedding model re-indexes documents automatically. - Ingest jobs wait without spending attempts while every embedding model rests, fail at once on errors retrying can't fix, and keep passages searchable by words when the model refuses. - Settings > Knowledgebases shows each knowledgebase's and document's index state: spinner, warning, grey check (words only) or green check. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-26 16:41:01 +01:00
parent 2259b6895f
commit e7b23b7454
111 changed files with 4573 additions and 148 deletions
@@ -0,0 +1,49 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import { rmSync } from "node:fs";
+import mediaFilePath from "@/src/utils/media-file-path";
+import grabKnowledgebaseDocumentRow from "./grab-knowledgebase-document-row";
+
+type Params = {
+    document_id: number | string;
+};
+
+/**
+ * Deletes a knowledgebase document: its vectors, chunks (and their
+ * full-text index rows), pending ingest,
+ * row and (for a file) its bytes and `media` row. Children are deleted here
+ * rather than left to the schema's cascade, since foreign keys are off
+ * outside the server process.
+ */
+export default async function deleteKnowledgebaseDocument({ document_id }: Params) {
+    const document = await grabKnowledgebaseDocumentRow({ document_id });
+
+    await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_vectors WHERE document_id = ?`, values: [document.id] });
+    await BunSQLite.sql({
+        sql: `DELETE FROM knowledgebase_chunks_fts WHERE document_id = ?`,
+        values: [document.id],
+    }).catch(() => {});
+    await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_chunks WHERE document_id = ?`, values: [document.id] });
+    await BunSQLite.sql({
+        sql: `DELETE FROM file_ingest_queue WHERE document_id = ? AND status IN ('pending', 'processing')`,
+        values: [document.id],
+    }).catch(() => {});
+    await BunSQLite.sql({ sql: `DELETE FROM knowledgebase_documents WHERE id = ?`, values: [document.id] });
+
+    if (document.media_id) {
+        const media = await BunSQLite.sql<{ media_write_path?: string; media_thumbnail_write_path?: string }>({
+            sql: `SELECT media_write_path, media_thumbnail_write_path FROM media WHERE id = ? LIMIT 1`,
+            values: [document.media_id],
+        }).then((res) => res.payload?.[0]);
+
+        for (const write_path of [media?.media_write_path, media?.media_thumbnail_write_path]) {
+            const absolute = mediaFilePath({ write_path });
+
+            if (absolute) rmSync(absolute, { force: true });
+        }
+
+        await BunSQLite.sql({ sql: `DELETE FROM media_paradigms WHERE media_id = ?`, values: [document.media_id] });
+        await BunSQLite.sql({ sql: `DELETE FROM media WHERE id = ?`, values: [document.media_id] });
+    }
+
+    return document;
+}