Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 16 Wiki Activity

Add knowledgebases

Browse Source 
Knowledgebases give bots RAG over documents, scoped like extensions: to
particular bots, to a workspace, or to every bot, with their own button
beside Tools in the composer. Documents are chunked and embedded by the
file-ingest worker with the install's embedding model, which is now an
install-wide setting rather than a per-bot one.

- Word search runs on an FTS5 index ranked by BM25, with a scored LIKE
  fallback for scripts written without spaces.
- Changing the embedding model re-indexes documents automatically.
- Ingest jobs wait without spending attempts while every embedding model
  rests, fail at once on errors retrying can't fix, and keep passages
  searchable by words when the model refuses.
- Settings > Knowledgebases shows each knowledgebase's and document's
  index state: spinner, warning, grey check (words only) or green check.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-26 16:41:01 +01:00
co-authored by Claude Opus 5.5
parent 2259b6895f
commit e7b23b7454
111 changed files with 4573 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/schema.ts
+285 -1
View File
@@ -1,6 +1,7 @@
import { LogCategories } from "@/src/dict/log-categories";
import { BotTypes } from "@/src/dict/bot-types-dict";
import { MediaParadigms, MediaTypes } from "@/src/dict/media-dict";
import { KnowledgebaseDocumentSources } from "@/src/dict/knowledgebase-dict";
import {
BotFaces,
BotHues,
@@ -3005,6 +3006,280 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
},
],
},
{
tableName: "knowledgebases",
tableDescription: `Collections of documents bots search before they answer (Settings, under Knowledgebases): retrieval over the install's one embedding model. Per bot (bot_knowledgebases), per workspace (workspace_knowledgebases) or every bot (all_bots)`,
fields: [
{
fieldName: "name",
fieldDescription: `What the settings page and the composer call it, e.g. "Product handbook"`,
dataType: "TEXT",
},
{
fieldName: "description",
fieldDescription: `The one line a bot reads to know what it holds and when to search it`,
dataType: "TEXT",
},
{
fieldName: "all_bots",
fieldDescription: `1 gives it to every bot, the coordinator included`,
dataType: "INTEGER",
options: [0, 1],
defaultValue: 0,
},
{
fieldName: "enabled",
dataType: "INTEGER",
options: [0, 1],
defaultValue: 1,
},
{
fieldName: "user_id",
fieldDescription: `Who created it. Not a foreign key: it belongs to the install`,
dataType: "INTEGER",
},
],
},
{
tableName: "knowledgebase_documents",
tableDescription: `One document in a knowledgebase: an uploaded file (bytes in private media) or text written in. Chunked and embedded by the ingest worker`,
fields: [
{
fieldName: "knowledgebase_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "knowledgebases",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "knowledgebase_documents_knowledgebases_fk",
},
},
{
fieldName: "name",
dataType: "TEXT",
},
{
fieldName: "source",
dataType: "TEXT",
options: KnowledgebaseDocumentSources.map((s) => s.value),
defaultValue: "file",
},
{
fieldName: "media_id",
fieldDescription: `The uploaded file's private \`media\` row (a file document). Not a foreign key`,
dataType: "INTEGER",
},
{
fieldName: "content",
fieldDescription: `The text of a text document, markdown`,
dataType: "TEXT",
},
{
fieldName: "mime_type",
dataType: "TEXT",
},
{
fieldName: "size",
fieldDescription: `Bytes`,
dataType: "INTEGER",
defaultValue: 0,
},
{
fieldName: "chunk_count",
dataType: "INTEGER",
defaultValue: 0,
},
{
fieldName: "embedding_model",
fieldDescription: `The model its vectors came from, null when it's searchable by words only. A change of the install's embedding model calls for a re-index`,
dataType: "TEXT",
},
{
fieldName: "ingested_at",
fieldDescription: `Unix ms of its last ingest, null while it waits`,
dataType: "INTEGER",
},
{
fieldName: "ingest_error",
dataType: "TEXT",
},
{
fieldName: "user_id",
fieldDescription: `Who added it. Not a foreign key`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_knowledgebase_documents_knowledgebase_id",
indexTableFields: ["knowledgebase_id"],
},
],
},
{
tableName: "knowledgebase_chunks",
tableDescription: `A piece of a knowledgebase document's text, the unit a search returns. Replaced whole when the document is ingested again`,
fields: [
{
fieldName: "document_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "knowledgebase_documents",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "knowledgebase_chunks_documents_fk",
},
},
{
fieldName: "knowledgebase_id",
fieldDescription: `Copied from the document, so a search filters without a join`,
dataType: "INTEGER",
},
{
fieldName: "chunk_index",
dataType: "INTEGER",
},
{
fieldName: "text",
dataType: "TEXT",
},
{
fieldName: "token_estimate",
fieldDescription: `A rough count (characters / 4)`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_knowledgebase_chunks_document_id",
indexTableFields: ["document_id", "chunk_index"],
},
{
indexName: "idx_knowledgebase_chunks_knowledgebase_id",
indexTableFields: ["knowledgebase_id"],
},
],
},
{
tableName: "knowledgebase_vectors",
tableDescription: `sqlite-vec vectors of knowledgebase chunks, \`AppData.EmbeddingDimensions\` wide, from the install's embedding model. A vec0 table is only recreated with \`bunx bun-sqlite schema --vector\`, so the entrypoint never passes it`,
isVector: true,
vectorType: "vec0",
fields: [
{
fieldName: "embedding",
fieldDescription: `Normalized, padded or cut to \`AppData.EmbeddingDimensions\``,
dataType: "BLOB",
isVector: true,
vectorSize: AppData.EmbeddingDimensions,
},
{
fieldName: "chunk_id",
dataType: "INTEGER",
sideCar: true,
},
{
fieldName: "document_id",
dataType: "INTEGER",
sideCar: true,
},
{
fieldName: "knowledgebase_id",
dataType: "INTEGER",
sideCar: true,
},
],
},
{
tableName: "bot_knowledgebases",
tableDescription: `A bot that searches a knowledgebase everywhere it works`,
fields: [
{
fieldName: "bot_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "bots",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "bot_knowledgebases_bots_fk",
},
},
{
fieldName: "knowledgebase_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "knowledgebases",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "bot_knowledgebases_knowledgebases_fk",
},
},
],
uniqueConstraints: [
{
alias: "one_knowledgebase_per_bot",
constraintName: "A bot gets a knowledgebase once",
constraintTableFields: [
{ value: "bot_id" },
{ value: "knowledgebase_id" },
],
},
],
indexes: [
{
indexName: "idx_bot_knowledgebases_knowledgebase_id",
indexTableFields: ["knowledgebase_id"],
},
],
},
{
tableName: "workspace_knowledgebases",
tableDescription: `Every bot in the workspace searches the knowledgebase there`,
fields: [
{
fieldName: "workspace_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "workspaces",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "workspace_knowledgebases_workspaces_fk",
},
},
{
fieldName: "knowledgebase_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "knowledgebases",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "workspace_knowledgebases_knowledgebases_fk",
},
},
],
uniqueConstraints: [
{
alias: "one_knowledgebase_per_workspace",
constraintName: "A workspace gets a knowledgebase once",
constraintTableFields: [
{ value: "workspace_id" },
{ value: "knowledgebase_id" },
],
},
],
indexes: [
{
indexName: "idx_workspace_knowledgebases_knowledgebase_id",
indexTableFields: ["knowledgebase_id"],
},
],
},
{
tableName: "logs",
fields: [
@@ -3594,7 +3869,7 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
},
{
tableName: "file_ingest_queue",
tableDescription: `Files waiting to have their text chunked and embedded. A queue, so a restart never strands a file and an upload never waits on a model. A finished job is deleted: \`bot_files.ingested_at\` and \`ingest_error\` keep the outcome`,
tableDescription: `Files (and knowledgebase documents) waiting to have their text chunked and embedded. A queue, so a restart never strands a file and an upload never waits on a model. A finished job is deleted: \`bot_files.ingested_at\` and \`ingest_error\` keep the outcome`,
isQueue: {
maxAttempts: 3,
lockTimeout: 10 * 60 * 1000,
@@ -3607,12 +3882,21 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
fieldDescription: `The \`bot_files\` row to (re)ingest`,
dataType: "INTEGER",
},
{
fieldName: "document_id",
fieldDescription: `The \`knowledgebase_documents\` row to (re)ingest, instead of a file`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_file_ingest_queue_file_id",
indexTableFields: ["file_id"],
},
{
indexName: "idx_file_ingest_queue_document_id",
indexTableFields: ["document_id"],
},
],
},
{
db/types/db.ts
+171 -2
View File
@@ -48,6 +48,12 @@ export const BunSQLiteTables = [
"workspace_custom_tools",
"bot_skills",
"workspace_skills",
"knowledgebases",
"knowledgebase_documents",
"knowledgebase_chunks",
"knowledgebase_vectors",
"bot_knowledgebases",
"workspace_knowledgebases",
"logs",
"user_presence",
"push_subscriptions",
@@ -176,7 +182,7 @@ export type BUN_SQLITE_COBBLE_MEDIA_PARADIGMS = {
*/
updated_at?: number | "";
media_id?: number | "";
media_paradigm?: "user-profile-image" | "generic" | "event" | "bot-attachment" | "bot-image" | "workspace-image" | "bot-file" | "task-media" | "";
media_paradigm?: "user-profile-image" | "generic" | "event" | "bot-attachment" | "bot-image" | "workspace-image" | "bot-file" | "task-media" | "knowledgebase-document" | "";
}
export type BUN_SQLITE_COBBLE_BOTS = {
@@ -1819,6 +1825,165 @@ export type BUN_SQLITE_COBBLE_WORKSPACE_SKILLS = {
skill_id?: number | "";
}
export type BUN_SQLITE_COBBLE_KNOWLEDGEBASES = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
/**
* What the settings page and the composer call it, e.g. "Product handbook"
*/
name?: string;
/**
* The one line a bot reads to know what it holds and when to search it
*/
description?: string;
/**
* 1 gives it to every bot, the coordinator included
*/
all_bots?: 0 | 1 | "";
enabled?: 0 | 1 | "";
/**
* Who created it. Not a foreign key: it belongs to the install
*/
user_id?: number | "";
}
export type BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
knowledgebase_id?: number | "";
name?: string;
source?: "file" | "text" | "";
/**
* The uploaded file's private `media` row (a file document). Not a foreign key
*/
media_id?: number | "";
/**
* The text of a text document, markdown
*/
content?: string;
mime_type?: string;
/**
* Bytes
*/
size?: number | "";
chunk_count?: number | "";
/**
* The model its vectors came from, null when it's searchable by words only. A change of the install's embedding model calls for a re-index
*/
embedding_model?: string;
/**
* Unix ms of its last ingest, null while it waits
*/
ingested_at?: number | "";
ingest_error?: string;
/**
* Who added it. Not a foreign key
*/
user_id?: number | "";
}
export type BUN_SQLITE_COBBLE_KNOWLEDGEBASE_CHUNKS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
document_id?: number | "";
/**
* Copied from the document, so a search filters without a join
*/
knowledgebase_id?: number | "";
chunk_index?: number | "";
text?: string;
/**
* A rough count (characters / 4)
*/
token_estimate?: number | "";
}
export type BUN_SQLITE_COBBLE_KNOWLEDGEBASE_VECTORS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
/**
* Normalized, padded or cut to `AppData.EmbeddingDimensions`
*/
embedding?: Float32Array<ArrayBuffer> | Buffer<ArrayBuffer> | null;
chunk_id?: number | "";
document_id?: number | "";
knowledgebase_id?: number | "";
}
export type BUN_SQLITE_COBBLE_BOT_KNOWLEDGEBASES = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
bot_id?: number | "";
knowledgebase_id?: number | "";
}
export type BUN_SQLITE_COBBLE_WORKSPACE_KNOWLEDGEBASES = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
workspace_id?: number | "";
knowledgebase_id?: number | "";
}
export type BUN_SQLITE_COBBLE_LOGS = {
/**
* The unique identifier of the record.
@@ -2195,6 +2360,10 @@ export type BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE = {
* The `bot_files` row to (re)ingest
*/
file_id?: number | "";
/**
* The `knowledgebase_documents` row to (re)ingest, instead of a file
*/
document_id?: number | "";
/**
* The state of the job. (pending | processing | completed | failed)
*/
@@ -2238,4 +2407,4 @@ export type BUN_SQLITE_COBBLE_APP_SETTINGS = {
value?: string;
}
export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_TASKS & BUN_SQLITE_COBBLE_TASK_MEDIA & BUN_SQLITE_COBBLE_TASK_NOTES & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_BOT_HOOKS & BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS & BUN_SQLITE_COBBLE_HOOK_PROPOSALS & BUN_SQLITE_COBBLE_SECRET_REQUESTS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_BOT_TYPES & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_SKILLS & BUN_SQLITE_COBBLE_EXTENSION_PROPOSALS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_SKILLS & BUN_SQLITE_COBBLE_WORKSPACE_SKILLS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS & BUN_SQLITE_COBBLE_BOT_FILES & BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS & BUN_SQLITE_COBBLE_TOKEN_USAGES & BUN_SQLITE_COBBLE_BOT_FILE_VECTORS & BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE & BUN_SQLITE_COBBLE_APP_SETTINGS
export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_TASKS & BUN_SQLITE_COBBLE_TASK_MEDIA & BUN_SQLITE_COBBLE_TASK_NOTES & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_BOT_HOOKS & BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS & BUN_SQLITE_COBBLE_HOOK_PROPOSALS & BUN_SQLITE_COBBLE_SECRET_REQUESTS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_BOT_TYPES & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_SKILLS & BUN_SQLITE_COBBLE_EXTENSION_PROPOSALS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_SKILLS & BUN_SQLITE_COBBLE_WORKSPACE_SKILLS & BUN_SQLITE_COBBLE_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_CHUNKS & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_VECTORS & BUN_SQLITE_COBBLE_BOT_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_WORKSPACE_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS & BUN_SQLITE_COBBLE_BOT_FILES & BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS & BUN_SQLITE_COBBLE_TOKEN_USAGES & BUN_SQLITE_COBBLE_BOT_FILE_VECTORS & BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE & BUN_SQLITE_COBBLE_APP_SETTINGS