Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 28 Wiki Activity

Keep searchable text only in knowledgebases

Browse Source 
Every bot file was chunked into bot_file_chunks and embedded into
bot_file_vectors, and uploads kept their extracted text in
media.text_content, which together took most of the database. Bot
files are no longer indexed: bots find them by name or path with
list_files and read them with read_file, and search_files, ingest_file
and the files modal's re-read button are gone. Attachments keep no
text; a reply reads a document from the file. Only knowledgebase
documents keep text.

The schema drops both tables and the ingest columns. At boot,
dropStoredFileText removes what older versions stored (file jobs,
upload text) and vacuums once much of the database is free. The
media:prune-text script it replaces is removed.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-28 04:44:46 +01:00
co-authored by Claude Opus 5.5
parent 2896e76256
commit ac3ebc43d4
55 changed files with 271 additions and 1712 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/schema.ts
+2 -106
View File
@@ -3792,16 +3792,6 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
options: BotFileSources.map((s) => s.value),
defaultValue: "bot-write",
},
{
fieldName: "ingested_at",
fieldDescription: `When its text was chunked and embedded. Null: not ingested (yet)`,
dataType: "INTEGER",
},
{
fieldName: "ingest_error",
fieldDescription: `Why the last ingestion failed, when it did`,
dataType: "TEXT",
},
],
uniqueConstraints: [
{
@@ -3825,56 +3815,6 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
},
],
},
{
tableName: "bot_file_chunks",
tableDescription: `A piece of a bot file's text, the unit a search returns. Replaced whole when the file is ingested again`,
fields: [
{
fieldName: "file_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "bot_files",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "bot_file_chunks_file_fk",
},
},
{
fieldName: "scope",
dataType: "TEXT",
options: BotFileScopes.map((s) => s.value),
},
{
fieldName: "owner_id",
fieldDescription: `The bot or workspace the file belongs to, so a search filters without a join`,
dataType: "INTEGER",
},
{
fieldName: "chunk_index",
dataType: "INTEGER",
},
{
fieldName: "text",
dataType: "TEXT",
},
{
fieldName: "token_estimate",
fieldDescription: `A rough count (characters / 4), for trimming results`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_bot_file_chunks_file_id",
indexTableFields: ["file_id", "chunk_index"],
},
{
indexName: "idx_bot_file_chunks_owner",
indexTableFields: ["scope", "owner_id"],
},
],
},
{
tableName: "token_usages",
tableDescription: `One model call that reported its usage: the tokens it spent, on which model and provider, for whom and for what. A bot message's total is the sum of the calls it took; the usage page sums them by model, provider and period`,
@@ -4023,44 +3963,9 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
},
],
},
{
tableName: "bot_file_vectors",
tableDescription: `sqlite-vec vectors of a file's chunks, at \`AppData.EmbeddingDimensions\` wide whatever the model returns. A vec0 table is only recreated with \`bunx bun-sqlite schema --vector\`, so the entrypoint never passes it`,
isVector: true,
vectorType: "vec0",
fields: [
{
fieldName: "embedding",
fieldDescription: `Normalized, padded or cut to \`AppData.EmbeddingDimensions\``,
dataType: "BLOB",
isVector: true,
vectorSize: AppData.EmbeddingDimensions,
},
{
fieldName: "chunk_id",
dataType: "INTEGER",
sideCar: true,
},
{
fieldName: "file_id",
dataType: "INTEGER",
sideCar: true,
},
{
fieldName: "scope",
dataType: "TEXT",
sideCar: true,
},
{
fieldName: "owner_id",
dataType: "INTEGER",
sideCar: true,
},
],
},
{
tableName: "file_ingest_queue",
tableDescription: `Files (and knowledgebase documents) waiting to have their text chunked and embedded. A queue, so a restart never strands a file and an upload never waits on a model. A finished job is deleted: \`bot_files.ingested_at\` and \`ingest_error\` keep the outcome`,
tableDescription: `Knowledgebase documents waiting to have their text chunked and embedded (bot files are never indexed). A queue, so a restart never strands a document and an upload never waits on a model. A finished job is deleted: \`knowledgebase_documents.ingested_at\` and \`ingest_error\` keep the outcome`,
isQueue: {
maxAttempts: 3,
lockTimeout: 10 * 60 * 1000,
@@ -4068,22 +3973,13 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
deleteOnComplete: true,
},
fields: [
{
fieldName: "file_id",
fieldDescription: `The \`bot_files\` row to (re)ingest`,
dataType: "INTEGER",
},
{
fieldName: "document_id",
fieldDescription: `The \`knowledgebase_documents\` row to (re)ingest, instead of a file`,
fieldDescription: `The \`knowledgebase_documents\` row to (re)ingest`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_file_ingest_queue_file_id",
indexTableFields: ["file_id"],
},
{
indexName: "idx_file_ingest_queue_document_id",
indexTableFields: ["document_id"],
db/types/db.ts
+2 -66
View File
@@ -61,9 +61,7 @@ export const BunSQLiteTables = [
"push_subscriptions",
"envs",
"bot_files",
"bot_file_chunks",
"token_usages",
"bot_file_vectors",
"file_ingest_queue",
"app_settings",
] as const
@@ -2335,41 +2333,6 @@ export type BUN_SQLITE_COBBLE_BOT_FILES = {
* How it got here: the user attached it to a message, the bot wrote it, or a tool made it
*/
source?: "chat-upload" | "bot-write" | "tool" | "";
/**
* When its text was chunked and embedded. Null: not ingested (yet)
*/
ingested_at?: number | "";
/**
* Why the last ingestion failed, when it did
*/
ingest_error?: string;
}
export type BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
file_id?: number | "";
scope?: "bot" | "workspace" | "";
/**
* The bot or workspace the file belongs to, so a search filters without a join
*/
owner_id?: number | "";
chunk_index?: number | "";
text?: string;
/**
* A rough count (characters / 4), for trimming results
*/
token_estimate?: number | "";
}
export type BUN_SQLITE_COBBLE_TOKEN_USAGES = {
@@ -2447,29 +2410,6 @@ export type BUN_SQLITE_COBBLE_TOKEN_USAGES = {
settings?: string;
}
export type BUN_SQLITE_COBBLE_BOT_FILE_VECTORS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
/**
* Normalized, padded or cut to `AppData.EmbeddingDimensions`
*/
embedding?: Float32Array<ArrayBuffer> | Buffer<ArrayBuffer> | null;
chunk_id?: number | "";
file_id?: number | "";
scope?: string;
owner_id?: number | "";
}
export type BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE = {
/**
* The unique identifier of the record.
@@ -2484,11 +2424,7 @@ export type BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE = {
*/
updated_at?: number | "";
/**
* The `bot_files` row to (re)ingest
*/
file_id?: number | "";
/**
* The `knowledgebase_documents` row to (re)ingest, instead of a file
* The `knowledgebase_documents` row to (re)ingest
*/
document_id?: number | "";
/**
@@ -2534,4 +2470,4 @@ export type BUN_SQLITE_COBBLE_APP_SETTINGS = {
value?: string;
}
export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_CHATROOMS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_TASKS & BUN_SQLITE_COBBLE_TASK_MEDIA & BUN_SQLITE_COBBLE_TASK_NOTES & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_BOT_HOOKS & BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS & BUN_SQLITE_COBBLE_HOOK_PROPOSALS & BUN_SQLITE_COBBLE_SECRET_REQUESTS & BUN_SQLITE_COBBLE_BOT_GRAPHICS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_BOT_TYPES & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_SKILLS & BUN_SQLITE_COBBLE_EXTENSION_PROPOSALS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_SKILLS & BUN_SQLITE_COBBLE_WORKSPACE_SKILLS & BUN_SQLITE_COBBLE_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_CHUNKS & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_VECTORS & BUN_SQLITE_COBBLE_BOT_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_WORKSPACE_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS & BUN_SQLITE_COBBLE_BOT_FILES & BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS & BUN_SQLITE_COBBLE_TOKEN_USAGES & BUN_SQLITE_COBBLE_BOT_FILE_VECTORS & BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE & BUN_SQLITE_COBBLE_APP_SETTINGS
export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_CHATROOMS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_TASKS & BUN_SQLITE_COBBLE_TASK_MEDIA & BUN_SQLITE_COBBLE_TASK_NOTES & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_BOT_HOOKS & BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS & BUN_SQLITE_COBBLE_HOOK_PROPOSALS & BUN_SQLITE_COBBLE_SECRET_REQUESTS & BUN_SQLITE_COBBLE_BOT_GRAPHICS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_BOT_TYPES & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_SKILLS & BUN_SQLITE_COBBLE_EXTENSION_PROPOSALS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_SKILLS & BUN_SQLITE_COBBLE_WORKSPACE_SKILLS & BUN_SQLITE_COBBLE_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_DOCUMENTS & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_CHUNKS & BUN_SQLITE_COBBLE_KNOWLEDGEBASE_VECTORS & BUN_SQLITE_COBBLE_BOT_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_WORKSPACE_KNOWLEDGEBASES & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS & BUN_SQLITE_COBBLE_BOT_FILES & BUN_SQLITE_COBBLE_TOKEN_USAGES & BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE & BUN_SQLITE_COBBLE_APP_SETTINGS