Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 9 Wiki Activity

Add bots files handling logic

Browse Source
This commit is contained in:
tben
2026-10-21 17:56:09 +01:00
parent b0af4db1bf
commit d17a4a7bdd
97 changed files with 5843 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/schema.ts
+228
View File
@@ -18,6 +18,7 @@ import { Harnesses } from "@/src/dict/harness-dict";
import { HarnessProviderKinds } from "@/src/dict/harness-provider-dict";
import { BotJobKinds, BotJobUpdateKinds } from "@/src/dict/bot-queue-dict";
import { AppData } from "@/src/data/app-data";
import { BotFilePurposes, BotFileScopes, BotFileSources } from "@/src/dict/bot-file-dict";
import type { BUN_SQLITE_DatabaseSchemaType } from "@moduletrace/bun-sqlite/types";
import { EnvParadigms } from "@/src/dict/env-dict";
import { McpServerStatuses, McpTransports } from "@/src/dict/mcp-dict";
@@ -2322,6 +2323,233 @@ const schema: BUN_SQLITE_DatabaseSchemaType = {
},
],
},
{
tableName: "bot_files",
tableDescription: `A file a bot saved or the user gave it: the catalog row for bytes on the data volume, under \`DATA_DIR/bots/<id>\` or \`DATA_DIR/workspaces/<id>\`. The bytes are a private \`media\` row (paradigm \`bot-file\`), so they're served like other media and deleted with the row. Never on a computer's filesystem`,
fields: [
{
fieldName: "bot_id",
fieldDescription: `The bot that wrote or received it. Goes with the bot`,
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "bots",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "bot_files_bot_fk",
},
},
{
fieldName: "scope",
fieldDescription: `Which directory it's in: a bot's own or a workspace's shared one`,
dataType: "TEXT",
options: BotFileScopes.map((s) => s.value),
defaultValue: "bot",
},
{
fieldName: "owner_id",
fieldDescription: `The bot id (scope bot) or workspace id (scope workspace) owning the directory. Not a foreign key, so a row outlives a workspace but its files are cleaned up with it`,
dataType: "INTEGER",
},
{
fieldName: "media_id",
fieldDescription: `The private \`media\` row holding the bytes. Not a foreign key: the catalog row can outlive it (a failed cleanup) and is then listed as missing`,
dataType: "INTEGER",
},
{
fieldName: "path",
fieldDescription: `Path inside the directory, forward slashes, e.g. reports/q3.md`,
dataType: "TEXT",
},
{
fieldName: "name",
dataType: "TEXT",
},
{
fieldName: "folder",
fieldDescription: `The folder part of \`path\`, empty at the top`,
dataType: "TEXT",
},
{
fieldName: "mime_type",
dataType: "TEXT",
},
{
fieldName: "media_type",
dataType: "TEXT",
options: MediaTypes.map((it) => it.value),
},
{
fieldName: "size",
fieldDescription: `Bytes on disk`,
dataType: "INTEGER",
},
{
fieldName: "purpose",
fieldDescription: `What it is for: an upload, something the bot wrote, a generated image or PDF`,
dataType: "TEXT",
options: BotFilePurposes.map((p) => p.value),
defaultValue: "writing",
},
{
fieldName: "source",
fieldDescription: `How it got here: the user attached it to a message, the bot wrote it, or a tool made it`,
dataType: "TEXT",
options: BotFileSources.map((s) => s.value),
defaultValue: "bot-write",
},
{
fieldName: "ingested_at",
fieldDescription: `When its text was chunked and embedded. Null: not ingested (yet)`,
dataType: "INTEGER",
},
{
fieldName: "ingest_error",
fieldDescription: `Why the last ingestion failed, when it did`,
dataType: "TEXT",
},
],
uniqueConstraints: [
{
alias: "one_file_per_path",
constraintName: "One file per directory and path",
constraintTableFields: [{ value: "scope" }, { value: "owner_id" }, { value: "path" }],
},
],
indexes: [
{
indexName: "idx_bot_files_owner",
indexTableFields: ["scope", "owner_id", "folder"],
},
{
indexName: "idx_bot_files_media_id",
indexTableFields: ["media_id"],
},
],
},
{
tableName: "bot_file_chunks",
tableDescription: `A piece of a bot file's text, the unit a search returns. Replaced whole when the file is ingested again`,
fields: [
{
fieldName: "file_id",
dataType: "INTEGER",
foreignKey: {
cascadeDelete: true,
destinationTableName: "bot_files",
destinationTableColumnName: "id",
destinationTableColumnType: "INTEGER",
foreignKeyName: "bot_file_chunks_file_fk",
},
},
{
fieldName: "scope",
dataType: "TEXT",
options: BotFileScopes.map((s) => s.value),
},
{
fieldName: "owner_id",
fieldDescription: `The bot or workspace the file belongs to, so a search filters without a join`,
dataType: "INTEGER",
},
{
fieldName: "chunk_index",
dataType: "INTEGER",
},
{
fieldName: "text",
dataType: "TEXT",
},
{
fieldName: "token_estimate",
fieldDescription: `A rough count (characters / 4), for trimming results`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_bot_file_chunks_file_id",
indexTableFields: ["file_id", "chunk_index"],
},
{
indexName: "idx_bot_file_chunks_owner",
indexTableFields: ["scope", "owner_id"],
},
],
},
{
tableName: "bot_file_vectors",
tableDescription: `sqlite-vec vectors of a file's chunks, at \`AppData.EmbeddingDimensions\` wide whatever the model returns. A vec0 table is only recreated with \`bunx bun-sqlite schema --vector\`, so the entrypoint never passes it`,
isVector: true,
vectorType: "vec0",
fields: [
{
fieldName: "embedding",
fieldDescription: `Normalized, padded or cut to \`AppData.EmbeddingDimensions\``,
dataType: "BLOB",
isVector: true,
vectorSize: AppData.EmbeddingDimensions,
},
{
fieldName: "chunk_id",
dataType: "INTEGER",
sideCar: true,
},
{
fieldName: "file_id",
dataType: "INTEGER",
sideCar: true,
},
{
fieldName: "scope",
dataType: "TEXT",
sideCar: true,
},
{
fieldName: "owner_id",
dataType: "INTEGER",
sideCar: true,
},
],
},
{
tableName: "file_ingest_queue",
tableDescription: `Files waiting to have their text chunked and embedded. A queue, so a restart never strands a file and an upload never waits on a model. A finished job is deleted: \`bot_files.ingested_at\` and \`ingest_error\` keep the outcome`,
isQueue: {
maxAttempts: 3,
lockTimeout: 10 * 60 * 1000,
backoff: 30_000,
deleteOnComplete: true,
},
fields: [
{
fieldName: "file_id",
fieldDescription: `The \`bot_files\` row to (re)ingest`,
dataType: "INTEGER",
},
],
indexes: [
{
indexName: "idx_file_ingest_queue_file_id",
indexTableFields: ["file_id"],
},
],
},
{
tableName: "app_settings",
tableDescription: `Install-wide settings the app reads that aren't envs: an env's value is handed to bots and computers, which a storage quota must never be`,
fields: [
{
fieldName: "key",
dataType: "TEXT",
unique: true,
},
{
fieldName: "value",
dataType: "TEXT",
},
],
},
],
};
db/types/db.ts
+182 -5
View File
@@ -40,6 +40,11 @@ export const BunSQLiteTables = [
"user_presence",
"push_subscriptions",
"envs",
"bot_files",
"bot_file_chunks",
"bot_file_vectors",
"file_ingest_queue",
"app_settings",
] as const
export type BUN_SQLITE_COBBLE_USERS = {
@@ -156,7 +161,7 @@ export type BUN_SQLITE_COBBLE_MEDIA_PARADIGMS = {
*/
updated_at?: number | "";
media_id?: number | "";
media_paradigm?: "user-profile-image" | "generic" | "event" | "bot-attachment" | "bot-image" | "workspace-image" | "";
media_paradigm?: "user-profile-image" | "generic" | "event" | "bot-attachment" | "bot-image" | "workspace-image" | "bot-file" | "";
}
export type BUN_SQLITE_COBBLE_BOTS = {
@@ -968,7 +973,7 @@ export type BUN_SQLITE_COBBLE_HARNESS_SETTINGS = {
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
harness?: "text" | "vision" | "transcription" | "speech" | "image" | "";
harness?: "text" | "vision" | "transcription" | "speech" | "image" | "embedding" | "";
/**
* harness_models row picked by a super admin. Null picks automatically. Not a foreign key, so removing a provider leaves the setting
*/
@@ -1008,7 +1013,7 @@ export type BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS = {
* The worker bot that overrides the default. The coordinator always uses the defaults
*/
bot_id?: number | "";
harness?: "text" | "vision" | "transcription" | "speech" | "image" | "";
harness?: "text" | "vision" | "transcription" | "speech" | "image" | "embedding" | "";
/**
* harness_models row picked for this bot. Null picks automatically. Not a foreign key, like harness_settings
*/
@@ -1048,7 +1053,7 @@ export type BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS = {
* The workspace
*/
workspace_id?: number | "";
harness?: "text" | "vision" | "transcription" | "speech" | "image" | "";
harness?: "text" | "vision" | "transcription" | "speech" | "image" | "embedding" | "";
/**
* harness_models row picked for this workspace. Null picks automatically. Not a foreign key, like harness_settings
*/
@@ -1418,4 +1423,176 @@ export type BUN_SQLITE_COBBLE_ENVS = {
expose_to_computers?: 0 | 1 | "";
}
export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS
export type BUN_SQLITE_COBBLE_BOT_FILES = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
/**
* The bot that wrote or received it. Goes with the bot
*/
bot_id?: number | "";
/**
* Which directory it's in: a bot's own or a workspace's shared one
*/
scope?: "bot" | "workspace" | "";
/**
* The bot id (scope bot) or workspace id (scope workspace) owning the directory. Not a foreign key, so a row outlives a workspace but its files are cleaned up with it
*/
owner_id?: number | "";
/**
* The private `media` row holding the bytes. Not a foreign key: the catalog row can outlive it (a failed cleanup) and is then listed as missing
*/
media_id?: number | "";
/**
* Path inside the directory, forward slashes, e.g. reports/q3.md
*/
path?: string;
name?: string;
/**
* The folder part of `path`, empty at the top
*/
folder?: string;
mime_type?: string;
media_type?: "image" | "video" | "document" | "audio" | "";
/**
* Bytes on disk
*/
size?: number | "";
/**
* What it is for: an upload, something the bot wrote, a generated image or PDF
*/
purpose?: "upload" | "writing" | "image" | "pdf" | "artifact" | "";
/**
* How it got here: the user attached it to a message, the bot wrote it, or a tool made it
*/
source?: "chat-upload" | "bot-write" | "tool" | "";
/**
* When its text was chunked and embedded. Null: not ingested (yet)
*/
ingested_at?: number | "";
/**
* Why the last ingestion failed, when it did
*/
ingest_error?: string;
}
export type BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
file_id?: number | "";
scope?: "bot" | "workspace" | "";
/**
* The bot or workspace the file belongs to, so a search filters without a join
*/
owner_id?: number | "";
chunk_index?: number | "";
text?: string;
/**
* A rough count (characters / 4), for trimming results
*/
token_estimate?: number | "";
}
export type BUN_SQLITE_COBBLE_BOT_FILE_VECTORS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
/**
* Normalized, padded or cut to `AppData.EmbeddingDimensions`
*/
embedding?: Float32Array<ArrayBuffer> | Buffer<ArrayBuffer> | null;
chunk_id?: number | "";
file_id?: number | "";
scope?: string;
owner_id?: number | "";
}
export type BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
/**
* The `bot_files` row to (re)ingest
*/
file_id?: number | "";
/**
* The state of the job. (pending | processing | completed | failed)
*/
status?: string;
/**
* The number of times the job has been claimed.
*/
attempts?: number | "";
/**
* The time from which the job can be claimed. (Unix Timestamp)
*/
available_at?: number | "";
/**
* The time when the job was claimed. (Unix Timestamp)
*/
locked_at?: number | "";
/**
* The worker that claimed the job.
*/
locked_by?: string;
/**
* The error from the last failed attempt.
*/
last_error?: string;
}
export type BUN_SQLITE_COBBLE_APP_SETTINGS = {
/**
* The unique identifier of the record.
*/
id?: number | "";
/**
* The time when the record was created. (Unix Timestamp)
*/
created_at?: number | "";
/**
* The time when the record was updated. (Unix Timestamp)
*/
updated_at?: number | "";
key?: string;
value?: string;
}
export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS & BUN_SQLITE_COBBLE_BOT_FILES & BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS & BUN_SQLITE_COBBLE_BOT_FILE_VECTORS & BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE & BUN_SQLITE_COBBLE_APP_SETTINGS