From d17a4a7bdd7494653a2b914fffd95f42ca058c83 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Wed, 21 Oct 2026 17:56:09 +0100 Subject: [PATCH] Add bots files handling logic --- AGENTS.md | 28 +- db/schema.ts | 228 +++++++++++++++ db/types/db.ts | 187 +++++++++++- src/bots-queue/process-bot-job.ts | 1 + src/bots-queue/run-bot-reply.ts | 22 +- src/bots-queue/run-bot-task.ts | 32 ++- src/bots-queue/types.ts | 5 + src/cron/index.ts | 8 + src/cron/ingest-pending-files/index.ts | 29 ++ src/data/app-data.ts | 59 ++++ src/data/harness-prompts.ts | 15 + src/dict/bot-file-dict.ts | 102 +++++++ src/dict/harness-dict.ts | 12 +- src/dict/media-dict.ts | 4 + src/file-ingest/enqueue-file-ingest.ts | 37 +++ src/file-ingest/index.ts | 82 ++++++ src/file-ingest/process-ingest-job.ts | 41 +++ src/file-ingest/wake-file-ingest.ts | 17 ++ .../db/app-settings/grab-app-setting.ts | 12 + .../db/app-settings/grab-app-settings.ts | 34 +++ .../grab-storage-quota-overview.ts | 25 ++ .../db/app-settings/save-app-setting.ts | 47 +++ .../db/bot-files/assert-storage-room.ts | 47 +++ .../backend/db/bot-files/chunk-text.test.ts | 54 ++++ .../backend/db/bot-files/chunk-text.ts | 148 ++++++++++ .../backend/db/bot-files/copy-bot-file.ts | 64 +++++ .../backend/db/bot-files/delete-bot-file.ts | 64 +++++ .../db/bot-files/extract-bot-file-text.ts | 91 ++++++ .../db/bot-files/grab-bot-file-media.ts | 51 ++++ .../db/bot-files/grab-bot-file-quota-bytes.ts | 24 ++ .../backend/db/bot-files/grab-bot-file.ts | 68 +++++ .../db/bot-files/grab-conversation-files.ts | 62 ++++ .../backend/db/bot-files/grab-storage-dir.ts | 33 +++ .../db/bot-files/grab-storage-usage.ts | 61 ++++ .../backend/db/bot-files/ingest-bot-file.ts | 248 ++++++++++++++++ .../backend/db/bot-files/list-bot-files.ts | 105 +++++++ .../db/bot-files/media-type-for-file.ts | 93 ++++++ .../backend/db/bot-files/move-bot-file.ts | 84 ++++++ .../backend/db/bot-files/purge-storage.ts | 86 ++++++ .../backend/db/bot-files/record-bot-file.ts | 259 +++++++++++++++++ .../backend/db/bot-files/search-bot-files.ts | 271 ++++++++++++++++++ .../db/bot-files/store-chat-attachments.ts | 93 ++++++ .../db/bot-files/vector-search.test.ts | 132 +++++++++ .../bot-sessions/grab-conversation-props.ts | 5 +- .../backend/db/bots-queue/complete-bot-job.ts | 6 + src/functions/backend/db/bots/delete-bot.ts | 14 + .../backend/db/bots/grab-bot-messages.ts | 19 +- .../backend/db/bots/insert-bot-message.ts | 13 + .../backend/db/bots/media-to-attachment.ts | 37 ++- .../backend/db/workspaces/delete-workspace.ts | 18 ++ .../server/media/handle-media-server.ts | 2 + src/harness/catalog/model-serves-harness.ts | 4 + src/harness/catalog/rank-auto-models.ts | 2 +- src/harness/embeddings/embed-text.ts | 84 ++++++ .../embeddings/fit-vector-dimensions.ts | 46 +++ .../providers/openai-compatible/embed.ts | 68 +++++ .../providers/openai-compatible/index.ts | 2 + src/harness/tools/storage/copy-file-tool.ts | 50 ++++ src/harness/tools/storage/delete-file-tool.ts | 27 ++ .../tools/storage/generate-image-tool.ts | 121 ++++++++ .../tools/storage/grab-bot-file-tools.ts | 50 ++++ .../tools/storage/grab-storage-tools.ts | 28 ++ src/harness/tools/storage/ingest-file-tool.ts | 44 +++ src/harness/tools/storage/list-files-tool.ts | 62 ++++ src/harness/tools/storage/make-pdf-tool.ts | 185 ++++++++++++ src/harness/tools/storage/move-file-tool.ts | 37 +++ src/harness/tools/storage/read-file-tool.ts | 95 ++++++ .../tools/storage/search-files-tool.ts | 49 ++++ .../tools/storage/storage-tool-context.ts | 110 +++++++ src/harness/tools/storage/write-file-tool.ts | 89 ++++++ src/harness/types.ts | 20 ++ src/layouts/bots/(partials)/bot-form.tsx | 38 ++- .../bots/[bot_id]/(hooks)/use-bot-files.ts | 64 +++++ .../[bot_id]/(partials)/bot-files-modal.tsx | 173 +++++++++++ .../(partials)/conversation-header-menu.tsx | 9 +- .../[bot_id]/(partials)/file-ref-chip.tsx | 38 +++ .../bots/[bot_id]/(partials)/file-row.tsx | 89 ++++++ .../[bot_id]/(partials)/markdown-link.tsx | 11 +- .../(functions)/grab-settings-server-props.ts | 2 + .../general/(partials)/storage-quota-form.tsx | 89 ++++++ .../general/(sections)/storage-section.tsx | 25 ++ src/pages/admin/settings/general/index.tsx | 2 + src/pages/api/admin/bots/delete-file.ts | 94 ++++++ src/pages/api/admin/bots/files.ts | 131 +++++++++ src/pages/api/admin/bots/ingest-file.ts | 82 ++++++ src/pages/api/admin/bots/send-message.ts | 12 + src/pages/api/admin/settings/storage-quota.ts | 52 ++++ src/schemas/bot-file-schema.ts | 23 ++ src/server.ts | 4 + src/types/index.ts | 52 ++++ src/utils/file-ref.test.ts | 36 +++ src/utils/file-ref.ts | 24 ++ src/utils/link-computer-refs.ts | 16 +- src/utils/resolve-bot-path.test.ts | 79 +++++ src/utils/resolve-bot-path.ts | 147 ++++++++++ src/utils/storage-access.test.ts | 60 ++++ src/utils/storage-access.ts | 80 ++++++ 97 files changed, 5843 insertions(+), 41 deletions(-) create mode 100644 src/cron/ingest-pending-files/index.ts create mode 100644 src/dict/bot-file-dict.ts create mode 100644 src/file-ingest/enqueue-file-ingest.ts create mode 100644 src/file-ingest/index.ts create mode 100644 src/file-ingest/process-ingest-job.ts create mode 100644 src/file-ingest/wake-file-ingest.ts create mode 100644 src/functions/backend/db/app-settings/grab-app-setting.ts create mode 100644 src/functions/backend/db/app-settings/grab-app-settings.ts create mode 100644 src/functions/backend/db/app-settings/grab-storage-quota-overview.ts create mode 100644 src/functions/backend/db/app-settings/save-app-setting.ts create mode 100644 src/functions/backend/db/bot-files/assert-storage-room.ts create mode 100644 src/functions/backend/db/bot-files/chunk-text.test.ts create mode 100644 src/functions/backend/db/bot-files/chunk-text.ts create mode 100644 src/functions/backend/db/bot-files/copy-bot-file.ts create mode 100644 src/functions/backend/db/bot-files/delete-bot-file.ts create mode 100644 src/functions/backend/db/bot-files/extract-bot-file-text.ts create mode 100644 src/functions/backend/db/bot-files/grab-bot-file-media.ts create mode 100644 src/functions/backend/db/bot-files/grab-bot-file-quota-bytes.ts create mode 100644 src/functions/backend/db/bot-files/grab-bot-file.ts create mode 100644 src/functions/backend/db/bot-files/grab-conversation-files.ts create mode 100644 src/functions/backend/db/bot-files/grab-storage-dir.ts create mode 100644 src/functions/backend/db/bot-files/grab-storage-usage.ts create mode 100644 src/functions/backend/db/bot-files/ingest-bot-file.ts create mode 100644 src/functions/backend/db/bot-files/list-bot-files.ts create mode 100644 src/functions/backend/db/bot-files/media-type-for-file.ts create mode 100644 src/functions/backend/db/bot-files/move-bot-file.ts create mode 100644 src/functions/backend/db/bot-files/purge-storage.ts create mode 100644 src/functions/backend/db/bot-files/record-bot-file.ts create mode 100644 src/functions/backend/db/bot-files/search-bot-files.ts create mode 100644 src/functions/backend/db/bot-files/store-chat-attachments.ts create mode 100644 src/functions/backend/db/bot-files/vector-search.test.ts create mode 100644 src/harness/embeddings/embed-text.ts create mode 100644 src/harness/embeddings/fit-vector-dimensions.ts create mode 100644 src/harness/providers/openai-compatible/embed.ts create mode 100644 src/harness/tools/storage/copy-file-tool.ts create mode 100644 src/harness/tools/storage/delete-file-tool.ts create mode 100644 src/harness/tools/storage/generate-image-tool.ts create mode 100644 src/harness/tools/storage/grab-bot-file-tools.ts create mode 100644 src/harness/tools/storage/grab-storage-tools.ts create mode 100644 src/harness/tools/storage/ingest-file-tool.ts create mode 100644 src/harness/tools/storage/list-files-tool.ts create mode 100644 src/harness/tools/storage/make-pdf-tool.ts create mode 100644 src/harness/tools/storage/move-file-tool.ts create mode 100644 src/harness/tools/storage/read-file-tool.ts create mode 100644 src/harness/tools/storage/search-files-tool.ts create mode 100644 src/harness/tools/storage/storage-tool-context.ts create mode 100644 src/harness/tools/storage/write-file-tool.ts create mode 100644 src/pages/admin/bots/[bot_id]/(hooks)/use-bot-files.ts create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/bot-files-modal.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/file-ref-chip.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/file-row.tsx create mode 100644 src/pages/admin/settings/general/(partials)/storage-quota-form.tsx create mode 100644 src/pages/admin/settings/general/(sections)/storage-section.tsx create mode 100644 src/pages/api/admin/bots/delete-file.ts create mode 100644 src/pages/api/admin/bots/files.ts create mode 100644 src/pages/api/admin/bots/ingest-file.ts create mode 100644 src/pages/api/admin/settings/storage-quota.ts create mode 100644 src/schemas/bot-file-schema.ts create mode 100644 src/utils/file-ref.test.ts create mode 100644 src/utils/file-ref.ts create mode 100644 src/utils/resolve-bot-path.test.ts create mode 100644 src/utils/resolve-bot-path.ts create mode 100644 src/utils/storage-access.test.ts create mode 100644 src/utils/storage-access.ts diff --git a/AGENTS.md b/AGENTS.md index 1a6a8c5..2ab2b8f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -115,7 +115,7 @@ Inspect live data with `bun llm-code-assistant/query-db.ts ""` - a read-onl ### Installer and release -`install.sh` is the only supported install and update path: `curl -fsSL https://url.moduletrace.com/cobble | sudo bash`. It is idempotent, so the same script is the updater, and `--uninstall [--purge]` is the way back out. It installs Docker when missing (Linux), creates the `cobble` network, writes `$COBBLE_HOME/install.conf` and a generated `.env` (mode 600), and starts one `cobble` container. Existing secrets are never regenerated; the database is backed up to `$COBBLE_HOME/backups` before a container is replaced. The default home is `/opt/cobble`, configurable with `--home`, and everything that must survive an update lives in its `data/` directory (`db`, `media`, `computers`). +`install.sh` is the only supported install and update path: `curl -fsSL https://url.moduletrace.com/cobble | sudo bash`. It is idempotent, so the same script is the updater, and `--uninstall [--purge]` is the way back out. It installs Docker when missing (Linux), creates the `cobble` network, writes `$COBBLE_HOME/install.conf` and a generated `.env` (mode 600), and starts one `cobble` container. Existing secrets are never regenerated; the database is backed up to `$COBBLE_HOME/backups` before a container is replaced. The default home is `/opt/cobble`, configurable with `--home`, and everything that must survive an update lives in its `data/` directory (`db`, `media`, `computers`, `bots`, `workspaces`, `mcp`). The app serves plain HTTP only; TLS is the reverse proxy's or tunnel's job, and the installer, the image's `HEALTHCHECK` and the dev server all assume it (`Bun.serve` gets no `tls`, there is no certificate handling anywhere). `isSecureRequest` still reads the proxy's `X-Forwarded-Proto`, so the auth and CSRF cookies get `Secure` and the installed app, Web Push and voice keep their secure context behind HTTPS; the installer binds `0.0.0.0` by default, and `--bind 127.0.0.1` limits it to a local proxy. Updates drop any `TLS_*` or `HTTP_REDIRECT_PORT` entries an older install left in `.env` and recreate a container that still mounts `/app/certs`. @@ -142,23 +142,37 @@ Envs are set in Settings, under Environment (super admin only), not in `.env`. T - **Web Push:** `push_subscriptions` (one row per endpoint and user; the endpoint is unique and a capability URL, never sent to the client) and `user_presence` (one row per user, refreshed by the socket keepalive in `src/websocket/index.ts`). VAPID keys are app settings in the `envs` table (`VAPID_PUBLIC_KEY` not secret, `VAPID_PRIVATE_KEY` secret), generated by `ensureVapidKeys` on the main server's first start; the subject is `SiteData.SiteURL`. `pushBotNotice` is the one trigger: bot messages (never the user's) and a needs-you face change without a message. It is skipped while `user_presence.last_seen_at` is within `AppData.PushPresenceWindowMs`, so an app open anywhere wins. `src/dict/push-dict.ts` owns payload limits and tags (one conversation replaces its own notification); sending never blocks or fails the job, 404/410 removes the device, and other failures are recorded on the subscription. API in `api/admin/push/` (subscribe, unsubscribe, test, remove-device, overview); Settings > Notifications (`notifications/`) has the per-device switch, a test send, and every device this user has. Permission is only asked from the switch (a denial is hard to undo, and iOS requires the gesture); iOS also needs the installed app. Development has no service worker, so the page says notifications need the installed production app. - **Risks and notes:** push needs HTTPS (or localhost), so it depends on the proxy or tunnel in front. Cloudflare never caches `/sw.js` (no-store). The viewport rewrite is a workaround for bunext's fixed meta and stays defensive (no-op if the tag is gone); iOS Safari ignores `interactive-widget`, so its on-screen keyboard overlays the composer rather than resizing the page. Notification bodies are short message previews, so they can appear on a lock screen; there is no per-device mute beyond turning notifications off. Presence is per user, not per device: an open desktop suppresses a phone until it has been closed for `PushPresenceWindowMs` (deliberate). Notifications are best effort: no retry queue, and a failed send is logged and recorded on the subscription. Live events missed while disconnected are not replayed, except the conversation's reply catch-up for a pending reply; opening the conversation refetches. The drawer is not a focus trap. +### Bot files and folders + +Bots keep files on the app's data volume, never on a computer: `DATA_DIR/bots/` for a bot's own, `DATA_DIR/workspaces/` shared by the bots working there. The catalog is `bot_files` (scope `bot`|`workspace`, `owner_id`, `path`, `size`, `purpose`, `source`, `ingested_at`, `ingest_error`), its bytes are a private `media` row (paradigm `bot-file`) written by `recordBotFile` (`src/functions/backend/db/bot-files/record-bot-file.ts`), so they're served at `/media/private/` through `grabBotFileMedia` and deleted with the row. + +Who reaches what: `grabStorageScopes` (`src/utils/storage-access.ts`) gives a worker its own folder plus the workspace's while it works there, and the coordinator every bot's and every workspace's. Enforced in the tools, never left to the prompt. `resolveBotPath` (`src/utils/resolve-bot-path.ts`) keeps every path inside its root (no absolute paths, `..`, NUL, symlink escapes) and the quota (`app_settings` key `bot_files_quota_mb`, set on Settings > General, `grabBotFileQuotaBytes`) is checked by `assertStorageRoom` before a write. + +Tools in `src/harness/tools/storage/` (`grabBotFileTools`, every run: chat, tasks, routines): `list_files`, `read_file`, `write_file`, `search_files`, `move_file`, `delete_file`, `ingest_file`, `copy_file` (only when the run reaches more than one folder), `generate_image` and `make_pdf`. A file written with `attach` rides on the reply: the outcome's `attachments` become `bot_message_media` rows on the result (`completeBotJob`). Bots name files as `[file:]` (`src/utils/file-ref.ts`, linked by `linkComputerRefs`, rendered by `file-ref-chip.tsx`). Chat uploads are copied into the conversation's folder (`storeChatAttachments`) so a later run can search them. + +Ingestion (`bot_file_chunks`, `bot_file_vectors` a vec0 table, `file_ingest_queue`): `ingestBotFile` extracts text (`extractBotFileText`, reusing `fileToText`; images and audio only on request), chunks it (`chunkText`), embeds it through the `embedding` harness and fits every vector to the fixed `AppData.EmbeddingDimensions` (`fitVectorDimensions`, pad or cut then normalize). The `file-ingest` worker (`src/file-ingest/`, started from `server.ts` next to the bots queue) works the queue; `ingest-pending-files` sweeps up anything stranded. With no embedding model the chunks are still stored and `searchBotFiles` falls back to word matching, always saying which it did. + +Two sqlite-vec limits the code works around: a vec0 table's metadata columns are NOT NULL (a raw insert fills `created_at`/`updated_at` itself), and a side-car column can't be filtered in the same query as `MATCH` (the search pulls the nearest `k` and filters scopes in code). Never pass `--vector` to `bunx bun-sqlite schema` in `entrypoint.sh`: it would wipe every vector on each boot. Changing `AppData.EmbeddingDimensions` needs a manual `--vector` run plus a re-ingest. + +A bot's or workspace's folder is removed with it (`purgeStorage`, called from `deleteBot` and `deleteWorkspace`, which read the media ids first). + ### Harness (the model layer) `src/harness/` is the brain: every model call goes through it. Install-wide, managed by super admins at `/admin/settings/models`. -- **Harnesses** (`Harnesses` in `src/dict/harness-dict.ts`): `text` (chats, plans, calls tools, drives computers; always on), `vision` ("Media reading": images/scans/screenshots for a text model that can't see), `transcription` (speech to text: speech-to-text models only, never chat models that hear audio), `speech` (text to speech: text-to-speech models only), `image` (image generation). Each has one `harness_settings` row (no row = automatic, on, with fallback): a `provider_id` (a `harness_providers` row the harness is limited to, for its pick and its automatic picks; null for any provider; with no row, the default provider, OpenRouter, `grabDefaultHarnessProviderId`), a pinned `model_id` (a `harness_models` row from that provider, not a foreign key) or null for automatic, `fallback` (also try the automatic picks), `enabled`, `options` (`voice`). -- **Providers** (`HarnessProviderKinds` in `src/dict/harness-provider-dict.ts`): OpenRouter (default, created by `ensureDefaultHarnessProvider`), OpenAI, Anthropic, Gemini, DeepSeek, Groq, Mistral, xAI, Ollama (local, its models count as free), any OpenAI-compatible server. Rows in `harness_providers` (a private table: the key is encrypted with `src/utils/crypt.ts`, never sent to the client; falls back to the preset's env, e.g. `OPENROUTER_API_KEY`, via `grabEnv`). One adapter today, `src/harness/providers/openai-compatible/` (`chat`, `transcribe`, `speak`, `generateImage`, `listModels`), chosen in `grab-harness-adapter.ts`; a provider needing a native API gets its own adapter there (`HarnessAdapter` in `src/harness/types.ts`). +- **Harnesses** (`Harnesses` in `src/dict/harness-dict.ts`): `text` (chats, plans, calls tools, drives computers; always on), `vision` ("Media reading": images/scans/screenshots for a text model that can't see), `transcription` (speech to text: speech-to-text models only, never chat models that hear audio), `speech` (text to speech: text-to-speech models only), `image` (image generation), `embedding` (turns bot file chunks into vectors for `search_files`: embedding models only, never auto-picked, so it stays unavailable until a super admin picks one, and search falls back to words without it). Each has one `harness_settings` row (no row = automatic, on, with fallback): a `provider_id` (a `harness_providers` row the harness is limited to, for its pick and its automatic picks; null for any provider; with no row, the default provider, OpenRouter, `grabDefaultHarnessProviderId`), a pinned `model_id` (a `harness_models` row from that provider, not a foreign key) or null for automatic, `fallback` (also try the automatic picks), `enabled`, `options` (`voice`). +- **Providers** (`HarnessProviderKinds` in `src/dict/harness-provider-dict.ts`): OpenRouter (default, created by `ensureDefaultHarnessProvider`), OpenAI, Anthropic, Gemini, DeepSeek, Groq, Mistral, xAI, Ollama (local, its models count as free), any OpenAI-compatible server. Rows in `harness_providers` (a private table: the key is encrypted with `src/utils/crypt.ts`, never sent to the client; falls back to the preset's env, e.g. `OPENROUTER_API_KEY`, via `grabEnv`). One adapter today, `src/harness/providers/openai-compatible/` (`chat`, `transcribe`, `speak`, `generateImage`, `embed` (`POST /embeddings`, batched), `listModels`), chosen in `grab-harness-adapter.ts`; a provider needing a native API gets its own adapter there (`HarnessAdapter` in `src/harness/types.ts`). - **Catalog:** `harness_models` per provider (modalities, tools support, voices, Artificial Analysis scores, `free`, `available`). The `sync-harness-models` cron (hourly, on start) refreshes every enabled provider (`syncAllHarnessModels`); unlisted models are kept but marked unavailable. OpenRouter lists every modality with capabilities; other providers list ids, so capabilities are guessed (`guess-model-modalities`). Free on OpenRouter means `:free` or `openrouter/free`, never zero token prices. - **Automatic picks** (`rank-auto-models`): only for the `automatic` harnesses in `Harnesses` (text, vision, image); transcription and speech have none and are not available until a model is picked (the model select reads "Not available", no fallback checkbox). Free models that can do the job (on the setting's provider when it has one; a provider without free models needs a picked model) (`model-serves-harness`: text needs tools), by vendor `AutoModelVendorPreference` (DeepSeek, Qwen, NVIDIA, Liquid), then other vendors that look competent (`AppData.HarnessAutoMin*`), then routers; within a vendor by intelligence index, then full-size before small, newest first. -- **Calling:** `resolveHarnessCandidates` (pinned, then automatic) -> `withCandidates` tries them in order with process-wide cooldowns (`resolve/harness-cooldowns.ts`): rate limits and outages rest the model (until `Retry-After` when given), 401/402 rest the whole provider, 403/404 rest the model for an hour. Errors are `HarnessError` (`code` provider|unavailable|aborted, `retryable`), classified in `to-harness-error.ts`. Entry points: `text/generate-text.ts` (one turn), `text/run-text-loop.ts` (tool loop, sticks to one model, wraps up after `HarnessMaxToolSteps`, trims the oldest tool output before each turn to fit the model's context, `fit-transcript-to-context.ts`), `media/read-image.ts`, `media/read-media.ts` (documents locally via `fileToText`, images and audio through their harnesses), `audio/transcribe-audio.ts`, `audio/speak-text.ts` (returns the provider's audio as a stream, falling back to the next model only until one answers), `image/generate-image.ts`. A text model that can't take an image or audio part gets it read by the vision/transcription harness first (`prepare-messages-for-model.ts`). +- **Calling:** `resolveHarnessCandidates` (pinned, then automatic) -> `withCandidates` tries them in order with process-wide cooldowns (`resolve/harness-cooldowns.ts`): rate limits and outages rest the model (until `Retry-After` when given), 401/402 rest the whole provider, 403/404 rest the model for an hour. Errors are `HarnessError` (`code` provider|unavailable|aborted, `retryable`), classified in `to-harness-error.ts`. Entry points: `text/generate-text.ts` (one turn), `text/run-text-loop.ts` (tool loop, sticks to one model, wraps up after `HarnessMaxToolSteps`, trims the oldest tool output before each turn to fit the model's context, `fit-transcript-to-context.ts`), `media/read-image.ts`, `media/read-media.ts` (documents locally via `fileToText`, images and audio through their harnesses), `audio/transcribe-audio.ts`, `audio/speak-text.ts` (returns the provider's audio as a stream, falling back to the next model only until one answers), `image/generate-image.ts`, `embeddings/embed-text.ts` (batches, then `fit-vector-dimensions.ts` pads or cuts each vector to `AppData.EmbeddingDimensions` and normalizes it, since a provider's width varies). A text model that can't take an image or audio part gets it read by the vision/transcription harness first (`prepare-messages-for-model.ts`). - **Per-bot models:** in a workspace, its `workspace_harness_settings` row wins (`source: "workspace"`). Otherwise a worker bot inherits the defaults per harness unless it has a `bot_harness_settings` row (same fields as `harness_settings`, unique per bot and harness). `grabHarnessSetting({ harness, bot_id })` returns the bot's row (`source: "bot"`) or the default; the coordinator always gets the default. Every entry point takes `bot_id` and passes it on (text loop -> media reading/transcription), so a bot's run uses its own models throughout. Saved by `saveBotHarnessSetting` (`inherit` deletes the row); the page data is `grabBotHarnessOverview`. A bot's chat header has a ghost model icon (`(partials)/header-icon-button.tsx`; the text model and provider show under the bot's name) that opens a modal with every harness (`(partials)/bot-models-modal.tsx`); the modal and `/admin/bots/[bot_id]/models` both use `(partials)/bot-harness-form.tsx`. On the coordinator the modal edits the defaults themselves (`api/admin/harness/save-settings`, the same settings as Settings, under Models, without the providers): super admins only, the icon is disabled for everyone else. Every one of these wraps `settings/models/(partials)/harness-settings-form.tsx`: per harness a provider select first (`harness-provider-select.tsx`: any provider, one provider, or on a bot "Same as the default"), then the model select narrowed to it. Removing a provider (`deleteHarnessProvider`) puts default and bot settings that picked it or its models back on any provider (`save` prop; "Same as the default" shows when the overview item has `default`). -- **Tools:** `HarnessTool` (built per run with its bot bound): own tools validate with a zod `schema`; tools described elsewhere pass a JSON schema as `parameters` and validate themselves. `run-tool-call.ts` reports bad calls and failures back to the model as `Error: ...`, and caps every tool call at `AppData.HarnessToolTimeoutMs` (aborted, its result handed back as an error the model can work around). Bot tools in `src/harness/tools/bot/`: `update_memory`, `update_instructions` (`grabBotTools`, every run), `start_task` and, for the coordinator with workers, `delegate_task` (`grabBotChatTools`, chat replies only; the job's `requested_by_bot_id` is the coordinator, and `relayJobToRequester` posts the worker's result or failure in its conversation, pushed as `relay` by `pushBotNotice`), `ask_user_to_take_over` (see Take-over), plus for the coordinator `create_workspace` and `update_workspace` (a new workspace with its shared context, or changing one; both also in its tasks, so a task that gathers material can write it into the workspace), `create_bot` (add a worker with its name, role and instructions, and join it to a workspace straight away) and, in its own chat, `add_bot_to_workspace` (put an existing worker in one of the user's workspaces) and `remove_bot_from_workspace` (take it out, keeping its conversation there), `assign_computer` (grant or revoke a worker's direct access, `add/remove-bot-to/from-computer`; access through a workspace stays and the result says so), `update_bot` (change a worker's name, role or core instructions, `src/functions/backend/db/bots/update-bot.ts`), `find_bot` and `find_workspace` (look the crew, or the live workspaces, up by id, name, role/description or instructions/context), `delete_bot` (a worker with its conversations, tasks, schedules and workspace places; the coordinator can't go) and `delete_workspace` (its conversations, tasks, schedules and links; its bots and computers stay). `search_computers` (`src/harness/tools/computer/search-computers-tool.ts`, chat replies and tasks) runs one read-only command on all of a bot's computers at once over the sandbox REST API (`run-computer-command.ts`), each output under its `[computer:ID]`, secrets redacted, capped by `AppData.ComputerSearch*`. +- **Tools:** `HarnessTool` (built per run with its bot bound): own tools validate with a zod `schema`; tools described elsewhere pass a JSON schema as `parameters` and validate themselves. `run-tool-call.ts` reports bad calls and failures back to the model as `Error: ...`, and caps every tool call at `AppData.HarnessToolTimeoutMs` (aborted, its result handed back as an error the model can work around). Bot tools in `src/harness/tools/storage/`: `list_files`, `read_file`, `write_file`, `search_files`, `move_file`, `delete_file`, `ingest_file`, `copy_file`, `generate_image`, `make_pdf` (`grabBotFileTools`, every run, see Bot files and folders). Bot tools in `src/harness/tools/bot/`: `update_memory`, `update_instructions` (`grabBotTools`, every run), `start_task` and, for the coordinator with workers, `delegate_task` (`grabBotChatTools`, chat replies only; the job's `requested_by_bot_id` is the coordinator, and `relayJobToRequester` posts the worker's result or failure in its conversation, pushed as `relay` by `pushBotNotice`), `ask_user_to_take_over` (see Take-over), plus for the coordinator `create_workspace` and `update_workspace` (a new workspace with its shared context, or changing one; both also in its tasks, so a task that gathers material can write it into the workspace), `create_bot` (add a worker with its name, role and instructions, and join it to a workspace straight away) and, in its own chat, `add_bot_to_workspace` (put an existing worker in one of the user's workspaces) and `remove_bot_from_workspace` (take it out, keeping its conversation there), `assign_computer` (grant or revoke a worker's direct access, `add/remove-bot-to/from-computer`; access through a workspace stays and the result says so), `update_bot` (change a worker's name, role or core instructions, `src/functions/backend/db/bots/update-bot.ts`), `find_bot` and `find_workspace` (look the crew, or the live workspaces, up by id, name, role/description or instructions/context), `delete_bot` (a worker with its conversations, tasks, schedules and workspace places; the coordinator can't go) and `delete_workspace` (its conversations, tasks, schedules and links; its bots and computers stay). `search_computers` (`src/harness/tools/computer/search-computers-tool.ts`, chat replies and tasks) runs one read-only command on all of a bot's computers at once over the sandbox REST API (`run-computer-command.ts`), each output under its `[computer:ID]`, secrets redacted, capped by `AppData.ComputerSearch*`. - **MCP:** `src/harness/mcp/`: `connectMcpServer` (Streamable HTTP, SSE fallback), `grabMcpTools` (each server tool becomes a `HarnessTool` with its JSON schema), `mcpResultToText` (text as is, images read by the vision harness, `isError` thrown). Computers are reached this way: `openComputerTools` (`src/harness/tools/computer/`) connects to the computer's AIO sandbox at `grabComputerMcpUrl` (the `MAIN_COMPUTER_URL` app setting for the main computer, for a dev server outside docker, e.g. `http://localhost:8170`, else `http://:/mcp`, or its IP address when the hostname can't be reached), leaves out `ComputerMcpExcludedTools`, and sets the computer's envs (see Envs). - **MCPs and custom tools (extensions):** installs add their own MCP servers (`mcp_servers`, Settings, under MCPs) and tools (`custom_tools`, Settings, under Tools). A bot gets the ones given to it (`bot_mcp_servers`, `bot_custom_tools`) and, in a workspace, that workspace's (`workspace_mcp_servers`, `workspace_custom_tools`); `botExtensionsSql` (`src/utils/bot-extensions-sql.ts`, tested) resolves a run's, `grabBotExtensions` reads them. `all_bots` ("every bot by default") only decides what bots are given: turning it on gives it to every bot (`attachToAllBots`, the coordinator included) and every new bot gets it (`attachDefaultExtensions`, from `createBot` and `ensureCoordinatorBot`); turning it off takes it from no one, and a bot that drops one keeps it dropped (`src/scripts/attach-every-bot-extensions.ts`, safe to rerun, moved installs from the old "reaches every bot" meaning). Super admins define them. Super admins and admins (the super admin alone for the coordinator, `assertBotExtensionManager`) give and take them per bot on the bot's settings page and under its chat's composer (`composer-tools.tsx`: one "N tools" button (`composer-tools-modal.tsx`) counting everything it can use there, which opens a modal listing them (a workspace's not removable) with the add picker, so ten of them don't fill the composer; beside it `composer-computer-button.tsx` opens the bot's current computer in the viewer), and per workspace on the workspace's settings page; the settings pages share `settings/(partials)/extensions-group.tsx`, all through `useExtensions` (`src/hooks/`), `grabBotExtensionsOverview` / `grabWorkspaceExtensionsOverview` (`bot_extensions` / `workspace_extensions` page props, both on `grabExtensionsOverview`) and `api/admin/bots/` and `api/admin/workspaces/` `add-extension`, `remove-extension` (`setBotExtension`, `setWorkspaceExtension`); the MCPs and Tools pages' Access modal sets bots and workspaces in bulk (`setExtensionAccess`, an admin never touches the coordinator's or `all_bots`). Helpers in `src/functions/backend/db/extensions/`, API in `api/admin/mcps/` and `api/admin/tools/`, zod in `src/schemas/extension-schema.ts`. `openBotExtensions` (`src/harness/tools/extensions/`) turns them into harness tools for chat replies and tasks (built-ins win a name clash; `close()` in the runners' `finally`; `note` tells the bot about a server it can't use). Templates (`fillTemplate`, `src/utils/fill-template.ts`): `{{args.x}}` and `{{env.KEY}}`, envs from `grabRunEnvs` (workspace beats general, app settings never, no need to be set on computers); secret values are redacted from every result. - **Custom tools** (`CustomToolKinds` in `src/dict/custom-tool-dict.ts`): a name (`checkCustomToolName`: snake_case, never a built-in's, `ReservedToolNames`), a description, a JSON-schema `parameters` and a `config` per kind, built by `buildCustomTool` (`src/harness/tools/custom/`): `http` (`runHttpTool`: the app fetches, arguments URL-encoded in the URL, no body template sends the arguments as JSON), `script` (`runScriptTool`: bash/python/javascript on the run's computer, the task's or in chat the bot's current one, over the sandbox REST API in a subshell, computer envs set, arguments as JSON in `COBBLE_ARGS` and on stdin; left out without a computer), `prompt` (`runPromptTool`: one `generateText` turn with the bot's models). Super admins try one from the page (`testCustomTool`, the main computer for scripts): a saved one from its row (Try), or the form as it is, saved or not (`draft`, nothing recorded), from the Test panel in its modal (`custom-tool-test-panel.tsx`). Every run (a bot's call or a try; not bad arguments or a cancel) is saved as the tool's `status`, `last_error` and `last_checked_at` (`recordCustomToolRun`); editing resets it to `untested`. - **MCP servers** (`McpTransports` in `src/dict/mcp-dict.ts`): `remote` (URL plus templated headers, `connectMcpServer`) or `stdio` (a command run with the host's shell, `hostShell`, in `DATA_DIR/mcp/` (`grabMcpDir`), after its `setup_command`, which `runMcpSetup` reruns whenever it changes (marker file) or on Reinstall; its env is only its own `env` map plus the shell basics, `grabMcpStdioEnv`, never the app's). Their tools are listed into `tools_cache` (`syncMcpServer`: on save, Refresh, and hourly for remote ones by the `sync-mcp-servers` cron) and named `__` (`mcpToolName`); the modal's Test connection checks the form as it is, saved or not, outside the pool (`testMcpDraft`, `testMcpServer`, `api/admin/mcps/test-mcp`, a new stdio server's setup runs in `mcp/0`); a run that can't connect, or connects again after failing, updates its `status` too (`recordMcpServerConnection`); runs build tools from the cache (`mcpHarnessTool`) and connect on the first call (`openMcpServer`). stdio servers live in a per-process pool (`src/harness/mcp/mcp-stdio-pool.ts`), stopped after `AppData.McpStdioIdleMs` idle, on save or delete. - **Check before and after acting:** every reply and task gets `ActionCheckNote` (`src/data/harness-prompts.ts`): look for an existing record before creating one (update it instead), and verify every action took effect before saying it's done. The tools back it: `update_memory` and `update_instructions` read the save back (`checkSavedText`, `grabBotSavedText`) and fail if it didn't take; `create_bot` refuses a name already on the crew, `create_schedule` a twin of one of the bot's schedules (`findDuplicateCronJob`: same place and same title, or same timing and brief), and `start_task`/`delegate_task` a task with the same title still open in that session (`grabOpenBotTask`); `create_workspace` already refuses a name in use. -- **Replies and tasks:** `run-bot-reply.ts` answers from the last `BotReplyHistoryMessages` messages (`botMessagesToHarness`) with `ChatReplyNote`, the bot's computers (`compose-computers-note.ts`: what they are and that they're reached through `start_task`) and the chat tools, at most `BotReplyMaxToolSteps` steps, and finishes silently if the bot already had the last word (`enqueueBotReply` reuses a pending reply, so a burst of messages gets one). `run-bot-task.ts` runs the brief (`compose-job-brief.ts`) with `TaskRunNote`, the bot tools and the job's computer tools, closing the MCP connection after. Retryable harness errors go back to the queue's backoff, the rest are `BotJobFatalError`. +- **Replies and tasks:** `run-bot-reply.ts` answers from the last `BotReplyHistoryMessages` messages (`botMessagesToHarness`) with `ChatReplyNote`, the bot's computers (`compose-computers-note.ts`: what they are and that they're reached through `start_task`) and the chat tools, at most `BotReplyMaxToolSteps` steps, and finishes silently if the bot already had the last word (`enqueueBotReply` reuses a pending reply, so a burst of messages gets one). `run-bot-task.ts` runs the brief (`compose-job-brief.ts`) with `TaskRunNote`, the bot tools, the file tools and the job's computer tools, closing the MCP connection after. Both add `BotFilesNote` (where a bot's files live, that they persist, and to search before saying it doesn't know) and collect `attachments` for files the bot wrote with `attach`, which `completeBotJob` links to the result message. Retryable harness errors go back to the queue's backoff, the rest are `BotJobFatalError`. - **API:** `api/admin/harness/` (`save-provider` also syncs, `sync-provider`, `delete-provider`, `save-settings`), super admin only, each returning the fresh `grabHarnessOverview()` page data. ## Admin Route Hierarchy @@ -171,7 +185,7 @@ Live routes under `/admin` (see `AdminSections`): `/admin/bots`, `/admin/workspa - `/admin/help` (live) - familiar problems and their fixes, one `SettingsGroup` per topic (`(data)/help-topics.ts`, markdown rendered with the chat's `MessageMarkdownComponents`, the topic id as the group's anchor). First topic: signing in to Google and other strict sites with Cookie-Editor over the Desktop view (`HelpSignInTopicId` in `src/dict/help-dict.ts`; `ask_user_to_take_over` points bots to it) - Routines, Activity and Help are the `RailPages` (`src/dict/rail-pages-dict.ts`), opened from the rail's user popover (which lists them, then Settings and Sign out). On them the rail shows its own aside (`(partials)/rail-pages-list.tsx`): a back link to the bots, then those pages, like settings - `/admin/settings/backups` - super admin only (listed in `SettingsPages`): create/list/download database snapshots in `db/.backups`, plus `restore/` (restore a listed backup, or upload a SQLite database file which is saved to `.backups` then restored). Shells out to the `bun-sqlite` CLI (`backup`, and `import ` since `restore` is interactive) via `src/functions/backend/backups/`; API routes in `api/admin/backups/` -- `/admin/settings` (live) - redirects to `SettingsDefaultPath` (`/admin/settings/general`). Sub pages: `general/` (theme: System/Light/Dark per device via `useThemePreference`, stored in `localStorage[AppData.ThemeStorageKey]`, the same key `ThemeInitScript` and twui's `ColorSchemeSelector` read; applies instantly, no save bar), `profile/` (current user profile), `notifications/` (this device's Web Push switch, a test send, and every device the user subscribed), `change-password/` (set/change own password) `models/` (super admin only, `super_admin_only` in the dict hides it from the rail: model providers and which model each harness uses) and `environment/` (super admin only: app settings and the envs bots and computers use, per scope), each wrapped in `SettingsShell` (`(partials)/settings-shell.tsx`: 64px pane header like the conversation header, then one centered column). Content is built from `SettingsGroup` blocks (label + description left, controls right, hairline-separated, no cards) and ends with the sticky `SettingsSaveBar` (Save enabled only when dirty, Discard, one-line status). Saves don't reload the page. Keep save notices in component state: `useFormInit` clears `status` whenever the form changes. `bots/` (the crew: chat, edit with `BotForm`, models, delete a worker via `api/admin/bots/delete-bot`), `workspaces/` (open, create through the Add flow, delete), `computers/` (state and resources, live; super admins create, edit with `AddComputerForm`, retry, delete), `mcps/` and `tools/` (see MCPs and custom tools: super admins add, edit, refresh or try, delete; admins only set Access, `extension-access-modal.tsx`; add, edit, access and try open in modals (`AddModalShell`)), each row with a status line under it (`extension-status-line.tsx`: connected or not, last run worked or failed, when, and why), those five for `canManageCrew` only (`grab-crew-settings-server-props.ts`, `can_view` in the dict) and `users/` (for `canListUsers`: super admins and admins; the dict's `can_view` gates it in the rail; add with a password (`canCreateUser`), edit (`canUpdateUser`), both with the one `UserForm` (`admin/settings/users/(partials)/user-form/`, `useFormInit`, photo included, saved in one request), revoke (adds `revoked`, `canRevokeUser`), delete (`canRevokeUser`'s rules, never yourself), pick their user types from `UserTypes` (`user-types-select.tsx`: only the ones `filterAssignableUserTypes` allows, never `UneditableUserTypes`; at least one, since login needs a type; `saveUser` checks each grant and removal with `userTypeAssignment`; the super admin's types are never shown or changed there, and they can't be revoked; the password is entered twice, `SaveUserRequestSchema` checks they match), delete with everything theirs; `api/admin/users/save-user`, `delete-user`). The lists share `SettingsList`, `SettingsListRow`, `SettingsRowAction` and `use-settings-list`. Sub pages are declared once in `src/dict/settings-pages-dict.ts`; on every `/admin/settings/*` page the rail swaps the crew for a back link plus that list. Add a sub page by adding its folder and a dict entry +- `/admin/settings` (live) - redirects to `SettingsDefaultPath` (`/admin/settings/general`). Sub pages: `general/` (theme: System/Light/Dark per device via `useThemePreference`, stored in `localStorage[AppData.ThemeStorageKey]`, the same key `ThemeInitScript` and twui's `ColorSchemeSelector` read; applies instantly, no save bar; and, for `canManageCrew`, the bot file quota: `storage-section.tsx`, `storage-quota-form.tsx` saving `api/admin/settings/storage-quota` into `app_settings`), `profile/` (current user profile), `notifications/` (this device's Web Push switch, a test send, and every device the user subscribed), `change-password/` (set/change own password) `models/` (super admin only, `super_admin_only` in the dict hides it from the rail: model providers and which model each harness uses) and `environment/` (super admin only: app settings and the envs bots and computers use, per scope), each wrapped in `SettingsShell` (`(partials)/settings-shell.tsx`: 64px pane header like the conversation header, then one centered column). Content is built from `SettingsGroup` blocks (label + description left, controls right, hairline-separated, no cards) and ends with the sticky `SettingsSaveBar` (Save enabled only when dirty, Discard, one-line status). Saves don't reload the page. Keep save notices in component state: `useFormInit` clears `status` whenever the form changes. `bots/` (the crew: chat, edit with `BotForm`, models, delete a worker via `api/admin/bots/delete-bot`), `workspaces/` (open, create through the Add flow, delete), `computers/` (state and resources, live; super admins create, edit with `AddComputerForm`, retry, delete), `mcps/` and `tools/` (see MCPs and custom tools: super admins add, edit, refresh or try, delete; admins only set Access, `extension-access-modal.tsx`; add, edit, access and try open in modals (`AddModalShell`)), each row with a status line under it (`extension-status-line.tsx`: connected or not, last run worked or failed, when, and why), those five for `canManageCrew` only (`grab-crew-settings-server-props.ts`, `can_view` in the dict) and `users/` (for `canListUsers`: super admins and admins; the dict's `can_view` gates it in the rail; add with a password (`canCreateUser`), edit (`canUpdateUser`), both with the one `UserForm` (`admin/settings/users/(partials)/user-form/`, `useFormInit`, photo included, saved in one request), revoke (adds `revoked`, `canRevokeUser`), delete (`canRevokeUser`'s rules, never yourself), pick their user types from `UserTypes` (`user-types-select.tsx`: only the ones `filterAssignableUserTypes` allows, never `UneditableUserTypes`; at least one, since login needs a type; `saveUser` checks each grant and removal with `userTypeAssignment`; the super admin's types are never shown or changed there, and they can't be revoked; the password is entered twice, `SaveUserRequestSchema` checks they match), delete with everything theirs; `api/admin/users/save-user`, `delete-user`). The lists share `SettingsList`, `SettingsListRow`, `SettingsRowAction` and `use-settings-list`. Sub pages are declared once in `src/dict/settings-pages-dict.ts`; on every `/admin/settings/*` page the rail swaps the crew for a back link plus that list. Add a sub page by adding its folder and a dict entry Every `/admin/*` page renders in the bots layout (`src/layouts/bots/`); there is no separate admin layout or aside. The rail header's Add icon (super admins and admins only, like the rail's "+" buttons) opens the Add flow (`(partials)/add-flow.tsx`, `AddFlowContext`): a bot, a workspace or (super admins) a computer, each form in a modal stacked over the choice, so closing it goes back there (the Bots and Workspaces labels have a "+" that opens the form directly; in a workspace's rail, "Workspace bots" has a "+" that opens `link-bots-modal.tsx` instead: the crew's workers not in it yet (`api/admin/workspaces/linkable-bots`), each added with `add-bot`, which pushes `server:bot-created`, and "New bot" stacking the bot form over it); on a workspace's pages new bots and computers join it. The rail shows the crew (`(partials)/rail-crew-list.tsx`, needs `bots` and `workspaces` in page props): the coordinator, the latest `AppData.RailRecentBots` workers (plus any open, unread or needing you, `pick-rail-bots`), "See all bots" with a few stones opening every bot in a modal, then Workspaces with a "+". On `/admin/workspaces/*` it shows the workspace's crew instead (`(partials)/rail-workspace-list.tsx`: back link, the workspace, then only the bots attached to it, "Workspace bots"; the coordinator stays in the main crew rail, and on a workspace bot's page the back link returns to the workspace page), except on `/admin/settings/*` where it shows "Back to bots" plus the settings sub pages (`(partials)/rail-settings-list.tsx`). Page heroes use the shared `AdminHero` component (`src/components/general/admin-hero.tsx`). diff --git a/db/schema.ts b/db/schema.ts index 0da378d..6f4aafb 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -18,6 +18,7 @@ import { Harnesses } from "@/src/dict/harness-dict"; import { HarnessProviderKinds } from "@/src/dict/harness-provider-dict"; import { BotJobKinds, BotJobUpdateKinds } from "@/src/dict/bot-queue-dict"; import { AppData } from "@/src/data/app-data"; +import { BotFilePurposes, BotFileScopes, BotFileSources } from "@/src/dict/bot-file-dict"; import type { BUN_SQLITE_DatabaseSchemaType } from "@moduletrace/bun-sqlite/types"; import { EnvParadigms } from "@/src/dict/env-dict"; import { McpServerStatuses, McpTransports } from "@/src/dict/mcp-dict"; @@ -2322,6 +2323,233 @@ const schema: BUN_SQLITE_DatabaseSchemaType = { }, ], }, + { + tableName: "bot_files", + tableDescription: `A file a bot saved or the user gave it: the catalog row for bytes on the data volume, under \`DATA_DIR/bots/\` or \`DATA_DIR/workspaces/\`. The bytes are a private \`media\` row (paradigm \`bot-file\`), so they're served like other media and deleted with the row. Never on a computer's filesystem`, + fields: [ + { + fieldName: "bot_id", + fieldDescription: `The bot that wrote or received it. Goes with the bot`, + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "bots", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "bot_files_bot_fk", + }, + }, + { + fieldName: "scope", + fieldDescription: `Which directory it's in: a bot's own or a workspace's shared one`, + dataType: "TEXT", + options: BotFileScopes.map((s) => s.value), + defaultValue: "bot", + }, + { + fieldName: "owner_id", + fieldDescription: `The bot id (scope bot) or workspace id (scope workspace) owning the directory. Not a foreign key, so a row outlives a workspace but its files are cleaned up with it`, + dataType: "INTEGER", + }, + { + fieldName: "media_id", + fieldDescription: `The private \`media\` row holding the bytes. Not a foreign key: the catalog row can outlive it (a failed cleanup) and is then listed as missing`, + dataType: "INTEGER", + }, + { + fieldName: "path", + fieldDescription: `Path inside the directory, forward slashes, e.g. reports/q3.md`, + dataType: "TEXT", + }, + { + fieldName: "name", + dataType: "TEXT", + }, + { + fieldName: "folder", + fieldDescription: `The folder part of \`path\`, empty at the top`, + dataType: "TEXT", + }, + { + fieldName: "mime_type", + dataType: "TEXT", + }, + { + fieldName: "media_type", + dataType: "TEXT", + options: MediaTypes.map((it) => it.value), + }, + { + fieldName: "size", + fieldDescription: `Bytes on disk`, + dataType: "INTEGER", + }, + { + fieldName: "purpose", + fieldDescription: `What it is for: an upload, something the bot wrote, a generated image or PDF`, + dataType: "TEXT", + options: BotFilePurposes.map((p) => p.value), + defaultValue: "writing", + }, + { + fieldName: "source", + fieldDescription: `How it got here: the user attached it to a message, the bot wrote it, or a tool made it`, + dataType: "TEXT", + options: BotFileSources.map((s) => s.value), + defaultValue: "bot-write", + }, + { + fieldName: "ingested_at", + fieldDescription: `When its text was chunked and embedded. Null: not ingested (yet)`, + dataType: "INTEGER", + }, + { + fieldName: "ingest_error", + fieldDescription: `Why the last ingestion failed, when it did`, + dataType: "TEXT", + }, + ], + uniqueConstraints: [ + { + alias: "one_file_per_path", + constraintName: "One file per directory and path", + constraintTableFields: [{ value: "scope" }, { value: "owner_id" }, { value: "path" }], + }, + ], + indexes: [ + { + indexName: "idx_bot_files_owner", + indexTableFields: ["scope", "owner_id", "folder"], + }, + { + indexName: "idx_bot_files_media_id", + indexTableFields: ["media_id"], + }, + ], + }, + { + tableName: "bot_file_chunks", + tableDescription: `A piece of a bot file's text, the unit a search returns. Replaced whole when the file is ingested again`, + fields: [ + { + fieldName: "file_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "bot_files", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "bot_file_chunks_file_fk", + }, + }, + { + fieldName: "scope", + dataType: "TEXT", + options: BotFileScopes.map((s) => s.value), + }, + { + fieldName: "owner_id", + fieldDescription: `The bot or workspace the file belongs to, so a search filters without a join`, + dataType: "INTEGER", + }, + { + fieldName: "chunk_index", + dataType: "INTEGER", + }, + { + fieldName: "text", + dataType: "TEXT", + }, + { + fieldName: "token_estimate", + fieldDescription: `A rough count (characters / 4), for trimming results`, + dataType: "INTEGER", + }, + ], + indexes: [ + { + indexName: "idx_bot_file_chunks_file_id", + indexTableFields: ["file_id", "chunk_index"], + }, + { + indexName: "idx_bot_file_chunks_owner", + indexTableFields: ["scope", "owner_id"], + }, + ], + }, + { + tableName: "bot_file_vectors", + tableDescription: `sqlite-vec vectors of a file's chunks, at \`AppData.EmbeddingDimensions\` wide whatever the model returns. A vec0 table is only recreated with \`bunx bun-sqlite schema --vector\`, so the entrypoint never passes it`, + isVector: true, + vectorType: "vec0", + fields: [ + { + fieldName: "embedding", + fieldDescription: `Normalized, padded or cut to \`AppData.EmbeddingDimensions\``, + dataType: "BLOB", + isVector: true, + vectorSize: AppData.EmbeddingDimensions, + }, + { + fieldName: "chunk_id", + dataType: "INTEGER", + sideCar: true, + }, + { + fieldName: "file_id", + dataType: "INTEGER", + sideCar: true, + }, + { + fieldName: "scope", + dataType: "TEXT", + sideCar: true, + }, + { + fieldName: "owner_id", + dataType: "INTEGER", + sideCar: true, + }, + ], + }, + { + tableName: "file_ingest_queue", + tableDescription: `Files waiting to have their text chunked and embedded. A queue, so a restart never strands a file and an upload never waits on a model. A finished job is deleted: \`bot_files.ingested_at\` and \`ingest_error\` keep the outcome`, + isQueue: { + maxAttempts: 3, + lockTimeout: 10 * 60 * 1000, + backoff: 30_000, + deleteOnComplete: true, + }, + fields: [ + { + fieldName: "file_id", + fieldDescription: `The \`bot_files\` row to (re)ingest`, + dataType: "INTEGER", + }, + ], + indexes: [ + { + indexName: "idx_file_ingest_queue_file_id", + indexTableFields: ["file_id"], + }, + ], + }, + { + tableName: "app_settings", + tableDescription: `Install-wide settings the app reads that aren't envs: an env's value is handed to bots and computers, which a storage quota must never be`, + fields: [ + { + fieldName: "key", + dataType: "TEXT", + unique: true, + }, + { + fieldName: "value", + dataType: "TEXT", + }, + ], + }, ], }; diff --git a/db/types/db.ts b/db/types/db.ts index 0932b46..e60f694 100644 --- a/db/types/db.ts +++ b/db/types/db.ts @@ -40,6 +40,11 @@ export const BunSQLiteTables = [ "user_presence", "push_subscriptions", "envs", + "bot_files", + "bot_file_chunks", + "bot_file_vectors", + "file_ingest_queue", + "app_settings", ] as const export type BUN_SQLITE_COBBLE_USERS = { @@ -156,7 +161,7 @@ export type BUN_SQLITE_COBBLE_MEDIA_PARADIGMS = { */ updated_at?: number | ""; media_id?: number | ""; - media_paradigm?: "user-profile-image" | "generic" | "event" | "bot-attachment" | "bot-image" | "workspace-image" | ""; + media_paradigm?: "user-profile-image" | "generic" | "event" | "bot-attachment" | "bot-image" | "workspace-image" | "bot-file" | ""; } export type BUN_SQLITE_COBBLE_BOTS = { @@ -968,7 +973,7 @@ export type BUN_SQLITE_COBBLE_HARNESS_SETTINGS = { * The time when the record was updated. (Unix Timestamp) */ updated_at?: number | ""; - harness?: "text" | "vision" | "transcription" | "speech" | "image" | ""; + harness?: "text" | "vision" | "transcription" | "speech" | "image" | "embedding" | ""; /** * harness_models row picked by a super admin. Null picks automatically. Not a foreign key, so removing a provider leaves the setting */ @@ -1008,7 +1013,7 @@ export type BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS = { * The worker bot that overrides the default. The coordinator always uses the defaults */ bot_id?: number | ""; - harness?: "text" | "vision" | "transcription" | "speech" | "image" | ""; + harness?: "text" | "vision" | "transcription" | "speech" | "image" | "embedding" | ""; /** * harness_models row picked for this bot. Null picks automatically. Not a foreign key, like harness_settings */ @@ -1048,7 +1053,7 @@ export type BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS = { * The workspace */ workspace_id?: number | ""; - harness?: "text" | "vision" | "transcription" | "speech" | "image" | ""; + harness?: "text" | "vision" | "transcription" | "speech" | "image" | "embedding" | ""; /** * harness_models row picked for this workspace. Null picks automatically. Not a foreign key, like harness_settings */ @@ -1418,4 +1423,176 @@ export type BUN_SQLITE_COBBLE_ENVS = { expose_to_computers?: 0 | 1 | ""; } -export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS \ No newline at end of file +export type BUN_SQLITE_COBBLE_BOT_FILES = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + /** + * The bot that wrote or received it. Goes with the bot + */ + bot_id?: number | ""; + /** + * Which directory it's in: a bot's own or a workspace's shared one + */ + scope?: "bot" | "workspace" | ""; + /** + * The bot id (scope bot) or workspace id (scope workspace) owning the directory. Not a foreign key, so a row outlives a workspace but its files are cleaned up with it + */ + owner_id?: number | ""; + /** + * The private `media` row holding the bytes. Not a foreign key: the catalog row can outlive it (a failed cleanup) and is then listed as missing + */ + media_id?: number | ""; + /** + * Path inside the directory, forward slashes, e.g. reports/q3.md + */ + path?: string; + name?: string; + /** + * The folder part of `path`, empty at the top + */ + folder?: string; + mime_type?: string; + media_type?: "image" | "video" | "document" | "audio" | ""; + /** + * Bytes on disk + */ + size?: number | ""; + /** + * What it is for: an upload, something the bot wrote, a generated image or PDF + */ + purpose?: "upload" | "writing" | "image" | "pdf" | "artifact" | ""; + /** + * How it got here: the user attached it to a message, the bot wrote it, or a tool made it + */ + source?: "chat-upload" | "bot-write" | "tool" | ""; + /** + * When its text was chunked and embedded. Null: not ingested (yet) + */ + ingested_at?: number | ""; + /** + * Why the last ingestion failed, when it did + */ + ingest_error?: string; +} + +export type BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + file_id?: number | ""; + scope?: "bot" | "workspace" | ""; + /** + * The bot or workspace the file belongs to, so a search filters without a join + */ + owner_id?: number | ""; + chunk_index?: number | ""; + text?: string; + /** + * A rough count (characters / 4), for trimming results + */ + token_estimate?: number | ""; +} + +export type BUN_SQLITE_COBBLE_BOT_FILE_VECTORS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + /** + * Normalized, padded or cut to `AppData.EmbeddingDimensions` + */ + embedding?: Float32Array | Buffer | null; + chunk_id?: number | ""; + file_id?: number | ""; + scope?: string; + owner_id?: number | ""; +} + +export type BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + /** + * The `bot_files` row to (re)ingest + */ + file_id?: number | ""; + /** + * The state of the job. (pending | processing | completed | failed) + */ + status?: string; + /** + * The number of times the job has been claimed. + */ + attempts?: number | ""; + /** + * The time from which the job can be claimed. (Unix Timestamp) + */ + available_at?: number | ""; + /** + * The time when the job was claimed. (Unix Timestamp) + */ + locked_at?: number | ""; + /** + * The worker that claimed the job. + */ + locked_by?: string; + /** + * The error from the last failed attempt. + */ + last_error?: string; +} + +export type BUN_SQLITE_COBBLE_APP_SETTINGS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + key?: string; + value?: string; +} + +export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_MESSAGE_REACTIONS & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_USER_PRESENCE & BUN_SQLITE_COBBLE_PUSH_SUBSCRIPTIONS & BUN_SQLITE_COBBLE_ENVS & BUN_SQLITE_COBBLE_BOT_FILES & BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS & BUN_SQLITE_COBBLE_BOT_FILE_VECTORS & BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE & BUN_SQLITE_COBBLE_APP_SETTINGS \ No newline at end of file diff --git a/src/bots-queue/process-bot-job.ts b/src/bots-queue/process-bot-job.ts index 8156533..9923b19 100644 --- a/src/bots-queue/process-bot-job.ts +++ b/src/bots-queue/process-bot-job.ts @@ -196,6 +196,7 @@ export default async function processBotJob({ result: outcome.result, needs_you: outcome.needs_you, silent: outcome.silent, + attachments: outcome.attachments, }); await pushBotNotice({ bot_id, ...completion }); diff --git a/src/bots-queue/run-bot-reply.ts b/src/bots-queue/run-bot-reply.ts index 6132d0f..c3bd978 100644 --- a/src/bots-queue/run-bot-reply.ts +++ b/src/bots-queue/run-bot-reply.ts @@ -2,6 +2,7 @@ import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db"; import { AppData } from "@/src/data/app-data"; import { ActionCheckNote, + BotFilesNote, ChatReplyNote, CoordinatorReplyNote, CoordinatorSoloNote, @@ -12,6 +13,7 @@ import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt"; import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers"; import runTextLoop from "@/src/harness/text/run-text-loop"; import grabBotChatTools from "@/src/harness/tools/bot/grab-bot-chat-tools"; +import grabBotFileTools from "@/src/harness/tools/storage/grab-bot-file-tools"; import openBotExtensions from "@/src/harness/tools/extensions/open-bot-extensions"; import botMessagesToHarness from "./bot-messages-to-harness"; import composeComputersNote from "./compose-computers-note"; @@ -76,6 +78,18 @@ export default async function runBotReply({ }).catch(() => ""); const take_overs: string[] = []; + /** + * Files the bot wrote and asked to send with its reply. + */ + const attachments: number[] = []; + + const file_tools = await grabBotFileTools({ + bot, + user_id, + workspace_id, + session_id, + onAttach: ({ media_id }) => attachments.push(media_id), + }); const tools = await grabBotChatTools({ bot, @@ -111,6 +125,7 @@ export default async function runBotReply({ prompt, carry_over, ChatReplyNote, + BotFilesNote, ActionCheckNote, workspace_id ? WorkspaceLearningNote : "", composeComputersNote({ @@ -132,7 +147,7 @@ export default async function runBotReply({ user_id, }), }), - tools: [...tools, ...extensions.tools], + tools: [...tools, ...file_tools, ...extensions.tools], max_steps: AppData.BotReplyMaxToolSteps, bot_id: bot.id, workspace_id, @@ -140,7 +155,10 @@ export default async function runBotReply({ onStep: report, }); - return withTakeOverRefs({ text: result.text, refs: take_overs }); + return { + ...withTakeOverRefs({ text: result.text, refs: take_overs }), + attachments: attachments.length ? attachments : undefined, + }; } finally { await extensions.close(); } diff --git a/src/bots-queue/run-bot-task.ts b/src/bots-queue/run-bot-task.ts index cb066ba..90659e4 100644 --- a/src/bots-queue/run-bot-task.ts +++ b/src/bots-queue/run-bot-task.ts @@ -1,5 +1,11 @@ import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db"; -import { ActionCheckNote, ScheduledRunNote, TaskRunNote, WorkspaceLearningNote } from "@/src/data/harness-prompts"; +import { + ActionCheckNote, + BotFilesNote, + ScheduledRunNote, + TaskRunNote, + WorkspaceLearningNote, +} from "@/src/data/harness-prompts"; import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers"; import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt"; import grabUserWorkspaces from "@/src/functions/backend/db/workspaces/grab-user-workspaces"; @@ -11,6 +17,7 @@ import createWorkspaceTool from "@/src/harness/tools/bot/create-workspace-tool"; import updateWorkspaceTool from "@/src/harness/tools/bot/update-workspace-tool"; import askToTakeOverTool from "@/src/harness/tools/computer/ask-to-take-over-tool"; import searchComputersTool from "@/src/harness/tools/computer/search-computers-tool"; +import grabBotFileTools from "@/src/harness/tools/storage/grab-bot-file-tools"; import reportRunFailedTool from "@/src/harness/tools/bot/report-run-failed-tool"; import computerRef from "@/src/utils/computer-ref"; import composeJobBrief from "./compose-job-brief"; @@ -81,6 +88,10 @@ export default async function runBotTask({ signal, }).catch(() => ""); const take_overs: string[] = []; + /** + * Files the bot wrote and asked to send with its report. + */ + const attachments: number[] = []; const scheduled = Boolean(params.schedule_id); let failure = null as string | null; @@ -120,6 +131,13 @@ export default async function runBotTask({ : []), ...(scheduled ? [reportRunFailedTool({ onFail: ({ reason }) => (failure = reason) })] : []), ...(computer?.tools || []), + ...(await grabBotFileTools({ + bot, + user_id, + workspace_id, + session_id, + onAttach: ({ media_id }) => attachments.push(media_id), + })), ]; extensions = await openBotExtensions({ @@ -136,6 +154,7 @@ export default async function runBotTask({ carry_over, TaskRunNote, scheduled ? ScheduledRunNote : "", + BotFilesNote, ActionCheckNote, workspace_id ? WorkspaceLearningNote : "", bot.kind == "coordinator" @@ -180,10 +199,13 @@ export default async function runBotTask({ throw new ScheduledRunFailedError(computer ? computer.redact(failure) : failure); } - return withTakeOverRefs({ - text: computer ? computer.redact(result.text) : result.text, - refs: take_overs, - }); + return { + ...withTakeOverRefs({ + text: computer ? computer.redact(result.text) : result.text, + refs: take_overs, + }), + attachments: attachments.length ? attachments : undefined, + }; } finally { await extensions?.close(); await computer?.close(); diff --git a/src/bots-queue/types.ts b/src/bots-queue/types.ts index e6f92a8..e77315d 100644 --- a/src/bots-queue/types.ts +++ b/src/bots-queue/types.ts @@ -40,6 +40,11 @@ export type BotJobOutcome = { * Finish without posting `result` (a reply with nothing left to answer). */ silent?: boolean; + /** + * Files the bot wrote and asked to send with its message (`media` + * ids), attached to the result message. + */ + attachments?: number[]; }; export type BotJobRunner = (context: BotJobContext) => Promise; diff --git a/src/cron/index.ts b/src/cron/index.ts index cfa9467..b4090d1 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -5,6 +5,7 @@ import cronSyncHarnessModels from "./sync-harness-models"; import syncComputers from "./sync-computers"; import syncMcpServers from "./sync-mcp-servers"; import runCronJobs from "./run-cron-jobs"; +import ingestPendingFiles from "./ingest-pending-files"; import registerCronJob from "./register-cron-job"; export default function cron() { @@ -54,4 +55,11 @@ export default function cron() { run_on_start: true, quiet: true, }); + + registerCronJob({ + title: "ingest-pending-files", + schedule: "*/10 * * * *", + task: ingestPendingFiles, + quiet: true, + }); } diff --git a/src/cron/ingest-pending-files/index.ts b/src/cron/ingest-pending-files/index.ts new file mode 100644 index 0000000..04a4d04 --- /dev/null +++ b/src/cron/ingest-pending-files/index.ts @@ -0,0 +1,29 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db"; +import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest"; + +/** + * Queues any bot file that never got ingested (an upload that happened + * while the worker was down, a restart mid-ingest), so nothing is left + * unsearchable. Runs every ten minutes. + */ +export default async function ingestPendingFiles() { + const res = await BunSQLite.sql({ + sql: `SELECT f.id FROM bot_files f + WHERE f.ingested_at IS NULL + AND f.ingest_error IS NULL + AND f.created_at > ? + AND NOT EXISTS ( + SELECT 1 FROM file_ingest_queue q + WHERE q.file_id = f.id AND q.status IN ('pending', 'processing') + ) + LIMIT 25`, + values: [Date.now() - 24 * 60 * 60 * 1000], + }); + + for (const file of res.payload || []) { + if (file.id) await enqueueFileIngest({ file_id: file.id }); + } + + return res.payload?.length || 0; +} diff --git a/src/data/app-data.ts b/src/data/app-data.ts index 7455588..e328e21 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -373,6 +373,65 @@ export const AppData = { * Most of a bot message read aloud; the rest is cut. */ SpeechMaxChars: 4000, + /** + * Bot files: a bot's own directory and a workspace's shared one live + * under `DATA_DIR` (`DATA_DIR/bots/`, `DATA_DIR/workspaces/`), + * on the data volume, never on a computer. + */ + BotDataDir: "bots", + WorkspaceDataDir: "workspaces", + /** + * The per-owner quota when the General settings page has none saved. + */ + BotFileQuotaMb: 1024, + /** + * One file a bot writes in one call, and the most of a file a read + * hands back at once (the rest comes with an offset). + */ + BotFileWriteMaxChars: 200_000, + BotFileReadMaxChars: 40_000, + BotFileListMaxItems: 200, + BotFileNameMaxChars: 120, + BotFilePathMaxChars: 400, + /** + * Most folders deep a path may go, so nothing buries a file. + */ + BotFilePathMaxDepth: 8, + /** + * The files modal and the bot's file list: one page of them. + */ + BotFilesPageSize: 50, + /** + * Ingesting a bot file: how much text one chunk holds, how much of the + * previous chunk it repeats, and the largest file it reads at all. + */ + ChunkTargetChars: 1200, + ChunkOverlapChars: 200, + IngestMaxBytes: 20 * 1024 * 1024, + /** + * The fixed width of every vector, whatever the embedding model + * returns: shorter vectors are padded with zeros and longer ones cut, + * then all are normalized. Changing it needs + * `bunx bun-sqlite schema --vector` and a full re-ingest. + */ + EmbeddingDimensions: 1536, + /** + * Texts sent to the embedding model per call, and per run of the + * ingestion worker. + */ + EmbeddingBatchSize: 32, + EmbeddingMaxInputChars: 8000, + /** + * How many chunks a file search returns, and how many more it asks + * the vector index for before filtering to the run's scopes. + */ + RagTopChunks: 8, + RagOverFetch: 4, + RagMaxK: 60, + FileIngestPollMs: 5000, + FileIngestConcurrency: 2, + FileIngestWorker: "ingest", + FileIngestMaxAttempts: 2, /** * What a speech model reads when a voice is tried in the model settings. */ diff --git a/src/data/harness-prompts.ts b/src/data/harness-prompts.ts index 00fd166..1174769 100644 --- a/src/data/harness-prompts.ts +++ b/src/data/harness-prompts.ts @@ -31,11 +31,26 @@ You are chatting with the user. Answer their latest messages directly and briefl - To add a worker to the crew, use create_bot once the user has asked for it or agreed, and add_bot_to_workspace to place it. A bot exists only when create_bot made it: never describe one as if it did, and never write it into a workspace's context. - Never claim you did something you didn't do. - The user can mention a bot as [bot:ID] and a workspace as [workspace:ID]; a [Mentions: ...] line after their message says who and what those are. Write the same refs when you name them back. +- Documents, images and reports you make are files: save them with write_file, make_pdf or generate_image, and send one to the user with it ("attach"). When you name a file, write its ref, e.g. [file:12], so the user can open it. ## Keep the work moving Be proactive. Follow where the conversation is heading and, once you've answered, offer the next step: one concrete suggestion or a short question about what to do next, e.g. "Want me to draft the reply as well?" or "Should I check the other two accounts the same way?". Base it on what the user is trying to get done, not on generic options. Offer one step, not a menu. Skip it when you've already asked the user something in this reply, when you've just started a task (its result will suggest what's next), or when the user is clearly wrapping up.`; +/** + * Added to every run, chat replies and tasks alike: where its files live, + * that they persist, and how to search them. + */ +export const BotFilesNote = `# Your files + +You have a folder of your own, kept with the app (not on a computer), and it stays between runs and conversations. + +- list_files shows what's in it; read_file opens one (by its [file:ID] ref or path); write_file saves a text file, make_pdf a PDF, generate_image an image. Move and delete with move_file and delete_file when something is out of date, so the folder stays useful. +- Text files are chunked and searchable as soon as they're saved. search_files finds the passages that answer a question, by meaning when an embedding model is set and by words otherwise. Search your files before saying you don't know something they may hold, and when the user asks about a document they gave you. +- Uploaded files land in the same folder, so what the user sent you earlier is searchable later too. +- When you name a file, write its [file:ID] ref: the user sees a link to it. +- A file you only described is not saved. Save it first, then say it's saved.`; + /** * Added to every run, chat replies and tasks alike: check before you * create, and check after you act. diff --git a/src/dict/bot-file-dict.ts b/src/dict/bot-file-dict.ts new file mode 100644 index 0000000..db72baa --- /dev/null +++ b/src/dict/bot-file-dict.ts @@ -0,0 +1,102 @@ +/** + * Where a bot file lives: a bot's own directory (`DATA_DIR/bots/`) or + * a workspace's shared one (`DATA_DIR/workspaces/`), which the bots + * working there and the coordinator reach. + */ +export const BotFileScopes = [ + { + title: "Bot", + value: "bot", + }, + { + title: "Workspace", + value: "workspace", + }, +] as const; + +export type BotFileScope = (typeof BotFileScopes)[number]["value"]; + +/** + * What a file is for. Only the display and the icon use it; the bot picks + * it when it writes one. + */ +export const BotFilePurposes = [ + { + title: "Upload", + value: "upload", + }, + { + title: "Writing", + value: "writing", + }, + { + title: "Image", + value: "image", + }, + { + title: "PDF", + value: "pdf", + }, + { + title: "Artifact", + value: "artifact", + }, +] as const; + +export type BotFilePurpose = (typeof BotFilePurposes)[number]["value"]; + +/** + * How a file got into the bot's storage. + */ +export const BotFileSources = [ + { + title: "Chat upload", + value: "chat-upload", + }, + { + title: "Bot write", + value: "bot-write", + }, + { + title: "Tool", + value: "tool", + }, +] as const; + +export type BotFileSource = (typeof BotFileSources)[number]["value"]; + +/** + * The `app_settings` key holding the per-owner storage quota, in MB. + */ +export const BotFileQuotaSettingKey = "bot_files_quota_mb"; + +/** + * The top folder chat uploads land in, under the conversation's scope. + */ +export const BotUploadFolder = "uploads"; + +/** + * Extensions a bot's text extraction reads without a model: documents + * that are already text once unpacked. + */ +export const BotTextFileExtensions = [ + "txt", + "md", + "markdown", + "csv", + "json", + "html", + "htm", + "xml", + "yml", + "yaml", + "log", + "tsv", + "pdf", + "docx", +] as const; + +/** + * What a file's extension says it is, for the storage row and the icon. + */ +export const BotImageFileExtensions = ["png", "jpg", "jpeg", "webp", "gif", "bmp", "svg"]; diff --git a/src/dict/harness-dict.ts b/src/dict/harness-dict.ts index 9e25ca5..0b55fc1 100644 --- a/src/dict/harness-dict.ts +++ b/src/dict/harness-dict.ts @@ -2,8 +2,9 @@ * The harnesses: what a model is used for. Each harness has one model * setting (`harness_settings`). Text is the brain and can't be turned off; * the others are helpers it calls on. `automatic` ones pick a free model - * themselves when none is picked; the rest (transcription, speech) are - * not available until one is, since free picks there rarely work. + * themselves when none is picked; the rest (transcription, speech, + * embeddings) are not available until one is, since free picks there + * rarely work. */ export const Harnesses = [ { @@ -41,6 +42,13 @@ export const Harnesses = [ description: `Makes images when a bot is asked for one.`, required: false, }, + { + title: "Embeddings", + value: "embedding", + automatic: false, + description: `Turns the files bots save into vectors, so they can search them. Without one, searching falls back to plain text matching.`, + required: false, + }, ] as const; export type HarnessKind = (typeof Harnesses)[number]["value"]; diff --git a/src/dict/media-dict.ts b/src/dict/media-dict.ts index a51dac2..88380f0 100644 --- a/src/dict/media-dict.ts +++ b/src/dict/media-dict.ts @@ -42,4 +42,8 @@ export const MediaParadigms = [ title: "Workspace Image", value: "workspace-image", }, + { + title: "Bot File", + value: "bot-file", + }, ] as const; diff --git a/src/file-ingest/enqueue-file-ingest.ts b/src/file-ingest/enqueue-file-ingest.ts new file mode 100644 index 0000000..6369c25 --- /dev/null +++ b/src/file-ingest/enqueue-file-ingest.ts @@ -0,0 +1,37 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE } from "@/db/types/db"; +import wakeFileIngest from "./wake-file-ingest"; + +type Params = { + file_id: number | string; +}; + +/** + * Queues a file to be chunked and embedded, unless it's already waiting. + * Never fails the caller: a file that can't be queued is picked up by the + * `ingest-pending-files` cron instead. + */ +export default async function enqueueFileIngest({ file_id }: Params) { + try { + const open = await BunSQLite.sql({ + sql: `SELECT id FROM file_ingest_queue WHERE file_id = ? AND status IN ('pending', 'processing') LIMIT 1`, + values: [file_id], + }); + + if (open.payload?.[0]?.id) { + wakeFileIngest(); + return; + } + + await BunSQLite.queue.add({ + table: "file_ingest_queue", + data: [{ file_id: Number(file_id) }], + }); + + wakeFileIngest(); + } catch { + /** + * The cron sweeps it up. + */ + } +} diff --git a/src/file-ingest/index.ts b/src/file-ingest/index.ts new file mode 100644 index 0000000..6967d2e --- /dev/null +++ b/src/file-ingest/index.ts @@ -0,0 +1,82 @@ +import bunext from "@moduletrace/bunext"; +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE } from "@/db/types/db"; +import { AppData } from "@/src/data/app-data"; +import { logError } from "@/src/functions/backend/logger"; +import processIngestJob from "./process-ingest-job"; +import { setFileIngestWaker } from "./wake-file-ingest"; + +const worker = AppData.FileIngestWorker; +let running = 0; +let polling = false; +let timer: ReturnType | undefined; + +/** + * Starts the file ingestion worker: polls every `AppData.FileIngestPollMs` + * for `file_ingest_queue` jobs and runs up to + * `AppData.FileIngestConcurrency` at once (chunking and embedding a bot's + * file). Gated in production to the `MAIN_SERVER` container, like cron and + * the bots queue. + */ +export default async function fileIngest() { + if (process.env.NODE_ENV == "production" && !process.env.MAIN_SERVER) { + return; + } + + bunext.bunextLog.watch(`Running File Ingest Queue ...`); + + /** + * A restart's cut-off jobs go back to waiting (their lock timeout + * would, but this is immediate). + */ + await BunSQLite.queue.release_stale({ table: "file_ingest_queue" }).catch((error) => + logError({ title: "File ingest couldn't release stale jobs", error }).catch(() => {}), + ); + + setFileIngestWaker({ waker: wake }); + poll(); +} + +function wake() { + if (polling) return; + + clearTimeout(timer); + poll(); +} + +async function poll() { + polling = true; + + try { + const free = AppData.FileIngestConcurrency - running; + + if (free > 0) { + const { payload } = await BunSQLite.queue.claim({ + table: "file_ingest_queue", + worker, + limit: free, + }); + + for (const job of payload || []) { + running++; + + processIngestJob({ job }) + .catch((error) => + logError({ + title: `File ingest job #${job.id} crashed`, + error, + }).catch(() => {}), + ) + .finally(() => { + running--; + }); + } + } + } catch (error) { + await logError({ title: "File ingest poll failed", error }).catch(() => {}); + } finally { + polling = false; + clearTimeout(timer); + timer = setTimeout(poll, AppData.FileIngestPollMs); + } +} diff --git a/src/file-ingest/process-ingest-job.ts b/src/file-ingest/process-ingest-job.ts new file mode 100644 index 0000000..51aabf8 --- /dev/null +++ b/src/file-ingest/process-ingest-job.ts @@ -0,0 +1,41 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE } from "@/db/types/db"; +import ingestBotFile from "@/src/functions/backend/db/bot-files/ingest-bot-file"; +import { logError } from "@/src/functions/backend/logger"; + +type Params = { + job: BUN_SQLITE_COBBLE_FILE_INGEST_QUEUE; +}; + +/** + * Ingests one queued file and completes (or fails) its job. A file that + * was deleted meanwhile is simply done. + */ +export default async function processIngestJob({ job }: Params) { + if (!job.id || !job.file_id) return; + + try { + const result = await ingestBotFile({ file_id: job.file_id }); + + await BunSQLite.queue.complete({ table: "file_ingest_queue", id: job.id }); + + if (result.status == "ingested") { + BunSQLite.sql({ + sql: `UPDATE bot_files SET ingest_error = NULL WHERE id = ?`, + values: [job.file_id], + }).catch(() => {}); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + + await BunSQLite.queue + .fail({ table: "file_ingest_queue", id: job.id, error: message.slice(0, 500) }) + .catch(() => {}); + + await logError({ + title: `Bot file #${job.file_id} couldn't be ingested`, + error, + metadata: { file_id: job.file_id }, + }).catch(() => {}); + } +} diff --git a/src/file-ingest/wake-file-ingest.ts b/src/file-ingest/wake-file-ingest.ts new file mode 100644 index 0000000..03cc6cd --- /dev/null +++ b/src/file-ingest/wake-file-ingest.ts @@ -0,0 +1,17 @@ +let wake: (() => void) | undefined; + +/** + * Lets the running ingest worker register how to poll at once, kept apart + * from the handler so an upload can wake it without bundling it. + */ +export function setFileIngestWaker({ waker }: { waker: () => void }) { + wake = waker; +} + +/** + * Polls now instead of at the next tick, so a fresh file is readable + * moments after it lands. Does nothing where the worker doesn't run. + */ +export default function wakeFileIngest() { + wake?.(); +} diff --git a/src/functions/backend/db/app-settings/grab-app-setting.ts b/src/functions/backend/db/app-settings/grab-app-setting.ts new file mode 100644 index 0000000..e8c0b60 --- /dev/null +++ b/src/functions/backend/db/app-settings/grab-app-setting.ts @@ -0,0 +1,12 @@ +import grabAppSettings from "./grab-app-settings"; + +type Params = { + key: string; +}; + +/** + * One install-wide setting, or undefined when it was never saved. + */ +export default async function grabAppSetting({ key }: Params) { + return (await grabAppSettings()).get(key); +} diff --git a/src/functions/backend/db/app-settings/grab-app-settings.ts b/src/functions/backend/db/app-settings/grab-app-settings.ts new file mode 100644 index 0000000..8a66463 --- /dev/null +++ b/src/functions/backend/db/app-settings/grab-app-settings.ts @@ -0,0 +1,34 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_APP_SETTINGS } from "@/db/types/db"; + +let cache: Map | null = null; + +/** + * Every install-wide setting, cached per process (like `grabAllEnvs`). + * `saveAppSetting` and `deleteAppSetting` clear it. + */ +export default async function grabAppSettings() { + if (cache) return cache; + + const res = await BunSQLite.sql({ + sql: `SELECT key, value FROM app_settings`, + values: [], + }); + + const map = new Map(); + + for (const row of res.payload || []) { + if (row.key) map.set(row.key, row.value || ""); + } + + cache = map; + + return map; +} + +/** + * Drops the cache, so the next read sees the database. + */ +export function clearAppSettingsCache() { + cache = null; +} diff --git a/src/functions/backend/db/app-settings/grab-storage-quota-overview.ts b/src/functions/backend/db/app-settings/grab-storage-quota-overview.ts new file mode 100644 index 0000000..23b8ecb --- /dev/null +++ b/src/functions/backend/db/app-settings/grab-storage-quota-overview.ts @@ -0,0 +1,25 @@ +import { BotFileQuotaSettingKey } from "@/src/dict/bot-file-dict"; +import { grabBotFileQuotaMb } from "../bot-files/grab-bot-file-quota-bytes"; +import { grabStorageUsageByOwner, grabTotalStorageUsage } from "../bot-files/grab-storage-usage"; + +/** + * The General settings' bot file storage: the quota each folder gets, and + * what is used right now. + */ +export default async function grabStorageQuotaOverview() { + const [quota_mb, total, by_owner] = await Promise.all([ + grabBotFileQuotaMb(), + grabTotalStorageUsage(), + grabStorageUsageByOwner(), + ]); + + return { + setting_key: BotFileQuotaSettingKey, + quota_mb, + total_bytes: total.bytes, + total_files: total.files, + folders: by_owner.length, + }; +} + +export type StorageQuotaOverview = Awaited>; diff --git a/src/functions/backend/db/app-settings/save-app-setting.ts b/src/functions/backend/db/app-settings/save-app-setting.ts new file mode 100644 index 0000000..b818ff3 --- /dev/null +++ b/src/functions/backend/db/app-settings/save-app-setting.ts @@ -0,0 +1,47 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_APP_SETTINGS, BunSQLiteTables } from "@/db/types/db"; +import { clearAppSettingsCache } from "./grab-app-settings"; + +type Params = { + key: string; + /** + * Null or empty removes it, so the code's default applies again. + */ + value?: string | null; +}; + +/** + * Saves (or removes) one install-wide setting and clears the cache, so the + * next read sees it. + */ +export default async function saveAppSetting({ key, value }: Params) { + const existing = await BunSQLite.sql({ + sql: `SELECT id FROM app_settings WHERE key = ? LIMIT 1`, + values: [key], + }).then((res) => res.payload?.[0]); + + if (!value) { + await BunSQLite.sql({ + sql: `DELETE FROM app_settings WHERE key = ?`, + values: [key], + }); + clearAppSettingsCache(); + return null; + } + + if (existing?.id) { + await BunSQLite.sql({ + sql: `UPDATE app_settings SET value = ?, updated_at = ? WHERE id = ?`, + values: [value, Date.now(), existing.id], + }); + } else { + await BunSQLite.insert({ + table: "app_settings", + data: [{ key, value }], + }); + } + + clearAppSettingsCache(); + + return value; +} diff --git a/src/functions/backend/db/bot-files/assert-storage-room.ts b/src/functions/backend/db/bot-files/assert-storage-room.ts new file mode 100644 index 0000000..d660a7c --- /dev/null +++ b/src/functions/backend/db/bot-files/assert-storage-room.ts @@ -0,0 +1,47 @@ +import type { BotFileScope } from "@/src/dict/bot-file-dict"; +import AppError from "@/src/utils/app-error"; +import grabBotFileQuotaBytes from "./grab-bot-file-quota-bytes"; +import grabStorageUsage from "./grab-storage-usage"; + +type Params = { + scope: BotFileScope; + owner_id: number | string; + /** + * How many bytes are about to be added. A rewrite of an existing file + * passes only the growth. + */ + incoming_bytes: number; +}; + +/** + * Refuses a write that would push a directory past its quota, naming the + * limit and the current usage so the bot can clear space first. + */ +export default async function assertStorageRoom({ + scope, + owner_id, + incoming_bytes, +}: Params) { + const quota = await grabBotFileQuotaBytes(); + + if (incoming_bytes <= 0) return; + + const usage = await grabStorageUsage({ scope, owner_id }); + + if (usage.bytes + incoming_bytes <= quota) return; + + throw new AppError( + `No room left: ${formatMb({ bytes: usage.bytes })} of ${formatMb({ bytes: quota })} used in ${scope == "workspace" ? "this workspace" : "your files"}. Delete or move something first.`, + ); +} + +/** + * A size in MB, for messages: "12.4 MB". + */ +export function formatMb({ bytes }: { bytes: number }) { + const mb = bytes / (1024 * 1024); + + if (mb >= 10) return `${Math.round(mb)} MB`; + + return `${mb.toFixed(1)} MB`; +} diff --git a/src/functions/backend/db/bot-files/chunk-text.test.ts b/src/functions/backend/db/bot-files/chunk-text.test.ts new file mode 100644 index 0000000..b75576d --- /dev/null +++ b/src/functions/backend/db/bot-files/chunk-text.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import chunkText from "./chunk-text"; + +describe("chunkText", () => { + test("returns nothing for empty text", () => { + expect(chunkText({ text: " \n\n " })).toEqual([]); + }); + + test("keeps a short document in one chunk", () => { + expect(chunkText({ text: "One paragraph.\n\nAnother one." })).toEqual([ + "One paragraph.\n\nAnother one.", + ]); + }); + + test("splits at paragraph boundaries, never mid-word", () => { + const paragraph = "word ".repeat(40).trim(); + const chunks = chunkText({ text: [1, 2, 3, 4, 5].map(() => paragraph).join("\n\n"), target: 300, overlap: 0 }); + + expect(chunks.length).toBeGreaterThan(1); + + for (const chunk of chunks) { + expect(chunk.split(/\s+/).every((word) => word == "word")).toBe(true); + } + }); + + test("carries the heading into the chunk it starts", () => { + const text = "# Report\n\nThe numbers are up.\n\n## Details\n\nEverything is fine."; + const chunks = chunkText({ text, target: 120, overlap: 0 }); + + expect(chunks[0]).toContain("# Report"); + }); + + test("overlaps consecutive chunks of a long paragraph", () => { + const text = Array.from({ length: 200 }, (_, i) => `word${i}`).join(" "); + const chunks = chunkText({ text, target: 200, overlap: 40 }); + + expect(chunks.length).toBeGreaterThan(1); + + const first_tail = chunks[0]!.split(" ").slice(-2).join(" "); + + expect(chunks[1]!.startsWith(first_tail.split(" ")[0]!) || chunks[1]!.includes(first_tail)).toBe(true); + }); + + test("keeps a word longer than the target whole", () => { + const chunks = chunkText({ text: "a".repeat(400), target: 100, overlap: 0 }); + + expect(chunks.join("")).toContain("a".repeat(400).slice(0, 10)); + expect(chunks.length).toBeGreaterThan(0); + }); + + test("collapses runs of blank lines", () => { + expect(chunkText({ text: "One.\n\n\n\n\nTwo." })).toEqual(["One.\n\nTwo."]); + }); +}); diff --git a/src/functions/backend/db/bot-files/chunk-text.ts b/src/functions/backend/db/bot-files/chunk-text.ts new file mode 100644 index 0000000..fe68086 --- /dev/null +++ b/src/functions/backend/db/bot-files/chunk-text.ts @@ -0,0 +1,148 @@ +import { AppData } from "@/src/data/app-data"; + +type Params = { + text: string; + /** + * How much text one chunk holds, in characters. + */ + target?: number; + /** + * How much of the previous chunk the next one repeats, so a sentence + * cut in two is still whole somewhere. + */ + overlap?: number; +}; + +/** + * Splits a document's text into chunks for embedding: paragraphs packed up + * to the target size, a heading carried into the chunk that follows it, + * and long paragraphs cut at word boundaries with an overlap. Never cuts + * mid-word (a word longer than the target is the one exception, since + * there's nowhere else to cut). + */ +export default function chunkText({ + text, + target = AppData.ChunkTargetChars, + overlap = AppData.ChunkOverlapChars, +}: Params): string[] { + const normalized = text.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); + + if (!normalized) return []; + + const blocks = normalized.split(/\n{2,}/).map((block) => block.trim()).filter(Boolean); + const chunks: string[] = []; + /** + * The current chunk's blocks, then (after a flush) only the overlap + * carried over: `fresh` says whether anything new joined it, so a + * carry on its own is never pushed as a chunk. + */ + let current: string[] = []; + let current_length = 0; + let fresh = false; + let heading = ""; + + const flush = () => { + const body = current.join("\n\n").trim(); + + if (body && fresh) { + chunks.push(heading && !body.startsWith(heading) ? `${heading}\n\n${body}` : body); + } + + const carry = overlap > 0 ? tail({ text: body, size: overlap }) : ""; + + current = carry ? [carry] : []; + current_length = carry.length; + fresh = false; + }; + + for (const block of blocks) { + const is_heading = /^#{1,6}\s/.test(block); + + if (is_heading) { + /** + * A heading starts a new chunk with it, so a chunk says what it + * is about. + */ + if (current_length > 0) flush(); + + heading = block; + continue; + } + + if (block.length >= target) { + if (current_length > 0) flush(); + + for (const piece of splitLongBlock({ block, target, overlap })) { + chunks.push(heading ? `${heading}\n\n${piece}` : piece); + } + + current = []; + current_length = 0; + fresh = false; + continue; + } + + if (current_length + block.length + 2 > target) flush(); + + current.push(block); + current_length += block.length + 2; + fresh = true; + } + + if (current_length > 0) flush(); + + return chunks.filter((chunk) => chunk.length > 0); +} + +/** + * Cuts a block that is longer than one chunk at word boundaries, keeping + * the overlap between pieces. + */ +function splitLongBlock({ + block, + target, + overlap, +}: { + block: string; + target: number; + overlap: number; +}) { + const words = block.split(/\s+/); + const pieces: string[] = []; + let current: string[] = []; + let length = 0; + + for (const word of words) { + if (length + word.length + 1 > target && current.length) { + pieces.push(current.join(" ")); + const carry = tail({ text: current.join(" "), size: overlap }); + + current = carry ? carry.split(" ") : []; + length = carry.length; + } + + if (!word) continue; + + current.push(word); + length += word.length + 1; + } + + if (current.length) pieces.push(current.join(" ")); + + return pieces; +} + +/** + * The last `size` characters of a text, starting at a word boundary so the + * overlap doesn't begin mid-word. + */ +function tail({ text, size }: { text: string; size: number }) { + if (!text || size <= 0) return ""; + + if (text.length <= size) return text; + + const cut = text.slice(text.length - size); + const boundary = cut.search(/\s/); + + return boundary == -1 ? cut : cut.slice(boundary).trim(); +} diff --git a/src/functions/backend/db/bot-files/copy-bot-file.ts b/src/functions/backend/db/bot-files/copy-bot-file.ts new file mode 100644 index 0000000..4bf6b0e --- /dev/null +++ b/src/functions/backend/db/bot-files/copy-bot-file.ts @@ -0,0 +1,64 @@ +import AppError from "@/src/utils/app-error"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import type { StorageScope } from "@/src/utils/storage-access"; +import { canReachScope } from "@/src/utils/storage-access"; +import grabBotFile from "./grab-bot-file"; +import recordBotFile from "./record-bot-file"; + +type Params = { + scopes: StorageScope[]; + file_id: number | string; + /** + * The bot making the copy (it owns the new row). + */ + bot_id: number | string; + /** + * Where the copy goes: one of the run's directories. + */ + scope: StorageScope; + /** + * Its path there. Defaults to the source's path. + */ + path?: string | null; + user_id?: number | string; +}; + +/** + * Copies a file into another of the run's directories (the coordinator + * publishing a worker's artifact into its own, or into a workspace). The + * bytes are copied, so deleting one leaves the other. + */ +export default async function copyBotFile({ + scopes, + file_id, + bot_id, + scope, + path, + user_id, +}: Params) { + if (!canReachScope({ scopes, scope: scope.scope, owner_id: scope.owner_id })) { + throw new AppError(`You can't write to that folder.`); + } + + const file = await grabBotFile({ scopes, file_id }); + + if (!file?.id || !file.media?.media_write_path) { + throw new AppError(`That file isn't in your files.`); + } + + const { DATA_DIR } = grabDirNames(); + const source = await Bun.file(`${DATA_DIR}/${file.media.media_write_path}`).arrayBuffer(); + + return await recordBotFile({ + bot_id, + scope: scope.scope, + owner_id: scope.owner_id, + path: path || file.path || file.name || `copy-${file.id}`, + data: new Uint8Array(source), + purpose: (file.purpose as any) || "writing", + source: "tool", + media_type: (file.media_type as any) || undefined, + mime_type: file.mime_type || undefined, + user_id, + }); +} diff --git a/src/functions/backend/db/bot-files/delete-bot-file.ts b/src/functions/backend/db/bot-files/delete-bot-file.ts new file mode 100644 index 0000000..e58c32c --- /dev/null +++ b/src/functions/backend/db/bot-files/delete-bot-file.ts @@ -0,0 +1,64 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db"; +import { rmSync } from "node:fs"; +import path from "node:path"; +import AppError from "@/src/utils/app-error"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import type { StorageScope } from "@/src/utils/storage-access"; +import grabBotFile from "./grab-bot-file"; +import grabStorageDir from "./grab-storage-dir"; + +type Params = { + scopes: StorageScope[]; + file_id: number | string; +}; + +/** + * Deletes a bot file: the catalog row (its chunks go with it), its bytes + * and its `media` row. Refuses anything outside the run's directories. + */ +export default async function deleteBotFile({ scopes, file_id }: Params) { + const file = await grabBotFile({ scopes, file_id }); + + if (!file?.id) { + throw new AppError(`That file isn't in your files.`); + } + + const { DATA_DIR } = grabDirNames(); + const root = grabStorageDir({ scope: file.scope as any, owner_id: file.owner_id! }); + const absolute = path.join(root, file.path || ""); + + if (absolute.startsWith(path.resolve(root))) { + rmSync(absolute, { force: true }); + } + + await BunSQLite.sql({ sql: `DELETE FROM bot_file_vectors WHERE file_id = ?`, values: [file.id] }); + await BunSQLite.sql({ sql: `DELETE FROM bot_files WHERE id = ?`, values: [file.id] }); + /** + * Its pending ingestion would only find nothing left. + */ + await BunSQLite.sql({ + sql: `DELETE FROM file_ingest_queue WHERE file_id = ? AND status IN ('pending', 'processing')`, + values: [file.id], + }).catch(() => {}); + + if (file.media_id) { + const media = await BunSQLite.sql<{ media_write_path?: string; media_thumbnail_write_path?: string }>({ + sql: `SELECT media_write_path, media_thumbnail_write_path FROM media WHERE id = ? LIMIT 1`, + values: [file.media_id], + }).then((res) => res.payload?.[0]); + + if (media?.media_write_path) { + rmSync(path.join(DATA_DIR, media.media_write_path), { force: true }); + } + + if (media?.media_thumbnail_write_path) { + rmSync(path.join(DATA_DIR, media.media_thumbnail_write_path), { force: true }); + } + + await BunSQLite.sql({ sql: `DELETE FROM media_paradigms WHERE media_id = ?`, values: [file.media_id] }); + await BunSQLite.sql({ sql: `DELETE FROM media WHERE id = ?`, values: [file.media_id] }); + } + + return file as BUN_SQLITE_COBBLE_BOT_FILES; +} diff --git a/src/functions/backend/db/bot-files/extract-bot-file-text.ts b/src/functions/backend/db/bot-files/extract-bot-file-text.ts new file mode 100644 index 0000000..9d3f5b2 --- /dev/null +++ b/src/functions/backend/db/bot-files/extract-bot-file-text.ts @@ -0,0 +1,91 @@ +import { AppData } from "@/src/data/app-data"; +import { + BotImageFileExtensions, + BotTextFileExtensions, +} from "@/src/dict/bot-file-dict"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import fileToText from "../users/media/file-to-text"; +import { extensionOf } from "./media-type-for-file"; + +type Params = { + /** + * The `media` row's write path (relative to `DATA_DIR`). + */ + write_path?: string | null; + /** + * Its text, when a previous upload already extracted it. + */ + text_content?: string | null; + name?: string | null; + mime_type?: string | null; + /** + * Read images and audio too, through the media harness. Off by + * default: only a bot asking for it pays for that. + */ + read_media?: boolean; +}; + +/** + * A file's text, for chunking: documents unpacked locally (PDF, DOCX, + * plain text, markdown, CSV, JSON, HTML), images and audio only when + * `read_media` is set. Null when the file has no text to read. + */ +export default async function extractBotFileText({ + write_path, + text_content, + name, + mime_type, + read_media, +}: Params): Promise { + if (text_content?.trim()) return text_content.trim(); + + if (!write_path) return null; + + const ext = extensionOf({ name, mime_type }); + + if (!ext) return null; + + const { DATA_DIR } = grabDirNames(); + const file_path = `${DATA_DIR}/${write_path}`; + + if ((BotTextFileExtensions as readonly string[]).includes(ext)) { + /** + * Markdown, CSV and the other plain text types read as they are: + * no unpacking needed, and no size limit spent on it. + */ + if (!["pdf", "docx"].includes(ext)) { + const file = Bun.file(file_path); + + if (file.size > AppData.IngestMaxBytes) return null; + + return await file.text(); + } + + const file = Bun.file(file_path); + + if (file.size > AppData.IngestMaxBytes) return null; + + return (await fileToText({ file_path })) || null; + } + + if (read_media) { + const file = Bun.file(file_path); + + if (file.size > AppData.IngestMaxBytes) return null; + + const { default: readMedia } = await import("@/src/harness/media/read-media"); + const media_type = (BotImageFileExtensions as readonly string[]).includes(ext) + ? `image/${ext == "jpg" ? "jpeg" : ext}` + : mime_type || "audio/mpeg"; + + return await readMedia({ + data: new Uint8Array(await file.arrayBuffer()), + media_type, + name: name || undefined, + }) + .then((text) => text?.trim() || null) + .catch(() => null); + } + + return null; +} diff --git a/src/functions/backend/db/bot-files/grab-bot-file-media.ts b/src/functions/backend/db/bot-files/grab-bot-file-media.ts new file mode 100644 index 0000000..6f2dfed --- /dev/null +++ b/src/functions/backend/db/bot-files/grab-bot-file-media.ts @@ -0,0 +1,51 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { + BUN_SQLITE_COBBLE_BOT_FILES, + BUN_SQLITE_COBBLE_MEDIA, +} from "@/db/types/db"; +import grabCrewRole from "../access/grab-crew-role"; +import grabUserBot from "../bots/grab-user-bot"; +import grabUserWorkspace from "../workspaces/grab-user-workspace"; + +type Params = { + media_id: number | string; + user_id: number | string; +}; + +/** + * A bot file's bytes (its private `media` row) for a user who reaches the + * directory it's in: the bot's own chat (or any worker, for super admins + * and admins; the coordinator only for the super admin), or the workspace + * it's shared in. Files belong to the install, like the crew, so whoever + * sees the bot sees what it saved. + */ +export default async function grabBotFileMedia({ + media_id, + user_id, +}: Params): Promise { + const res = await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE media_id = ? LIMIT 1`, + values: [media_id], + }); + + const file = res.payload?.[0]; + + if (!file?.id) return null; + + const role = await grabCrewRole({ user_id }); + const reaches = + file.scope == "workspace" + ? Boolean(await grabUserWorkspace({ workspace_id: file.owner_id!, user_id })) + : role == "super_admin" + ? true + : Boolean(await grabUserBot({ bot_id: file.owner_id!, user_id })); + + if (!reaches) return null; + + const media = await BunSQLite.sql({ + sql: `SELECT * FROM media WHERE id = ? LIMIT 1`, + values: [media_id], + }); + + return media.payload?.[0] || null; +} diff --git a/src/functions/backend/db/bot-files/grab-bot-file-quota-bytes.ts b/src/functions/backend/db/bot-files/grab-bot-file-quota-bytes.ts new file mode 100644 index 0000000..ed8a0a7 --- /dev/null +++ b/src/functions/backend/db/bot-files/grab-bot-file-quota-bytes.ts @@ -0,0 +1,24 @@ +import { AppData } from "@/src/data/app-data"; +import { BotFileQuotaSettingKey } from "@/src/dict/bot-file-dict"; +import grabAppSetting from "../app-settings/grab-app-setting"; + +/** + * The storage limit one directory may hold, from the General settings + * (`bot_files_quota_mb`), else `AppData.BotFileQuotaMb`. A workspace's + * shared directory gets the same limit as a bot's own. + */ +export default async function grabBotFileQuotaBytes() { + const saved = Number(await grabAppSetting({ key: BotFileQuotaSettingKey })); + const mb = Number.isFinite(saved) && saved > 0 ? saved : AppData.BotFileQuotaMb; + + return Math.round(mb * 1024 * 1024); +} + +/** + * The same limit in MB, for the settings page. + */ +export async function grabBotFileQuotaMb() { + const saved = Number(await grabAppSetting({ key: BotFileQuotaSettingKey })); + + return Number.isFinite(saved) && saved > 0 ? saved : AppData.BotFileQuotaMb; +} diff --git a/src/functions/backend/db/bot-files/grab-bot-file.ts b/src/functions/backend/db/bot-files/grab-bot-file.ts new file mode 100644 index 0000000..d73e8c0 --- /dev/null +++ b/src/functions/backend/db/bot-files/grab-bot-file.ts @@ -0,0 +1,68 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { + BUN_SQLITE_COBBLE_BOT_FILES, + BUN_SQLITE_COBBLE_MEDIA, +} from "@/db/types/db"; +import { canReachScope, type StorageScope } from "@/src/utils/storage-access"; + +type Params = { + /** + * The directories the run may touch (`grabStorageScopes`). + */ + scopes: StorageScope[]; + /** + * The catalog row's id. + */ + file_id?: number | string | null; + /** + * Or its path, with the directory it's in. + */ + scope?: string | null; + owner_id?: number | string | null; + path?: string | null; +}; + +export type BotFileWithMedia = BUN_SQLITE_COBBLE_BOT_FILES & { + media?: BUN_SQLITE_COBBLE_MEDIA | null; +}; + +/** + * One bot file, only if it's in one of the run's directories, with its + * `media` row when that still exists. + */ +export default async function grabBotFile({ + scopes, + file_id, + scope, + owner_id, + path, +}: Params): Promise { + if (!file_id && !(scope && owner_id != null && path)) return null; + + const res = file_id + ? await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE id = ? LIMIT 1`, + values: [file_id], + }) + : await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE scope = ? AND owner_id = ? AND path = ? LIMIT 1`, + values: [String(scope), Number(owner_id), String(path)], + }); + + const file = res.payload?.[0]; + + if (!file?.id) return null; + + if (!canReachScope({ scopes, scope: file.scope as any, owner_id: file.owner_id! })) { + return null; + } + + const media = file.media_id + ? await BunSQLite.sql({ + sql: `SELECT * FROM media WHERE id = ? LIMIT 1`, + values: [file.media_id], + }).then((r) => r.payload?.[0] || null) + : null; + + return { ...file, media }; +} diff --git a/src/functions/backend/db/bot-files/grab-conversation-files.ts b/src/functions/backend/db/bot-files/grab-conversation-files.ts new file mode 100644 index 0000000..ca10fe8 --- /dev/null +++ b/src/functions/backend/db/bot-files/grab-conversation-files.ts @@ -0,0 +1,62 @@ +import { AppData } from "@/src/data/app-data"; +import type { BotFileScopesPayload } from "@/src/types"; +import grabMediaURL from "../../media/grab-media-url"; +import grabBotFileQuotaBytes, { grabBotFileQuotaMb } from "./grab-bot-file-quota-bytes"; +import grabStorageUsage from "./grab-storage-usage"; +import listBotFiles from "./list-bot-files"; + +type Params = { + bot_id: number; + /** + * The workspace the conversation is in: its shared folder, else the + * bot's own. + */ + workspace_id?: number | string | null; +}; + +/** + * A conversation's files, for its page props: the folder it works in + * (a workspace's shared one, else the bot's own), its latest + * `AppData.BotFilesPageSize` of them, and the usage against the quota. + * The full list (folders, search, older files) loads in the files modal. + */ +export default async function grabConversationFiles({ + bot_id, + workspace_id, +}: Params): Promise { + const scope = workspace_id + ? ({ scope: "workspace", owner_id: Number(workspace_id) } as const) + : ({ scope: "bot", owner_id: Number(bot_id) } as const); + + const [files, usage, quota_bytes, quota_mb] = await Promise.all([ + listBotFiles({ scopes: [scope], limit: AppData.BotFilesPageSize }), + grabStorageUsage(scope), + grabBotFileQuotaBytes(), + grabBotFileQuotaMb(), + ]); + + return { + files: files.map((file) => { + const { media_url } = grabMediaURL({ media: { id: file.media_id } as any }); + + return { + id: Number(file.id), + name: file.name || "", + path: file.path || "", + size: Number(file.size) || 0, + media_type: String(file.media_type || "document"), + purpose: String(file.purpose || "writing"), + scope: String(file.scope || "bot"), + owner_id: Number(file.owner_id) || 0, + ingested_at: file.ingested_at ? Number(file.ingested_at) : undefined, + ingest_error: file.ingest_error || undefined, + created_at: Number(file.created_at) || 0, + url: media_url || "", + }; + }), + folders: [], + usage, + quota_bytes, + quota_mb, + }; +} diff --git a/src/functions/backend/db/bot-files/grab-storage-dir.ts b/src/functions/backend/db/bot-files/grab-storage-dir.ts new file mode 100644 index 0000000..47311f0 --- /dev/null +++ b/src/functions/backend/db/bot-files/grab-storage-dir.ts @@ -0,0 +1,33 @@ +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import { AppData } from "@/src/data/app-data"; +import type { BotFileScope } from "@/src/dict/bot-file-dict"; +import grabDirNames from "@/src/utils/grab-dir-names"; + +type Params = { + scope: BotFileScope; + /** + * The bot id (scope bot) or workspace id (scope workspace). + */ + owner_id: number | string; + /** + * Create it when it's missing (the directories are made on demand, so + * an install from before this has no backfill to do). + */ + create?: boolean; +}; + +/** + * A storage directory on the data volume: `DATA_DIR/bots/` for a bot, + * `DATA_DIR/workspaces/` for a workspace. Files never live on a + * computer, so they survive one being rebuilt. + */ +export default function grabStorageDir({ scope, owner_id, create = true }: Params) { + const { DATA_DIR } = grabDirNames(); + const folder = scope == "workspace" ? AppData.WorkspaceDataDir : AppData.BotDataDir; + const dir = path.join(DATA_DIR, folder, String(owner_id)); + + if (create) mkdirSync(dir, { recursive: true }); + + return dir; +} diff --git a/src/functions/backend/db/bot-files/grab-storage-usage.ts b/src/functions/backend/db/bot-files/grab-storage-usage.ts new file mode 100644 index 0000000..f1c2db1 --- /dev/null +++ b/src/functions/backend/db/bot-files/grab-storage-usage.ts @@ -0,0 +1,61 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BotFileScope } from "@/src/dict/bot-file-dict"; + +type Params = { + scope: BotFileScope; + owner_id: number | string; +}; + +type UsageRow = { + total: number | null; + files: number | null; +}; + +/** + * How many bytes (and files) a directory holds. + */ +export default async function grabStorageUsage({ scope, owner_id }: Params) { + const res = await BunSQLite.sql({ + sql: `SELECT COALESCE(SUM(size), 0) AS total, COUNT(*) AS files FROM bot_files WHERE scope = ? AND owner_id = ?`, + values: [scope, owner_id], + }); + + const row = res.payload?.[0]; + + return { + bytes: Number(row?.total) || 0, + files: Number(row?.files) || 0, + }; +} + +/** + * Every directory's usage at once, for the settings page. + */ +export async function grabStorageUsageByOwner() { + const res = await BunSQLite.sql<{ scope: string; owner_id: number; total: number; files: number }>({ + sql: `SELECT scope, owner_id, SUM(size) AS total, COUNT(*) AS files FROM bot_files GROUP BY scope, owner_id`, + values: [], + }); + + return (res.payload || []).map((row) => ({ + scope: row.scope as BotFileScope, + owner_id: Number(row.owner_id), + bytes: Number(row.total) || 0, + files: Number(row.files) || 0, + })); +} + +/** + * Total bytes across every directory. + */ +export async function grabTotalStorageUsage() { + const res = await BunSQLite.sql<{ total: number | null; files: number | null }>({ + sql: `SELECT COALESCE(SUM(size), 0) AS total, COUNT(*) AS files FROM bot_files`, + values: [], + }); + + return { + bytes: Number(res.payload?.[0]?.total) || 0, + files: Number(res.payload?.[0]?.files) || 0, + }; +} diff --git a/src/functions/backend/db/bot-files/ingest-bot-file.ts b/src/functions/backend/db/bot-files/ingest-bot-file.ts new file mode 100644 index 0000000..ad5caa3 --- /dev/null +++ b/src/functions/backend/db/bot-files/ingest-bot-file.ts @@ -0,0 +1,248 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { + BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS, + BUN_SQLITE_COBBLE_BOT_FILES, + BunSQLiteTables, +} from "@/db/types/db"; +import { AppData } from "@/src/data/app-data"; +import embedText from "@/src/harness/embeddings/embed-text"; +import HarnessError from "@/src/harness/harness-error"; +import chunkText from "./chunk-text"; +import extractBotFileText from "./extract-bot-file-text"; + +type Params = { + file_id: number | string; + /** + * Read images and audio too (a bot asking for it). + */ + read_media?: boolean; + signal?: AbortSignal; +}; + +export type IngestResult = { + /** + * `ingested`, `empty` (no text to read), `skipped` (too large) or + * `unavailable` (no embedding model), which keeps the file's text but + * leaves it searchable by words only. + */ + status: "ingested" | "empty" | "skipped" | "unavailable"; + chunks: number; +}; + +/** + * Reads a bot file's text, splits it into chunks and embeds them into + * `bot_file_vectors`, replacing whatever was stored before. The chunks + * stay in `bot_file_chunks` even when there's no embedding model, so a + * search can still match words in them. + * + * Failures are recorded on the file (`ingest_error`) and rethrown, so the + * queue can retry. + */ +export default async function ingestBotFile({ + file_id, + read_media, + signal, +}: Params): Promise { + const file = await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE id = ? LIMIT 1`, + values: [file_id], + }).then((res) => res.payload?.[0]); + + if (!file?.id) return { status: "empty", chunks: 0 }; + + const media = file.media_id + ? await BunSQLite.sql<{ media_write_path?: string; text_content?: string }>({ + sql: `SELECT media_write_path, text_content FROM media WHERE id = ? LIMIT 1`, + values: [file.media_id], + }).then((res) => res.payload?.[0]) + : undefined; + + if (Number(file.size) > AppData.IngestMaxBytes && !media?.text_content) { + await BunSQLite.sql({ + sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`, + values: [`Too large to read (over ${AppData.IngestMaxBytes / (1024 * 1024)} MB).`, Date.now(), file.id], + }); + + return { status: "skipped", chunks: 0 }; + } + + try { + const text = await extractBotFileText({ + write_path: media?.media_write_path, + text_content: media?.text_content, + name: file.name, + mime_type: file.mime_type, + read_media, + }); + + await clearChunks({ file_id: file.id }); + + if (!text?.trim()) { + await markIngested({ file_id: file.id, chunks: 0 }); + + return { status: "empty", chunks: 0 }; + } + + const chunks = chunkText({ text }); + + if (!chunks.length) { + await markIngested({ file_id: file.id, chunks: 0 }); + + return { status: "empty", chunks: 0 }; + } + + const chunk_ids = await insertChunks({ file, chunks }); + let embedded = 0; + + try { + /** + * A big file's chunks go in batches, so one provider refusal + * doesn't throw away the whole file. + */ + for (let start = 0; start < chunks.length; start += AppData.EmbeddingBatchSize) { + const batch = chunks.slice(start, start + AppData.EmbeddingBatchSize); + const { vectors } = await embedText({ + input: batch, + bot_id: file.bot_id!, + workspace_id: file.scope == "workspace" ? file.owner_id : null, + signal, + }); + + for (let i = 0; i < vectors.length; i++) { + await insertVector({ + vector: vectors[i]!, + chunk_id: chunk_ids[start + i]!, + file_id: file.id, + scope: String(file.scope), + owner_id: Number(file.owner_id), + }); + + embedded++; + } + } + } catch (error) { + if (error instanceof HarnessError && error.code == "unavailable") { + /** + * No embedding model: the chunks still let a search match + * words, so the file isn't a failure. + */ + await markIngested({ + file_id: file.id, + chunks: chunks.length, + error: "No embedding model is set, so this file is searchable by words only.", + }); + + return { status: "unavailable", chunks: chunks.length }; + } + + throw error; + } + + await markIngested({ file_id: file.id, chunks: chunks.length }); + + return { status: "ingested", chunks: embedded }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + + await BunSQLite.sql({ + sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`, + values: [message.slice(0, 500), Date.now(), file.id], + }).catch(() => {}); + + throw error; + } +} + +async function clearChunks({ file_id }: { file_id: number }) { + await BunSQLite.sql({ sql: `DELETE FROM bot_file_vectors WHERE file_id = ?`, values: [file_id] }); + await BunSQLite.sql({ sql: `DELETE FROM bot_file_chunks WHERE file_id = ?`, values: [file_id] }); +} + +async function insertChunks({ + file, + chunks, +}: { + file: BUN_SQLITE_COBBLE_BOT_FILES; + chunks: string[]; +}) { + const ids: number[] = []; + + for (let index = 0; index < chunks.length; index++) { + const text = chunks[index]!; + + const res = await BunSQLite.insert< + BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS, + (typeof BunSQLiteTables)[number] + >({ + table: "bot_file_chunks", + data: [ + { + file_id: file.id, + scope: file.scope as any, + owner_id: Number(file.owner_id), + chunk_index: index, + text, + token_estimate: Math.ceil(text.length / 4), + }, + ], + }); + + const id = res.postInsertReturn?.insertId; + + if (id) ids.push(id); + } + + return ids; +} + +async function insertVector({ + vector, + chunk_id, + file_id, + scope, + owner_id, +}: { + vector: Float32Array; + chunk_id: number; + file_id: number; + scope: string; + owner_id: number; +}) { + /** + * vec0's metadata columns (its id and timestamps) are NOT NULL, so a + * raw insert has to fill them; `BunSQLite.insert` would add them but + * the vector needs `vec_f32`. + */ + const now = Date.now(); + + const res = await BunSQLite.sql({ + sql: `INSERT INTO bot_file_vectors (created_at, updated_at, embedding, chunk_id, file_id, scope, owner_id) VALUES (?, ?, vec_f32(?), ?, ?, ?, ?)`, + values: [now, now, vector as any, chunk_id, file_id, scope, owner_id], + }); + + if (!res.success) { + throw new Error(`Couldn't store an embedding: ${res.error || res.msg}`); + } +} + +async function markIngested({ + file_id, + chunks, + error, +}: { + file_id: number; + chunks: number; + error?: string; +}) { + const now = Date.now(); + + await BunSQLite.sql({ + sql: `UPDATE bot_files SET ingested_at = ?, ingest_error = ?, updated_at = ? WHERE id = ?`, + values: [now, error || null, now, file_id], + }); + + await BunSQLite.sql({ + sql: `UPDATE media SET is_rag_ingested = ?, updated_at = ? WHERE id = (SELECT media_id FROM bot_files WHERE id = ?)`, + values: [chunks > 0 ? 1 : 0, now, file_id], + }).catch(() => {}); +} diff --git a/src/functions/backend/db/bot-files/list-bot-files.ts b/src/functions/backend/db/bot-files/list-bot-files.ts new file mode 100644 index 0000000..d684b1d --- /dev/null +++ b/src/functions/backend/db/bot-files/list-bot-files.ts @@ -0,0 +1,105 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db"; +import { AppData } from "@/src/data/app-data"; +import type { StorageScope } from "@/src/utils/storage-access"; + +type Params = { + scopes: StorageScope[]; + /** + * Only this directory's files (a folder path), or, with `recursive`, + * everything under it. + */ + folder?: string | null; + recursive?: boolean; + /** + * Match the name or the path, case-insensitive. + */ + search?: string | null; + limit?: number; +}; + +/** + * The files in the run's directories, newest first: one folder's, or + * everything under it, optionally matching a name. + */ +export default async function listBotFiles({ + scopes, + folder, + recursive, + search, + limit = AppData.BotFileListMaxItems, +}: Params): Promise { + if (!scopes.length) return []; + + const target = (folder || "").trim().replace(/\\/g, "/").replace(/^\/+|\/+$/g, ""); + + const owner_sql = scopes.map(() => `(scope = ? AND owner_id = ?)`).join(" OR "); + const owner_values = scopes.flatMap((s) => [s.scope, s.owner_id]); + + const where = [owner_sql]; + const values: (string | number)[] = [...owner_values]; + + if (target) { + where.push( + recursive + ? `(folder = ? OR folder LIKE ?)` + : `folder = ?`, + ); + + values.push(target); + + if (recursive) values.push(`${target}/%`); + } else if (!recursive) { + where.push(`(folder = '' OR folder IS NULL)`); + } + + if (search?.trim()) { + where.push(`(name LIKE ? OR path LIKE ?)`); + values.push(`%${search.trim()}%`, `%${search.trim()}%`); + } + + const res = await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE ${where.join(" AND ")} ORDER BY updated_at DESC, id DESC LIMIT ?`, + values: [...values, limit], + }); + + return res.payload || []; +} + +/** + * The folders across the run's directories: a file's folder and every + * parent of it, so an empty folder still shows (folders are the paths + * that exist). + */ +export async function listBotFolders({ + scopes, + limit = AppData.BotFileListMaxItems, +}: { + scopes: StorageScope[]; + limit?: number; +}) { + const files = await listBotFiles({ scopes, recursive: true, limit: 5000 }); + const folders = new Map(); + + for (const file of files) { + const segments = (file.folder || "").split("/").filter(Boolean); + + for (let i = 1; i <= segments.length; i++) { + const path = segments.slice(0, i).join("/"); + const key = `${file.scope}:${file.owner_id}:${path}`; + + if (!folders.has(key)) { + folders.set(key, { + scope: String(file.scope), + owner_id: Number(file.owner_id), + path, + }); + } + } + + if (folders.size >= limit) break; + } + + return [...folders.values()].sort((a, b) => a.path.localeCompare(b.path)); +} + diff --git a/src/functions/backend/db/bot-files/media-type-for-file.ts b/src/functions/backend/db/bot-files/media-type-for-file.ts new file mode 100644 index 0000000..6a287c3 --- /dev/null +++ b/src/functions/backend/db/bot-files/media-type-for-file.ts @@ -0,0 +1,93 @@ +import type { MediaType } from "@/src/types"; +import { BotImageFileExtensions } from "@/src/dict/bot-file-dict"; + +type Params = { + /** + * A file name or an extension, e.g. `report.pdf` or `pdf`. + */ + name?: string | null; + mime_type?: string | null; +}; + +const AudioExtensions = ["mp3", "wav", "m4a", "ogg", "flac", "webm"]; +const VideoExtensions = ["mp4", "mov", "mkv", "avi"]; + +/** + * What a file's `media_type` is, from its extension (else its MIME type): + * an image, a document, audio or video. + */ +export default function mediaTypeForFile({ name, mime_type }: Params): MediaType { + const ext = extensionOf({ name, mime_type }); + + if (ext && BotImageFileExtensions.includes(ext)) return "image"; + if (ext && AudioExtensions.includes(ext)) return "audio"; + if (ext && VideoExtensions.includes(ext)) return "video"; + + return "document"; +} + +/** + * Its extension, lower case, without the dot. + */ +export function extensionOf({ name, mime_type }: Params) { + const from_name = name?.includes(".") ? name.split(".").pop() : undefined; + + if (from_name) return from_name.toLowerCase().replace(/[^a-z0-9]/g, ""); + + const from_mime = mime_type && !mime_type.includes("octet-stream") + ? mime_type.split("/").pop() + : undefined; + + return from_mime ? from_mime.toLowerCase().replace(/[^a-z0-9]/g, "") : undefined; +} + +/** + * A MIME type for the row, from the extension when there isn't one. + */ +export function mimeTypeForFile({ name, mime_type }: Params) { + if (mime_type && mime_type != "application/octet-stream") return mime_type; + + const ext = extensionOf({ name }); + + return (ext && MediaTypesByExtension[ext]) || "application/octet-stream"; +} + +/** + * The types worth knowing, so nothing has to guess (SNIFFING a file for + * this would mean reading it twice). + */ +const MediaTypesByExtension: Record = { + txt: "text/plain", + text: "text/plain", + log: "text/plain", + md: "text/markdown", + markdown: "text/markdown", + csv: "text/csv", + tsv: "text/tab-separated-values", + json: "application/json", + html: "text/html", + htm: "text/html", + xml: "application/xml", + yml: "application/yaml", + yaml: "application/yaml", + pdf: "application/pdf", + docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + doc: "application/msword", + xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + webp: "image/webp", + gif: "image/gif", + bmp: "image/bmp", + svg: "image/svg+xml", + mp3: "audio/mpeg", + wav: "audio/wav", + m4a: "audio/mp4", + ogg: "audio/ogg", + flac: "audio/flac", + mp4: "video/mp4", + mov: "video/quicktime", + webm: "video/webm", + zip: "application/zip", +}; diff --git a/src/functions/backend/db/bot-files/move-bot-file.ts b/src/functions/backend/db/bot-files/move-bot-file.ts new file mode 100644 index 0000000..c07f32b --- /dev/null +++ b/src/functions/backend/db/bot-files/move-bot-file.ts @@ -0,0 +1,84 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { existsSync, mkdirSync, renameSync, rmSync, copyFileSync } from "node:fs"; +import path from "node:path"; +import AppError from "@/src/utils/app-error"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import resolveBotPath from "@/src/utils/resolve-bot-path"; +import type { StorageScope } from "@/src/utils/storage-access"; +import { canReachScope } from "@/src/utils/storage-access"; +import grabBotFile from "./grab-bot-file"; +import grabStorageDir from "./grab-storage-dir"; +import { grabBotFileRow } from "./record-bot-file"; + +type Params = { + scopes: StorageScope[]; + file_id: number | string; + /** + * The new path, in the file's own directory. + */ + path: string; +}; + +/** + * Renames a file or moves it into another folder, inside its own + * directory. The bytes move on disk and the row follows; an existing file + * at the target is refused, never overwritten by a rename. + */ +export default async function moveBotFile({ scopes, file_id, path: target }: Params) { + const file = await grabBotFile({ scopes, file_id }); + + if (!file?.id) { + throw new AppError(`That file isn't in your files.`); + } + + if (!canReachScope({ scopes, scope: file.scope as any, owner_id: file.owner_id! })) { + throw new AppError(`That file isn't in your files.`); + } + + const root = grabStorageDir({ scope: file.scope as any, owner_id: file.owner_id! }); + const resolved = resolveBotPath({ root, path: target }); + + if (resolved.relative == file.path) return file; + + const taken = await grabBotFileRow({ + scope: file.scope as any, + owner_id: file.owner_id!, + path: resolved.relative, + }); + + if (taken?.id) { + throw new AppError(`There's already a file at ${resolved.relative}.`); + } + + const { DATA_DIR } = grabDirNames(); + const from = path.join(root, file.path || ""); + + mkdirSync(path.dirname(resolved.absolute), { recursive: true }); + + if (existsSync(from)) { + try { + renameSync(from, resolved.absolute); + } catch { + copyFileSync(from, resolved.absolute); + rmSync(from, { force: true }); + } + } + + await BunSQLite.sql({ + sql: `UPDATE bot_files SET path = ?, name = ?, folder = ?, updated_at = ? WHERE id = ?`, + values: [resolved.relative, resolved.name, resolved.folder, Date.now(), file.id], + }); + + if (file.media_id) { + const write_path = resolved.absolute.startsWith(DATA_DIR) + ? resolved.absolute.slice(DATA_DIR.length).replace(/^[\\/]/, "").split("\\").join("/") + : resolved.relative; + + await BunSQLite.sql({ + sql: `UPDATE media SET media_write_path = ?, updated_at = ? WHERE id = ?`, + values: [write_path, Date.now(), file.media_id], + }); + } + + return { ...file, path: resolved.relative, name: resolved.name, folder: resolved.folder }; +} diff --git a/src/functions/backend/db/bot-files/purge-storage.ts b/src/functions/backend/db/bot-files/purge-storage.ts new file mode 100644 index 0000000..5d4c91d --- /dev/null +++ b/src/functions/backend/db/bot-files/purge-storage.ts @@ -0,0 +1,86 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db"; +import { rmSync } from "node:fs"; +import path from "node:path"; +import type { BotFileScope } from "@/src/dict/bot-file-dict"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import { logError } from "@/src/functions/backend/logger"; +import grabStorageDir from "./grab-storage-dir"; + +type Params = { + scope: BotFileScope; + owner_id: number | string; + /** + * Its files' `media` ids, read before the rows go. Left out, they are + * read here (only useful when the catalog rows still exist). + */ + media_ids?: number[]; +}; + +/** + * The `media` ids of a directory's files, read before the bot or workspace + * (and with it, the catalog rows) is deleted. + */ +export async function grabStorageMediaIds({ scope, owner_id }: { scope: BotFileScope; owner_id: number | string }) { + const res = await BunSQLite.sql({ + sql: `SELECT media_id FROM bot_files WHERE scope = ? AND owner_id = ?`, + values: [scope, owner_id], + }); + + return (res.payload || []).map((file) => Number(file.media_id)).filter((id) => id > 0); +} + +/** + * Removes a directory's files for good when the bot or workspace it + * belongs to goes: the disk folder, every file's `media` row, and what is + * left of the catalog. A failure is logged, never thrown: deleting the + * bot must not stop on a stray file. + */ +export default async function purgeStorage({ scope, owner_id, media_ids }: Params) { + try { + const ids = media_ids || (await grabStorageMediaIds({ scope, owner_id })); + + if (ids.length) { + const placeholders = ids.map(() => "?").join(", "); + + await BunSQLite.sql({ + sql: `DELETE FROM media_paradigms WHERE media_id IN (${placeholders})`, + values: ids, + }).catch(() => {}); + + await BunSQLite.sql({ + sql: `DELETE FROM media WHERE id IN (${placeholders})`, + values: ids, + }).catch(() => {}); + } + + await BunSQLite.sql({ + sql: `DELETE FROM bot_file_vectors WHERE scope = ? AND owner_id = ?`, + values: [scope, owner_id], + }).catch(() => {}); + + /** + * Usually gone already, with the bot or workspace (the cascade). + */ + await BunSQLite.sql({ + sql: `DELETE FROM bot_files WHERE scope = ? AND owner_id = ?`, + values: [scope, owner_id], + }); + + const dir = grabStorageDir({ scope, owner_id, create: false }); + const { DATA_DIR } = grabDirNames(); + + /** + * Never outside the data volume, whatever the path says. + */ + if (path.resolve(dir).startsWith(path.resolve(DATA_DIR))) { + rmSync(dir, { recursive: true, force: true }); + } + } catch (error) { + await logError({ + title: `${scope} ${owner_id}'s files couldn't be cleaned up`, + error, + metadata: { scope, owner_id }, + }).catch(() => {}); + } +} diff --git a/src/functions/backend/db/bot-files/record-bot-file.ts b/src/functions/backend/db/bot-files/record-bot-file.ts new file mode 100644 index 0000000..05b332d --- /dev/null +++ b/src/functions/backend/db/bot-files/record-bot-file.ts @@ -0,0 +1,259 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { + BUN_SQLITE_COBBLE_BOT_FILES, + BUN_SQLITE_COBBLE_MEDIA, + BunSQLiteTables, +} from "@/db/types/db"; +import type { BotFilePurpose, BotFileScope, BotFileSource } from "@/src/dict/bot-file-dict"; +import type { MediaType } from "@/src/types"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import resolveBotPath from "@/src/utils/resolve-bot-path"; +import assertStorageRoom from "./assert-storage-room"; +import grabStorageDir from "./grab-storage-dir"; +import mediaTypeForFile, { mimeTypeForFile } from "./media-type-for-file"; + +type Params = { + /** + * The bot saving it (who `bot_files.bot_id` names). + */ + bot_id: number | string; + scope: BotFileScope; + /** + * The bot or workspace owning the directory. + */ + owner_id: number | string; + /** + * Path inside the directory, e.g. `reports/q3.md`. + */ + path: string; + data: Uint8Array | string; + purpose?: BotFilePurpose; + source?: BotFileSource; + /** + * Overrides the type read from the name. + */ + media_type?: MediaType; + mime_type?: string; + /** + * Who the `media` row says uploaded it: the user the run is for. + */ + user_id?: number | string; +}; + +/** + * Saves a file into a storage directory: the bytes on the data volume and + * the catalog row holding them, as one step. Every writer goes through + * here (a bot's `write_file`, an upload, a generated image, a PDF), so + * disk and database never disagree. + * + * Writing over an existing path keeps its `bot_files` row and its `media` + * row, so a message that attached the file now shows the new bytes. + */ +export default async function recordBotFile({ + bot_id, + scope, + owner_id, + path, + data, + purpose = "writing", + source = "bot-write", + media_type, + mime_type, + user_id, +}: Params): Promise { + const root = grabStorageDir({ scope, owner_id }); + const resolved = resolveBotPath({ root, path }); + const bytes = typeof data == "string" ? Buffer.from(data, "utf8") : data; + const media_type_resolved = media_type || mediaTypeForFile({ name: resolved.name, mime_type }); + const mime = mimeTypeForFile({ name: resolved.name, mime_type }); + const { DATA_DIR } = grabDirNames(); + + const existing = await grabBotFileRow({ scope, owner_id, path: resolved.relative }); + const previous_size = Number(existing?.size) || 0; + + await assertStorageRoom({ + scope, + owner_id, + incoming_bytes: bytes.byteLength - previous_size, + }); + + const absolute = resolved.absolute; + const write_path = absolute.startsWith(DATA_DIR) + ? absolute.slice(DATA_DIR.length).replace(/^[\\/]/, "").split("\\").join("/") + : resolved.relative; + + await Bun.write(absolute, bytes); + + const media_id = existing?.media_id + ? await updateMediaRow({ + media_id: existing.media_id, + write_path, + mime, + media_type: media_type_resolved, + }) + : await insertMediaRow({ + write_path, + mime, + media_type: media_type_resolved, + title: resolved.name, + user_id, + }); + + const now = Date.now(); + + if (existing?.id) { + await BunSQLite.sql({ + sql: `UPDATE bot_files SET media_id = ?, name = ?, folder = ?, mime_type = ?, media_type = ?, size = ?, + purpose = ?, source = ?, ingested_at = NULL, ingest_error = NULL, updated_at = ? WHERE id = ?`, + values: [ + media_id, + resolved.name, + resolved.folder, + mime, + media_type_resolved, + bytes.byteLength, + purpose, + source, + now, + existing.id, + ], + }); + + return { + ...existing, + media_id, + name: resolved.name, + folder: resolved.folder, + mime_type: mime, + media_type: media_type_resolved as any, + size: bytes.byteLength, + purpose, + source, + ingested_at: undefined, + ingest_error: undefined, + updated_at: now, + }; + } + + const res = await BunSQLite.insert({ + table: "bot_files", + data: [ + { + bot_id: Number(bot_id), + scope, + owner_id: Number(owner_id), + media_id, + path: resolved.relative, + name: resolved.name, + folder: resolved.folder, + mime_type: mime, + media_type: media_type_resolved as any, + size: bytes.byteLength, + purpose, + source, + }, + ], + }); + + const id = res.postInsertReturn?.insertId; + + if (!res.success || !id) { + throw new Error(`Couldn't record the file: ${res.msg || res.error}`); + } + + return { + id, + bot_id: Number(bot_id), + scope, + owner_id: Number(owner_id), + media_id, + path: resolved.relative, + name: resolved.name, + folder: resolved.folder, + mime_type: mime, + media_type: media_type_resolved as any, + size: bytes.byteLength, + purpose, + source, + }; +} + +/** + * The catalog row at a path, if there is one. + */ +export async function grabBotFileRow({ + scope, + owner_id, + path, +}: { + scope: BotFileScope; + owner_id: number | string; + path: string; +}) { + const res = await BunSQLite.sql({ + sql: `SELECT * FROM bot_files WHERE scope = ? AND owner_id = ? AND path = ? LIMIT 1`, + values: [scope, owner_id, path], + }); + + return res.payload?.[0] || null; +} + +async function insertMediaRow({ + write_path, + mime, + media_type, + title, + user_id, +}: { + write_path: string; + mime: string; + media_type: MediaType; + title: string; + user_id?: number | string; +}) { + const res = await BunSQLite.insert({ + table: "media", + data: [ + { + user_id: Number(user_id) || 0, + title, + media_type: media_type as any, + media_write_path: write_path, + mime_type: mime, + is_private: 1, + }, + ], + }); + + const id = res.postInsertReturn?.insertId; + + if (!res.success || !id) { + throw new Error(`Couldn't record the file's bytes: ${res.msg || res.error}`); + } + + await BunSQLite.insert({ + table: "media_paradigms", + data: [{ media_id: id, media_paradigm: "bot-file" as any }], + }); + + return id; +} + +async function updateMediaRow({ + media_id, + write_path, + mime, + media_type, +}: { + media_id: number | string; + write_path: string; + mime: string; + media_type: MediaType; +}): Promise { + await BunSQLite.sql({ + sql: `UPDATE media SET media_write_path = ?, mime_type = ?, media_type = ?, is_rag_ingested = 0, updated_at = ? WHERE id = ?`, + values: [write_path, mime, media_type, Date.now(), media_id], + }); + + return Number(media_id); +} diff --git a/src/functions/backend/db/bot-files/search-bot-files.ts b/src/functions/backend/db/bot-files/search-bot-files.ts new file mode 100644 index 0000000..4aa7d7f --- /dev/null +++ b/src/functions/backend/db/bot-files/search-bot-files.ts @@ -0,0 +1,271 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { AppData } from "@/src/data/app-data"; +import { embedQuery } from "@/src/harness/embeddings/embed-text"; +import HarnessError from "@/src/harness/harness-error"; +import type { StorageScope } from "@/src/utils/storage-access"; + +type Params = { + scopes: StorageScope[]; + query: string; + /** + * Only this file's chunks. + */ + file_id?: number | string | null; + limit?: number; + signal?: AbortSignal; + /** + * Whose embedding model to use. + */ + bot_id?: number | string; + workspace_id?: number | string | null; +}; + +export type BotFileSearchHit = { + file_id: number; + chunk_id: number; + chunk_index: number; + name: string; + path: string; + scope: string; + /** + * Cosine distance when the vector index answered, null for a word + * match. + */ + distance: number | null; + text: string; +}; + +export type BotFileSearchResult = { + hits: BotFileSearchHit[]; + /** + * How the search ran: `vector`, `words` (no embedding model, or nothing + * matched) or `none` (no files at all). + */ + mode: "vector" | "words" | "none"; + /** + * Said plainly for the bot's own reading of what it got back. + */ + note: string; +}; + +/** + * Searches a run's files: by meaning through the embedding vectors when a + * model is set, else (or when nothing matches) by words over the stored + * chunks. Never throws for an unavailable model: the bot always gets + * something it can act on. + */ +export default async function searchBotFiles({ + scopes, + query, + file_id, + limit = AppData.RagTopChunks, + signal, + bot_id, + workspace_id, +}: Params): Promise { + const text = query.trim(); + + if (!scopes.length || !text) { + return { hits: [], mode: "none", note: "No files to search." }; + } + + try { + const vector = await embedQuery({ + text, + signal, + bot_id, + workspace_id, + }); + + const hits = await vectorSearch({ + scopes, + query_vector: vector, + file_id, + limit, + seek: Math.min(limit * AppData.RagOverFetch, AppData.RagMaxK), + }); + + if (hits.length) { + return { + hits, + mode: "vector", + note: `${hits.length} passage(s), found by meaning.`, + }; + } + } catch (error) { + if (!(error instanceof HarnessError)) throw error; + } + + const hits = await wordSearch({ scopes, query: text, file_id, limit }); + + return { + hits, + mode: hits.length ? "words" : "none", + note: hits.length + ? `${hits.length} passage(s), found by matching words (no embedding model, or nothing matched by meaning).` + : `Nothing in your files matched "${text}".`, + }; +} + +/** + * The nearest chunks by vector. sqlite-vec refuses a filter on a vec0 + * side-car column in the same query as the match, so the index is asked + * for its nearest `seek` and the run's scopes are applied to that list + * here, before the chunk text is read. + */ +async function vectorSearch({ + scopes, + query_vector, + file_id, + limit, + seek, +}: { + scopes: StorageScope[]; + query_vector: Float32Array; + file_id?: number | string | null; + limit: number; + seek: number; +}): Promise { + const nearest = await BunSQLite.sql<{ + chunk_id: number; + file_id: number; + scope: string; + owner_id: number; + distance: number; + }>({ + sql: `SELECT chunk_id, file_id, scope, owner_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?) AND k = ? ORDER BY distance`, + values: [query_vector as any, seek], + }); + + /** + * `BunSQLite.sql` resolves with `success: false` instead of throwing, so + * a broken vector query would otherwise read as "nothing matched" and + * silently fall back to words. A failure here is a real error. + */ + if (!nearest.success) { + throw new Error(`The file search index couldn't be read: ${nearest.error || nearest.msg}`); + } + + const allowed = nearest.payload?.filter((row) => + scopes.some((scope) => scope.scope == row.scope && Number(scope.owner_id) == Number(row.owner_id)), + ); + + const picked = (allowed || []) + .filter((row) => !file_id || Number(row.file_id) == Number(file_id)) + .slice(0, limit); + + if (!picked.length) return []; + + const chunk_ids = picked.map((row) => Number(row.chunk_id)); + const placeholders = chunk_ids.map(() => "?").join(", "); + + const chunks = await BunSQLite.sql<{ + id: number; + text: string; + chunk_index: number; + name: string; + path: string; + }>({ + sql: `SELECT c.id, c.text, c.chunk_index, f.name, f.path FROM bot_file_chunks c + JOIN bot_files f ON f.id = c.file_id + WHERE c.id IN (${placeholders})`, + values: chunk_ids, + }); + + const by_id = new Map((chunks.payload || []).map((row) => [Number(row.id), row])); + + return picked.flatMap((row) => { + const chunk = by_id.get(Number(row.chunk_id)); + + if (!chunk) return []; + + return [ + toHit({ + chunk_id: row.chunk_id, + file_id: row.file_id, + scope: row.scope, + distance: Number(row.distance), + text: chunk.text, + chunk_index: chunk.chunk_index, + name: chunk.name, + path: chunk.path, + }), + ]; + }); +} + +/** + * Chunks whose text has the query's words, newest files first. Used when + * there's no embedding model, and as the fallback when the vectors + * answered nothing. + */ +async function wordSearch({ + scopes, + query, + file_id, + limit, +}: { + scopes: StorageScope[]; + query: string; + file_id?: number | string | null; + limit: number; +}): Promise { + const words = [ + ...new Set( + query + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((word) => word.length > 2), + ), + ].slice(0, 6); + + if (!words.length) return []; + + const scope_sql = scopes.map(() => `(c.scope = ? AND c.owner_id = ?)`).join(" OR "); + const scope_values = scopes.flatMap((s) => [s.scope, s.owner_id]); + const word_sql = words.map(() => `lower(c.text) LIKE ?`).join(" OR "); + const word_values = words.map((word) => `%${word}%`); + const file_clause = file_id ? "AND c.file_id = ?" : ""; + + const res = await BunSQLite.sql<{ + chunk_id: number; + file_id: number; + text: string; + chunk_index: number; + name: string; + path: string; + scope: string; + }>({ + sql: `SELECT c.id AS chunk_id, c.file_id, c.text, c.chunk_index, c.scope, f.name, f.path + FROM bot_file_chunks c + JOIN bot_files f ON f.id = c.file_id + WHERE (${scope_sql}) ${file_clause} AND (${word_sql}) + ORDER BY f.updated_at DESC, c.chunk_index + LIMIT ?`, + values: [...scope_values, ...(file_id ? [file_id] : []), ...word_values, limit], + }); + + return (res.payload || []).map((row) => toHit({ ...row, distance: null })); +} + +function toHit(row: { + chunk_id: number; + file_id: number; + text: string; + chunk_index: number; + name: string; + path: string; + scope: string; + distance: number | null; +}): BotFileSearchHit { + return { + file_id: Number(row.file_id), + chunk_id: Number(row.chunk_id), + chunk_index: Number(row.chunk_index), + name: row.name || "", + path: row.path || "", + scope: row.scope || "bot", + distance: row.distance == null ? null : Number(row.distance), + text: row.text || "", + }; +} diff --git a/src/functions/backend/db/bot-files/store-chat-attachments.ts b/src/functions/backend/db/bot-files/store-chat-attachments.ts new file mode 100644 index 0000000..e5384e1 --- /dev/null +++ b/src/functions/backend/db/bot-files/store-chat-attachments.ts @@ -0,0 +1,93 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db"; +import { BotUploadFolder } from "@/src/dict/bot-file-dict"; +import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest"; +import { logError } from "@/src/functions/backend/logger"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import recordBotFile from "./record-bot-file"; + +type Params = { + bot_id: number | string; + workspace_id?: number | string | null; + user_id: number | string; + media: BUN_SQLITE_COBBLE_MEDIA[]; +}; + +/** + * Copies the files the user attached to a message into the conversation's + * folder (`uploads//`), so they stay searchable after the + * message scrolls away and a later run can find them. Best effort: a + * failure is logged, never shown, and the message still goes out. + */ +export default async function storeChatAttachments({ + bot_id, + workspace_id, + user_id, + media, +}: Params) { + if (!media.length) return; + + const { DATA_DIR } = grabDirNames(); + const folder = `${BotUploadFolder}/${new Date().toISOString().slice(0, 7)}`; + + for (const file of media) { + try { + if (!file.media_write_path) continue; + + const source = Bun.file(`${DATA_DIR}/${file.media_write_path}`); + + if (!(await source.exists())) continue; + + /** + * Its name, made safe and unique enough that a second upload + * of "invoice.pdf" doesn't overwrite the first. + */ + const name = safeName({ name: file.title || `file-${file.id}` }); + + const saved = await recordBotFile({ + bot_id, + scope: workspace_id ? "workspace" : "bot", + owner_id: workspace_id || bot_id, + path: `${folder}/${name}`, + data: new Uint8Array(await source.arrayBuffer()), + purpose: "upload", + source: "chat-upload", + media_type: (file.media_type as any) || undefined, + mime_type: file.mime_type || undefined, + user_id, + }); + + /** + * The same text the upload already extracted, so a PDF isn't + * unpacked a second time. + */ + if (file.text_content && saved.media_id) { + await BunSQLite.sql({ + sql: `UPDATE media SET text_content = ? WHERE id = ?`, + values: [file.text_content, saved.media_id], + }).catch(() => {}); + } + + await enqueueFileIngest({ file_id: saved.id! }); + } catch (error) { + await logError({ + title: `A chat attachment couldn't be saved to the bot's files`, + error, + metadata: { media_id: file.id, bot_id }, + }).catch(() => {}); + } + } +} + +/** + * A file name that is safe on disk. + */ +function safeName({ name }: { name: string }) { + const cleaned = name + .replace(/[<>:"|?*/\\]/g, "-") + .replace(/\s+/g, " ") + .trim() + .slice(0, 100); + + return cleaned || "file"; +} diff --git a/src/functions/backend/db/bot-files/vector-search.test.ts b/src/functions/backend/db/bot-files/vector-search.test.ts new file mode 100644 index 0000000..ab12555 --- /dev/null +++ b/src/functions/backend/db/bot-files/vector-search.test.ts @@ -0,0 +1,132 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import * as sqliteVec from "sqlite-vec"; +import fitVectorDimensions from "@/src/harness/embeddings/fit-vector-dimensions"; + +/** + * The exact insert and nearest-neighbour shape `ingestBotFile` and + * `searchBotFiles` use, on a scratch database: vec0's metadata columns are + * NOT NULL, and a side-car filter can't share the match query. + */ +function createDb() { + const db = new Database(":memory:"); + sqliteVec.load(db); + db.run(`CREATE VIRTUAL TABLE bot_file_vectors USING vec0( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at INTEGER, + updated_at INTEGER, + embedding float[8], + +chunk_id INTEGER, + +file_id INTEGER, + +scope TEXT, + +owner_id INTEGER + )`); + + return db; +} + +const vector = (values: number[]) => fitVectorDimensions({ vector: values, dimensions: 8 }); + +const insert = (db: Database, row: { chunk_id: number; file_id: number; scope: string; owner_id: number; v: number[] }) => { + const now = Date.now(); + + db.run( + `INSERT INTO bot_file_vectors (created_at, updated_at, embedding, chunk_id, file_id, scope, owner_id) VALUES (?, ?, vec_f32(?), ?, ?, ?, ?)`, + [now, now, vector(row.v), row.chunk_id, row.file_id, row.scope, row.owner_id], + ); +}; + +describe("bot file vectors", () => { + test("stores a vector and matches the nearest one", () => { + const db = createDb(); + + insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0, 0, 0] }); + insert(db, { chunk_id: 2, file_id: 11, scope: "bot", owner_id: 1, v: [0, 1, 0, 0] }); + + const rows = db + .query( + `SELECT chunk_id, file_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 2 ORDER BY distance`, + ) + .all(vector([0.9, 0.1, 0, 0])) as { chunk_id: number; distance: number }[]; + + expect(rows.length).toBe(2); + expect(rows[0]!.chunk_id).toBe(1); + expect(rows[0]!.distance).toBeLessThan(rows[1]!.distance); + }); + + test("filters scopes in the app, not in the match query", () => { + const db = createDb(); + + insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] }); + insert(db, { chunk_id: 2, file_id: 20, scope: "bot", owner_id: 2, v: [1, 0.05] }); + insert(db, { chunk_id: 3, file_id: 30, scope: "workspace", owner_id: 3, v: [1, 0.1] }); + + const nearest = db + .query(`SELECT chunk_id, scope, owner_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 10 ORDER BY distance`) + .all(vector([1, 0])) as { chunk_id: number; scope: string; owner_id: number }[]; + + expect(nearest.length).toBe(3); + + const mine = nearest.filter((row) => row.scope == "bot" && row.owner_id == 1); + + expect(mine.map((row) => row.chunk_id)).toEqual([1]); + }); + + test("a side-car filter in the match query is refused", () => { + const db = createDb(); + + insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] }); + + expect(() => + db + .query( + `SELECT chunk_id FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 2 AND scope = 'bot'`, + ) + .all(vector([1, 0])), + ).toThrow(/illegal WHERE constraint/); + }); + + test("an insert without vec0's metadata columns is refused", () => { + const db = createDb(); + + expect(() => + db.run( + `INSERT INTO bot_file_vectors (embedding, chunk_id, file_id, scope, owner_id) VALUES (vec_f32(?1), 1, 10, 'bot', 1)`, + [vector([1, 0])], + ), + ).toThrow(); + }); + + test("deletes a file's vectors by its side-car column", () => { + const db = createDb(); + + insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] }); + + db.run(`DELETE FROM bot_file_vectors WHERE file_id = ?`, [10]); + + const count = db.query(`SELECT COUNT(*) AS n FROM bot_file_vectors`).get() as { n: number }; + + expect(count.n).toBe(0); + }); +}); + +describe("fitVectorDimensions", () => { + test("pads a short vector and normalizes it", () => { + const out = fitVectorDimensions({ vector: [3, 4], dimensions: 4 }); + const norm = Math.sqrt([...out].reduce((sum, value) => sum + value * value, 0)); + + expect(out.length).toBe(4); + expect(Math.round(norm * 1000) / 1000).toBe(1); + expect(Math.round(out[0]! * 1000) / 1000).toBe(0.6); + }); + + test("cuts a long vector", () => { + expect(fitVectorDimensions({ vector: [1, 1, 1, 1, 1, 1], dimensions: 2 }).length).toBe(2); + }); + + test("leaves a zero vector as zeros", () => { + expect([...fitVectorDimensions({ vector: [0, 0], dimensions: 2 })]).toEqual([0, 0]); + }); +}); + +afterAll(() => {}); diff --git a/src/functions/backend/db/bot-sessions/grab-conversation-props.ts b/src/functions/backend/db/bot-sessions/grab-conversation-props.ts index 0662e44..645ded0 100644 --- a/src/functions/backend/db/bot-sessions/grab-conversation-props.ts +++ b/src/functions/backend/db/bot-sessions/grab-conversation-props.ts @@ -7,6 +7,7 @@ import grabBots from "../bots/grab-bots"; import grabActiveBotTask from "../bots/grab-active-bot-task"; import grabBotMessages from "../bots/grab-bot-messages"; import grabBotExtensionsOverview from "../extensions/grab-bot-extensions-overview"; +import grabConversationFiles from "../bot-files/grab-conversation-files"; import grabBotComputers from "../computers/grab-bot-computers"; import resolveHarnessCandidates from "@/src/harness/resolve/resolve-harness-candidates"; import grabUserWorkspaces from "../workspaces/grab-user-workspaces"; @@ -67,7 +68,7 @@ export default async function grabConversationProps({ bot.unread = 0; - const [bot_messages, bot_text_model, bot_computers, bot_task, bot_extensions, transcription, speech] = await Promise.all([ + const [bot_messages, bot_text_model, bot_computers, bot_task, bot_extensions, bot_files, transcription, speech] = await Promise.all([ grabBotMessages({ session_id: session.id!, user_id, @@ -77,6 +78,7 @@ export default async function grabConversationProps({ grabBotComputers({ bot_id: bot.id, workspace_id: scope }), grabActiveBotTask({ bot_id: bot.id, session_id: session.id }), grabBotExtensionsOverview({ user_id, bot_id: bot.id, workspace_id: scope }), + grabConversationFiles({ bot_id: bot.id, workspace_id: scope }), hasHarnessModel({ harness: "transcription", bot_id: bot.id, workspace_id: scope }), hasHarnessModel({ harness: "speech", bot_id: bot.id, workspace_id: scope }), ]); @@ -95,6 +97,7 @@ export default async function grabConversationProps({ bot_computers, bot_task, bot_extensions, + bot_files, bot_voice: { transcription, speech }, }; } diff --git a/src/functions/backend/db/bots-queue/complete-bot-job.ts b/src/functions/backend/db/bots-queue/complete-bot-job.ts index 9999cef..1629755 100644 --- a/src/functions/backend/db/bots-queue/complete-bot-job.ts +++ b/src/functions/backend/db/bots-queue/complete-bot-job.ts @@ -26,6 +26,10 @@ type Params = { * Finish without posting a message. */ silent?: boolean; + /** + * Files the bot saved and asked to send with it (`media` ids). + */ + attachments?: number[]; }; /** @@ -43,6 +47,7 @@ export default async function completeBotJob({ result, needs_you, silent, + attachments, }: Params) { const job = await grabLockedBotJob({ job_id, worker }); @@ -81,6 +86,7 @@ export default async function completeBotJob({ workspace_id: job.workspace_id || null, text: result, needs_you, + media_ids: attachments, }); const user_id = await grabJobUserId({ job }); diff --git a/src/functions/backend/db/bots/delete-bot.ts b/src/functions/backend/db/bots/delete-bot.ts index d1ca964..5c733c2 100644 --- a/src/functions/backend/db/bots/delete-bot.ts +++ b/src/functions/backend/db/bots/delete-bot.ts @@ -1,5 +1,6 @@ import BunSQLite from "@moduletrace/bun-sqlite"; import deleteMedia from "@/src/functions/backend/db/media/delete-media"; +import purgeStorage, { grabStorageMediaIds } from "@/src/functions/backend/db/bot-files/purge-storage"; import AppError from "@/src/utils/app-error"; import grabUserBot from "./grab-user-bot"; import assertCrewManager from "@/src/functions/backend/db/access/assert-crew-manager"; @@ -27,6 +28,13 @@ export default async function deleteBot({ user_id, bot_id }: Params) { throw new AppError(`The coordinator can't be deleted.`); } + /** + * Read before the delete: the files' rows go with the bot. + */ + const file_media_ids = await grabStorageMediaIds({ scope: "bot", owner_id: bot.id! }).catch( + () => [], + ); + const res = await BunSQLite.sql({ sql: `DELETE FROM bots WHERE id = ?`, values: [bot.id] }); if (!res.success) { @@ -37,5 +45,11 @@ export default async function deleteBot({ user_id, bot_id }: Params) { await deleteMedia({ id: bot.image_media_id, user_id: Number(user_id) }).catch(() => {}); } + /** + * Its folder goes with it: the bytes, their `media` rows and the search + * vectors. + */ + await purgeStorage({ scope: "bot", owner_id: bot.id!, media_ids: file_media_ids }); + return bot; } diff --git a/src/functions/backend/db/bots/grab-bot-messages.ts b/src/functions/backend/db/bots/grab-bot-messages.ts index 1837e2b..09d918b 100644 --- a/src/functions/backend/db/bots/grab-bot-messages.ts +++ b/src/functions/backend/db/bots/grab-bot-messages.ts @@ -4,7 +4,7 @@ import BunSQLite from "@moduletrace/bun-sqlite"; import grabMessagesMedia from "./grab-messages-media"; import grabMessagesReactions from "./grab-messages-reactions"; import grabMessagesReplies from "./grab-messages-replies"; -import mediaToAttachment from "./media-to-attachment"; +import mediaToAttachment, { grabAttachmentFileIds } from "./media-to-attachment"; type Params = { session_id: number | string; @@ -61,6 +61,14 @@ export default async function grabBotMessages({ : Promise.resolve(new Map()), ]); + /** + * Files that live in a bot's folder carry their `bot_files` id, so a + * chip opens the file rather than only its bytes. + */ + const file_ids = await grabAttachmentFileIds({ + media_ids: [...media.values()].flat().map((file) => Number(file.id)), + }); + return messages.map((message) => { const id = Number(message.id); const files = media.get(id); @@ -69,7 +77,14 @@ export default async function grabBotMessages({ return { ...message, ...(files - ? { attachments: files.map((file) => mediaToAttachment({ media: file })) } + ? { + attachments: files.map((file) => + mediaToAttachment({ + media: file, + file_id: file_ids.get(Number(file.id)), + }), + ), + } : {}), ...(reply_to ? { reply_to } : {}), ...(user_id ? { reactions: reactions.get(id) || [] } : {}), diff --git a/src/functions/backend/db/bots/insert-bot-message.ts b/src/functions/backend/db/bots/insert-bot-message.ts index 87c9978..c1be864 100644 --- a/src/functions/backend/db/bots/insert-bot-message.ts +++ b/src/functions/backend/db/bots/insert-bot-message.ts @@ -13,6 +13,11 @@ type Params = { workspace_id?: number | null; text: string; needs_you?: boolean; + /** + * Files the bot saved and sent with it (`media` ids): a PDF it made, a + * generated image, a document it wrote. + */ + media_ids?: number[]; }; /** @@ -25,6 +30,7 @@ export default async function insertBotMessage({ workspace_id, text, needs_you, + media_ids = [], }: Params): Promise { const now = Date.now(); const needs_you_flag = needs_you ? 1 : 0; @@ -52,6 +58,13 @@ export default async function insertBotMessage({ throw new Error(`Failed to save message: ${res.msg}`); } + for (const media_id of media_ids) { + await BunSQLite.sql({ + sql: `INSERT INTO bot_message_media (created_at, updated_at, message_id, media_id) VALUES (?, ?, ?, ?)`, + values: [now, now, id, media_id], + }).catch(() => {}); + } + await BunSQLite.sql({ sql: `UPDATE bot_sessions SET last_activity_at = ?, unread = 1 WHERE id = ?`, values: [now, session_id], diff --git a/src/functions/backend/db/bots/media-to-attachment.ts b/src/functions/backend/db/bots/media-to-attachment.ts index bacf229..3dcf088 100644 --- a/src/functions/backend/db/bots/media-to-attachment.ts +++ b/src/functions/backend/db/bots/media-to-attachment.ts @@ -1,16 +1,23 @@ -import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db"; +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOT_FILES, BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db"; import grabMediaURL from "@/src/functions/backend/media/grab-media-url"; import type { BotMessageAttachment, MediaType } from "@/src/types"; type Params = { media: BUN_SQLITE_COBBLE_MEDIA; + /** + * The bot file it is, when the caller already knows (avoids a lookup + * per attachment). + */ + file_id?: number | null; }; /** * A `media` row as a chat attachment: its original name (`title`), type - * and the private URLs it's served at. + * and the private URLs it's served at. A file saved in a bot's folder + * carries its `file_id`, so the chip can open the file's page. */ -export default function mediaToAttachment({ media }: Params): BotMessageAttachment { +export default function mediaToAttachment({ media, file_id }: Params): BotMessageAttachment { const { media_url, media_thumbnail_url } = grabMediaURL({ media: { ...media } }); return { @@ -19,5 +26,29 @@ export default function mediaToAttachment({ media }: Params): BotMessageAttachme media_type: (media.media_type || "document") as MediaType, url: media_url || "", thumbnail_url: media_thumbnail_url, + file_id: file_id || undefined, }; } + +/** + * The `bot_files` rows behind a message's media, keyed by media id, so the + * attachments can carry their file ids without a query each. + */ +export async function grabAttachmentFileIds({ media_ids }: { media_ids: number[] }) { + const ids = media_ids.filter((id) => id > 0); + + if (!ids.length) return new Map(); + + const placeholders = ids.map(() => "?").join(", "); + + const res = await BunSQLite.sql({ + sql: `SELECT id, media_id FROM bot_files WHERE media_id IN (${placeholders})`, + values: ids, + }); + + return new Map( + (res.payload || []) + .filter((file) => file.media_id) + .map((file) => [Number(file.media_id), Number(file.id)]), + ); +} diff --git a/src/functions/backend/db/workspaces/delete-workspace.ts b/src/functions/backend/db/workspaces/delete-workspace.ts index efc2f37..102142d 100644 --- a/src/functions/backend/db/workspaces/delete-workspace.ts +++ b/src/functions/backend/db/workspaces/delete-workspace.ts @@ -1,4 +1,5 @@ import deleteMedia from "@/src/functions/backend/db/media/delete-media"; +import purgeStorage, { grabStorageMediaIds } from "@/src/functions/backend/db/bot-files/purge-storage"; import BunSQLite from "@moduletrace/bun-sqlite"; import AppError from "@/src/utils/app-error"; import grabUserWorkspace from "./grab-user-workspace"; @@ -22,6 +23,14 @@ export default async function deleteWorkspace({ user_id, workspace_id }: Params) throw new AppError(`Workspace not found.`); } + /** + * Read before the delete: the shared files' rows go with the workspace. + */ + const file_media_ids = await grabStorageMediaIds({ + scope: "workspace", + owner_id: workspace.id!, + }).catch(() => []); + const res = await BunSQLite.sql({ sql: `DELETE FROM workspaces WHERE id = ?`, values: [workspace.id], @@ -35,5 +44,14 @@ export default async function deleteWorkspace({ user_id, workspace_id }: Params) await deleteMedia({ id: workspace.image_media_id, user_id: Number(user_id) }).catch(() => {}); } + /** + * The bots' shared folder there goes with it. + */ + await purgeStorage({ + scope: "workspace", + owner_id: workspace.id!, + media_ids: file_media_ids, + }); + return workspace; } diff --git a/src/functions/server/media/handle-media-server.ts b/src/functions/server/media/handle-media-server.ts index 7f87748..05eef02 100644 --- a/src/functions/server/media/handle-media-server.ts +++ b/src/functions/server/media/handle-media-server.ts @@ -5,6 +5,7 @@ import { existsSync } from "node:fs"; import path from "node:path"; import userAuth from "../../backend/auth/user-auth"; import grabSharedPictureMedia from "@/src/functions/backend/db/access/grab-shared-picture-media"; +import grabBotFileMedia from "@/src/functions/backend/db/bot-files/grab-bot-file-media"; const { DATA_DIR } = grabDirNames(); @@ -52,6 +53,7 @@ export default async function handleMediaServer({ media = media_res.singleRes || (media_id ? await grabSharedPictureMedia({ media_id, user_id: user.id! }) : null) || + (media_id ? await grabBotFileMedia({ media_id, user_id: user.id! }) : null) || undefined; } else { const media_res = await BunSQLite.select< diff --git a/src/harness/catalog/model-serves-harness.ts b/src/harness/catalog/model-serves-harness.ts index 5bc80ed..b90db29 100644 --- a/src/harness/catalog/model-serves-harness.ts +++ b/src/harness/catalog/model-serves-harness.ts @@ -17,6 +17,7 @@ type Params = { * audio: free ones often drop it) * - speech: a text-to-speech model * - image: makes images + * - embedding: turns text into vectors (never a chat model) */ export default function modelServesHarness({ model, harness }: Params) { const input = new Set(model.input_modalities); @@ -39,5 +40,8 @@ export default function modelServesHarness({ model, harness }: Params) { case "image": return output.has("image"); + + case "embedding": + return output.has("embeddings"); } } diff --git a/src/harness/catalog/rank-auto-models.ts b/src/harness/catalog/rank-auto-models.ts index bf08174..44c5d45 100644 --- a/src/harness/catalog/rank-auto-models.ts +++ b/src/harness/catalog/rank-auto-models.ts @@ -28,7 +28,7 @@ type Params = { * * Within a group, benchmarked models come first by intelligence index, * then the rest: full-size before small variants, newest first. None for - * a harness that isn't `automatic` (transcription, speech). + * a harness that isn't `automatic` (transcription, speech, embeddings). */ export default function rankAutoModels({ models, diff --git a/src/harness/embeddings/embed-text.ts b/src/harness/embeddings/embed-text.ts new file mode 100644 index 0000000..da98f3e --- /dev/null +++ b/src/harness/embeddings/embed-text.ts @@ -0,0 +1,84 @@ +import grabHarnessAdapter from "../providers/grab-harness-adapter"; +import resolveHarnessCandidates from "../resolve/resolve-harness-candidates"; +import withCandidates from "../resolve/with-candidates"; +import fitVectorDimensions from "./fit-vector-dimensions"; + +type Params = { + /** + * The texts to embed, in order. Empty strings are embedded as they + * are (a provider may refuse them; callers skip empty chunks). + */ + input: string[]; + signal?: AbortSignal; + /** + * Uses this bot's embedding model when it has its own. + */ + bot_id?: number | string; + /** + * The workspace the call is in: its models win over the bot's. + */ + workspace_id?: number | string | null; +}; + +export type EmbeddingResult = { + /** + * One vector per input, `AppData.EmbeddingDimensions` wide, normalized. + */ + vectors: Float32Array[]; + /** + * The width the model actually returned, before fitting. Recorded so a + * change of model is visible. + */ + model_dimensions: number; +}; + +/** + * Embeds texts with the embedding harness, fitted to the vector table's + * fixed width. Throws `HarnessError` when no embedding model is + * configured or every candidate failed; callers fall back to text + * matching then. + */ +export default async function embedText({ + input, + signal, + bot_id, + workspace_id, +}: Params): Promise { + if (!input.length) return { vectors: [], model_dimensions: 0 }; + + const candidates = await resolveHarnessCandidates({ + harness: "embedding", + bot_id, + workspace_id, + }); + + const { result } = await withCandidates({ + harness: "embedding", + candidates, + run: async (candidate) => + await grabHarnessAdapter({ kind: candidate.provider.kind }).embed({ + candidate, + input, + signal: signal || new AbortController().signal, + }), + }); + + return { + vectors: result.vectors.map((vector) => fitVectorDimensions({ vector })), + model_dimensions: result.vectors[0]?.length || 0, + }; +} + +/** + * The same for one text, for a search query. + */ +export async function embedQuery({ + text, + signal, + bot_id, + workspace_id, +}: Omit & { text: string }) { + const { vectors } = await embedText({ input: [text], signal, bot_id, workspace_id }); + + return vectors[0] as Float32Array; +} diff --git a/src/harness/embeddings/fit-vector-dimensions.ts b/src/harness/embeddings/fit-vector-dimensions.ts new file mode 100644 index 0000000..078f74b --- /dev/null +++ b/src/harness/embeddings/fit-vector-dimensions.ts @@ -0,0 +1,46 @@ +import { AppData } from "@/src/data/app-data"; + +type Params = { + vector: number[]; + /** + * Defaults to `AppData.EmbeddingDimensions`, the width of the fixed + * `bot_file_vectors` table. + */ + dimensions?: number; +}; + +/** + * Makes a provider's vector the width the vector table takes: cut when + * it's wider, padded with zeros when narrower, then normalized to unit + * length so cosine distance means the same for every model. + */ +export default function fitVectorDimensions({ + vector, + dimensions = AppData.EmbeddingDimensions, +}: Params): Float32Array { + const out = new Float32Array(new ArrayBuffer(dimensions * 4)); + const length = Math.min(vector.length, dimensions); + + let sum = 0; + + for (let i = 0; i < length; i++) { + const value = Number(vector[i]) || 0; + + out[i] = value; + sum += value * value; + } + + /** + * A zero vector can't be normalized; leave it as zeros (it then + * matches nothing). + */ + const norm = Math.sqrt(sum); + + if (norm > 0) { + for (let i = 0; i < dimensions; i++) { + out[i] = out[i]! / norm; + } + } + + return out; +} diff --git a/src/harness/providers/openai-compatible/embed.ts b/src/harness/providers/openai-compatible/embed.ts new file mode 100644 index 0000000..c581b94 --- /dev/null +++ b/src/harness/providers/openai-compatible/embed.ts @@ -0,0 +1,68 @@ +import { AppData } from "@/src/data/app-data"; +import HarnessError from "../../harness-error"; +import type { HarnessAdapter } from "../../types"; +import grabProviderPreset from "../grab-provider-preset"; +import providerRequest from "./provider-request"; + +/** + * Embeddings at the provider's `embeddings_path` (the OpenAI `/embeddings` + * shape). Batched, since a large file's chunks would otherwise go in one + * request the provider may refuse. + */ +const embed: HarnessAdapter["embed"] = async ({ candidate, input, signal }) => { + const { provider, model } = candidate; + const preset = grabProviderPreset({ kind: provider.kind }); + const vectors: number[][] = []; + + /** + * Every OpenAI-compatible API puts embeddings at `/embeddings`. + */ + const path = "/embeddings"; + + for (let start = 0; start < input.length; start += AppData.EmbeddingBatchSize) { + const batch = input + .slice(start, start + AppData.EmbeddingBatchSize) + .map((text) => text.slice(0, AppData.EmbeddingMaxInputChars)); + + const res = await providerRequest({ + provider, + path, + json: { model: model.slug, input: batch }, + signal, + candidate_key: candidate.key, + }); + + const json = await res.json().catch(() => null); + const items: any[] = Array.isArray(json?.data) + ? json.data + : Array.isArray(json?.embeddings) + ? json.embeddings + : []; + + if (items.length != batch.length) { + throw new HarnessError({ + message: `${provider.name || preset.title}: ${items.length} of ${batch.length} texts got a vector.`, + retryable: true, + candidate_key: candidate.key, + }); + } + + for (const item of items) { + const vector = Array.isArray(item) ? item : item?.embedding; + + if (!Array.isArray(vector) || !vector.length) { + throw new HarnessError({ + message: `${provider.name || preset.title} sent no embedding back.`, + retryable: true, + candidate_key: candidate.key, + }); + } + + vectors.push(vector.map(Number)); + } + } + + return { vectors }; +}; + +export default embed; diff --git a/src/harness/providers/openai-compatible/index.ts b/src/harness/providers/openai-compatible/index.ts index 936f7ca..978933e 100644 --- a/src/harness/providers/openai-compatible/index.ts +++ b/src/harness/providers/openai-compatible/index.ts @@ -1,5 +1,6 @@ import type { HarnessAdapter } from "../../types"; import chat from "./chat"; +import embed from "./embed"; import generateImage from "./generate-image"; import listModels from "./list-models"; import speak from "./speak"; @@ -15,6 +16,7 @@ const OpenAICompatibleAdapter: HarnessAdapter = { transcribe, speak, generateImage, + embed, listModels, }; diff --git a/src/harness/tools/storage/copy-file-tool.ts b/src/harness/tools/storage/copy-file-tool.ts new file mode 100644 index 0000000..e9f7d9f --- /dev/null +++ b/src/harness/tools/storage/copy-file-tool.ts @@ -0,0 +1,50 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import copyBotFile from "@/src/functions/backend/db/bot-files/copy-bot-file"; +import type { HarnessTool } from "../../types"; +import { fileRef, findNamedScope, type StorageToolContext } from "./storage-tool-context"; + +type Args = { + file_id: number; + scope: string; + path?: string; +}; + +/** + * Copies a file into another of the bot's directories: the coordinator + * bringing a worker's file into its own or into a workspace, or the other + * way round. Only available when the run reaches more than one. + */ +export default function copyFileTool(context: StorageToolContext): HarnessTool { + return { + name: "copy_file", + description: `Copy one of your files into another directory, by its ref id: ${context.scopes + .map((s) => `${s.scope}:${s.owner_id}`) + .join(", ")}. The copy is separate: deleting one leaves the other.`, + schema: z.object({ + file_id: z.number().int().positive(), + scope: z.string().describe(`Where the copy goes, e.g. "bot:2" or "workspace:1"`), + path: z.string().max(AppData.BotFilePathMaxChars).optional(), + }), + run: async ({ args }) => { + const target = findNamedScope({ scopes: context.scopes, name: args.scope }); + + if (!target) { + return `You can't write there. Your directories: ${context.scopes + .map((s) => `${s.scope}:${s.owner_id}`) + .join(", ")}.`; + } + + const file = await copyBotFile({ + scopes: context.scopes, + file_id: args.file_id, + bot_id: context.bot_id, + scope: target, + path: args.path, + user_id: context.user_id, + }); + + return `Copied to ${target.scope} ${target.owner_id} as ${file.path}. ${fileRef({ file_id: file.id! })}`; + }, + }; +} diff --git a/src/harness/tools/storage/delete-file-tool.ts b/src/harness/tools/storage/delete-file-tool.ts new file mode 100644 index 0000000..5b71042 --- /dev/null +++ b/src/harness/tools/storage/delete-file-tool.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; +import deleteBotFile from "@/src/functions/backend/db/bot-files/delete-bot-file"; +import type { HarnessTool } from "../../types"; +import type { StorageToolContext } from "./storage-tool-context"; + +type Args = { + file_id: number; +}; + +/** + * Deletes a file the bot saved, with its searchable text. Its bytes go for + * good, so a file the user may still want is moved, not deleted. + */ +export default function deleteFileTool(context: StorageToolContext): HarnessTool { + return { + name: "delete_file", + description: `Delete one of your files for good, by its ref id. Use it to clear space, never for something the user still needs.`, + schema: z.object({ + file_id: z.number().int().positive(), + }), + run: async ({ args }) => { + const file = await deleteBotFile({ scopes: context.scopes, file_id: args.file_id }); + + return `${file.path} is deleted.`; + }, + }; +} diff --git a/src/harness/tools/storage/generate-image-tool.ts b/src/harness/tools/storage/generate-image-tool.ts new file mode 100644 index 0000000..b3d5d7d --- /dev/null +++ b/src/harness/tools/storage/generate-image-tool.ts @@ -0,0 +1,121 @@ +import { z } from "zod"; +import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest"; +import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file"; +import generateImage from "@/src/harness/image/generate-image"; +import type { HarnessTool } from "../../types"; +import { + describeScope, + fileRef, + findNamedScope, + formatBytes, + type StorageToolContext, +} from "./storage-tool-context"; + +type Args = { + prompt: string; + path?: string; + size?: string; + attach?: boolean; + scope?: string; +}; + +/** + * Makes an image with the image generation harness and saves it in the + * bot's files, so it can be sent, shown again later or used by another + * run. The bytes stay in the app's storage, never on a computer. + */ +export default function generateImageTool(context: StorageToolContext): HarnessTool { + const own = context.scopes[0]; + + return { + name: "generate_image", + description: `Make an image from a description and save it in your files. Describe what should be in it, in one clear prompt. The image is saved (so you can send it now and show it later) and can ride on this reply with "attach".`, + schema: z.object({ + prompt: z.string().min(3).max(4000), + path: z + .string() + .max(300) + .optional() + .describe(`Where to save it, e.g. "images/cover.png"; one is named for you otherwise`), + size: z + .string() + .max(20) + .optional() + .describe(`e.g. "1024x1024", when the model takes one`), + attach: z.boolean().optional().describe(`Send it to the user with this reply`), + scope: z.string().optional().describe(`"bot:" or "workspace:"`), + }), + run: async ({ args, signal }) => { + const named = args.scope + ? findNamedScope({ scopes: context.scopes, name: args.scope }) + : own; + + if (!named) { + return `You can't save there. Your directories: ${context.scopes + .map((s) => `${s.scope}:${s.owner_id}`) + .join(", ")}.`; + } + + const images = await generateImage({ + prompt: args.prompt, + size: args.size, + signal, + bot_id: context.bot_id, + workspace_id: context.workspace_id, + }).catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Image generation isn't working: ${message}`); + }); + + if (!images.length) { + return `The image model sent nothing back. Tell the user, and try a simpler prompt.`; + } + + const saved: string[] = []; + + for (let index = 0; index < images.length; index++) { + const image = images[index]!; + const ext = image.media_type.includes("png") + ? "png" + : image.media_type.includes("webp") + ? "webp" + : "jpg"; + const name = + args.path && images.length == 1 + ? args.path + : `${stripExtension({ path: args.path || "image" })}${images.length > 1 ? `-${index + 1}` : ""}.${ext}`; + + const file = await recordBotFile({ + bot_id: context.bot_id, + scope: named.scope, + owner_id: named.owner_id, + path: name, + data: image.data, + purpose: "image", + source: "tool", + media_type: "image", + mime_type: image.media_type, + user_id: context.user_id, + }); + + await enqueueFileIngest({ file_id: file.id! }); + + if (args.attach && file.media_id) { + context.onAttach?.({ media_id: Number(file.media_id), name: file.name || "" }); + } + + saved.push( + `${file.path} (${formatBytes({ bytes: Number(file.size) || 0 })}) ${fileRef({ file_id: file.id! })}`, + ); + } + + return `Made and saved ${saved.length} image(s) in ${describeScope(named)}:\n${saved.join("\n")}${ + args.attach ? "" : "\n\nIt isn't attached to this reply; the user sees it in your files." + }`; + }, + }; +} + +function stripExtension({ path }: { path: string }) { + return path.replace(/\.[a-zA-Z0-9]{1,5}$/, ""); +} diff --git a/src/harness/tools/storage/grab-bot-file-tools.ts b/src/harness/tools/storage/grab-bot-file-tools.ts new file mode 100644 index 0000000..239ac22 --- /dev/null +++ b/src/harness/tools/storage/grab-bot-file-tools.ts @@ -0,0 +1,50 @@ +import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db"; +import grabUserWorkspaces from "@/src/functions/backend/db/workspaces/grab-user-workspaces"; +import type { HarnessTool } from "../../types"; +import generateImageTool from "./generate-image-tool"; +import grabStorageTools from "./grab-storage-tools"; +import makePdfTool from "./make-pdf-tool"; +import storageToolContext from "./storage-tool-context"; + +type Params = { + bot: Pick; + user_id: number | string; + workspace_id?: number | string | null; + session_id?: number | string; + /** + * Called for a file the bot wrote with "attach" set, so the reply + * carries it. + */ + onAttach?: (params: { media_id: number; name: string }) => void; +}; + +/** + * The tools a bot uses to keep and search files (`src/harness/tools/ + * storage/`): listing, reading, writing, moving, deleting and searching + * its own directory and, in a workspace or as the coordinator, the shared + * ones, plus making images and PDFs into them. Every run gets them, so a + * chat reply, a task and a scheduled run all work with the same files. + */ +export default async function grabBotFileTools({ + bot, + user_id, + workspace_id, + session_id, + onAttach, +}: Params): Promise { + const all_workspace_ids = + bot.kind == "coordinator" + ? (await grabUserWorkspaces({ user_id })).filter((w) => !w.archived).map((w) => Number(w.id)) + : []; + + const context = storageToolContext({ + bot, + user_id, + workspace_id, + session_id, + all_workspace_ids, + onAttach, + }); + + return [...grabStorageTools(context), generateImageTool(context), makePdfTool(context)]; +} diff --git a/src/harness/tools/storage/grab-storage-tools.ts b/src/harness/tools/storage/grab-storage-tools.ts new file mode 100644 index 0000000..52612d8 --- /dev/null +++ b/src/harness/tools/storage/grab-storage-tools.ts @@ -0,0 +1,28 @@ +import type { HarnessTool } from "../../types"; +import copyFileTool from "./copy-file-tool"; +import deleteFileTool from "./delete-file-tool"; +import ingestFileTool from "./ingest-file-tool"; +import listFilesTool from "./list-files-tool"; +import moveFileTool from "./move-file-tool"; +import readFileTool from "./read-file-tool"; +import searchFilesTool from "./search-files-tool"; +import type { StorageToolContext } from "./storage-tool-context"; +import writeFileTool from "./write-file-tool"; + +/** + * The file tools a bot works with on every run, bound to its own + * directories (and, for the coordinator, every bot's and workspace's). + * `copy_file` only shows when the run reaches more than one directory. + */ +export default function grabStorageTools(context: StorageToolContext): HarnessTool[] { + return [ + listFilesTool(context), + readFileTool(context), + writeFileTool(context), + searchFilesTool(context), + moveFileTool(context), + deleteFileTool(context), + ingestFileTool(context), + ...(context.scopes.length > 1 ? [copyFileTool(context)] : []), + ]; +} diff --git a/src/harness/tools/storage/ingest-file-tool.ts b/src/harness/tools/storage/ingest-file-tool.ts new file mode 100644 index 0000000..0726d9c --- /dev/null +++ b/src/harness/tools/storage/ingest-file-tool.ts @@ -0,0 +1,44 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import ingestBotFile from "@/src/functions/backend/db/bot-files/ingest-bot-file"; +import type { HarnessTool } from "../../types"; +import type { StorageToolContext } from "./storage-tool-context"; + +type Args = { + file_id: number; + read_media?: boolean; +}; + +/** + * Ingests a file again: reads its text, chunks it and embeds it, so a + * changed or previously failed file is searchable. Images and audio are + * only read when asked for, since that costs a model call. + */ +export default function ingestFileTool(_context: StorageToolContext): HarnessTool { + return { + name: "ingest_file", + description: `Make one of your files searchable again (a course correction when search misses something you know is in it), by its ref id. Images and scans are only read when "read_media" is true.`, + schema: z.object({ + file_id: z.number().int().positive(), + read_media: z.boolean().optional(), + }), + run: async ({ args, signal }) => { + const result = await ingestBotFile({ + file_id: args.file_id, + read_media: args.read_media, + signal, + }); + + switch (result.status) { + case "ingested": + return `Ingested: ${result.chunks} passage(s), searchable now.`; + case "unavailable": + return `Its text is stored and searchable by words, but no embedding model is set, so meaning search is off.`; + case "skipped": + return `It's too large to read (over ${Math.round(AppData.IngestMaxBytes / (1024 * 1024))} MB).`; + default: + return `It has no text to read.`; + } + }, + }; +} diff --git a/src/harness/tools/storage/list-files-tool.ts b/src/harness/tools/storage/list-files-tool.ts new file mode 100644 index 0000000..8ca8ada --- /dev/null +++ b/src/harness/tools/storage/list-files-tool.ts @@ -0,0 +1,62 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import listBotFiles from "@/src/functions/backend/db/bot-files/list-bot-files"; +import type { HarnessTool } from "../../types"; +import { fileRef, formatBytes, type StorageToolContext } from "./storage-tool-context"; + +type Args = { + folder?: string; + search?: string; + recursive?: boolean; + scope?: string; +}; + +/** + * Lists a bot's files: one folder's, or everything under it, or whatever + * matches a name. Each file comes with its ref so the bot can read it. + */ +export default function listFilesTool(context: StorageToolContext): HarnessTool { + return { + name: "list_files", + description: `List the files you have saved, newest first: one folder's with "folder", everything with "recursive", or the ones whose name or path has "search". Each line has its path, size and ref (e.g. [file:12]) to read it.`, + schema: z.object({ + folder: z.string().max(AppData.BotFilePathMaxChars).optional(), + search: z.string().max(120).optional(), + recursive: z.boolean().optional(), + scope: z + .string() + .optional() + .describe(`"bot:" or "workspace:", to look in one of your directories only`), + }), + run: async ({ args }) => { + const folder = args.folder?.trim() || ""; + + const files = await listBotFiles({ + scopes: context.scopes, + folder, + recursive: args.recursive, + search: args.search, + }); + + if (!files.length) { + return folder + ? `Nothing in ${folder}.` + : `You have no files yet. Write one with write_file.`; + } + + const lines = files.map((file) => { + const where = file.scope == "workspace" ? ` (workspace ${file.owner_id})` : ""; + const ingested = file.ingested_at ? "" : " - not searchable yet"; + + return `- ${file.path} - ${formatBytes({ bytes: Number(file.size) || 0 })}${where}${ingested} ${fileRef({ file_id: file.id! })}`; + }); + + const more = + files.length >= AppData.BotFileListMaxItems + ? `\n(Showing the first ${AppData.BotFileListMaxItems}. Narrow it with "folder" or "search".)` + : ""; + + return `${files.length} file(s):\n${lines.join("\n")}${more}`; + }, + }; +} diff --git a/src/harness/tools/storage/make-pdf-tool.ts b/src/harness/tools/storage/make-pdf-tool.ts new file mode 100644 index 0000000..832fb08 --- /dev/null +++ b/src/harness/tools/storage/make-pdf-tool.ts @@ -0,0 +1,185 @@ +import { z } from "zod"; +import { PDFDocument, StandardFonts, rgb } from "pdf-lib"; +import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest"; +import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file"; +import type { HarnessTool } from "../../types"; +import { + describeScope, + fileRef, + findNamedScope, + formatBytes, + type StorageToolContext, +} from "./storage-tool-context"; + +type Args = { + path: string; + title?: string; + content: string; + attach?: boolean; + scope?: string; +}; + +const PageWidth = 595.28; +const PageHeight = 841.89; +const Margin = 56; +const BodySize = 11; +const Leading = 15; + +/** + * Turns text into a PDF (headings, paragraphs and bullets) and saves it in + * the bot's files. Made in the app with `pdf-lib`, so it needs no + * computer. + */ +export default function makePdfTool(context: StorageToolContext): HarnessTool { + const own = context.scopes[0]; + + return { + name: "make_pdf", + description: `Make a PDF from text (markdown: "# " headings, "- " bullets, blank lines between paragraphs) and save it in your files. Send it to the user with "attach".`, + schema: z.object({ + path: z.string().min(1).max(300).describe(`e.g. "reports/summary.pdf"`), + title: z.string().max(200).optional(), + content: z.string().min(1).max(100_000), + attach: z.boolean().optional(), + scope: z.string().optional(), + }), + run: async ({ args }) => { + const named = args.scope + ? findNamedScope({ scopes: context.scopes, name: args.scope }) + : own; + + if (!named) { + return `You can't save there. Your directories: ${context.scopes + .map((s) => `${s.scope}:${s.owner_id}`) + .join(", ")}.`; + } + + const bytes = await renderPdf({ title: args.title, content: args.content }); + const path = args.path.toLowerCase().endsWith(".pdf") ? args.path : `${args.path}.pdf`; + + const file = await recordBotFile({ + bot_id: context.bot_id, + scope: named.scope, + owner_id: named.owner_id, + path, + data: bytes, + purpose: "pdf", + source: "tool", + mime_type: "application/pdf", + user_id: context.user_id, + }); + + await enqueueFileIngest({ file_id: file.id! }); + + if (args.attach && file.media_id) { + context.onAttach?.({ media_id: Number(file.media_id), name: file.name || "" }); + } + + return `Made ${file.path} (${formatBytes({ bytes: bytes.byteLength })}) in ${describeScope(named)}. ${fileRef({ file_id: file.id! })}`; + }, + }; +} + +/** + * Lays the text out on A4 pages. Fonts are the PDF standard ones, so + * nothing has to be embedded. + */ +async function renderPdf({ title, content }: { title?: string; content: string }) { + const pdf = await PDFDocument.create(); + const font = await pdf.embedFont(StandardFonts.Helvetica); + const bold = await pdf.embedFont(StandardFonts.HelveticaBold); + + let page = pdf.addPage([PageWidth, PageHeight]); + let y = PageHeight - Margin; + + const newPage = () => { + page = pdf.addPage([PageWidth, PageHeight]); + y = PageHeight - Margin; + }; + + const write = ({ + text, + size, + use_bold, + indent = 0, + spacing = 0, + }: { + text: string; + size: number; + use_bold?: boolean; + indent?: number; + spacing?: number; + }) => { + const used = use_bold ? bold : font; + const width = PageWidth - Margin * 2 - indent; + const words = text.split(/\s+/); + let line = ""; + + const emit = (value: string) => { + if (y - size < Margin) newPage(); + + page.drawText(value, { + x: Margin + indent, + y: y - size, + size, + font: used, + color: rgb(0.09, 0.09, 0.11), + }); + + y -= size + 4; + }; + + for (const word of words) { + const candidate = line ? `${line} ${word}` : word; + + if (used.widthOfTextAtSize(candidate, size) > width && line) { + emit(line); + line = word; + } else { + line = candidate; + } + } + + if (line) emit(line); + + y -= spacing; + }; + + if (title) { + write({ text: title, size: 20, use_bold: true, spacing: 10 }); + } + + for (const block of content.replace(/\r\n/g, "\n").split(/\n{2,}/)) { + const trimmed = block.trim(); + + if (!trimmed) continue; + + const heading = trimmed.match(/^(#{1,6})\s+(.*)$/); + + if (heading) { + const level = heading[1]!.length; + + write({ + text: heading[2]!, + size: level == 1 ? 16 : level == 2 ? 14 : 12, + use_bold: true, + spacing: 8, + }); + continue; + } + + for (const line of trimmed.split("\n")) { + const bullet = line.match(/^\s*[-*+]\s+(.*)$/); + + if (bullet) { + write({ text: `- ${bullet[1]}`, size: BodySize, indent: 12 }); + } else { + write({ text: line.trim(), size: BodySize }); + } + } + + y -= Leading / 2; + } + + return new Uint8Array(await pdf.save()); +} diff --git a/src/harness/tools/storage/move-file-tool.ts b/src/harness/tools/storage/move-file-tool.ts new file mode 100644 index 0000000..d482a37 --- /dev/null +++ b/src/harness/tools/storage/move-file-tool.ts @@ -0,0 +1,37 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import moveBotFile from "@/src/functions/backend/db/bot-files/move-bot-file"; +import type { HarnessTool } from "../../types"; +import { fileRef, type StorageToolContext } from "./storage-tool-context"; + +type Args = { + file_id: number; + path: string; +}; + +/** + * Renames or moves a file into another folder, inside its own directory. + */ +export default function moveFileTool(context: StorageToolContext): HarnessTool { + return { + name: "move_file", + description: `Rename a file or move it into another folder, by its ref id. The new path stays inside the same directory (yours, or the workspace's).`, + schema: z.object({ + file_id: z.number().int().positive(), + path: z + .string() + .min(1) + .max(AppData.BotFilePathMaxChars) + .describe(`The new path, e.g. "archive/q3.md"`), + }), + run: async ({ args }) => { + const file = await moveBotFile({ + scopes: context.scopes, + file_id: args.file_id, + path: args.path, + }); + + return `Moved to ${file.path}. ${fileRef({ file_id: file.id! })}`; + }, + }; +} diff --git a/src/harness/tools/storage/read-file-tool.ts b/src/harness/tools/storage/read-file-tool.ts new file mode 100644 index 0000000..780f07d --- /dev/null +++ b/src/harness/tools/storage/read-file-tool.ts @@ -0,0 +1,95 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import readMedia from "@/src/harness/media/read-media"; +import type { HarnessTool } from "../../types"; +import { fileRef, type StorageToolContext } from "./storage-tool-context"; + +type Args = { + file_id?: number; + path?: string; + offset?: number; + scope?: string; +}; + +const TextExtensions = [ + "txt", + "md", + "markdown", + "csv", + "json", + "html", + "htm", + "xml", + "yml", + "yaml", + "log", + "tsv", +]; + +/** + * Reads one of the bot's files: text as it is (a window at a time when it + * is long), an image or audio through the media harness, and a PDF or DOCX + * through the text extractor. + */ +export default function readFileTool(context: StorageToolContext): HarnessTool { + return { + name: "read_file", + description: `Read a file you saved, by its ref id or its path. Text comes back in a window of about ${Math.round(AppData.BotFileReadMaxChars / 1000)}k characters; use "offset" for the next part. Images and audio are read for you, and PDFs and Word documents are turned into text.`, + schema: z.object({ + file_id: z.number().int().positive().optional(), + path: z.string().max(AppData.BotFilePathMaxChars).optional(), + offset: z.number().int().min(0).optional(), + scope: z.string().optional().describe(`"bot:" or "workspace:", with "path"`), + }), + run: async ({ args }) => { + const [scope_name, owner] = (args.scope || "").split(":"); + + const file = await grabBotFile({ + scopes: context.scopes, + file_id: args.file_id, + scope: scope_name, + owner_id: owner, + path: args.path, + }); + + if (!file?.id || !file.media?.media_write_path) { + return `No file matches that. Use list_files to see what you have.`; + } + + const { DATA_DIR } = grabDirNames(); + const absolute = `${DATA_DIR}/${file.media.media_write_path}`; + const ext = (file.name || "").split(".").pop()?.toLowerCase() || ""; + + if (TextExtensions.includes(ext)) { + const text = await Bun.file(absolute).text(); + const offset = args.offset || 0; + const window = text.slice(offset, offset + AppData.BotFileReadMaxChars); + const end = offset + window.length; + + return [ + `${file.path} ${fileRef({ file_id: file.id })}`, + window, + end < text.length + ? `\n[Read ${offset}-${end} of ${text.length} characters. Call read_file with offset ${end} for the rest.]` + : "", + ] + .filter(Boolean) + .join("\n\n"); + } + + const bytes = new Uint8Array(await Bun.file(absolute).arrayBuffer()); + + const text = await readMedia({ + data: bytes, + media_type: file.mime_type || "application/octet-stream", + name: file.name || undefined, + bot_id: context.bot_id, + workspace_id: context.workspace_id, + }); + + return `${file.path} ${fileRef({ file_id: file.id })}\n\n${text.slice(0, AppData.BotFileReadMaxChars)}`; + }, + }; +} diff --git a/src/harness/tools/storage/search-files-tool.ts b/src/harness/tools/storage/search-files-tool.ts new file mode 100644 index 0000000..083cc81 --- /dev/null +++ b/src/harness/tools/storage/search-files-tool.ts @@ -0,0 +1,49 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import searchBotFiles from "@/src/functions/backend/db/bot-files/search-bot-files"; +import type { HarnessTool } from "../../types"; +import { fileRef, type StorageToolContext } from "./storage-tool-context"; + +type Args = { + query: string; + file_id?: number; + limit?: number; +}; + +/** + * Searches the bot's saved files by meaning (through the embedding + * vectors), falling back to words, and returns the passages with their + * file's ref so the bot can read more. + */ +export default function searchFilesTool(context: StorageToolContext): HarnessTool { + return { + name: "search_files", + description: `Search the files you have saved (documents the user gave you and what you wrote) and get the passages that answer "query", with their file's ref. Search before saying you don't know something that may be in a file, and before writing a document from memory.`, + schema: z.object({ + query: z.string().min(1).max(500), + file_id: z.number().int().positive().optional().describe(`Only this file's passages`), + limit: z.number().int().min(1).max(20).optional(), + }), + run: async ({ args, signal }) => { + const result = await searchBotFiles({ + scopes: context.scopes, + query: args.query, + file_id: args.file_id, + limit: args.limit || AppData.RagTopChunks, + signal, + bot_id: context.bot_id, + workspace_id: context.workspace_id, + }); + + if (!result.hits.length) return result.note; + + const passages = result.hits.map((hit, index) => { + const where = hit.scope == "workspace" ? ` (workspace ${hit.scope})` : ""; + + return `## ${index + 1}. ${hit.name}${where} ${fileRef({ file_id: hit.file_id })}\n${hit.text.trim()}`; + }); + + return `${result.note}\n\n${passages.join("\n\n")}`; + }, + }; +} diff --git a/src/harness/tools/storage/storage-tool-context.ts b/src/harness/tools/storage/storage-tool-context.ts new file mode 100644 index 0000000..b413d1b --- /dev/null +++ b/src/harness/tools/storage/storage-tool-context.ts @@ -0,0 +1,110 @@ +import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db"; +import grabStorageScopes, { type StorageScope } from "@/src/utils/storage-access"; + +export type StorageToolContext = { + /** + * The user the run is for. + */ + user_id: number | string; + bot_id: number | string; + /** + * The directories the run may touch, itself first. + */ + scopes: StorageScope[]; + /** + * The session being answered, when a file can ride on its reply. + */ + session_id?: number | string; + /** + * The workspace the run is in, for embeddings. + */ + workspace_id?: number | string | null; + /** + * Called with a file's media id when a tool wrote it and the bot asked + * for it to be attached to the reply. + */ + onAttach?: (params: { media_id: number; name: string }) => void; +}; + +type Params = { + bot: Pick; + user_id: number | string; + workspace_id?: number | string | null; + session_id?: number | string; + /** + * Every workspace, for the coordinator (it reaches them all). + */ + all_workspace_ids?: number[]; + onAttach?: StorageToolContext["onAttach"]; +}; + +/** + * The directories one run may touch, from the bot it's for. The + * coordinator reaches every bot's and workspace's files; a worker its own, + * plus its workspace's shared one while it works there. + */ +export default function storageToolContext({ + bot, + user_id, + workspace_id, + session_id, + all_workspace_ids, + onAttach, +}: Params): StorageToolContext { + return { + user_id, + bot_id: bot.id || 0, + scopes: grabStorageScopes({ + bot_id: bot.id || 0, + bot_kind: bot.kind, + workspace_id, + all_workspace_ids, + }), + session_id, + workspace_id, + onAttach, + }; +} + +/** + * A ref the chat renders as a chip, used in tool results so a file the bot + * names links to it. + */ +export function fileRef({ file_id }: { file_id: number | string }) { + return `[file:${file_id}]`; +} + +/** + * A human size: "12.4 KB". + */ +export function formatBytes({ bytes }: { bytes: number }) { + if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`; + + return `${bytes} B`; +} + +/** + * The scope label a tool result shows: "your files" or "the workspace's + * shared files". + */ +export function describeScope({ scope, owner_id }: StorageScope) { + return scope == "workspace" ? `the workspace's shared files (#${owner_id})` : "your files"; +} + +/** + * Finds a scope the bot named, e.g. "workspace:3" or "bot:2". + */ +export function findNamedScope({ + scopes, + name, +}: { + scopes: StorageScope[]; + name?: string | null; +}) { + if (!name?.trim()) return undefined; + + const [scope, owner] = name.trim().split(":"); + + return scopes.find((s) => s.scope == scope && String(s.owner_id) == owner); +} diff --git a/src/harness/tools/storage/write-file-tool.ts b/src/harness/tools/storage/write-file-tool.ts new file mode 100644 index 0000000..1bac708 --- /dev/null +++ b/src/harness/tools/storage/write-file-tool.ts @@ -0,0 +1,89 @@ +import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; +import type { BotFilePurpose } from "@/src/dict/bot-file-dict"; +import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest"; +import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file"; +import type { HarnessTool } from "../../types"; +import { + describeScope, + fileRef, + findNamedScope, + formatBytes, + type StorageToolContext, +} from "./storage-tool-context"; + +type Args = { + path: string; + content: string; + scope?: string; + attach?: boolean; + purpose?: BotFilePurpose; +}; + +/** + * Writes a text file (markdown, CSV, JSON, code, plain text) into one of + * the bot's directories, making folders as needed. The file stays for + * later runs and becomes searchable. + */ +export default function writeFileTool(context: StorageToolContext): HarnessTool { + const own = context.scopes[0]; + + return { + name: "write_file", + description: `Save a text file for later: markdown, CSV, JSON, code or plain text. Give it a path like "reports/q3.md"; folders are made as needed. Files belong to ${ + own?.scope == "workspace" ? "the workspace you're working in" : "you" + } and stay between runs: write a file before telling the user something is saved, and search_files to find one you wrote before. It is readable and searchable at once.${ + context.scopes.length > 1 + ? ` "scope" picks a directory: ${context.scopes.map((s) => `${s.scope}:${s.owner_id}`).join(", ")}.` + : "" + }`, + schema: z.object({ + path: z + .string() + .min(1) + .max(AppData.BotFilePathMaxChars) + .describe(`Where it goes, e.g. "notes.md" or "reports/q3.md"`), + content: z.string().max(AppData.BotFileWriteMaxChars), + scope: z.string().optional().describe(`"bot:" or "workspace:"`), + attach: z + .boolean() + .optional() + .describe(`Also send it to the user with this reply, as an attachment`), + purpose: z + .enum(["writing", "artifact"]) + .optional() + .describe(`"writing" for notes and text, "artifact" for a finished file the user asked for`), + }), + run: async ({ args }) => { + const named = args.scope ? findNamedScope({ scopes: context.scopes, name: args.scope }) : own; + + if (!named) { + return `You can't write there. Your directories: ${context.scopes + .map((s) => `${s.scope}:${s.owner_id}`) + .join(", ")}.`; + } + + const file = await recordBotFile({ + bot_id: context.bot_id, + scope: named.scope, + owner_id: named.owner_id, + path: args.path, + data: args.content, + purpose: args.purpose || "writing", + source: "bot-write", + user_id: context.user_id, + }); + + await enqueueFileIngest({ file_id: file.id! }); + + if (args.attach && file.media_id) { + context.onAttach?.({ media_id: Number(file.media_id), name: file.name || "" }); + } + + return [ + `Saved ${file.path} (${formatBytes({ bytes: Number(file.size) || 0 })}) in ${describeScope(named)}. ${fileRef({ file_id: file.id! })}`, + "Tell the user the file's name and what's in it.", + ].join("\n\n"); + }, + }; +} diff --git a/src/harness/types.ts b/src/harness/types.ts index 09c1744..b325d33 100644 --- a/src/harness/types.ts +++ b/src/harness/types.ts @@ -203,6 +203,25 @@ export type HarnessImageOutput = { media_type: string; }; +export type HarnessEmbedParams = { + candidate: HarnessCandidate; + /** + * The texts to embed, in order. Their vectors come back in the same + * order, at whatever width the model produces (the caller fits them + * to `AppData.EmbeddingDimensions`). + */ + input: string[]; + signal: AbortSignal; +}; + +export type HarnessEmbedResult = { + vectors: number[][]; + /** + * The provider's usage, when it reports it. + */ + usage?: HarnessUsage; +}; + /** * How the harness talks to one kind of provider. Every call takes the * candidate it runs on and an abort signal. @@ -214,6 +233,7 @@ export type HarnessAdapter = { generateImage: ( params: HarnessImageParams, ) => Promise<{ images: HarnessImageOutput[] }>; + embed: (params: HarnessEmbedParams) => Promise; listModels: (params: { provider: HarnessProviderConfig; signal?: AbortSignal; diff --git a/src/layouts/bots/(partials)/bot-form.tsx b/src/layouts/bots/(partials)/bot-form.tsx index 3c93217..0a7cca2 100644 --- a/src/layouts/bots/(partials)/bot-form.tsx +++ b/src/layouts/bots/(partials)/bot-form.tsx @@ -47,7 +47,9 @@ export default function BotForm({ bot, onCancel, onSaved }: Props) { const [name, setName] = useState(bot?.name || ""); const [role, setRole] = useState(bot?.role || ""); const [hue, setHue] = useState( - () => (bot?.hue as BotHue) || pickFreeHue({ taken: crew.map((b) => b.hue) }), + () => + (bot?.hue as BotHue) || + pickFreeHue({ taken: crew.map((b) => b.hue) }), ); const [system_prompt, setSystemPrompt] = useState(bot?.system_prompt || ""); const [join, setJoin] = useState(!bot && Boolean(workspace?.id)); @@ -57,7 +59,14 @@ export default function BotForm({ bot, onCancel, onSaved }: Props) { const submit = async () => { if (loading) return; - if (!window.confirm(bot?.id ? `Save changes to ${bot.name || "this bot"}?` : `Add ${name || "this bot"} to your crew?`)) return; + if ( + !window.confirm( + bot?.id + ? `Save changes to ${bot.name || "this bot"}?` + : `Add ${name || "this bot"} to your crew?`, + ) + ) + return; setLoading(true); setError(undefined); @@ -66,13 +75,21 @@ export default function BotForm({ bot, onCancel, onSaved }: Props) { const res = bot?.id ? await postAdminApi({ path: "bots/update-bot", - body: { bot_id: bot.id, name: name || bot.name, role, hue, system_prompt }, + body: { + bot_id: bot.id, + name: name || bot.name, + role, + hue, + system_prompt, + }, }) : await postAdminApi<{ bot_id: number }>({ path: "bots/create-bot", body: { name, role, hue, system_prompt, workspace_id }, }); - const bot_id = bot?.id || (res.singleRes as { bot_id?: number } | undefined)?.bot_id; + const bot_id = + bot?.id || + (res.singleRes as { bot_id?: number } | undefined)?.bot_id; if (!res.success || !bot_id) { setLoading(false); @@ -91,7 +108,10 @@ export default function BotForm({ bot, onCancel, onSaved }: Props) { if (!image_res?.success) { setLoading(false); - setError(image_res?.msg || "The bot is saved, but its picture isn't. Try another image."); + setError( + image_res?.msg || + "The bot is saved, but its picture isn't. Try another image.", + ); return; } @@ -162,12 +182,16 @@ export default function BotForm({ bot, onCancel, onSaved }: Props) { info="One line, shown under its name." onChange={(e) => setRole(e.target.value)} /> - + )} ( + pageProps?.bot_files || undefined, + ); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(); + + const load = useCallback( + async (params?: { folder?: string; search?: string }) => { + setLoading(true); + setError(undefined); + + const query = new URLSearchParams(); + + if (workspace_id) query.set("workspace_id", String(workspace_id)); + else query.set("bot_id", String(bot_id)); + + if (params?.folder) query.set("folder", params.folder); + if (params?.search) query.set("search", params.search); + + const res = await fetchApi<{}, APIResponseObject>( + `/api/admin/bots/files?${query.toString()}`, + { method: "GET" }, + ).catch(() => null); + + setLoading(false); + + if (!res?.success || !res.singleRes) { + setError(res?.msg || `Couldn't load the files.`); + return; + } + + setData(res.singleRes); + }, + [bot_id, workspace_id], + ); + + useEffect(() => { + if (enabled) load().catch(() => {}); + }, [enabled, load]); + + return { data, loading, error, reload: load, setData }; +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/bot-files-modal.tsx b/src/pages/admin/bots/[bot_id]/(partials)/bot-files-modal.tsx new file mode 100644 index 0000000..8976ff2 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/bot-files-modal.tsx @@ -0,0 +1,173 @@ +import { useContext, useEffect, useState } from "react"; +import Modal from "@moduletrace/twui/elements/Modal"; +import H2 from "@moduletrace/twui/layout/H2"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import Input from "@moduletrace/twui/form/Input"; +import { FolderOpen } from "lucide-react"; +import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi"; +import type { APIResponseObject } from "@moduletrace/bunext/types"; +import type { BUN_SQLITE_COBBLE_WORKSPACES } from "@/db/types/db"; +import type { BotWithLastMessage } from "@/src/types"; +import HeaderIconButton from "@/src/components/general/header-icon-button"; +import { AppContext } from "@/src/pages/__root"; +import canManageCrew from "@/src/utils/rbac/can-manage-crew"; +import useBotFiles from "../(hooks)/use-bot-files"; +import FileRow from "./file-row"; +import HeaderMenuItem from "./header-menu-item"; + +type Props = { + bot: BotWithLastMessage; + workspace?: BUN_SQLITE_COBBLE_WORKSPACES | null; + /** + * A row in the conversation header's menu instead of an icon beside it. + */ + menu?: boolean; + on_open?: () => void; +}; + +/** + * The files a conversation works with: a bot's own folder, or the + * workspace's shared one. Each can be opened, read again so it can be + * searched, or deleted (a crew manager only). The list loads when the + * modal opens, and search goes through the same endpoint, so the page + * props stay small. + */ +export default function BotFilesModal({ bot, workspace, menu, on_open }: Props) { + const { pageProps } = useContext(AppContext); + const [open, setOpen] = useState(false); + const [search, setSearch] = useState(""); + const [busy_id, setBusyId] = useState(); + const [status, setStatus] = useState(); + const { data, loading, error, reload } = useBotFiles({ + bot_id: Number(bot.id), + workspace_id: workspace?.id ? Number(workspace.id) : null, + enabled: open, + }); + + const can_manage = canManageCrew({ user_types: pageProps?.user_types }); + const name = bot.name || "This bot"; + const title = workspace?.id ? `${workspace.name || "Workspace"} files` : `${name}'s files`; + + useEffect(() => { + if (open) on_open?.(); + }, [open]); + + const run = async ({ + url, + payload, + file_id, + done, + }: { + url: string; + payload: Record; + file_id: number; + done: string; + }) => { + setBusyId(file_id); + setStatus(undefined); + + const res = await fetchApi<{}, APIResponseObject>(url, { + method: "POST", + body: payload, + }).catch(() => null); + + setBusyId(undefined); + setStatus(res?.success ? done : res?.msg || `That didn't work.`); + + if (res?.success) await reload({ search }); + }; + + const files = data?.files || []; + const usage = data?.usage; + + return ( + + ) : ( + + ) + } + targetWrapperProps={{ className: menu ? "w-full" : "shrink-0" }} + open={open} + setOpen={setOpen} + className="max-w-xl bg-paper p-6" + > + +

{title}

+ + {workspace?.id + ? `Files every bot here can read and search. They're kept with the app, not on a computer.` + : `What ${name} has saved: documents you sent, files it wrote and images it made. They're kept with the app, not on a computer.${can_manage ? " Deleting one removes it for good." : ""}`} + + {usage ? ( + + {`${usage.files} file(s), ${(usage.bytes / (1024 * 1024)).toFixed(1)} MB of ${ + data?.quota_mb || 0 + } MB used in this folder.`} + + ) : null} + + + + setSearch(event.target.value)} + onKeyDown={(event) => { + if (event.key == "Enter") reload({ search }).catch(() => {}); + }} + clearInputProps={{ onClick: () => reload({}).catch(() => {}) }} + /> + + + + {files.length ? ( + files.map((file) => ( + + run({ + url: `/api/admin/bots/ingest-file`, + payload: { file_id: file.id }, + file_id: file.id, + done: `${file.name} will be read again shortly.`, + }) + } + onDelete={() => + run({ + url: `/api/admin/bots/delete-file`, + payload: { file_id: file.id }, + file_id: file.id, + done: `${file.name} is deleted.`, + }) + } + /> + )) + ) : ( + + {loading + ? "Loading files..." + : error || (search ? `Nothing matches "${search}".` : "No files yet.")} + + )} + + + {status ? ( + + {status} + + ) : null} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/conversation-header-menu.tsx b/src/pages/admin/bots/[bot_id]/(partials)/conversation-header-menu.tsx index b36cffa..a3626f2 100644 --- a/src/pages/admin/bots/[bot_id]/(partials)/conversation-header-menu.tsx +++ b/src/pages/admin/bots/[bot_id]/(partials)/conversation-header-menu.tsx @@ -2,7 +2,7 @@ import { useContext } from "react"; import Paper from "@moduletrace/twui/elements/Paper"; import Stack from "@moduletrace/twui/layout/Stack"; import Divider from "@moduletrace/twui/layout/Divider"; -import { History, Settings } from "lucide-react"; +import { FolderOpen, History, Settings } from "lucide-react"; import type { BUN_SQLITE_COBBLE_WORKSPACES } from "@/db/types/db"; import type { BotWithLastMessage, HarnessOverview } from "@/src/types"; import BotSessionsModal from "@/src/components/general/bot-sessions-modal"; @@ -10,6 +10,7 @@ import { AppContext } from "@/src/pages/__root"; import canManageCrew from "@/src/utils/rbac/can-manage-crew"; import isPassThroughUser from "@/src/utils/rbac/is-pass-through-user"; import BotComputersModal from "./bot-computers-modal"; +import BotFilesModal from "./bot-files-modal"; import BotModelsModal from "./bot-models-modal"; import HeaderMenuItem from "./header-menu-item"; import NewSessionButton from "./new-session-button"; @@ -94,6 +95,12 @@ export default function ConversationHeaderMenu({ workspace_id={workspace?.id || null} on_open={on_open} /> + it.id == file_id); + + if (!file) { + return {`file ${file_id}`}; + } + + return ( + + + {file.name} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/file-row.tsx b/src/pages/admin/bots/[bot_id]/(partials)/file-row.tsx new file mode 100644 index 0000000..ed66a76 --- /dev/null +++ b/src/pages/admin/bots/[bot_id]/(partials)/file-row.tsx @@ -0,0 +1,89 @@ +import Link from "@moduletrace/twui/layout/Link"; +import Row from "@moduletrace/twui/layout/Row"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import Button from "@moduletrace/twui/layout/Button"; +import { RefreshCw, Trash2 } from "lucide-react"; +import type { MediaType, PublicBotFile } from "@/src/types"; +import relativeTime from "@/src/utils/relative-time"; +import AttachmentIcon from "./attachment-icon"; + +type Props = { + file: PublicBotFile; + /** + * A crew manager may delete and re-ingest. + */ + can_manage: boolean; + busy?: boolean; + onIngest: () => void; + onDelete: () => void; +}; + +/** + * One file in the files modal: its type, path, size and age, and what can + * be done with it. A file that isn't searchable says why. + */ +export default function FileRow({ file, can_manage, busy, onIngest, onDelete }: Props) { + return ( + + + + + + + {file.path || file.name} + + + {[ + file.scope == "workspace" ? "Workspace" : "Bot files", + `${Math.max(1, Math.round(file.size / 1024))} KB`, + file.created_at ? relativeTime({ at: file.created_at }) : "", + file.ingest_error + ? file.ingest_error + : file.ingested_at + ? "searchable" + : "not searchable yet", + ] + .filter(Boolean) + .join(" ยท ")} + + + {can_manage ? ( + + + + + ) : null} + + ); +} diff --git a/src/pages/admin/bots/[bot_id]/(partials)/markdown-link.tsx b/src/pages/admin/bots/[bot_id]/(partials)/markdown-link.tsx index 7ce8911..b256478 100644 --- a/src/pages/admin/bots/[bot_id]/(partials)/markdown-link.tsx +++ b/src/pages/admin/bots/[bot_id]/(partials)/markdown-link.tsx @@ -4,11 +4,13 @@ import { ComputerViewModes, type ComputerViewMode } from "@/src/dict/computer-di import { BotRefHrefPrefix, ComputerRefHrefPrefix, + FileRefHrefPrefix, TakeOverRefHrefPrefix, WorkspaceRefHrefPrefix, } from "@/src/utils/link-computer-refs"; import BotRefChip from "./bot-ref-chip"; import ComputerRefChip from "./computer-ref-chip"; +import FileRefChip from "./file-ref-chip"; import TakeOverChip from "./take-over-chip"; import WorkspaceRefChip from "./workspace-ref-chip"; @@ -16,7 +18,8 @@ import WorkspaceRefChip from "./workspace-ref-chip"; * A link in a bot message. A computer the bot named (`linkComputerRefs`) * becomes its switch button, a take-over request a button that opens the * computer, a bot it named a link to that bot's conversation, a workspace - * a link to that workspace; anything else opens in a new tab. + * a link to that workspace, a file it saved a chip that opens it; anything + * else opens in a new tab. */ export default function MarkdownLink({ href, children }: ComponentProps<"a">) { if (href?.startsWith(ComputerRefHrefPrefix)) { @@ -52,6 +55,12 @@ export default function MarkdownLink({ href, children }: ComponentProps<"a">) { if (workspace_id) return ; } + if (href?.startsWith(FileRefHrefPrefix)) { + const file_id = Number(href.slice(FileRefHrefPrefix.length)); + + if (file_id) return ; + } + return ( >[0]; @@ -40,6 +41,7 @@ export default async function grabSettingsServerProps({ req }: Params) { props: { person: sanitizeUserBio({ user: user_res.singleRes }), has_password, + storage_quota: await grabStorageQuotaOverview().catch(() => null), }, }; } diff --git a/src/pages/admin/settings/general/(partials)/storage-quota-form.tsx b/src/pages/admin/settings/general/(partials)/storage-quota-form.tsx new file mode 100644 index 0000000..dbcb1b0 --- /dev/null +++ b/src/pages/admin/settings/general/(partials)/storage-quota-form.tsx @@ -0,0 +1,89 @@ +import { useState } from "react"; +import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi"; +import type { APIResponseObject } from "@moduletrace/bunext/types"; +import Form from "@moduletrace/twui/form/Form"; +import Input from "@moduletrace/twui/form/Input"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import type { StorageQuotaOverview } from "@/src/functions/backend/db/app-settings/grab-storage-quota-overview"; +import SettingsSaveBar from "../../(partials)/settings-save-bar"; + +type Props = { + storage?: StorageQuotaOverview | null; +}; + +/** + * The per-folder storage quota for what bots save (a bot's own folder and + * each workspace's shared one). Saved on its own, without a reload. + */ +export default function StorageQuotaForm({ storage }: Props) { + const [overview, setOverview] = useState(storage || undefined); + const [dirty, setDirty] = useState(false); + const [value, setValue] = useState(String(storage?.quota_mb ?? "")); + const [loading, setLoading] = useState(false); + const [status, setStatus] = useState<{ msg?: string; error?: boolean; success?: boolean }>(); + + const save = async () => { + setLoading(true); + setStatus(undefined); + + const res = await fetchApi<{}, APIResponseObject>( + "/api/admin/settings/storage-quota", + { method: "POST", body: { quota_mb: Number(value) || 0 } }, + ).catch(() => null); + + setLoading(false); + + if (!res?.success || !res.singleRes) { + setStatus({ error: true, msg: res?.msg || `Couldn't save the quota.` }); + return; + } + + setOverview(res.singleRes); + setValue(String(res.singleRes.quota_mb)); + setDirty(false); + setStatus({ success: true, msg: `Saved: ${res.singleRes.quota_mb} MB per folder.` }); + }; + + return ( +
{ + event.preventDefault(); + save().catch(() => {}); + }} + className="w-full gap-3! items-stretch" + > + + { + setValue(event.target.value); + setDirty(true); + setStatus(undefined); + }} + info="A bot's own folder and each workspace's folder each get this much. A write past it is refused, so the bot clears space first." + /> + {overview ? ( + + {`${overview.total_files} file(s) in ${overview.folders} folder(s), ${(overview.total_bytes / (1024 * 1024)).toFixed(1)} MB in total.`} + + ) : null} + + { + setValue(String(overview?.quota_mb ?? "")); + setDirty(false); + setStatus(undefined); + }} + /> + + ); +} diff --git a/src/pages/admin/settings/general/(sections)/storage-section.tsx b/src/pages/admin/settings/general/(sections)/storage-section.tsx new file mode 100644 index 0000000..5809e4a --- /dev/null +++ b/src/pages/admin/settings/general/(sections)/storage-section.tsx @@ -0,0 +1,25 @@ +import { useContext } from "react"; +import { AppContext } from "@/src/pages/__root"; +import canManageCrew from "@/src/utils/rbac/can-manage-crew"; +import SettingsGroup from "../../(partials)/settings-group"; +import StorageQuotaForm from "../(partials)/storage-quota-form"; + +/** + * What bots keep on disk: the quota each of their folders gets. Only a + * crew manager sees it, since it's an install-wide limit. + */ +export default function StorageSection() { + const { pageProps } = useContext(AppContext); + + if (!canManageCrew({ user_types: pageProps?.user_types })) return null; + + return ( + + + + ); +} diff --git a/src/pages/admin/settings/general/index.tsx b/src/pages/admin/settings/general/index.tsx index 382a867..b21ba9d 100644 --- a/src/pages/admin/settings/general/index.tsx +++ b/src/pages/admin/settings/general/index.tsx @@ -1,4 +1,5 @@ import AppearanceSection from "./(sections)/appearance-section"; +import StorageSection from "./(sections)/storage-section"; import SettingsShell from "../(partials)/settings-shell"; import settingsPageMeta from "../(functions)/settings-page-meta"; @@ -9,6 +10,7 @@ export default function AdminSettingsGeneralPage() { description="How the app looks and behaves." > + ); } diff --git a/src/pages/api/admin/bots/delete-file.ts b/src/pages/api/admin/bots/delete-file.ts new file mode 100644 index 0000000..518ef6f --- /dev/null +++ b/src/pages/api/admin/bots/delete-file.ts @@ -0,0 +1,94 @@ +import userAuth from "@/src/functions/backend/auth/user-auth"; +import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file"; +import deleteBotFile from "@/src/functions/backend/db/bot-files/delete-bot-file"; +import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot"; +import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace"; +import logEvent, { grabRequestIP } from "@/src/functions/backend/logger"; +import { DeleteBotFileRequestSchema } from "@/src/schemas/bot-file-schema"; +import AppError from "@/src/utils/app-error"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import canManageCrew from "@/src/utils/rbac/can-manage-crew"; +import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import type { StorageScope } from "@/src/utils/storage-access"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * Deletes one of a conversation's files. A crew manager only, and never a + * read-only user. + */ +export const handler: BunextAPIRouteHandler = async ({ req, body }) => { + try { + if (req.method != "POST") throw new AppError(`Method not allowed`); + + const { user, user_types } = await userAuth({ req }); + + if (!user?.id) return { success: false, msg: `Unauthorized`, logoutUser: true }; + if (isReadOnlyUser({ user_types: user_types || [] })) { + throw new AppError(`This user can only read records.`); + } + if (!canManageCrew({ user_types: user_types || [] })) { + throw new AppError(`Only a crew manager can delete files.`); + } + + const { file_id } = parseWithSchema({ schema: DeleteBotFileRequestSchema, data: body }); + + const file = await grabBotFile({ + scopes: await reachableScopes({ user_id: user.id, file_id }), + file_id, + }); + + if (!file?.id) throw new AppError(`That file isn't here.`); + + await deleteBotFile({ + scopes: [{ scope: file.scope as any, owner_id: Number(file.owner_id) }], + file_id: file.id, + }); + + await logEvent({ + title: "Bot file deleted", + category: "system", + user_id: user.id, + plain_text: `User #${user.id} deleted ${file.path} from ${file.scope} ${file.owner_id}.`, + metadata: { file_id: file.id }, + ip_address: grabRequestIP(req), + }); + + return { success: true, msg: `${file.name || "The file"} is deleted.` }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; + +/** + * The directories the user reaches: every worker's for a crew manager, + * the coordinator's for the super admin, and every workspace it's in. + */ +async function reachableScopes({ + user_id, + file_id, +}: { + user_id: number; + file_id: number; +}): Promise { + const { default: BunSQLite } = await import("@moduletrace/bun-sqlite"); + + const res = await BunSQLite.sql<{ scope: string; owner_id: number }>({ + sql: `SELECT scope, owner_id FROM bot_files WHERE id = ? LIMIT 1`, + values: [file_id], + }); + + const file = res.payload?.[0]; + + if (!file?.owner_id) return []; + + if (file.scope == "workspace") { + const workspace = await grabUserWorkspace({ workspace_id: file.owner_id, user_id }); + + return workspace?.id ? [{ scope: "workspace", owner_id: Number(file.owner_id) }] : []; + } + + const bot = await grabUserBot({ bot_id: file.owner_id, user_id }); + + return bot?.id ? [{ scope: "bot", owner_id: Number(file.owner_id) }] : []; +} diff --git a/src/pages/api/admin/bots/files.ts b/src/pages/api/admin/bots/files.ts new file mode 100644 index 0000000..87545b3 --- /dev/null +++ b/src/pages/api/admin/bots/files.ts @@ -0,0 +1,131 @@ +import userAuth from "@/src/functions/backend/auth/user-auth"; +import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file"; +import grabBotFileQuotaBytes, { + grabBotFileQuotaMb, +} from "@/src/functions/backend/db/bot-files/grab-bot-file-quota-bytes"; +import grabStorageUsage from "@/src/functions/backend/db/bot-files/grab-storage-usage"; +import listBotFiles, { listBotFolders } from "@/src/functions/backend/db/bot-files/list-bot-files"; +import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot"; +import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace"; +import grabMediaURL from "@/src/functions/backend/media/grab-media-url"; +import type { BotFileScopesPayload } from "@/src/types"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import type { BotFileScope } from "@/src/dict/bot-file-dict"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * A conversation's files: the bot's own, or a workspace's shared ones. + * `?file_id=` returns one (for a message's file chip), else a folder's + * files. Only a user who reaches the bot or workspace sees them. + */ +export const handler: BunextAPIRouteHandler> = async ({ + req, +}) => { + try { + if (req.method != "GET") { + return { success: false, msg: `Method not allowed` }; + } + + const { user } = await userAuth({ req }); + + if (!user?.logged_in_status || !user.id) { + return { success: false, msg: `Unauthorized`, logoutUser: true }; + } + + const url = new URL(req.url); + const bot_id = Number(url.searchParams.get("bot_id")); + const workspace_id = Number(url.searchParams.get("workspace_id")); + const file_id = Number(url.searchParams.get("file_id")); + const folder = url.searchParams.get("folder") || ""; + const search = url.searchParams.get("search") || ""; + + const scope: { scope: BotFileScope; owner_id: number } | null = workspace_id + ? (await grabUserWorkspace({ workspace_id, user_id: user.id })) + ? { scope: "workspace", owner_id: workspace_id } + : null + : bot_id + ? (await grabUserBot({ bot_id, user_id: user.id })) + ? { scope: "bot", owner_id: bot_id } + : null + : null; + + if (!scope) { + return { success: false, msg: `Not found.` }; + } + + const scopes = [scope]; + + if (file_id) { + const file = await grabBotFile({ scopes, file_id }); + + if (!file?.id) return { success: false, msg: `Not found.` }; + + return { + success: true, + singleRes: { + files: [toPublicFile({ file })], + folders: [], + usage: { bytes: 0, files: 0 }, + quota_bytes: 0, + quota_mb: 0, + }, + }; + } + + const [files, folders, usage, quota_bytes, quota_mb] = await Promise.all([ + listBotFiles({ scopes, folder, search }), + listBotFolders({ scopes }), + grabStorageUsage(scope), + grabBotFileQuotaBytes(), + grabBotFileQuotaMb(), + ]); + + return { + success: true, + singleRes: { + files: files.map((file) => toPublicFile({ file })), + folders, + usage, + quota_bytes, + quota_mb, + }, + }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; + +function toPublicFile({ file }: { file: BotFileWithMediaLite }) { + const { media_url } = grabMediaURL({ media: { ...(file.media || {}), id: file.media_id } as any }); + + return { + id: Number(file.id), + name: file.name || "", + path: file.path || "", + size: Number(file.size) || 0, + media_type: file.media_type || "document", + purpose: file.purpose || "writing", + scope: file.scope || "bot", + owner_id: Number(file.owner_id) || 0, + ingested_at: file.ingested_at ? Number(file.ingested_at) : undefined, + ingest_error: file.ingest_error || undefined, + created_at: Number(file.created_at) || 0, + url: media_url || "", + }; +} + +type BotFileWithMediaLite = { + id?: number | ""; + media_id?: number | ""; + name?: string; + path?: string; + size?: number | ""; + media_type?: string; + purpose?: string; + scope?: string; + owner_id?: number | ""; + ingested_at?: number | ""; + ingest_error?: string; + created_at?: number | ""; + media?: Record | null; +}; diff --git a/src/pages/api/admin/bots/ingest-file.ts b/src/pages/api/admin/bots/ingest-file.ts new file mode 100644 index 0000000..ceeb4a4 --- /dev/null +++ b/src/pages/api/admin/bots/ingest-file.ts @@ -0,0 +1,82 @@ +import userAuth from "@/src/functions/backend/auth/user-auth"; +import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file"; +import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot"; +import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace"; +import { IngestBotFileRequestSchema } from "@/src/schemas/bot-file-schema"; +import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest"; +import AppError from "@/src/utils/app-error"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import canManageCrew from "@/src/utils/rbac/can-manage-crew"; +import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import type { StorageScope } from "@/src/utils/storage-access"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * Queues a conversation's file to be read again and re-embedded, for when + * a search misses something the user knows is in it. + */ +export const handler: BunextAPIRouteHandler = async ({ req, body }) => { + try { + if (req.method != "POST") throw new AppError(`Method not allowed`); + + const { user, user_types } = await userAuth({ req }); + + if (!user?.id) return { success: false, msg: `Unauthorized`, logoutUser: true }; + if (isReadOnlyUser({ user_types: user_types || [] })) { + throw new AppError(`This user can only read records.`); + } + if (!canManageCrew({ user_types: user_types || [] })) { + throw new AppError(`Only a crew manager can re-read files.`); + } + + const { file_id, read_media } = parseWithSchema({ + schema: IngestBotFileRequestSchema, + data: body, + }); + + const scopes = await reachableScopes({ user_id: user.id, file_id }); + + const file = await grabBotFile({ scopes, file_id }); + + if (!file?.id) throw new AppError(`That file isn't here.`); + + await enqueueFileIngest({ file_id: file.id }); + + return { success: true, msg: `${file.name || "The file"} will be read again shortly.` }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; + +/** + * The directories the user reaches for this file. + */ +async function reachableScopes({ + user_id, + file_id, +}: { + user_id: number; + file_id: number; +}): Promise { + const { default: BunSQLite } = await import("@moduletrace/bun-sqlite"); + + const res = await BunSQLite.sql<{ scope: string; owner_id: number }>({ + sql: `SELECT scope, owner_id FROM bot_files WHERE id = ? LIMIT 1`, + values: [file_id], + }); + + const file = res.payload?.[0]; + + if (!file?.owner_id) return []; + + if (file.scope == "workspace") { + const workspace = await grabUserWorkspace({ workspace_id: file.owner_id, user_id }); + + return workspace?.id ? [{ scope: "workspace", owner_id: Number(file.owner_id) }] : []; + } + + const bot = await grabUserBot({ bot_id: file.owner_id, user_id }); + + return bot?.id ? [{ scope: "bot", owner_id: Number(file.owner_id) }] : []; +} diff --git a/src/pages/api/admin/bots/send-message.ts b/src/pages/api/admin/bots/send-message.ts index 4b6f190..29a292f 100644 --- a/src/pages/api/admin/bots/send-message.ts +++ b/src/pages/api/admin/bots/send-message.ts @@ -4,6 +4,7 @@ import ensureBotSession from "@/src/functions/backend/db/bot-sessions/ensure-bot import insertUserMessage from "@/src/functions/backend/db/bots/insert-user-message"; import grabUserBotMessage from "@/src/functions/backend/db/bots/grab-user-bot-message"; import grabAttachableMedia from "@/src/functions/backend/db/bots/grab-attachable-media"; +import storeChatAttachments from "@/src/functions/backend/db/bot-files/store-chat-attachments"; import enqueueBotReply from "@/src/functions/backend/db/bots-queue/enqueue-bot-reply"; import pushBotNotice from "@/src/bots-queue/push-bot-notice"; import wakeBotsQueue from "@/src/bots-queue/wake-bots-queue"; @@ -93,6 +94,17 @@ export const handler: BunextAPIRouteHandler = async ({ media, }); + /** + * The files join the conversation's folder, so a later run can + * search what the user sent now. + */ + await storeChatAttachments({ + bot_id: bot.id, + workspace_id: conversation.workspace_id, + user_id: user.id, + media, + }); + await enqueueBotReply({ user_id: user.id, bot_id: bot.id, diff --git a/src/pages/api/admin/settings/storage-quota.ts b/src/pages/api/admin/settings/storage-quota.ts new file mode 100644 index 0000000..498164e --- /dev/null +++ b/src/pages/api/admin/settings/storage-quota.ts @@ -0,0 +1,52 @@ +import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user"; +import grabStorageQuotaOverview, { + type StorageQuotaOverview, +} from "@/src/functions/backend/db/app-settings/grab-storage-quota-overview"; +import saveAppSetting from "@/src/functions/backend/db/app-settings/save-app-setting"; +import logEvent, { grabRequestIP } from "@/src/functions/backend/logger"; +import { BotFileQuotaSettingKey } from "@/src/dict/bot-file-dict"; +import { SaveStorageQuotaRequestSchema } from "@/src/schemas/bot-file-schema"; +import AppError from "@/src/utils/app-error"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * Super admin only: the per-folder quota for what bots save, in MB. Zero + * puts the code's default back. + */ +export const handler: BunextAPIRouteHandler> = async ({ + req, + body, +}) => { + try { + if (req.method != "POST") throw new AppError(`Method not allowed`); + + const user = await grabSuperAdminUser({ req }); + + if (!user?.id) throw new AppError(`Only a super admin can change the storage quota.`); + + const { quota_mb } = parseWithSchema({ + schema: SaveStorageQuotaRequestSchema, + data: body, + }); + + await saveAppSetting({ + key: BotFileQuotaSettingKey, + value: quota_mb > 0 ? String(quota_mb) : null, + }); + + await logEvent({ + title: "Bot file quota changed", + category: "system", + user_id: user.id, + plain_text: `User #${user.id} set the bot file quota to ${quota_mb} MB per folder.`, + metadata: { quota_mb }, + ip_address: grabRequestIP(req), + }); + + return { success: true, singleRes: await grabStorageQuotaOverview() }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; diff --git a/src/schemas/bot-file-schema.ts b/src/schemas/bot-file-schema.ts new file mode 100644 index 0000000..d6c6ce9 --- /dev/null +++ b/src/schemas/bot-file-schema.ts @@ -0,0 +1,23 @@ +import { z } from "zod"; + +/** + * Delete one of a conversation's files. + */ +export const DeleteBotFileRequestSchema = z.object({ + file_id: z.coerce.number().int().positive(), +}); + +/** + * Read one file again so it can be searched. + */ +export const IngestBotFileRequestSchema = z.object({ + file_id: z.coerce.number().int().positive(), + read_media: z.boolean().optional(), +}); + +/** + * The General settings' storage quota, in MB. + */ +export const SaveStorageQuotaRequestSchema = z.object({ + quota_mb: z.coerce.number().int().min(0).max(1_000_000), +}); diff --git a/src/server.ts b/src/server.ts index 0f533d4..7b82f1f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,6 +6,7 @@ import userAuth from "./functions/backend/auth/user-auth"; import type { WebSocketSession } from "./websocket/types"; import { logError, grabRequestIP } from "./functions/backend/logger"; import botsQueue from "./bots-queue"; +import fileIngest from "./file-ingest"; import liveRelay from "./live-relay"; import enableForeignKeys from "./functions/backend/db/enable-foreign-keys"; import handleComputerProxy from "./functions/server/computers/handle-computer-proxy"; @@ -164,6 +165,9 @@ try { botsQueue().catch((error) => logError({ title: "Bots queue couldn't start", error }).catch(() => {}), ); +fileIngest().catch((error) => + logError({ title: "File ingest couldn't start", error }).catch(() => {}), +); liveRelay().catch((error) => logError({ title: "Live relay couldn't start", error }).catch(() => {}), ); diff --git a/src/types/index.ts b/src/types/index.ts index 805a54e..0561319 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -622,6 +622,38 @@ export type EnvsOverviewPayload = { known: KnownEnvStatus[]; }; +/** + * A bot file as the chat's files modal lists it. + */ +export type PublicBotFile = { + id: number; + name: string; + path: string; + size: number; + media_type: string; + purpose: string; + scope: string; + owner_id: number; + ingested_at?: number; + ingest_error?: string; + created_at: number; + /** + * Where the bytes are served (`/media/private/`). + */ + url: string; +}; + +/** + * A conversation's files, with the directory's usage against its quota. + */ +export type BotFileScopesPayload = { + files: PublicBotFile[]; + folders: { scope: string; owner_id: number; path: string }[]; + usage: { bytes: number; files: number }; + quota_bytes: number; + quota_mb: number; +}; + /** * A worker bot's model settings page data, also returned when it saves. */ @@ -634,6 +666,10 @@ export type BotMessageAttachment = { media_type: MediaType; url: string; thumbnail_url?: string; + /** + * The bot file it is, when it was saved in one of the bot's folders. + */ + file_id?: number; }; /** @@ -794,10 +830,26 @@ export type PagePropsType = { harness_overview?: HarnessOverviewPayload | null; bot_harness_overview?: BotHarnessOverviewPayload | null; envs_overview?: EnvsOverviewPayload | null; + /** + * What bots keep on disk: the per-folder quota and its usage (the + * General settings page). + */ + storage_quota?: { + setting_key: string; + quota_mb: number; + total_bytes: number; + total_files: number; + folders: number; + } | null; push_overview?: PushOverview | null; mcps_overview?: McpServersOverviewPayload | null; tools_overview?: CustomToolsOverviewPayload | null; bot_extensions?: BotExtensionsPayload | null; + /** + * The open conversation's files (a bot's own, or a workspace's shared + * ones), for the files modal and the chips that name one. + */ + bot_files?: BotFileScopesPayload | null; /** * Whether the open conversation's mic (transcription) and read-aloud * (speech) are turned on for the bot there. diff --git a/src/utils/file-ref.test.ts b/src/utils/file-ref.test.ts new file mode 100644 index 0000000..8eb9591 --- /dev/null +++ b/src/utils/file-ref.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test"; +import fileRef, { fileRefId } from "./file-ref"; +import linkComputerRefs from "./link-computer-refs"; + +describe("fileRef", () => { + test("writes the ref the chat links", () => { + expect(fileRef({ id: 12 })).toBe("[file:12]"); + }); + + test("reads its id back", () => { + expect(fileRefId({ href: "#file-12" })).toBe(12); + expect(fileRefId({ href: "#file-0" })).toBeNull(); + expect(fileRefId({ href: "#bot-3" })).toBeNull(); + expect(fileRefId({ href: undefined })).toBeNull(); + }); +}); + +describe("linkComputerRefs", () => { + test("turns a file ref into a link", () => { + expect(linkComputerRefs({ text: "Saved [file:12] there." })).toBe( + "Saved [file 12](#file-12) there.", + ); + }); + + test("leaves an existing markdown link alone", () => { + expect(linkComputerRefs({ text: "[file:12](#file-12)" })).toBe("[file:12](#file-12)"); + }); + + test("still links the other refs", () => { + const text = linkComputerRefs({ text: "[bot:3] [workspace:2] [computer:5]" }); + + expect(text).toContain("(#bot-3)"); + expect(text).toContain("(#workspace-2)"); + expect(text).toContain("(#computer-5)"); + }); +}); diff --git a/src/utils/file-ref.ts b/src/utils/file-ref.ts new file mode 100644 index 0000000..623818e --- /dev/null +++ b/src/utils/file-ref.ts @@ -0,0 +1,24 @@ +/** + * How a bot names one of its files in a message: `[file:12]`. The chat + * turns it into a link to the file (`linkComputerRefs`), which a chip + * renders with the file's name. + */ +export default function fileRef({ id }: { id: number | string }) { + return `[file:${id}]`; +} + +/** + * The link target a file ref becomes, e.g. `#file-12`. + */ +export const FileRefHrefPrefix = "#file-"; + +/** + * The id in a file ref's link target, or null. + */ +export function fileRefId({ href }: { href?: string | null }) { + if (!href?.startsWith(FileRefHrefPrefix)) return null; + + const id = Number(href.slice(FileRefHrefPrefix.length)); + + return Number.isFinite(id) && id > 0 ? id : null; +} diff --git a/src/utils/link-computer-refs.ts b/src/utils/link-computer-refs.ts index d576c41..343ae96 100644 --- a/src/utils/link-computer-refs.ts +++ b/src/utils/link-computer-refs.ts @@ -24,11 +24,18 @@ export const BotRefHrefPrefix = "#bot-"; */ export const WorkspaceRefHrefPrefix = "#workspace-"; +/** + * The link target a file ref (`fileRef`) becomes, e.g. `#file-12`. The + * chat renders these as a chip that opens the file. + */ +export const FileRefHrefPrefix = "#file-"; + /** * Turns every computer ref (`[computer:3]`) and take-over ref - * (`[take-over:3:browser]`), bot ref (`[bot:3]`) and workspace ref - * (`[workspace:2]`) in a message into a markdown link, so it - * survives markdown rendering as a link the chat can swap for a button. + * (`[take-over:3:browser]`), bot ref (`[bot:3]`), workspace ref + * (`[workspace:2]`) and file ref (`[file:12]`) in a message into a + * markdown link, so it survives markdown rendering as a link the chat can + * swap for a button or chip. */ export default function linkComputerRefs({ text }: { text: string }) { return text @@ -44,5 +51,6 @@ export default function linkComputerRefs({ text }: { text: string }) { .replace( /\[workspace:(\d+)\](?!\()/g, (_, id: string) => `[workspace ${id}](${WorkspaceRefHrefPrefix}${id})`, - ); + ) + .replace(/\[file:(\d+)\](?!\()/g, (_, id: string) => `[file ${id}](${FileRefHrefPrefix}${id})`); } diff --git a/src/utils/resolve-bot-path.test.ts b/src/utils/resolve-bot-path.test.ts new file mode 100644 index 0000000..4ee2e8d --- /dev/null +++ b/src/utils/resolve-bot-path.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import resolveBotPath from "./resolve-bot-path"; + +function makeRoot() { + return mkdtempSync(path.join(tmpdir(), "cobble-botfiles-")); +} + +describe("resolveBotPath", () => { + test("keeps a normal path inside the root", () => { + const root = makeRoot(); + const resolved = resolveBotPath({ root, path: "reports/q3.md" }); + + expect(resolved.relative).toBe("reports/q3.md"); + expect(resolved.folder).toBe("reports"); + expect(resolved.name).toBe("q3.md"); + expect(resolved.absolute.startsWith(root)).toBe(true); + }); + + test("normalizes backslashes and extra slashes", () => { + const resolved = resolveBotPath({ root: makeRoot(), path: "notes//today.md/" }); + + expect(resolved.relative).toBe("notes/today.md"); + }); + + test("refuses an absolute path", () => { + expect(() => resolveBotPath({ root: makeRoot(), path: "/etc/passwd" })).toThrow(); + expect(() => resolveBotPath({ root: makeRoot(), path: "C:/windows" })).toThrow(); + }); + + test("refuses going up out of the root", () => { + expect(() => resolveBotPath({ root: makeRoot(), path: "../secret.txt" })).toThrow(); + expect(() => resolveBotPath({ root: makeRoot(), path: "a/../../secret.txt" })).toThrow(); + }); + + test("refuses a NUL byte", () => { + expect(() => resolveBotPath({ root: makeRoot(), path: "a\0b.txt" })).toThrow(); + }); + + test("refuses a symlink that leads out of the root", () => { + const root = makeRoot(); + const outside = makeRoot(); + writeFileSync(path.join(outside, "secret.txt"), "no"); + + mkdirSync(path.join(root, "linked")); + symlinkSync(outside, path.join(root, "linked", "escape")); + + expect(() => + resolveBotPath({ root, path: "linked/escape/secret.txt" }), + ).toThrow(); + }); + + test("allows a symlink that stays inside", () => { + const root = makeRoot(); + mkdirSync(path.join(root, "real")); + writeFileSync(path.join(root, "real", "note.md"), "hi"); + symlinkSync(path.join(root, "real"), path.join(root, "alias")); + + expect(resolveBotPath({ root, path: "alias/note.md" }).relative).toBe("alias/note.md"); + }); + + test("refuses a name too long", () => { + expect(() => + resolveBotPath({ root: makeRoot(), path: `${"a".repeat(200)}.md` }), + ).toThrow(); + }); + + test("refuses a path too deep", () => { + const deep = Array.from({ length: 12 }, (_, i) => `d${i}`).join("/"); + + expect(() => resolveBotPath({ root: makeRoot(), path: `${deep}/file.md` })).toThrow(); + }); + + test("refuses a name with characters a disk won't take", () => { + expect(() => resolveBotPath({ root: makeRoot(), path: "a` or + * `DATA_DIR/workspaces/`), already absolute. + */ + root: string; + /** + * What the bot asked for: a path inside that directory, e.g. + * `reports/q3.md`. Never absolute. + */ + path?: string | null; +}; + +export type ResolvedBotPath = { + /** + * The normalized path inside the root, forward slashes, no leading + * slash: what `bot_files.path` holds. + */ + relative: string; + /** + * The absolute path on disk, inside `root`. + */ + absolute: string; + /** + * The folder part of `relative`, empty at the top. + */ + folder: string; + /** + * The last segment. + */ + name: string; +}; + +/** + * Turns a bot's path into one safely inside its root: rejects absolute + * paths, `..`, NUL bytes and anything too long or too deep, and, once the + * path exists, anything a symlink points outside the root. Throws an + * `AppError` with a message safe to show the user (the bot gets it back + * as the tool's error). + */ +export default function resolveBotPath({ root, path: requested }: Params): ResolvedBotPath { + const raw = (requested || "").trim().replace(/\\/g, "/"); + + if (raw.includes("\0")) { + throw new AppError(`That path isn't valid.`); + } + + if (raw.startsWith("/") || /^[a-zA-Z]:/.test(raw)) { + throw new AppError( + `Use a path inside your files folder, not an absolute one (e.g. "reports/q3.md").`, + ); + } + + const segments = raw.split("/").map((segment) => segment.trim()).filter(Boolean); + + for (const segment of segments) { + if (segment == "." || segment == "..") { + throw new AppError(`That path isn't valid: it must stay inside your files folder.`); + } + } + + if (segments.length > AppData.BotFilePathMaxDepth) { + throw new AppError( + `That path is too deep: at most ${AppData.BotFilePathMaxDepth} folders.`, + ); + } + + const name = segments.at(-1); + + if (!name) { + throw new AppError(`Give a file name, e.g. "notes.md".`); + } + + if (name.length > AppData.BotFileNameMaxChars) { + throw new AppError( + `That file name is too long: at most ${AppData.BotFileNameMaxChars} characters.`, + ); + } + + if (/[<>:"|?*]/.test(name)) { + throw new AppError(`A file name can't contain < > : " | ? or *.`); + } + + const relative = segments.join("/"); + + if (relative.length > AppData.BotFilePathMaxChars) { + throw new AppError(`That path is too long.`); + } + + const absolute = path.join(root, ...segments); + const inside = path.resolve(absolute); + + if (inside != path.resolve(root) && !inside.startsWith(`${path.resolve(root)}${path.sep}`)) { + throw new AppError(`That path isn't valid: it must stay inside your files folder.`); + } + + /** + * A symlink inside the root must not lead out of it: check the deepest + * part that exists. + */ + guardRealPath({ root, absolute: inside }); + + return { + relative, + absolute: inside, + folder: segments.slice(0, -1).join("/"), + name, + }; +} + +/** + * Refuses a path whose existing part resolves outside the root through a + * symlink (or any other reason). + */ +function guardRealPath({ root, absolute }: { root: string; absolute: string }) { + const real_root = existsSync(root) ? realpathSync(root) : path.resolve(root); + let probe = absolute; + + while (!existsSync(probe)) { + const parent = path.dirname(probe); + + if (parent == probe) break; + + probe = parent; + } + + const real_probe = realpathSync(probe); + + if (real_probe != real_root && !real_probe.startsWith(`${real_root}${path.sep}`)) { + throw new AppError(`That path isn't valid: it must stay inside your files folder.`); + } +} + +/** + * Whether a path is inside a root, before it exists (used for folders). + */ +export function isInsideRoot({ root, absolute }: { root: string; absolute: string }) { + const resolved_root = path.resolve(root); + const resolved = path.resolve(absolute); + + return resolved == resolved_root || resolved.startsWith(`${resolved_root}${path.sep}`); +} diff --git a/src/utils/storage-access.test.ts b/src/utils/storage-access.test.ts new file mode 100644 index 0000000..9fb3a2b --- /dev/null +++ b/src/utils/storage-access.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import grabStorageScopes from "./storage-access"; + +describe("grabStorageScopes", () => { + test("a worker in its own chat reaches only its folder", () => { + expect(grabStorageScopes({ bot_id: 4, bot_kind: "worker" })).toEqual([ + { scope: "bot", owner_id: 4 }, + ]); + }); + + test("a worker in a workspace also reaches the workspace's folder", () => { + expect( + grabStorageScopes({ bot_id: 4, bot_kind: "worker", workspace_id: 9 }), + ).toEqual([ + { scope: "bot", owner_id: 4 }, + { scope: "workspace", owner_id: 9 }, + ]); + }); + + test("the coordinator reaches every workspace, its own folder first", () => { + expect( + grabStorageScopes({ + bot_id: 1, + bot_kind: "coordinator", + all_workspace_ids: [9, 3], + }), + ).toEqual([ + { scope: "bot", owner_id: 1 }, + { scope: "workspace", owner_id: 9 }, + { scope: "workspace", owner_id: 3 }, + ]); + }); + + test("the coordinator in a workspace still reaches them all", () => { + const scopes = grabStorageScopes({ + bot_id: 1, + bot_kind: "coordinator", + workspace_id: 9, + all_workspace_ids: [9], + }); + + expect(scopes).toEqual([ + { scope: "bot", owner_id: 1 }, + { scope: "workspace", owner_id: 9 }, + ]); + }); + + test("ignores invalid workspace ids", () => { + expect( + grabStorageScopes({ + bot_id: 1, + bot_kind: "coordinator", + all_workspace_ids: [0, 2], + }), + ).toEqual([ + { scope: "bot", owner_id: 1 }, + { scope: "workspace", owner_id: 2 }, + ]); + }); +}); diff --git a/src/utils/storage-access.ts b/src/utils/storage-access.ts new file mode 100644 index 0000000..bd8d01b --- /dev/null +++ b/src/utils/storage-access.ts @@ -0,0 +1,80 @@ +import type { BotFileScope } from "@/src/dict/bot-file-dict"; + +/** + * One storage directory a run may read and write: a bot's own + * (`DATA_DIR/bots/`) or a workspace's shared one + * (`DATA_DIR/workspaces/`). + */ +export type StorageScope = { + scope: BotFileScope; + owner_id: number; +}; + +type Params = { + bot_id: number; + /** + * `coordinator`: it reaches every bot's and every workspace's files. + */ + bot_kind?: string | null; + /** + * The workspace the run is in, if any. A worker then reaches that + * workspace's shared files too. + */ + workspace_id?: number | string | null; + /** + * Every workspace, for the coordinator (it isn't a member of one, it + * reaches them all). Filled by the caller that already read them. + */ + all_workspace_ids?: number[]; +}; + +/** + * The directories a run may touch, itself first. A worker: its own, plus + * its workspace's shared one while it works there. The coordinator: every + * bot's and every workspace's, its own first. + * + * Enforced by every file tool, never left to the prompt: a bot that names + * a directory it doesn't reach is told so. + */ +export default function grabStorageScopes({ + bot_id, + bot_kind, + workspace_id, + all_workspace_ids = [], +}: Params): StorageScope[] { + const own: StorageScope = { scope: "bot", owner_id: Number(bot_id) }; + + if (bot_kind == "coordinator") { + return [ + own, + ...all_workspace_ids + .filter((id) => Number(id) > 0) + .map((id): StorageScope => ({ scope: "workspace", owner_id: Number(id) })), + ]; + } + + return workspace_id ? [own, { scope: "workspace", owner_id: Number(workspace_id) }] : [own]; +} + +/** + * Whether a scope is one of the run's. + */ +export function canReachScope({ + scopes, + scope, + owner_id, +}: { + scopes: StorageScope[]; + scope: BotFileScope; + owner_id: number | string; +}) { + return scopes.some((s) => s.scope == scope && s.owner_id == Number(owner_id)); +} + +/** + * A scope's label for tools and messages: "your files" for a bot's own, + * the workspace's name for a shared one. + */ +export function storageScopeKey({ scope, owner_id }: StorageScope) { + return `${scope}:${owner_id}`; +}