Add bots files handling logic
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
import grabAppSettings from "./grab-app-settings";
|
||||
|
||||
type Params = {
|
||||
key: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* One install-wide setting, or undefined when it was never saved.
|
||||
*/
|
||||
export default async function grabAppSetting({ key }: Params) {
|
||||
return (await grabAppSettings()).get(key);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_APP_SETTINGS } from "@/db/types/db";
|
||||
|
||||
let cache: Map<string, string> | null = null;
|
||||
|
||||
/**
|
||||
* Every install-wide setting, cached per process (like `grabAllEnvs`).
|
||||
* `saveAppSetting` and `deleteAppSetting` clear it.
|
||||
*/
|
||||
export default async function grabAppSettings() {
|
||||
if (cache) return cache;
|
||||
|
||||
const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_APP_SETTINGS>({
|
||||
sql: `SELECT key, value FROM app_settings`,
|
||||
values: [],
|
||||
});
|
||||
|
||||
const map = new Map<string, string>();
|
||||
|
||||
for (const row of res.payload || []) {
|
||||
if (row.key) map.set(row.key, row.value || "");
|
||||
}
|
||||
|
||||
cache = map;
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops the cache, so the next read sees the database.
|
||||
*/
|
||||
export function clearAppSettingsCache() {
|
||||
cache = null;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { BotFileQuotaSettingKey } from "@/src/dict/bot-file-dict";
|
||||
import { grabBotFileQuotaMb } from "../bot-files/grab-bot-file-quota-bytes";
|
||||
import { grabStorageUsageByOwner, grabTotalStorageUsage } from "../bot-files/grab-storage-usage";
|
||||
|
||||
/**
|
||||
* The General settings' bot file storage: the quota each folder gets, and
|
||||
* what is used right now.
|
||||
*/
|
||||
export default async function grabStorageQuotaOverview() {
|
||||
const [quota_mb, total, by_owner] = await Promise.all([
|
||||
grabBotFileQuotaMb(),
|
||||
grabTotalStorageUsage(),
|
||||
grabStorageUsageByOwner(),
|
||||
]);
|
||||
|
||||
return {
|
||||
setting_key: BotFileQuotaSettingKey,
|
||||
quota_mb,
|
||||
total_bytes: total.bytes,
|
||||
total_files: total.files,
|
||||
folders: by_owner.length,
|
||||
};
|
||||
}
|
||||
|
||||
export type StorageQuotaOverview = Awaited<ReturnType<typeof grabStorageQuotaOverview>>;
|
||||
@@ -0,0 +1,47 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_APP_SETTINGS, BunSQLiteTables } from "@/db/types/db";
|
||||
import { clearAppSettingsCache } from "./grab-app-settings";
|
||||
|
||||
type Params = {
|
||||
key: string;
|
||||
/**
|
||||
* Null or empty removes it, so the code's default applies again.
|
||||
*/
|
||||
value?: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Saves (or removes) one install-wide setting and clears the cache, so the
|
||||
* next read sees it.
|
||||
*/
|
||||
export default async function saveAppSetting({ key, value }: Params) {
|
||||
const existing = await BunSQLite.sql<BUN_SQLITE_COBBLE_APP_SETTINGS>({
|
||||
sql: `SELECT id FROM app_settings WHERE key = ? LIMIT 1`,
|
||||
values: [key],
|
||||
}).then((res) => res.payload?.[0]);
|
||||
|
||||
if (!value) {
|
||||
await BunSQLite.sql({
|
||||
sql: `DELETE FROM app_settings WHERE key = ?`,
|
||||
values: [key],
|
||||
});
|
||||
clearAppSettingsCache();
|
||||
return null;
|
||||
}
|
||||
|
||||
if (existing?.id) {
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE app_settings SET value = ?, updated_at = ? WHERE id = ?`,
|
||||
values: [value, Date.now(), existing.id],
|
||||
});
|
||||
} else {
|
||||
await BunSQLite.insert<BUN_SQLITE_COBBLE_APP_SETTINGS, (typeof BunSQLiteTables)[number]>({
|
||||
table: "app_settings",
|
||||
data: [{ key, value }],
|
||||
});
|
||||
}
|
||||
|
||||
clearAppSettingsCache();
|
||||
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { BotFileScope } from "@/src/dict/bot-file-dict";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabBotFileQuotaBytes from "./grab-bot-file-quota-bytes";
|
||||
import grabStorageUsage from "./grab-storage-usage";
|
||||
|
||||
type Params = {
|
||||
scope: BotFileScope;
|
||||
owner_id: number | string;
|
||||
/**
|
||||
* How many bytes are about to be added. A rewrite of an existing file
|
||||
* passes only the growth.
|
||||
*/
|
||||
incoming_bytes: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Refuses a write that would push a directory past its quota, naming the
|
||||
* limit and the current usage so the bot can clear space first.
|
||||
*/
|
||||
export default async function assertStorageRoom({
|
||||
scope,
|
||||
owner_id,
|
||||
incoming_bytes,
|
||||
}: Params) {
|
||||
const quota = await grabBotFileQuotaBytes();
|
||||
|
||||
if (incoming_bytes <= 0) return;
|
||||
|
||||
const usage = await grabStorageUsage({ scope, owner_id });
|
||||
|
||||
if (usage.bytes + incoming_bytes <= quota) return;
|
||||
|
||||
throw new AppError(
|
||||
`No room left: ${formatMb({ bytes: usage.bytes })} of ${formatMb({ bytes: quota })} used in ${scope == "workspace" ? "this workspace" : "your files"}. Delete or move something first.`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A size in MB, for messages: "12.4 MB".
|
||||
*/
|
||||
export function formatMb({ bytes }: { bytes: number }) {
|
||||
const mb = bytes / (1024 * 1024);
|
||||
|
||||
if (mb >= 10) return `${Math.round(mb)} MB`;
|
||||
|
||||
return `${mb.toFixed(1)} MB`;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import chunkText from "./chunk-text";
|
||||
|
||||
describe("chunkText", () => {
|
||||
test("returns nothing for empty text", () => {
|
||||
expect(chunkText({ text: " \n\n " })).toEqual([]);
|
||||
});
|
||||
|
||||
test("keeps a short document in one chunk", () => {
|
||||
expect(chunkText({ text: "One paragraph.\n\nAnother one." })).toEqual([
|
||||
"One paragraph.\n\nAnother one.",
|
||||
]);
|
||||
});
|
||||
|
||||
test("splits at paragraph boundaries, never mid-word", () => {
|
||||
const paragraph = "word ".repeat(40).trim();
|
||||
const chunks = chunkText({ text: [1, 2, 3, 4, 5].map(() => paragraph).join("\n\n"), target: 300, overlap: 0 });
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.split(/\s+/).every((word) => word == "word")).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test("carries the heading into the chunk it starts", () => {
|
||||
const text = "# Report\n\nThe numbers are up.\n\n## Details\n\nEverything is fine.";
|
||||
const chunks = chunkText({ text, target: 120, overlap: 0 });
|
||||
|
||||
expect(chunks[0]).toContain("# Report");
|
||||
});
|
||||
|
||||
test("overlaps consecutive chunks of a long paragraph", () => {
|
||||
const text = Array.from({ length: 200 }, (_, i) => `word${i}`).join(" ");
|
||||
const chunks = chunkText({ text, target: 200, overlap: 40 });
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
|
||||
const first_tail = chunks[0]!.split(" ").slice(-2).join(" ");
|
||||
|
||||
expect(chunks[1]!.startsWith(first_tail.split(" ")[0]!) || chunks[1]!.includes(first_tail)).toBe(true);
|
||||
});
|
||||
|
||||
test("keeps a word longer than the target whole", () => {
|
||||
const chunks = chunkText({ text: "a".repeat(400), target: 100, overlap: 0 });
|
||||
|
||||
expect(chunks.join("")).toContain("a".repeat(400).slice(0, 10));
|
||||
expect(chunks.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("collapses runs of blank lines", () => {
|
||||
expect(chunkText({ text: "One.\n\n\n\n\nTwo." })).toEqual(["One.\n\nTwo."]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
|
||||
type Params = {
|
||||
text: string;
|
||||
/**
|
||||
* How much text one chunk holds, in characters.
|
||||
*/
|
||||
target?: number;
|
||||
/**
|
||||
* How much of the previous chunk the next one repeats, so a sentence
|
||||
* cut in two is still whole somewhere.
|
||||
*/
|
||||
overlap?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Splits a document's text into chunks for embedding: paragraphs packed up
|
||||
* to the target size, a heading carried into the chunk that follows it,
|
||||
* and long paragraphs cut at word boundaries with an overlap. Never cuts
|
||||
* mid-word (a word longer than the target is the one exception, since
|
||||
* there's nowhere else to cut).
|
||||
*/
|
||||
export default function chunkText({
|
||||
text,
|
||||
target = AppData.ChunkTargetChars,
|
||||
overlap = AppData.ChunkOverlapChars,
|
||||
}: Params): string[] {
|
||||
const normalized = text.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
||||
|
||||
if (!normalized) return [];
|
||||
|
||||
const blocks = normalized.split(/\n{2,}/).map((block) => block.trim()).filter(Boolean);
|
||||
const chunks: string[] = [];
|
||||
/**
|
||||
* The current chunk's blocks, then (after a flush) only the overlap
|
||||
* carried over: `fresh` says whether anything new joined it, so a
|
||||
* carry on its own is never pushed as a chunk.
|
||||
*/
|
||||
let current: string[] = [];
|
||||
let current_length = 0;
|
||||
let fresh = false;
|
||||
let heading = "";
|
||||
|
||||
const flush = () => {
|
||||
const body = current.join("\n\n").trim();
|
||||
|
||||
if (body && fresh) {
|
||||
chunks.push(heading && !body.startsWith(heading) ? `${heading}\n\n${body}` : body);
|
||||
}
|
||||
|
||||
const carry = overlap > 0 ? tail({ text: body, size: overlap }) : "";
|
||||
|
||||
current = carry ? [carry] : [];
|
||||
current_length = carry.length;
|
||||
fresh = false;
|
||||
};
|
||||
|
||||
for (const block of blocks) {
|
||||
const is_heading = /^#{1,6}\s/.test(block);
|
||||
|
||||
if (is_heading) {
|
||||
/**
|
||||
* A heading starts a new chunk with it, so a chunk says what it
|
||||
* is about.
|
||||
*/
|
||||
if (current_length > 0) flush();
|
||||
|
||||
heading = block;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (block.length >= target) {
|
||||
if (current_length > 0) flush();
|
||||
|
||||
for (const piece of splitLongBlock({ block, target, overlap })) {
|
||||
chunks.push(heading ? `${heading}\n\n${piece}` : piece);
|
||||
}
|
||||
|
||||
current = [];
|
||||
current_length = 0;
|
||||
fresh = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current_length + block.length + 2 > target) flush();
|
||||
|
||||
current.push(block);
|
||||
current_length += block.length + 2;
|
||||
fresh = true;
|
||||
}
|
||||
|
||||
if (current_length > 0) flush();
|
||||
|
||||
return chunks.filter((chunk) => chunk.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cuts a block that is longer than one chunk at word boundaries, keeping
|
||||
* the overlap between pieces.
|
||||
*/
|
||||
function splitLongBlock({
|
||||
block,
|
||||
target,
|
||||
overlap,
|
||||
}: {
|
||||
block: string;
|
||||
target: number;
|
||||
overlap: number;
|
||||
}) {
|
||||
const words = block.split(/\s+/);
|
||||
const pieces: string[] = [];
|
||||
let current: string[] = [];
|
||||
let length = 0;
|
||||
|
||||
for (const word of words) {
|
||||
if (length + word.length + 1 > target && current.length) {
|
||||
pieces.push(current.join(" "));
|
||||
const carry = tail({ text: current.join(" "), size: overlap });
|
||||
|
||||
current = carry ? carry.split(" ") : [];
|
||||
length = carry.length;
|
||||
}
|
||||
|
||||
if (!word) continue;
|
||||
|
||||
current.push(word);
|
||||
length += word.length + 1;
|
||||
}
|
||||
|
||||
if (current.length) pieces.push(current.join(" "));
|
||||
|
||||
return pieces;
|
||||
}
|
||||
|
||||
/**
|
||||
* The last `size` characters of a text, starting at a word boundary so the
|
||||
* overlap doesn't begin mid-word.
|
||||
*/
|
||||
function tail({ text, size }: { text: string; size: number }) {
|
||||
if (!text || size <= 0) return "";
|
||||
|
||||
if (text.length <= size) return text;
|
||||
|
||||
const cut = text.slice(text.length - size);
|
||||
const boundary = cut.search(/\s/);
|
||||
|
||||
return boundary == -1 ? cut : cut.slice(boundary).trim();
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import type { StorageScope } from "@/src/utils/storage-access";
|
||||
import { canReachScope } from "@/src/utils/storage-access";
|
||||
import grabBotFile from "./grab-bot-file";
|
||||
import recordBotFile from "./record-bot-file";
|
||||
|
||||
type Params = {
|
||||
scopes: StorageScope[];
|
||||
file_id: number | string;
|
||||
/**
|
||||
* The bot making the copy (it owns the new row).
|
||||
*/
|
||||
bot_id: number | string;
|
||||
/**
|
||||
* Where the copy goes: one of the run's directories.
|
||||
*/
|
||||
scope: StorageScope;
|
||||
/**
|
||||
* Its path there. Defaults to the source's path.
|
||||
*/
|
||||
path?: string | null;
|
||||
user_id?: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Copies a file into another of the run's directories (the coordinator
|
||||
* publishing a worker's artifact into its own, or into a workspace). The
|
||||
* bytes are copied, so deleting one leaves the other.
|
||||
*/
|
||||
export default async function copyBotFile({
|
||||
scopes,
|
||||
file_id,
|
||||
bot_id,
|
||||
scope,
|
||||
path,
|
||||
user_id,
|
||||
}: Params) {
|
||||
if (!canReachScope({ scopes, scope: scope.scope, owner_id: scope.owner_id })) {
|
||||
throw new AppError(`You can't write to that folder.`);
|
||||
}
|
||||
|
||||
const file = await grabBotFile({ scopes, file_id });
|
||||
|
||||
if (!file?.id || !file.media?.media_write_path) {
|
||||
throw new AppError(`That file isn't in your files.`);
|
||||
}
|
||||
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const source = await Bun.file(`${DATA_DIR}/${file.media.media_write_path}`).arrayBuffer();
|
||||
|
||||
return await recordBotFile({
|
||||
bot_id,
|
||||
scope: scope.scope,
|
||||
owner_id: scope.owner_id,
|
||||
path: path || file.path || file.name || `copy-${file.id}`,
|
||||
data: new Uint8Array(source),
|
||||
purpose: (file.purpose as any) || "writing",
|
||||
source: "tool",
|
||||
media_type: (file.media_type as any) || undefined,
|
||||
mime_type: file.mime_type || undefined,
|
||||
user_id,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db";
|
||||
import { rmSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import type { StorageScope } from "@/src/utils/storage-access";
|
||||
import grabBotFile from "./grab-bot-file";
|
||||
import grabStorageDir from "./grab-storage-dir";
|
||||
|
||||
type Params = {
|
||||
scopes: StorageScope[];
|
||||
file_id: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Deletes a bot file: the catalog row (its chunks go with it), its bytes
|
||||
* and its `media` row. Refuses anything outside the run's directories.
|
||||
*/
|
||||
export default async function deleteBotFile({ scopes, file_id }: Params) {
|
||||
const file = await grabBotFile({ scopes, file_id });
|
||||
|
||||
if (!file?.id) {
|
||||
throw new AppError(`That file isn't in your files.`);
|
||||
}
|
||||
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const root = grabStorageDir({ scope: file.scope as any, owner_id: file.owner_id! });
|
||||
const absolute = path.join(root, file.path || "");
|
||||
|
||||
if (absolute.startsWith(path.resolve(root))) {
|
||||
rmSync(absolute, { force: true });
|
||||
}
|
||||
|
||||
await BunSQLite.sql({ sql: `DELETE FROM bot_file_vectors WHERE file_id = ?`, values: [file.id] });
|
||||
await BunSQLite.sql({ sql: `DELETE FROM bot_files WHERE id = ?`, values: [file.id] });
|
||||
/**
|
||||
* Its pending ingestion would only find nothing left.
|
||||
*/
|
||||
await BunSQLite.sql({
|
||||
sql: `DELETE FROM file_ingest_queue WHERE file_id = ? AND status IN ('pending', 'processing')`,
|
||||
values: [file.id],
|
||||
}).catch(() => {});
|
||||
|
||||
if (file.media_id) {
|
||||
const media = await BunSQLite.sql<{ media_write_path?: string; media_thumbnail_write_path?: string }>({
|
||||
sql: `SELECT media_write_path, media_thumbnail_write_path FROM media WHERE id = ? LIMIT 1`,
|
||||
values: [file.media_id],
|
||||
}).then((res) => res.payload?.[0]);
|
||||
|
||||
if (media?.media_write_path) {
|
||||
rmSync(path.join(DATA_DIR, media.media_write_path), { force: true });
|
||||
}
|
||||
|
||||
if (media?.media_thumbnail_write_path) {
|
||||
rmSync(path.join(DATA_DIR, media.media_thumbnail_write_path), { force: true });
|
||||
}
|
||||
|
||||
await BunSQLite.sql({ sql: `DELETE FROM media_paradigms WHERE media_id = ?`, values: [file.media_id] });
|
||||
await BunSQLite.sql({ sql: `DELETE FROM media WHERE id = ?`, values: [file.media_id] });
|
||||
}
|
||||
|
||||
return file as BUN_SQLITE_COBBLE_BOT_FILES;
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import {
|
||||
BotImageFileExtensions,
|
||||
BotTextFileExtensions,
|
||||
} from "@/src/dict/bot-file-dict";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import fileToText from "../users/media/file-to-text";
|
||||
import { extensionOf } from "./media-type-for-file";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* The `media` row's write path (relative to `DATA_DIR`).
|
||||
*/
|
||||
write_path?: string | null;
|
||||
/**
|
||||
* Its text, when a previous upload already extracted it.
|
||||
*/
|
||||
text_content?: string | null;
|
||||
name?: string | null;
|
||||
mime_type?: string | null;
|
||||
/**
|
||||
* Read images and audio too, through the media harness. Off by
|
||||
* default: only a bot asking for it pays for that.
|
||||
*/
|
||||
read_media?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* A file's text, for chunking: documents unpacked locally (PDF, DOCX,
|
||||
* plain text, markdown, CSV, JSON, HTML), images and audio only when
|
||||
* `read_media` is set. Null when the file has no text to read.
|
||||
*/
|
||||
export default async function extractBotFileText({
|
||||
write_path,
|
||||
text_content,
|
||||
name,
|
||||
mime_type,
|
||||
read_media,
|
||||
}: Params): Promise<string | null> {
|
||||
if (text_content?.trim()) return text_content.trim();
|
||||
|
||||
if (!write_path) return null;
|
||||
|
||||
const ext = extensionOf({ name, mime_type });
|
||||
|
||||
if (!ext) return null;
|
||||
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const file_path = `${DATA_DIR}/${write_path}`;
|
||||
|
||||
if ((BotTextFileExtensions as readonly string[]).includes(ext)) {
|
||||
/**
|
||||
* Markdown, CSV and the other plain text types read as they are:
|
||||
* no unpacking needed, and no size limit spent on it.
|
||||
*/
|
||||
if (!["pdf", "docx"].includes(ext)) {
|
||||
const file = Bun.file(file_path);
|
||||
|
||||
if (file.size > AppData.IngestMaxBytes) return null;
|
||||
|
||||
return await file.text();
|
||||
}
|
||||
|
||||
const file = Bun.file(file_path);
|
||||
|
||||
if (file.size > AppData.IngestMaxBytes) return null;
|
||||
|
||||
return (await fileToText({ file_path })) || null;
|
||||
}
|
||||
|
||||
if (read_media) {
|
||||
const file = Bun.file(file_path);
|
||||
|
||||
if (file.size > AppData.IngestMaxBytes) return null;
|
||||
|
||||
const { default: readMedia } = await import("@/src/harness/media/read-media");
|
||||
const media_type = (BotImageFileExtensions as readonly string[]).includes(ext)
|
||||
? `image/${ext == "jpg" ? "jpeg" : ext}`
|
||||
: mime_type || "audio/mpeg";
|
||||
|
||||
return await readMedia({
|
||||
data: new Uint8Array(await file.arrayBuffer()),
|
||||
media_type,
|
||||
name: name || undefined,
|
||||
})
|
||||
.then((text) => text?.trim() || null)
|
||||
.catch(() => null);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type {
|
||||
BUN_SQLITE_COBBLE_BOT_FILES,
|
||||
BUN_SQLITE_COBBLE_MEDIA,
|
||||
} from "@/db/types/db";
|
||||
import grabCrewRole from "../access/grab-crew-role";
|
||||
import grabUserBot from "../bots/grab-user-bot";
|
||||
import grabUserWorkspace from "../workspaces/grab-user-workspace";
|
||||
|
||||
type Params = {
|
||||
media_id: number | string;
|
||||
user_id: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A bot file's bytes (its private `media` row) for a user who reaches the
|
||||
* directory it's in: the bot's own chat (or any worker, for super admins
|
||||
* and admins; the coordinator only for the super admin), or the workspace
|
||||
* it's shared in. Files belong to the install, like the crew, so whoever
|
||||
* sees the bot sees what it saved.
|
||||
*/
|
||||
export default async function grabBotFileMedia({
|
||||
media_id,
|
||||
user_id,
|
||||
}: Params): Promise<BUN_SQLITE_COBBLE_MEDIA | null> {
|
||||
const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT * FROM bot_files WHERE media_id = ? LIMIT 1`,
|
||||
values: [media_id],
|
||||
});
|
||||
|
||||
const file = res.payload?.[0];
|
||||
|
||||
if (!file?.id) return null;
|
||||
|
||||
const role = await grabCrewRole({ user_id });
|
||||
const reaches =
|
||||
file.scope == "workspace"
|
||||
? Boolean(await grabUserWorkspace({ workspace_id: file.owner_id!, user_id }))
|
||||
: role == "super_admin"
|
||||
? true
|
||||
: Boolean(await grabUserBot({ bot_id: file.owner_id!, user_id }));
|
||||
|
||||
if (!reaches) return null;
|
||||
|
||||
const media = await BunSQLite.sql<BUN_SQLITE_COBBLE_MEDIA>({
|
||||
sql: `SELECT * FROM media WHERE id = ? LIMIT 1`,
|
||||
values: [media_id],
|
||||
});
|
||||
|
||||
return media.payload?.[0] || null;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import { BotFileQuotaSettingKey } from "@/src/dict/bot-file-dict";
|
||||
import grabAppSetting from "../app-settings/grab-app-setting";
|
||||
|
||||
/**
|
||||
* The storage limit one directory may hold, from the General settings
|
||||
* (`bot_files_quota_mb`), else `AppData.BotFileQuotaMb`. A workspace's
|
||||
* shared directory gets the same limit as a bot's own.
|
||||
*/
|
||||
export default async function grabBotFileQuotaBytes() {
|
||||
const saved = Number(await grabAppSetting({ key: BotFileQuotaSettingKey }));
|
||||
const mb = Number.isFinite(saved) && saved > 0 ? saved : AppData.BotFileQuotaMb;
|
||||
|
||||
return Math.round(mb * 1024 * 1024);
|
||||
}
|
||||
|
||||
/**
|
||||
* The same limit in MB, for the settings page.
|
||||
*/
|
||||
export async function grabBotFileQuotaMb() {
|
||||
const saved = Number(await grabAppSetting({ key: BotFileQuotaSettingKey }));
|
||||
|
||||
return Number.isFinite(saved) && saved > 0 ? saved : AppData.BotFileQuotaMb;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type {
|
||||
BUN_SQLITE_COBBLE_BOT_FILES,
|
||||
BUN_SQLITE_COBBLE_MEDIA,
|
||||
} from "@/db/types/db";
|
||||
import { canReachScope, type StorageScope } from "@/src/utils/storage-access";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* The directories the run may touch (`grabStorageScopes`).
|
||||
*/
|
||||
scopes: StorageScope[];
|
||||
/**
|
||||
* The catalog row's id.
|
||||
*/
|
||||
file_id?: number | string | null;
|
||||
/**
|
||||
* Or its path, with the directory it's in.
|
||||
*/
|
||||
scope?: string | null;
|
||||
owner_id?: number | string | null;
|
||||
path?: string | null;
|
||||
};
|
||||
|
||||
export type BotFileWithMedia = BUN_SQLITE_COBBLE_BOT_FILES & {
|
||||
media?: BUN_SQLITE_COBBLE_MEDIA | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* One bot file, only if it's in one of the run's directories, with its
|
||||
* `media` row when that still exists.
|
||||
*/
|
||||
export default async function grabBotFile({
|
||||
scopes,
|
||||
file_id,
|
||||
scope,
|
||||
owner_id,
|
||||
path,
|
||||
}: Params): Promise<BotFileWithMedia | null> {
|
||||
if (!file_id && !(scope && owner_id != null && path)) return null;
|
||||
|
||||
const res = file_id
|
||||
? await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT * FROM bot_files WHERE id = ? LIMIT 1`,
|
||||
values: [file_id],
|
||||
})
|
||||
: await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT * FROM bot_files WHERE scope = ? AND owner_id = ? AND path = ? LIMIT 1`,
|
||||
values: [String(scope), Number(owner_id), String(path)],
|
||||
});
|
||||
|
||||
const file = res.payload?.[0];
|
||||
|
||||
if (!file?.id) return null;
|
||||
|
||||
if (!canReachScope({ scopes, scope: file.scope as any, owner_id: file.owner_id! })) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const media = file.media_id
|
||||
? await BunSQLite.sql<BUN_SQLITE_COBBLE_MEDIA>({
|
||||
sql: `SELECT * FROM media WHERE id = ? LIMIT 1`,
|
||||
values: [file.media_id],
|
||||
}).then((r) => r.payload?.[0] || null)
|
||||
: null;
|
||||
|
||||
return { ...file, media };
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import type { BotFileScopesPayload } from "@/src/types";
|
||||
import grabMediaURL from "../../media/grab-media-url";
|
||||
import grabBotFileQuotaBytes, { grabBotFileQuotaMb } from "./grab-bot-file-quota-bytes";
|
||||
import grabStorageUsage from "./grab-storage-usage";
|
||||
import listBotFiles from "./list-bot-files";
|
||||
|
||||
type Params = {
|
||||
bot_id: number;
|
||||
/**
|
||||
* The workspace the conversation is in: its shared folder, else the
|
||||
* bot's own.
|
||||
*/
|
||||
workspace_id?: number | string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* A conversation's files, for its page props: the folder it works in
|
||||
* (a workspace's shared one, else the bot's own), its latest
|
||||
* `AppData.BotFilesPageSize` of them, and the usage against the quota.
|
||||
* The full list (folders, search, older files) loads in the files modal.
|
||||
*/
|
||||
export default async function grabConversationFiles({
|
||||
bot_id,
|
||||
workspace_id,
|
||||
}: Params): Promise<BotFileScopesPayload> {
|
||||
const scope = workspace_id
|
||||
? ({ scope: "workspace", owner_id: Number(workspace_id) } as const)
|
||||
: ({ scope: "bot", owner_id: Number(bot_id) } as const);
|
||||
|
||||
const [files, usage, quota_bytes, quota_mb] = await Promise.all([
|
||||
listBotFiles({ scopes: [scope], limit: AppData.BotFilesPageSize }),
|
||||
grabStorageUsage(scope),
|
||||
grabBotFileQuotaBytes(),
|
||||
grabBotFileQuotaMb(),
|
||||
]);
|
||||
|
||||
return {
|
||||
files: files.map((file) => {
|
||||
const { media_url } = grabMediaURL({ media: { id: file.media_id } as any });
|
||||
|
||||
return {
|
||||
id: Number(file.id),
|
||||
name: file.name || "",
|
||||
path: file.path || "",
|
||||
size: Number(file.size) || 0,
|
||||
media_type: String(file.media_type || "document"),
|
||||
purpose: String(file.purpose || "writing"),
|
||||
scope: String(file.scope || "bot"),
|
||||
owner_id: Number(file.owner_id) || 0,
|
||||
ingested_at: file.ingested_at ? Number(file.ingested_at) : undefined,
|
||||
ingest_error: file.ingest_error || undefined,
|
||||
created_at: Number(file.created_at) || 0,
|
||||
url: media_url || "",
|
||||
};
|
||||
}),
|
||||
folders: [],
|
||||
usage,
|
||||
quota_bytes,
|
||||
quota_mb,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { mkdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import type { BotFileScope } from "@/src/dict/bot-file-dict";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
|
||||
type Params = {
|
||||
scope: BotFileScope;
|
||||
/**
|
||||
* The bot id (scope bot) or workspace id (scope workspace).
|
||||
*/
|
||||
owner_id: number | string;
|
||||
/**
|
||||
* Create it when it's missing (the directories are made on demand, so
|
||||
* an install from before this has no backfill to do).
|
||||
*/
|
||||
create?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* A storage directory on the data volume: `DATA_DIR/bots/<id>` for a bot,
|
||||
* `DATA_DIR/workspaces/<id>` for a workspace. Files never live on a
|
||||
* computer, so they survive one being rebuilt.
|
||||
*/
|
||||
export default function grabStorageDir({ scope, owner_id, create = true }: Params) {
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const folder = scope == "workspace" ? AppData.WorkspaceDataDir : AppData.BotDataDir;
|
||||
const dir = path.join(DATA_DIR, folder, String(owner_id));
|
||||
|
||||
if (create) mkdirSync(dir, { recursive: true });
|
||||
|
||||
return dir;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BotFileScope } from "@/src/dict/bot-file-dict";
|
||||
|
||||
type Params = {
|
||||
scope: BotFileScope;
|
||||
owner_id: number | string;
|
||||
};
|
||||
|
||||
type UsageRow = {
|
||||
total: number | null;
|
||||
files: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* How many bytes (and files) a directory holds.
|
||||
*/
|
||||
export default async function grabStorageUsage({ scope, owner_id }: Params) {
|
||||
const res = await BunSQLite.sql<UsageRow>({
|
||||
sql: `SELECT COALESCE(SUM(size), 0) AS total, COUNT(*) AS files FROM bot_files WHERE scope = ? AND owner_id = ?`,
|
||||
values: [scope, owner_id],
|
||||
});
|
||||
|
||||
const row = res.payload?.[0];
|
||||
|
||||
return {
|
||||
bytes: Number(row?.total) || 0,
|
||||
files: Number(row?.files) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Every directory's usage at once, for the settings page.
|
||||
*/
|
||||
export async function grabStorageUsageByOwner() {
|
||||
const res = await BunSQLite.sql<{ scope: string; owner_id: number; total: number; files: number }>({
|
||||
sql: `SELECT scope, owner_id, SUM(size) AS total, COUNT(*) AS files FROM bot_files GROUP BY scope, owner_id`,
|
||||
values: [],
|
||||
});
|
||||
|
||||
return (res.payload || []).map((row) => ({
|
||||
scope: row.scope as BotFileScope,
|
||||
owner_id: Number(row.owner_id),
|
||||
bytes: Number(row.total) || 0,
|
||||
files: Number(row.files) || 0,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Total bytes across every directory.
|
||||
*/
|
||||
export async function grabTotalStorageUsage() {
|
||||
const res = await BunSQLite.sql<{ total: number | null; files: number | null }>({
|
||||
sql: `SELECT COALESCE(SUM(size), 0) AS total, COUNT(*) AS files FROM bot_files`,
|
||||
values: [],
|
||||
});
|
||||
|
||||
return {
|
||||
bytes: Number(res.payload?.[0]?.total) || 0,
|
||||
files: Number(res.payload?.[0]?.files) || 0,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type {
|
||||
BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS,
|
||||
BUN_SQLITE_COBBLE_BOT_FILES,
|
||||
BunSQLiteTables,
|
||||
} from "@/db/types/db";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import embedText from "@/src/harness/embeddings/embed-text";
|
||||
import HarnessError from "@/src/harness/harness-error";
|
||||
import chunkText from "./chunk-text";
|
||||
import extractBotFileText from "./extract-bot-file-text";
|
||||
|
||||
type Params = {
|
||||
file_id: number | string;
|
||||
/**
|
||||
* Read images and audio too (a bot asking for it).
|
||||
*/
|
||||
read_media?: boolean;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
export type IngestResult = {
|
||||
/**
|
||||
* `ingested`, `empty` (no text to read), `skipped` (too large) or
|
||||
* `unavailable` (no embedding model), which keeps the file's text but
|
||||
* leaves it searchable by words only.
|
||||
*/
|
||||
status: "ingested" | "empty" | "skipped" | "unavailable";
|
||||
chunks: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads a bot file's text, splits it into chunks and embeds them into
|
||||
* `bot_file_vectors`, replacing whatever was stored before. The chunks
|
||||
* stay in `bot_file_chunks` even when there's no embedding model, so a
|
||||
* search can still match words in them.
|
||||
*
|
||||
* Failures are recorded on the file (`ingest_error`) and rethrown, so the
|
||||
* queue can retry.
|
||||
*/
|
||||
export default async function ingestBotFile({
|
||||
file_id,
|
||||
read_media,
|
||||
signal,
|
||||
}: Params): Promise<IngestResult> {
|
||||
const file = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT * FROM bot_files WHERE id = ? LIMIT 1`,
|
||||
values: [file_id],
|
||||
}).then((res) => res.payload?.[0]);
|
||||
|
||||
if (!file?.id) return { status: "empty", chunks: 0 };
|
||||
|
||||
const media = file.media_id
|
||||
? await BunSQLite.sql<{ media_write_path?: string; text_content?: string }>({
|
||||
sql: `SELECT media_write_path, text_content FROM media WHERE id = ? LIMIT 1`,
|
||||
values: [file.media_id],
|
||||
}).then((res) => res.payload?.[0])
|
||||
: undefined;
|
||||
|
||||
if (Number(file.size) > AppData.IngestMaxBytes && !media?.text_content) {
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`,
|
||||
values: [`Too large to read (over ${AppData.IngestMaxBytes / (1024 * 1024)} MB).`, Date.now(), file.id],
|
||||
});
|
||||
|
||||
return { status: "skipped", chunks: 0 };
|
||||
}
|
||||
|
||||
try {
|
||||
const text = await extractBotFileText({
|
||||
write_path: media?.media_write_path,
|
||||
text_content: media?.text_content,
|
||||
name: file.name,
|
||||
mime_type: file.mime_type,
|
||||
read_media,
|
||||
});
|
||||
|
||||
await clearChunks({ file_id: file.id });
|
||||
|
||||
if (!text?.trim()) {
|
||||
await markIngested({ file_id: file.id, chunks: 0 });
|
||||
|
||||
return { status: "empty", chunks: 0 };
|
||||
}
|
||||
|
||||
const chunks = chunkText({ text });
|
||||
|
||||
if (!chunks.length) {
|
||||
await markIngested({ file_id: file.id, chunks: 0 });
|
||||
|
||||
return { status: "empty", chunks: 0 };
|
||||
}
|
||||
|
||||
const chunk_ids = await insertChunks({ file, chunks });
|
||||
let embedded = 0;
|
||||
|
||||
try {
|
||||
/**
|
||||
* A big file's chunks go in batches, so one provider refusal
|
||||
* doesn't throw away the whole file.
|
||||
*/
|
||||
for (let start = 0; start < chunks.length; start += AppData.EmbeddingBatchSize) {
|
||||
const batch = chunks.slice(start, start + AppData.EmbeddingBatchSize);
|
||||
const { vectors } = await embedText({
|
||||
input: batch,
|
||||
bot_id: file.bot_id!,
|
||||
workspace_id: file.scope == "workspace" ? file.owner_id : null,
|
||||
signal,
|
||||
});
|
||||
|
||||
for (let i = 0; i < vectors.length; i++) {
|
||||
await insertVector({
|
||||
vector: vectors[i]!,
|
||||
chunk_id: chunk_ids[start + i]!,
|
||||
file_id: file.id,
|
||||
scope: String(file.scope),
|
||||
owner_id: Number(file.owner_id),
|
||||
});
|
||||
|
||||
embedded++;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof HarnessError && error.code == "unavailable") {
|
||||
/**
|
||||
* No embedding model: the chunks still let a search match
|
||||
* words, so the file isn't a failure.
|
||||
*/
|
||||
await markIngested({
|
||||
file_id: file.id,
|
||||
chunks: chunks.length,
|
||||
error: "No embedding model is set, so this file is searchable by words only.",
|
||||
});
|
||||
|
||||
return { status: "unavailable", chunks: chunks.length };
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
await markIngested({ file_id: file.id, chunks: chunks.length });
|
||||
|
||||
return { status: "ingested", chunks: embedded };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`,
|
||||
values: [message.slice(0, 500), Date.now(), file.id],
|
||||
}).catch(() => {});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function clearChunks({ file_id }: { file_id: number }) {
|
||||
await BunSQLite.sql({ sql: `DELETE FROM bot_file_vectors WHERE file_id = ?`, values: [file_id] });
|
||||
await BunSQLite.sql({ sql: `DELETE FROM bot_file_chunks WHERE file_id = ?`, values: [file_id] });
|
||||
}
|
||||
|
||||
async function insertChunks({
|
||||
file,
|
||||
chunks,
|
||||
}: {
|
||||
file: BUN_SQLITE_COBBLE_BOT_FILES;
|
||||
chunks: string[];
|
||||
}) {
|
||||
const ids: number[] = [];
|
||||
|
||||
for (let index = 0; index < chunks.length; index++) {
|
||||
const text = chunks[index]!;
|
||||
|
||||
const res = await BunSQLite.insert<
|
||||
BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS,
|
||||
(typeof BunSQLiteTables)[number]
|
||||
>({
|
||||
table: "bot_file_chunks",
|
||||
data: [
|
||||
{
|
||||
file_id: file.id,
|
||||
scope: file.scope as any,
|
||||
owner_id: Number(file.owner_id),
|
||||
chunk_index: index,
|
||||
text,
|
||||
token_estimate: Math.ceil(text.length / 4),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const id = res.postInsertReturn?.insertId;
|
||||
|
||||
if (id) ids.push(id);
|
||||
}
|
||||
|
||||
return ids;
|
||||
}
|
||||
|
||||
async function insertVector({
|
||||
vector,
|
||||
chunk_id,
|
||||
file_id,
|
||||
scope,
|
||||
owner_id,
|
||||
}: {
|
||||
vector: Float32Array;
|
||||
chunk_id: number;
|
||||
file_id: number;
|
||||
scope: string;
|
||||
owner_id: number;
|
||||
}) {
|
||||
/**
|
||||
* vec0's metadata columns (its id and timestamps) are NOT NULL, so a
|
||||
* raw insert has to fill them; `BunSQLite.insert` would add them but
|
||||
* the vector needs `vec_f32`.
|
||||
*/
|
||||
const now = Date.now();
|
||||
|
||||
const res = await BunSQLite.sql({
|
||||
sql: `INSERT INTO bot_file_vectors (created_at, updated_at, embedding, chunk_id, file_id, scope, owner_id) VALUES (?, ?, vec_f32(?), ?, ?, ?, ?)`,
|
||||
values: [now, now, vector as any, chunk_id, file_id, scope, owner_id],
|
||||
});
|
||||
|
||||
if (!res.success) {
|
||||
throw new Error(`Couldn't store an embedding: ${res.error || res.msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function markIngested({
|
||||
file_id,
|
||||
chunks,
|
||||
error,
|
||||
}: {
|
||||
file_id: number;
|
||||
chunks: number;
|
||||
error?: string;
|
||||
}) {
|
||||
const now = Date.now();
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE bot_files SET ingested_at = ?, ingest_error = ?, updated_at = ? WHERE id = ?`,
|
||||
values: [now, error || null, now, file_id],
|
||||
});
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE media SET is_rag_ingested = ?, updated_at = ? WHERE id = (SELECT media_id FROM bot_files WHERE id = ?)`,
|
||||
values: [chunks > 0 ? 1 : 0, now, file_id],
|
||||
}).catch(() => {});
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import type { StorageScope } from "@/src/utils/storage-access";
|
||||
|
||||
type Params = {
|
||||
scopes: StorageScope[];
|
||||
/**
|
||||
* Only this directory's files (a folder path), or, with `recursive`,
|
||||
* everything under it.
|
||||
*/
|
||||
folder?: string | null;
|
||||
recursive?: boolean;
|
||||
/**
|
||||
* Match the name or the path, case-insensitive.
|
||||
*/
|
||||
search?: string | null;
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* The files in the run's directories, newest first: one folder's, or
|
||||
* everything under it, optionally matching a name.
|
||||
*/
|
||||
export default async function listBotFiles({
|
||||
scopes,
|
||||
folder,
|
||||
recursive,
|
||||
search,
|
||||
limit = AppData.BotFileListMaxItems,
|
||||
}: Params): Promise<BUN_SQLITE_COBBLE_BOT_FILES[]> {
|
||||
if (!scopes.length) return [];
|
||||
|
||||
const target = (folder || "").trim().replace(/\\/g, "/").replace(/^\/+|\/+$/g, "");
|
||||
|
||||
const owner_sql = scopes.map(() => `(scope = ? AND owner_id = ?)`).join(" OR ");
|
||||
const owner_values = scopes.flatMap((s) => [s.scope, s.owner_id]);
|
||||
|
||||
const where = [owner_sql];
|
||||
const values: (string | number)[] = [...owner_values];
|
||||
|
||||
if (target) {
|
||||
where.push(
|
||||
recursive
|
||||
? `(folder = ? OR folder LIKE ?)`
|
||||
: `folder = ?`,
|
||||
);
|
||||
|
||||
values.push(target);
|
||||
|
||||
if (recursive) values.push(`${target}/%`);
|
||||
} else if (!recursive) {
|
||||
where.push(`(folder = '' OR folder IS NULL)`);
|
||||
}
|
||||
|
||||
if (search?.trim()) {
|
||||
where.push(`(name LIKE ? OR path LIKE ?)`);
|
||||
values.push(`%${search.trim()}%`, `%${search.trim()}%`);
|
||||
}
|
||||
|
||||
const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT * FROM bot_files WHERE ${where.join(" AND ")} ORDER BY updated_at DESC, id DESC LIMIT ?`,
|
||||
values: [...values, limit],
|
||||
});
|
||||
|
||||
return res.payload || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* The folders across the run's directories: a file's folder and every
|
||||
* parent of it, so an empty folder still shows (folders are the paths
|
||||
* that exist).
|
||||
*/
|
||||
export async function listBotFolders({
|
||||
scopes,
|
||||
limit = AppData.BotFileListMaxItems,
|
||||
}: {
|
||||
scopes: StorageScope[];
|
||||
limit?: number;
|
||||
}) {
|
||||
const files = await listBotFiles({ scopes, recursive: true, limit: 5000 });
|
||||
const folders = new Map<string, { scope: string; owner_id: number; path: string }>();
|
||||
|
||||
for (const file of files) {
|
||||
const segments = (file.folder || "").split("/").filter(Boolean);
|
||||
|
||||
for (let i = 1; i <= segments.length; i++) {
|
||||
const path = segments.slice(0, i).join("/");
|
||||
const key = `${file.scope}:${file.owner_id}:${path}`;
|
||||
|
||||
if (!folders.has(key)) {
|
||||
folders.set(key, {
|
||||
scope: String(file.scope),
|
||||
owner_id: Number(file.owner_id),
|
||||
path,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (folders.size >= limit) break;
|
||||
}
|
||||
|
||||
return [...folders.values()].sort((a, b) => a.path.localeCompare(b.path));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import type { MediaType } from "@/src/types";
|
||||
import { BotImageFileExtensions } from "@/src/dict/bot-file-dict";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* A file name or an extension, e.g. `report.pdf` or `pdf`.
|
||||
*/
|
||||
name?: string | null;
|
||||
mime_type?: string | null;
|
||||
};
|
||||
|
||||
const AudioExtensions = ["mp3", "wav", "m4a", "ogg", "flac", "webm"];
|
||||
const VideoExtensions = ["mp4", "mov", "mkv", "avi"];
|
||||
|
||||
/**
|
||||
* What a file's `media_type` is, from its extension (else its MIME type):
|
||||
* an image, a document, audio or video.
|
||||
*/
|
||||
export default function mediaTypeForFile({ name, mime_type }: Params): MediaType {
|
||||
const ext = extensionOf({ name, mime_type });
|
||||
|
||||
if (ext && BotImageFileExtensions.includes(ext)) return "image";
|
||||
if (ext && AudioExtensions.includes(ext)) return "audio";
|
||||
if (ext && VideoExtensions.includes(ext)) return "video";
|
||||
|
||||
return "document";
|
||||
}
|
||||
|
||||
/**
|
||||
* Its extension, lower case, without the dot.
|
||||
*/
|
||||
export function extensionOf({ name, mime_type }: Params) {
|
||||
const from_name = name?.includes(".") ? name.split(".").pop() : undefined;
|
||||
|
||||
if (from_name) return from_name.toLowerCase().replace(/[^a-z0-9]/g, "");
|
||||
|
||||
const from_mime = mime_type && !mime_type.includes("octet-stream")
|
||||
? mime_type.split("/").pop()
|
||||
: undefined;
|
||||
|
||||
return from_mime ? from_mime.toLowerCase().replace(/[^a-z0-9]/g, "") : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A MIME type for the row, from the extension when there isn't one.
|
||||
*/
|
||||
export function mimeTypeForFile({ name, mime_type }: Params) {
|
||||
if (mime_type && mime_type != "application/octet-stream") return mime_type;
|
||||
|
||||
const ext = extensionOf({ name });
|
||||
|
||||
return (ext && MediaTypesByExtension[ext]) || "application/octet-stream";
|
||||
}
|
||||
|
||||
/**
|
||||
* The types worth knowing, so nothing has to guess (SNIFFING a file for
|
||||
* this would mean reading it twice).
|
||||
*/
|
||||
const MediaTypesByExtension: Record<string, string> = {
|
||||
txt: "text/plain",
|
||||
text: "text/plain",
|
||||
log: "text/plain",
|
||||
md: "text/markdown",
|
||||
markdown: "text/markdown",
|
||||
csv: "text/csv",
|
||||
tsv: "text/tab-separated-values",
|
||||
json: "application/json",
|
||||
html: "text/html",
|
||||
htm: "text/html",
|
||||
xml: "application/xml",
|
||||
yml: "application/yaml",
|
||||
yaml: "application/yaml",
|
||||
pdf: "application/pdf",
|
||||
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
doc: "application/msword",
|
||||
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
png: "image/png",
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
webp: "image/webp",
|
||||
gif: "image/gif",
|
||||
bmp: "image/bmp",
|
||||
svg: "image/svg+xml",
|
||||
mp3: "audio/mpeg",
|
||||
wav: "audio/wav",
|
||||
m4a: "audio/mp4",
|
||||
ogg: "audio/ogg",
|
||||
flac: "audio/flac",
|
||||
mp4: "video/mp4",
|
||||
mov: "video/quicktime",
|
||||
webm: "video/webm",
|
||||
zip: "application/zip",
|
||||
};
|
||||
@@ -0,0 +1,84 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import { existsSync, mkdirSync, renameSync, rmSync, copyFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import resolveBotPath from "@/src/utils/resolve-bot-path";
|
||||
import type { StorageScope } from "@/src/utils/storage-access";
|
||||
import { canReachScope } from "@/src/utils/storage-access";
|
||||
import grabBotFile from "./grab-bot-file";
|
||||
import grabStorageDir from "./grab-storage-dir";
|
||||
import { grabBotFileRow } from "./record-bot-file";
|
||||
|
||||
type Params = {
|
||||
scopes: StorageScope[];
|
||||
file_id: number | string;
|
||||
/**
|
||||
* The new path, in the file's own directory.
|
||||
*/
|
||||
path: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Renames a file or moves it into another folder, inside its own
|
||||
* directory. The bytes move on disk and the row follows; an existing file
|
||||
* at the target is refused, never overwritten by a rename.
|
||||
*/
|
||||
export default async function moveBotFile({ scopes, file_id, path: target }: Params) {
|
||||
const file = await grabBotFile({ scopes, file_id });
|
||||
|
||||
if (!file?.id) {
|
||||
throw new AppError(`That file isn't in your files.`);
|
||||
}
|
||||
|
||||
if (!canReachScope({ scopes, scope: file.scope as any, owner_id: file.owner_id! })) {
|
||||
throw new AppError(`That file isn't in your files.`);
|
||||
}
|
||||
|
||||
const root = grabStorageDir({ scope: file.scope as any, owner_id: file.owner_id! });
|
||||
const resolved = resolveBotPath({ root, path: target });
|
||||
|
||||
if (resolved.relative == file.path) return file;
|
||||
|
||||
const taken = await grabBotFileRow({
|
||||
scope: file.scope as any,
|
||||
owner_id: file.owner_id!,
|
||||
path: resolved.relative,
|
||||
});
|
||||
|
||||
if (taken?.id) {
|
||||
throw new AppError(`There's already a file at ${resolved.relative}.`);
|
||||
}
|
||||
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const from = path.join(root, file.path || "");
|
||||
|
||||
mkdirSync(path.dirname(resolved.absolute), { recursive: true });
|
||||
|
||||
if (existsSync(from)) {
|
||||
try {
|
||||
renameSync(from, resolved.absolute);
|
||||
} catch {
|
||||
copyFileSync(from, resolved.absolute);
|
||||
rmSync(from, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE bot_files SET path = ?, name = ?, folder = ?, updated_at = ? WHERE id = ?`,
|
||||
values: [resolved.relative, resolved.name, resolved.folder, Date.now(), file.id],
|
||||
});
|
||||
|
||||
if (file.media_id) {
|
||||
const write_path = resolved.absolute.startsWith(DATA_DIR)
|
||||
? resolved.absolute.slice(DATA_DIR.length).replace(/^[\\/]/, "").split("\\").join("/")
|
||||
: resolved.relative;
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE media SET media_write_path = ?, updated_at = ? WHERE id = ?`,
|
||||
values: [write_path, Date.now(), file.media_id],
|
||||
});
|
||||
}
|
||||
|
||||
return { ...file, path: resolved.relative, name: resolved.name, folder: resolved.folder };
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db";
|
||||
import { rmSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { BotFileScope } from "@/src/dict/bot-file-dict";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import { logError } from "@/src/functions/backend/logger";
|
||||
import grabStorageDir from "./grab-storage-dir";
|
||||
|
||||
type Params = {
|
||||
scope: BotFileScope;
|
||||
owner_id: number | string;
|
||||
/**
|
||||
* Its files' `media` ids, read before the rows go. Left out, they are
|
||||
* read here (only useful when the catalog rows still exist).
|
||||
*/
|
||||
media_ids?: number[];
|
||||
};
|
||||
|
||||
/**
|
||||
* The `media` ids of a directory's files, read before the bot or workspace
|
||||
* (and with it, the catalog rows) is deleted.
|
||||
*/
|
||||
export async function grabStorageMediaIds({ scope, owner_id }: { scope: BotFileScope; owner_id: number | string }) {
|
||||
const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT media_id FROM bot_files WHERE scope = ? AND owner_id = ?`,
|
||||
values: [scope, owner_id],
|
||||
});
|
||||
|
||||
return (res.payload || []).map((file) => Number(file.media_id)).filter((id) => id > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a directory's files for good when the bot or workspace it
|
||||
* belongs to goes: the disk folder, every file's `media` row, and what is
|
||||
* left of the catalog. A failure is logged, never thrown: deleting the
|
||||
* bot must not stop on a stray file.
|
||||
*/
|
||||
export default async function purgeStorage({ scope, owner_id, media_ids }: Params) {
|
||||
try {
|
||||
const ids = media_ids || (await grabStorageMediaIds({ scope, owner_id }));
|
||||
|
||||
if (ids.length) {
|
||||
const placeholders = ids.map(() => "?").join(", ");
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `DELETE FROM media_paradigms WHERE media_id IN (${placeholders})`,
|
||||
values: ids,
|
||||
}).catch(() => {});
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `DELETE FROM media WHERE id IN (${placeholders})`,
|
||||
values: ids,
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `DELETE FROM bot_file_vectors WHERE scope = ? AND owner_id = ?`,
|
||||
values: [scope, owner_id],
|
||||
}).catch(() => {});
|
||||
|
||||
/**
|
||||
* Usually gone already, with the bot or workspace (the cascade).
|
||||
*/
|
||||
await BunSQLite.sql({
|
||||
sql: `DELETE FROM bot_files WHERE scope = ? AND owner_id = ?`,
|
||||
values: [scope, owner_id],
|
||||
});
|
||||
|
||||
const dir = grabStorageDir({ scope, owner_id, create: false });
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
|
||||
/**
|
||||
* Never outside the data volume, whatever the path says.
|
||||
*/
|
||||
if (path.resolve(dir).startsWith(path.resolve(DATA_DIR))) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
} catch (error) {
|
||||
await logError({
|
||||
title: `${scope} ${owner_id}'s files couldn't be cleaned up`,
|
||||
error,
|
||||
metadata: { scope, owner_id },
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type {
|
||||
BUN_SQLITE_COBBLE_BOT_FILES,
|
||||
BUN_SQLITE_COBBLE_MEDIA,
|
||||
BunSQLiteTables,
|
||||
} from "@/db/types/db";
|
||||
import type { BotFilePurpose, BotFileScope, BotFileSource } from "@/src/dict/bot-file-dict";
|
||||
import type { MediaType } from "@/src/types";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import resolveBotPath from "@/src/utils/resolve-bot-path";
|
||||
import assertStorageRoom from "./assert-storage-room";
|
||||
import grabStorageDir from "./grab-storage-dir";
|
||||
import mediaTypeForFile, { mimeTypeForFile } from "./media-type-for-file";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* The bot saving it (who `bot_files.bot_id` names).
|
||||
*/
|
||||
bot_id: number | string;
|
||||
scope: BotFileScope;
|
||||
/**
|
||||
* The bot or workspace owning the directory.
|
||||
*/
|
||||
owner_id: number | string;
|
||||
/**
|
||||
* Path inside the directory, e.g. `reports/q3.md`.
|
||||
*/
|
||||
path: string;
|
||||
data: Uint8Array | string;
|
||||
purpose?: BotFilePurpose;
|
||||
source?: BotFileSource;
|
||||
/**
|
||||
* Overrides the type read from the name.
|
||||
*/
|
||||
media_type?: MediaType;
|
||||
mime_type?: string;
|
||||
/**
|
||||
* Who the `media` row says uploaded it: the user the run is for.
|
||||
*/
|
||||
user_id?: number | string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Saves a file into a storage directory: the bytes on the data volume and
|
||||
* the catalog row holding them, as one step. Every writer goes through
|
||||
* here (a bot's `write_file`, an upload, a generated image, a PDF), so
|
||||
* disk and database never disagree.
|
||||
*
|
||||
* Writing over an existing path keeps its `bot_files` row and its `media`
|
||||
* row, so a message that attached the file now shows the new bytes.
|
||||
*/
|
||||
export default async function recordBotFile({
|
||||
bot_id,
|
||||
scope,
|
||||
owner_id,
|
||||
path,
|
||||
data,
|
||||
purpose = "writing",
|
||||
source = "bot-write",
|
||||
media_type,
|
||||
mime_type,
|
||||
user_id,
|
||||
}: Params): Promise<BUN_SQLITE_COBBLE_BOT_FILES> {
|
||||
const root = grabStorageDir({ scope, owner_id });
|
||||
const resolved = resolveBotPath({ root, path });
|
||||
const bytes = typeof data == "string" ? Buffer.from(data, "utf8") : data;
|
||||
const media_type_resolved = media_type || mediaTypeForFile({ name: resolved.name, mime_type });
|
||||
const mime = mimeTypeForFile({ name: resolved.name, mime_type });
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
|
||||
const existing = await grabBotFileRow({ scope, owner_id, path: resolved.relative });
|
||||
const previous_size = Number(existing?.size) || 0;
|
||||
|
||||
await assertStorageRoom({
|
||||
scope,
|
||||
owner_id,
|
||||
incoming_bytes: bytes.byteLength - previous_size,
|
||||
});
|
||||
|
||||
const absolute = resolved.absolute;
|
||||
const write_path = absolute.startsWith(DATA_DIR)
|
||||
? absolute.slice(DATA_DIR.length).replace(/^[\\/]/, "").split("\\").join("/")
|
||||
: resolved.relative;
|
||||
|
||||
await Bun.write(absolute, bytes);
|
||||
|
||||
const media_id = existing?.media_id
|
||||
? await updateMediaRow({
|
||||
media_id: existing.media_id,
|
||||
write_path,
|
||||
mime,
|
||||
media_type: media_type_resolved,
|
||||
})
|
||||
: await insertMediaRow({
|
||||
write_path,
|
||||
mime,
|
||||
media_type: media_type_resolved,
|
||||
title: resolved.name,
|
||||
user_id,
|
||||
});
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
if (existing?.id) {
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE bot_files SET media_id = ?, name = ?, folder = ?, mime_type = ?, media_type = ?, size = ?,
|
||||
purpose = ?, source = ?, ingested_at = NULL, ingest_error = NULL, updated_at = ? WHERE id = ?`,
|
||||
values: [
|
||||
media_id,
|
||||
resolved.name,
|
||||
resolved.folder,
|
||||
mime,
|
||||
media_type_resolved,
|
||||
bytes.byteLength,
|
||||
purpose,
|
||||
source,
|
||||
now,
|
||||
existing.id,
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
...existing,
|
||||
media_id,
|
||||
name: resolved.name,
|
||||
folder: resolved.folder,
|
||||
mime_type: mime,
|
||||
media_type: media_type_resolved as any,
|
||||
size: bytes.byteLength,
|
||||
purpose,
|
||||
source,
|
||||
ingested_at: undefined,
|
||||
ingest_error: undefined,
|
||||
updated_at: now,
|
||||
};
|
||||
}
|
||||
|
||||
const res = await BunSQLite.insert<BUN_SQLITE_COBBLE_BOT_FILES, (typeof BunSQLiteTables)[number]>({
|
||||
table: "bot_files",
|
||||
data: [
|
||||
{
|
||||
bot_id: Number(bot_id),
|
||||
scope,
|
||||
owner_id: Number(owner_id),
|
||||
media_id,
|
||||
path: resolved.relative,
|
||||
name: resolved.name,
|
||||
folder: resolved.folder,
|
||||
mime_type: mime,
|
||||
media_type: media_type_resolved as any,
|
||||
size: bytes.byteLength,
|
||||
purpose,
|
||||
source,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const id = res.postInsertReturn?.insertId;
|
||||
|
||||
if (!res.success || !id) {
|
||||
throw new Error(`Couldn't record the file: ${res.msg || res.error}`);
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
bot_id: Number(bot_id),
|
||||
scope,
|
||||
owner_id: Number(owner_id),
|
||||
media_id,
|
||||
path: resolved.relative,
|
||||
name: resolved.name,
|
||||
folder: resolved.folder,
|
||||
mime_type: mime,
|
||||
media_type: media_type_resolved as any,
|
||||
size: bytes.byteLength,
|
||||
purpose,
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The catalog row at a path, if there is one.
|
||||
*/
|
||||
export async function grabBotFileRow({
|
||||
scope,
|
||||
owner_id,
|
||||
path,
|
||||
}: {
|
||||
scope: BotFileScope;
|
||||
owner_id: number | string;
|
||||
path: string;
|
||||
}) {
|
||||
const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT * FROM bot_files WHERE scope = ? AND owner_id = ? AND path = ? LIMIT 1`,
|
||||
values: [scope, owner_id, path],
|
||||
});
|
||||
|
||||
return res.payload?.[0] || null;
|
||||
}
|
||||
|
||||
async function insertMediaRow({
|
||||
write_path,
|
||||
mime,
|
||||
media_type,
|
||||
title,
|
||||
user_id,
|
||||
}: {
|
||||
write_path: string;
|
||||
mime: string;
|
||||
media_type: MediaType;
|
||||
title: string;
|
||||
user_id?: number | string;
|
||||
}) {
|
||||
const res = await BunSQLite.insert<BUN_SQLITE_COBBLE_MEDIA, (typeof BunSQLiteTables)[number]>({
|
||||
table: "media",
|
||||
data: [
|
||||
{
|
||||
user_id: Number(user_id) || 0,
|
||||
title,
|
||||
media_type: media_type as any,
|
||||
media_write_path: write_path,
|
||||
mime_type: mime,
|
||||
is_private: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const id = res.postInsertReturn?.insertId;
|
||||
|
||||
if (!res.success || !id) {
|
||||
throw new Error(`Couldn't record the file's bytes: ${res.msg || res.error}`);
|
||||
}
|
||||
|
||||
await BunSQLite.insert({
|
||||
table: "media_paradigms",
|
||||
data: [{ media_id: id, media_paradigm: "bot-file" as any }],
|
||||
});
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
async function updateMediaRow({
|
||||
media_id,
|
||||
write_path,
|
||||
mime,
|
||||
media_type,
|
||||
}: {
|
||||
media_id: number | string;
|
||||
write_path: string;
|
||||
mime: string;
|
||||
media_type: MediaType;
|
||||
}): Promise<number> {
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE media SET media_write_path = ?, mime_type = ?, media_type = ?, is_rag_ingested = 0, updated_at = ? WHERE id = ?`,
|
||||
values: [write_path, mime, media_type, Date.now(), media_id],
|
||||
});
|
||||
|
||||
return Number(media_id);
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import { embedQuery } from "@/src/harness/embeddings/embed-text";
|
||||
import HarnessError from "@/src/harness/harness-error";
|
||||
import type { StorageScope } from "@/src/utils/storage-access";
|
||||
|
||||
type Params = {
|
||||
scopes: StorageScope[];
|
||||
query: string;
|
||||
/**
|
||||
* Only this file's chunks.
|
||||
*/
|
||||
file_id?: number | string | null;
|
||||
limit?: number;
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Whose embedding model to use.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
workspace_id?: number | string | null;
|
||||
};
|
||||
|
||||
export type BotFileSearchHit = {
|
||||
file_id: number;
|
||||
chunk_id: number;
|
||||
chunk_index: number;
|
||||
name: string;
|
||||
path: string;
|
||||
scope: string;
|
||||
/**
|
||||
* Cosine distance when the vector index answered, null for a word
|
||||
* match.
|
||||
*/
|
||||
distance: number | null;
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type BotFileSearchResult = {
|
||||
hits: BotFileSearchHit[];
|
||||
/**
|
||||
* How the search ran: `vector`, `words` (no embedding model, or nothing
|
||||
* matched) or `none` (no files at all).
|
||||
*/
|
||||
mode: "vector" | "words" | "none";
|
||||
/**
|
||||
* Said plainly for the bot's own reading of what it got back.
|
||||
*/
|
||||
note: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Searches a run's files: by meaning through the embedding vectors when a
|
||||
* model is set, else (or when nothing matches) by words over the stored
|
||||
* chunks. Never throws for an unavailable model: the bot always gets
|
||||
* something it can act on.
|
||||
*/
|
||||
export default async function searchBotFiles({
|
||||
scopes,
|
||||
query,
|
||||
file_id,
|
||||
limit = AppData.RagTopChunks,
|
||||
signal,
|
||||
bot_id,
|
||||
workspace_id,
|
||||
}: Params): Promise<BotFileSearchResult> {
|
||||
const text = query.trim();
|
||||
|
||||
if (!scopes.length || !text) {
|
||||
return { hits: [], mode: "none", note: "No files to search." };
|
||||
}
|
||||
|
||||
try {
|
||||
const vector = await embedQuery({
|
||||
text,
|
||||
signal,
|
||||
bot_id,
|
||||
workspace_id,
|
||||
});
|
||||
|
||||
const hits = await vectorSearch({
|
||||
scopes,
|
||||
query_vector: vector,
|
||||
file_id,
|
||||
limit,
|
||||
seek: Math.min(limit * AppData.RagOverFetch, AppData.RagMaxK),
|
||||
});
|
||||
|
||||
if (hits.length) {
|
||||
return {
|
||||
hits,
|
||||
mode: "vector",
|
||||
note: `${hits.length} passage(s), found by meaning.`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
if (!(error instanceof HarnessError)) throw error;
|
||||
}
|
||||
|
||||
const hits = await wordSearch({ scopes, query: text, file_id, limit });
|
||||
|
||||
return {
|
||||
hits,
|
||||
mode: hits.length ? "words" : "none",
|
||||
note: hits.length
|
||||
? `${hits.length} passage(s), found by matching words (no embedding model, or nothing matched by meaning).`
|
||||
: `Nothing in your files matched "${text}".`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The nearest chunks by vector. sqlite-vec refuses a filter on a vec0
|
||||
* side-car column in the same query as the match, so the index is asked
|
||||
* for its nearest `seek` and the run's scopes are applied to that list
|
||||
* here, before the chunk text is read.
|
||||
*/
|
||||
async function vectorSearch({
|
||||
scopes,
|
||||
query_vector,
|
||||
file_id,
|
||||
limit,
|
||||
seek,
|
||||
}: {
|
||||
scopes: StorageScope[];
|
||||
query_vector: Float32Array;
|
||||
file_id?: number | string | null;
|
||||
limit: number;
|
||||
seek: number;
|
||||
}): Promise<BotFileSearchHit[]> {
|
||||
const nearest = await BunSQLite.sql<{
|
||||
chunk_id: number;
|
||||
file_id: number;
|
||||
scope: string;
|
||||
owner_id: number;
|
||||
distance: number;
|
||||
}>({
|
||||
sql: `SELECT chunk_id, file_id, scope, owner_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?) AND k = ? ORDER BY distance`,
|
||||
values: [query_vector as any, seek],
|
||||
});
|
||||
|
||||
/**
|
||||
* `BunSQLite.sql` resolves with `success: false` instead of throwing, so
|
||||
* a broken vector query would otherwise read as "nothing matched" and
|
||||
* silently fall back to words. A failure here is a real error.
|
||||
*/
|
||||
if (!nearest.success) {
|
||||
throw new Error(`The file search index couldn't be read: ${nearest.error || nearest.msg}`);
|
||||
}
|
||||
|
||||
const allowed = nearest.payload?.filter((row) =>
|
||||
scopes.some((scope) => scope.scope == row.scope && Number(scope.owner_id) == Number(row.owner_id)),
|
||||
);
|
||||
|
||||
const picked = (allowed || [])
|
||||
.filter((row) => !file_id || Number(row.file_id) == Number(file_id))
|
||||
.slice(0, limit);
|
||||
|
||||
if (!picked.length) return [];
|
||||
|
||||
const chunk_ids = picked.map((row) => Number(row.chunk_id));
|
||||
const placeholders = chunk_ids.map(() => "?").join(", ");
|
||||
|
||||
const chunks = await BunSQLite.sql<{
|
||||
id: number;
|
||||
text: string;
|
||||
chunk_index: number;
|
||||
name: string;
|
||||
path: string;
|
||||
}>({
|
||||
sql: `SELECT c.id, c.text, c.chunk_index, f.name, f.path FROM bot_file_chunks c
|
||||
JOIN bot_files f ON f.id = c.file_id
|
||||
WHERE c.id IN (${placeholders})`,
|
||||
values: chunk_ids,
|
||||
});
|
||||
|
||||
const by_id = new Map((chunks.payload || []).map((row) => [Number(row.id), row]));
|
||||
|
||||
return picked.flatMap((row) => {
|
||||
const chunk = by_id.get(Number(row.chunk_id));
|
||||
|
||||
if (!chunk) return [];
|
||||
|
||||
return [
|
||||
toHit({
|
||||
chunk_id: row.chunk_id,
|
||||
file_id: row.file_id,
|
||||
scope: row.scope,
|
||||
distance: Number(row.distance),
|
||||
text: chunk.text,
|
||||
chunk_index: chunk.chunk_index,
|
||||
name: chunk.name,
|
||||
path: chunk.path,
|
||||
}),
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunks whose text has the query's words, newest files first. Used when
|
||||
* there's no embedding model, and as the fallback when the vectors
|
||||
* answered nothing.
|
||||
*/
|
||||
async function wordSearch({
|
||||
scopes,
|
||||
query,
|
||||
file_id,
|
||||
limit,
|
||||
}: {
|
||||
scopes: StorageScope[];
|
||||
query: string;
|
||||
file_id?: number | string | null;
|
||||
limit: number;
|
||||
}): Promise<BotFileSearchHit[]> {
|
||||
const words = [
|
||||
...new Set(
|
||||
query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter((word) => word.length > 2),
|
||||
),
|
||||
].slice(0, 6);
|
||||
|
||||
if (!words.length) return [];
|
||||
|
||||
const scope_sql = scopes.map(() => `(c.scope = ? AND c.owner_id = ?)`).join(" OR ");
|
||||
const scope_values = scopes.flatMap((s) => [s.scope, s.owner_id]);
|
||||
const word_sql = words.map(() => `lower(c.text) LIKE ?`).join(" OR ");
|
||||
const word_values = words.map((word) => `%${word}%`);
|
||||
const file_clause = file_id ? "AND c.file_id = ?" : "";
|
||||
|
||||
const res = await BunSQLite.sql<{
|
||||
chunk_id: number;
|
||||
file_id: number;
|
||||
text: string;
|
||||
chunk_index: number;
|
||||
name: string;
|
||||
path: string;
|
||||
scope: string;
|
||||
}>({
|
||||
sql: `SELECT c.id AS chunk_id, c.file_id, c.text, c.chunk_index, c.scope, f.name, f.path
|
||||
FROM bot_file_chunks c
|
||||
JOIN bot_files f ON f.id = c.file_id
|
||||
WHERE (${scope_sql}) ${file_clause} AND (${word_sql})
|
||||
ORDER BY f.updated_at DESC, c.chunk_index
|
||||
LIMIT ?`,
|
||||
values: [...scope_values, ...(file_id ? [file_id] : []), ...word_values, limit],
|
||||
});
|
||||
|
||||
return (res.payload || []).map((row) => toHit({ ...row, distance: null }));
|
||||
}
|
||||
|
||||
function toHit(row: {
|
||||
chunk_id: number;
|
||||
file_id: number;
|
||||
text: string;
|
||||
chunk_index: number;
|
||||
name: string;
|
||||
path: string;
|
||||
scope: string;
|
||||
distance: number | null;
|
||||
}): BotFileSearchHit {
|
||||
return {
|
||||
file_id: Number(row.file_id),
|
||||
chunk_id: Number(row.chunk_id),
|
||||
chunk_index: Number(row.chunk_index),
|
||||
name: row.name || "",
|
||||
path: row.path || "",
|
||||
scope: row.scope || "bot",
|
||||
distance: row.distance == null ? null : Number(row.distance),
|
||||
text: row.text || "",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
|
||||
import { BotUploadFolder } from "@/src/dict/bot-file-dict";
|
||||
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
|
||||
import { logError } from "@/src/functions/backend/logger";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import recordBotFile from "./record-bot-file";
|
||||
|
||||
type Params = {
|
||||
bot_id: number | string;
|
||||
workspace_id?: number | string | null;
|
||||
user_id: number | string;
|
||||
media: BUN_SQLITE_COBBLE_MEDIA[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Copies the files the user attached to a message into the conversation's
|
||||
* folder (`uploads/<yyyy-mm>/<name>`), so they stay searchable after the
|
||||
* message scrolls away and a later run can find them. Best effort: a
|
||||
* failure is logged, never shown, and the message still goes out.
|
||||
*/
|
||||
export default async function storeChatAttachments({
|
||||
bot_id,
|
||||
workspace_id,
|
||||
user_id,
|
||||
media,
|
||||
}: Params) {
|
||||
if (!media.length) return;
|
||||
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const folder = `${BotUploadFolder}/${new Date().toISOString().slice(0, 7)}`;
|
||||
|
||||
for (const file of media) {
|
||||
try {
|
||||
if (!file.media_write_path) continue;
|
||||
|
||||
const source = Bun.file(`${DATA_DIR}/${file.media_write_path}`);
|
||||
|
||||
if (!(await source.exists())) continue;
|
||||
|
||||
/**
|
||||
* Its name, made safe and unique enough that a second upload
|
||||
* of "invoice.pdf" doesn't overwrite the first.
|
||||
*/
|
||||
const name = safeName({ name: file.title || `file-${file.id}` });
|
||||
|
||||
const saved = await recordBotFile({
|
||||
bot_id,
|
||||
scope: workspace_id ? "workspace" : "bot",
|
||||
owner_id: workspace_id || bot_id,
|
||||
path: `${folder}/${name}`,
|
||||
data: new Uint8Array(await source.arrayBuffer()),
|
||||
purpose: "upload",
|
||||
source: "chat-upload",
|
||||
media_type: (file.media_type as any) || undefined,
|
||||
mime_type: file.mime_type || undefined,
|
||||
user_id,
|
||||
});
|
||||
|
||||
/**
|
||||
* The same text the upload already extracted, so a PDF isn't
|
||||
* unpacked a second time.
|
||||
*/
|
||||
if (file.text_content && saved.media_id) {
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE media SET text_content = ? WHERE id = ?`,
|
||||
values: [file.text_content, saved.media_id],
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
await enqueueFileIngest({ file_id: saved.id! });
|
||||
} catch (error) {
|
||||
await logError({
|
||||
title: `A chat attachment couldn't be saved to the bot's files`,
|
||||
error,
|
||||
metadata: { media_id: file.id, bot_id },
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A file name that is safe on disk.
|
||||
*/
|
||||
function safeName({ name }: { name: string }) {
|
||||
const cleaned = name
|
||||
.replace(/[<>:"|?*/\\]/g, "-")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 100);
|
||||
|
||||
return cleaned || "file";
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import * as sqliteVec from "sqlite-vec";
|
||||
import fitVectorDimensions from "@/src/harness/embeddings/fit-vector-dimensions";
|
||||
|
||||
/**
|
||||
* The exact insert and nearest-neighbour shape `ingestBotFile` and
|
||||
* `searchBotFiles` use, on a scratch database: vec0's metadata columns are
|
||||
* NOT NULL, and a side-car filter can't share the match query.
|
||||
*/
|
||||
function createDb() {
|
||||
const db = new Database(":memory:");
|
||||
sqliteVec.load(db);
|
||||
db.run(`CREATE VIRTUAL TABLE bot_file_vectors USING vec0(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at INTEGER,
|
||||
updated_at INTEGER,
|
||||
embedding float[8],
|
||||
+chunk_id INTEGER,
|
||||
+file_id INTEGER,
|
||||
+scope TEXT,
|
||||
+owner_id INTEGER
|
||||
)`);
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
const vector = (values: number[]) => fitVectorDimensions({ vector: values, dimensions: 8 });
|
||||
|
||||
const insert = (db: Database, row: { chunk_id: number; file_id: number; scope: string; owner_id: number; v: number[] }) => {
|
||||
const now = Date.now();
|
||||
|
||||
db.run(
|
||||
`INSERT INTO bot_file_vectors (created_at, updated_at, embedding, chunk_id, file_id, scope, owner_id) VALUES (?, ?, vec_f32(?), ?, ?, ?, ?)`,
|
||||
[now, now, vector(row.v), row.chunk_id, row.file_id, row.scope, row.owner_id],
|
||||
);
|
||||
};
|
||||
|
||||
describe("bot file vectors", () => {
|
||||
test("stores a vector and matches the nearest one", () => {
|
||||
const db = createDb();
|
||||
|
||||
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0, 0, 0] });
|
||||
insert(db, { chunk_id: 2, file_id: 11, scope: "bot", owner_id: 1, v: [0, 1, 0, 0] });
|
||||
|
||||
const rows = db
|
||||
.query(
|
||||
`SELECT chunk_id, file_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 2 ORDER BY distance`,
|
||||
)
|
||||
.all(vector([0.9, 0.1, 0, 0])) as { chunk_id: number; distance: number }[];
|
||||
|
||||
expect(rows.length).toBe(2);
|
||||
expect(rows[0]!.chunk_id).toBe(1);
|
||||
expect(rows[0]!.distance).toBeLessThan(rows[1]!.distance);
|
||||
});
|
||||
|
||||
test("filters scopes in the app, not in the match query", () => {
|
||||
const db = createDb();
|
||||
|
||||
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] });
|
||||
insert(db, { chunk_id: 2, file_id: 20, scope: "bot", owner_id: 2, v: [1, 0.05] });
|
||||
insert(db, { chunk_id: 3, file_id: 30, scope: "workspace", owner_id: 3, v: [1, 0.1] });
|
||||
|
||||
const nearest = db
|
||||
.query(`SELECT chunk_id, scope, owner_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 10 ORDER BY distance`)
|
||||
.all(vector([1, 0])) as { chunk_id: number; scope: string; owner_id: number }[];
|
||||
|
||||
expect(nearest.length).toBe(3);
|
||||
|
||||
const mine = nearest.filter((row) => row.scope == "bot" && row.owner_id == 1);
|
||||
|
||||
expect(mine.map((row) => row.chunk_id)).toEqual([1]);
|
||||
});
|
||||
|
||||
test("a side-car filter in the match query is refused", () => {
|
||||
const db = createDb();
|
||||
|
||||
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] });
|
||||
|
||||
expect(() =>
|
||||
db
|
||||
.query(
|
||||
`SELECT chunk_id FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 2 AND scope = 'bot'`,
|
||||
)
|
||||
.all(vector([1, 0])),
|
||||
).toThrow(/illegal WHERE constraint/);
|
||||
});
|
||||
|
||||
test("an insert without vec0's metadata columns is refused", () => {
|
||||
const db = createDb();
|
||||
|
||||
expect(() =>
|
||||
db.run(
|
||||
`INSERT INTO bot_file_vectors (embedding, chunk_id, file_id, scope, owner_id) VALUES (vec_f32(?1), 1, 10, 'bot', 1)`,
|
||||
[vector([1, 0])],
|
||||
),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
test("deletes a file's vectors by its side-car column", () => {
|
||||
const db = createDb();
|
||||
|
||||
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] });
|
||||
|
||||
db.run(`DELETE FROM bot_file_vectors WHERE file_id = ?`, [10]);
|
||||
|
||||
const count = db.query(`SELECT COUNT(*) AS n FROM bot_file_vectors`).get() as { n: number };
|
||||
|
||||
expect(count.n).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fitVectorDimensions", () => {
|
||||
test("pads a short vector and normalizes it", () => {
|
||||
const out = fitVectorDimensions({ vector: [3, 4], dimensions: 4 });
|
||||
const norm = Math.sqrt([...out].reduce((sum, value) => sum + value * value, 0));
|
||||
|
||||
expect(out.length).toBe(4);
|
||||
expect(Math.round(norm * 1000) / 1000).toBe(1);
|
||||
expect(Math.round(out[0]! * 1000) / 1000).toBe(0.6);
|
||||
});
|
||||
|
||||
test("cuts a long vector", () => {
|
||||
expect(fitVectorDimensions({ vector: [1, 1, 1, 1, 1, 1], dimensions: 2 }).length).toBe(2);
|
||||
});
|
||||
|
||||
test("leaves a zero vector as zeros", () => {
|
||||
expect([...fitVectorDimensions({ vector: [0, 0], dimensions: 2 })]).toEqual([0, 0]);
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {});
|
||||
@@ -7,6 +7,7 @@ import grabBots from "../bots/grab-bots";
|
||||
import grabActiveBotTask from "../bots/grab-active-bot-task";
|
||||
import grabBotMessages from "../bots/grab-bot-messages";
|
||||
import grabBotExtensionsOverview from "../extensions/grab-bot-extensions-overview";
|
||||
import grabConversationFiles from "../bot-files/grab-conversation-files";
|
||||
import grabBotComputers from "../computers/grab-bot-computers";
|
||||
import resolveHarnessCandidates from "@/src/harness/resolve/resolve-harness-candidates";
|
||||
import grabUserWorkspaces from "../workspaces/grab-user-workspaces";
|
||||
@@ -67,7 +68,7 @@ export default async function grabConversationProps({
|
||||
|
||||
bot.unread = 0;
|
||||
|
||||
const [bot_messages, bot_text_model, bot_computers, bot_task, bot_extensions, transcription, speech] = await Promise.all([
|
||||
const [bot_messages, bot_text_model, bot_computers, bot_task, bot_extensions, bot_files, transcription, speech] = await Promise.all([
|
||||
grabBotMessages({
|
||||
session_id: session.id!,
|
||||
user_id,
|
||||
@@ -77,6 +78,7 @@ export default async function grabConversationProps({
|
||||
grabBotComputers({ bot_id: bot.id, workspace_id: scope }),
|
||||
grabActiveBotTask({ bot_id: bot.id, session_id: session.id }),
|
||||
grabBotExtensionsOverview({ user_id, bot_id: bot.id, workspace_id: scope }),
|
||||
grabConversationFiles({ bot_id: bot.id, workspace_id: scope }),
|
||||
hasHarnessModel({ harness: "transcription", bot_id: bot.id, workspace_id: scope }),
|
||||
hasHarnessModel({ harness: "speech", bot_id: bot.id, workspace_id: scope }),
|
||||
]);
|
||||
@@ -95,6 +97,7 @@ export default async function grabConversationProps({
|
||||
bot_computers,
|
||||
bot_task,
|
||||
bot_extensions,
|
||||
bot_files,
|
||||
bot_voice: { transcription, speech },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -26,6 +26,10 @@ type Params = {
|
||||
* Finish without posting a message.
|
||||
*/
|
||||
silent?: boolean;
|
||||
/**
|
||||
* Files the bot saved and asked to send with it (`media` ids).
|
||||
*/
|
||||
attachments?: number[];
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -43,6 +47,7 @@ export default async function completeBotJob({
|
||||
result,
|
||||
needs_you,
|
||||
silent,
|
||||
attachments,
|
||||
}: Params) {
|
||||
const job = await grabLockedBotJob({ job_id, worker });
|
||||
|
||||
@@ -81,6 +86,7 @@ export default async function completeBotJob({
|
||||
workspace_id: job.workspace_id || null,
|
||||
text: result,
|
||||
needs_you,
|
||||
media_ids: attachments,
|
||||
});
|
||||
|
||||
const user_id = await grabJobUserId({ job });
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import deleteMedia from "@/src/functions/backend/db/media/delete-media";
|
||||
import purgeStorage, { grabStorageMediaIds } from "@/src/functions/backend/db/bot-files/purge-storage";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabUserBot from "./grab-user-bot";
|
||||
import assertCrewManager from "@/src/functions/backend/db/access/assert-crew-manager";
|
||||
@@ -27,6 +28,13 @@ export default async function deleteBot({ user_id, bot_id }: Params) {
|
||||
throw new AppError(`The coordinator can't be deleted.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read before the delete: the files' rows go with the bot.
|
||||
*/
|
||||
const file_media_ids = await grabStorageMediaIds({ scope: "bot", owner_id: bot.id! }).catch(
|
||||
() => [],
|
||||
);
|
||||
|
||||
const res = await BunSQLite.sql({ sql: `DELETE FROM bots WHERE id = ?`, values: [bot.id] });
|
||||
|
||||
if (!res.success) {
|
||||
@@ -37,5 +45,11 @@ export default async function deleteBot({ user_id, bot_id }: Params) {
|
||||
await deleteMedia({ id: bot.image_media_id, user_id: Number(user_id) }).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Its folder goes with it: the bytes, their `media` rows and the search
|
||||
* vectors.
|
||||
*/
|
||||
await purgeStorage({ scope: "bot", owner_id: bot.id!, media_ids: file_media_ids });
|
||||
|
||||
return bot;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import grabMessagesMedia from "./grab-messages-media";
|
||||
import grabMessagesReactions from "./grab-messages-reactions";
|
||||
import grabMessagesReplies from "./grab-messages-replies";
|
||||
import mediaToAttachment from "./media-to-attachment";
|
||||
import mediaToAttachment, { grabAttachmentFileIds } from "./media-to-attachment";
|
||||
|
||||
type Params = {
|
||||
session_id: number | string;
|
||||
@@ -61,6 +61,14 @@ export default async function grabBotMessages({
|
||||
: Promise.resolve(new Map()),
|
||||
]);
|
||||
|
||||
/**
|
||||
* Files that live in a bot's folder carry their `bot_files` id, so a
|
||||
* chip opens the file rather than only its bytes.
|
||||
*/
|
||||
const file_ids = await grabAttachmentFileIds({
|
||||
media_ids: [...media.values()].flat().map((file) => Number(file.id)),
|
||||
});
|
||||
|
||||
return messages.map((message) => {
|
||||
const id = Number(message.id);
|
||||
const files = media.get(id);
|
||||
@@ -69,7 +77,14 @@ export default async function grabBotMessages({
|
||||
return {
|
||||
...message,
|
||||
...(files
|
||||
? { attachments: files.map((file) => mediaToAttachment({ media: file })) }
|
||||
? {
|
||||
attachments: files.map((file) =>
|
||||
mediaToAttachment({
|
||||
media: file,
|
||||
file_id: file_ids.get(Number(file.id)),
|
||||
}),
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
...(reply_to ? { reply_to } : {}),
|
||||
...(user_id ? { reactions: reactions.get(id) || [] } : {}),
|
||||
|
||||
@@ -13,6 +13,11 @@ type Params = {
|
||||
workspace_id?: number | null;
|
||||
text: string;
|
||||
needs_you?: boolean;
|
||||
/**
|
||||
* Files the bot saved and sent with it (`media` ids): a PDF it made, a
|
||||
* generated image, a document it wrote.
|
||||
*/
|
||||
media_ids?: number[];
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -25,6 +30,7 @@ export default async function insertBotMessage({
|
||||
workspace_id,
|
||||
text,
|
||||
needs_you,
|
||||
media_ids = [],
|
||||
}: Params): Promise<BUN_SQLITE_COBBLE_BOT_MESSAGES> {
|
||||
const now = Date.now();
|
||||
const needs_you_flag = needs_you ? 1 : 0;
|
||||
@@ -52,6 +58,13 @@ export default async function insertBotMessage({
|
||||
throw new Error(`Failed to save message: ${res.msg}`);
|
||||
}
|
||||
|
||||
for (const media_id of media_ids) {
|
||||
await BunSQLite.sql({
|
||||
sql: `INSERT INTO bot_message_media (created_at, updated_at, message_id, media_id) VALUES (?, ?, ?, ?)`,
|
||||
values: [now, now, id, media_id],
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
await BunSQLite.sql({
|
||||
sql: `UPDATE bot_sessions SET last_activity_at = ?, unread = 1 WHERE id = ?`,
|
||||
values: [now, session_id],
|
||||
|
||||
@@ -1,16 +1,23 @@
|
||||
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import type { BUN_SQLITE_COBBLE_BOT_FILES, BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
|
||||
import grabMediaURL from "@/src/functions/backend/media/grab-media-url";
|
||||
import type { BotMessageAttachment, MediaType } from "@/src/types";
|
||||
|
||||
type Params = {
|
||||
media: BUN_SQLITE_COBBLE_MEDIA;
|
||||
/**
|
||||
* The bot file it is, when the caller already knows (avoids a lookup
|
||||
* per attachment).
|
||||
*/
|
||||
file_id?: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* A `media` row as a chat attachment: its original name (`title`), type
|
||||
* and the private URLs it's served at.
|
||||
* and the private URLs it's served at. A file saved in a bot's folder
|
||||
* carries its `file_id`, so the chip can open the file's page.
|
||||
*/
|
||||
export default function mediaToAttachment({ media }: Params): BotMessageAttachment {
|
||||
export default function mediaToAttachment({ media, file_id }: Params): BotMessageAttachment {
|
||||
const { media_url, media_thumbnail_url } = grabMediaURL({ media: { ...media } });
|
||||
|
||||
return {
|
||||
@@ -19,5 +26,29 @@ export default function mediaToAttachment({ media }: Params): BotMessageAttachme
|
||||
media_type: (media.media_type || "document") as MediaType,
|
||||
url: media_url || "",
|
||||
thumbnail_url: media_thumbnail_url,
|
||||
file_id: file_id || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The `bot_files` rows behind a message's media, keyed by media id, so the
|
||||
* attachments can carry their file ids without a query each.
|
||||
*/
|
||||
export async function grabAttachmentFileIds({ media_ids }: { media_ids: number[] }) {
|
||||
const ids = media_ids.filter((id) => id > 0);
|
||||
|
||||
if (!ids.length) return new Map<number, number>();
|
||||
|
||||
const placeholders = ids.map(() => "?").join(", ");
|
||||
|
||||
const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
|
||||
sql: `SELECT id, media_id FROM bot_files WHERE media_id IN (${placeholders})`,
|
||||
values: ids,
|
||||
});
|
||||
|
||||
return new Map(
|
||||
(res.payload || [])
|
||||
.filter((file) => file.media_id)
|
||||
.map((file) => [Number(file.media_id), Number(file.id)]),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import deleteMedia from "@/src/functions/backend/db/media/delete-media";
|
||||
import purgeStorage, { grabStorageMediaIds } from "@/src/functions/backend/db/bot-files/purge-storage";
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import AppError from "@/src/utils/app-error";
|
||||
import grabUserWorkspace from "./grab-user-workspace";
|
||||
@@ -22,6 +23,14 @@ export default async function deleteWorkspace({ user_id, workspace_id }: Params)
|
||||
throw new AppError(`Workspace not found.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read before the delete: the shared files' rows go with the workspace.
|
||||
*/
|
||||
const file_media_ids = await grabStorageMediaIds({
|
||||
scope: "workspace",
|
||||
owner_id: workspace.id!,
|
||||
}).catch(() => []);
|
||||
|
||||
const res = await BunSQLite.sql({
|
||||
sql: `DELETE FROM workspaces WHERE id = ?`,
|
||||
values: [workspace.id],
|
||||
@@ -35,5 +44,14 @@ export default async function deleteWorkspace({ user_id, workspace_id }: Params)
|
||||
await deleteMedia({ id: workspace.image_media_id, user_id: Number(user_id) }).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* The bots' shared folder there goes with it.
|
||||
*/
|
||||
await purgeStorage({
|
||||
scope: "workspace",
|
||||
owner_id: workspace.id!,
|
||||
media_ids: file_media_ids,
|
||||
});
|
||||
|
||||
return workspace;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import userAuth from "../../backend/auth/user-auth";
|
||||
import grabSharedPictureMedia from "@/src/functions/backend/db/access/grab-shared-picture-media";
|
||||
import grabBotFileMedia from "@/src/functions/backend/db/bot-files/grab-bot-file-media";
|
||||
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
|
||||
@@ -52,6 +53,7 @@ export default async function handleMediaServer({
|
||||
media =
|
||||
media_res.singleRes ||
|
||||
(media_id ? await grabSharedPictureMedia({ media_id, user_id: user.id! }) : null) ||
|
||||
(media_id ? await grabBotFileMedia({ media_id, user_id: user.id! }) : null) ||
|
||||
undefined;
|
||||
} else {
|
||||
const media_res = await BunSQLite.select<
|
||||
|
||||
Reference in New Issue
Block a user