Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 32 Wiki Activity

Keep searchable text only in knowledgebases

Browse Source 
Every bot file was chunked into bot_file_chunks and embedded into
bot_file_vectors, and uploads kept their extracted text in
media.text_content, which together took most of the database. Bot
files are no longer indexed: bots find them by name or path with
list_files and read them with read_file, and search_files, ingest_file
and the files modal's re-read button are gone. Attachments keep no
text; a reply reads a document from the file. Only knowledgebase
documents keep text.

The schema drops both tables and the ingest columns. At boot,
dropStoredFileText removes what older versions stored (file jobs,
upload text) and vacuums once much of the database is free. The
media:prune-text script it replaces is removed.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-28 04:44:46 +01:00
co-authored by Claude Opus 5.5
parent 2896e76256
commit ac3ebc43d4
55 changed files with 271 additions and 1712 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/functions/backend/database/build-table-catalog-sql.ts
+1 -1
View File
@@ -5,7 +5,7 @@
* table name as an argument the way a table-valued function does.
*
* Left out: SQLite's own `sqlite_*` tables, and the shadow tables a
* virtual table keeps its internals in (the `bot_file_vectors_*` tables
* virtual table keeps its internals in (the `knowledgebase_vectors_*` tables
* behind the sqlite-vec index), which are storage, not rows anyone
* browses.
*/
src/functions/backend/db/bot-files/delete-bot-file.ts
+1 -16
View File
@@ -15,7 +15,7 @@ type Params = {
};
/**
* Deletes a bot file: the catalog row (its chunks go with it), its bytes
* Deletes a bot file: the catalog row, its bytes
* and its `media` row. Refuses anything outside the run's directories.
*/
export default async function deleteBotFile({ scopes, file_id }: Params) {
@@ -32,22 +32,7 @@ export default async function deleteBotFile({ scopes, file_id }: Params) {
rmSync(absolute, { force: true });
}
await BunSQLite.sql({ sql: `DELETE FROM bot_file_vectors WHERE file_id = ?`, values: [file.id] });
/**
* Deleted here rather than left to the schema's cascade: SQLite has
* foreign keys off per connection unless `enableForeignKeys` ran, and
* a script or CLI process that deletes a file would otherwise leave
* its chunks behind, which then fail the next schema sync.
*/
await BunSQLite.sql({ sql: `DELETE FROM bot_file_chunks WHERE file_id = ?`, values: [file.id] });
await BunSQLite.sql({ sql: `DELETE FROM bot_files WHERE id = ?`, values: [file.id] });
/**
* Its pending ingestion would only find nothing left.
*/
await BunSQLite.sql({
sql: `DELETE FROM file_ingest_queue WHERE file_id = ? AND status IN ('pending', 'processing')`,
values: [file.id],
}).catch(() => {});
if (file.media_id) {
const media = await BunSQLite.sql<{ media_write_path?: string; media_thumbnail_write_path?: string }>({
src/functions/backend/db/bot-files/grab-conversation-files.ts
-2
View File
@@ -53,8 +53,6 @@ export default async function grabConversationFiles({
purpose: String(file.purpose || "writing"),
scope: String(file.scope || "bot"),
owner_id: Number(file.owner_id) || 0,
ingested_at: file.ingested_at ? Number(file.ingested_at) : undefined,
ingest_error: file.ingest_error || undefined,
created_at: Number(file.created_at) || 0,
url: media_url || "",
};
src/functions/backend/db/bot-files/ingest-bot-file.ts
-329
View File
@@ -1,329 +0,0 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import type {
BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS,
BUN_SQLITE_COBBLE_BOT_FILES,
BunSQLiteTables,
} from "@/db/types/db";
import { AppData } from "@/src/data/app-data";
import embedText from "@/src/harness/embeddings/embed-text";
import HarnessError from "@/src/harness/harness-error";
import { grabHarnessUsageScope } from "@/src/harness/usage/harness-usage-scope";
import type { HarnessUsageScope } from "@/src/harness/usage/types";
import runWithFallbackScope from "@/src/harness/usage/run-with-fallback-scope";
import grabUsageOwnerUserId from "@/src/functions/backend/db/usage/grab-usage-owner";
import chunkText from "./chunk-text";
import extractBotFileText from "./extract-bot-file-text";
type Params = {
file_id: number | string;
/**
* Read images and audio too (a bot asking for it).
*/
read_media?: boolean;
signal?: AbortSignal;
};
export type IngestResult = {
/**
* `ingested`, `empty` (no text to read), `skipped` (too large) or
* `unavailable` (no embedding model), which keeps the file's text but
* leaves it searchable by words only.
*/
status: "ingested" | "empty" | "skipped" | "unavailable";
chunks: number;
};
/**
* Reads a bot file's text, splits it into chunks and embeds them into
* `bot_file_vectors`, replacing whatever was stored before. The chunks
* stay in `bot_file_chunks` even when there's no embedding model, so a
* search can still match words in them.
*
* Failures are recorded on the file (`ingest_error`) and rethrown, so the
* queue can retry.
*/
export default async function ingestBotFile({
file_id,
read_media,
signal,
}: Params): Promise<IngestResult> {
const file = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
sql: `SELECT * FROM bot_files WHERE id = ? LIMIT 1`,
values: [file_id],
}).then((res) => res.payload?.[0]);
if (!file?.id) return { status: "empty", chunks: 0 };
const media = file.media_id
? await BunSQLite.sql<{ media_write_path?: string; text_content?: string }>({
sql: `SELECT media_write_path, text_content FROM media WHERE id = ? LIMIT 1`,
values: [file.media_id],
}).then((res) => res.payload?.[0])
: undefined;
/**
* Decided on the file's size alone. A cached `text_content` is not a
* reason to read an oversized file: it was only ever a slice of it,
* and taking it as the whole document made a file too big to unpack
* look already read.
*/
if (Number(file.size) > AppData.IngestMaxBytes) {
await BunSQLite.sql({
sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`,
values: [`Too large to read (over ${AppData.IngestMaxBytes / (1024 * 1024)} MB).`, Date.now(), file.id],
});
return { status: "skipped", chunks: 0 };
}
/**
* A bot's own run already carries its user. The ingest queue has no run
* behind it, so everything this file costs - reading an image or an
* audio file, embedding its chunks - is entered under the owner's scope
* instead: it spends real tokens and must show on the usage page.
*/
const owner_scope: HarnessUsageScope | null = grabHarnessUsageScope()?.user_id
? null
: await resolveOwnerScope({ file });
try {
const text = await runWithFallbackScope({
scope: owner_scope,
task: () =>
extractBotFileText({
write_path: media?.media_write_path,
text_content: media?.text_content,
name: file.name,
mime_type: file.mime_type,
read_media,
}),
});
await clearChunks({ file_id: file.id });
if (!text?.trim()) {
await markIngested({ file_id: file.id, chunks: 0 });
return { status: "empty", chunks: 0 };
}
const chunks = chunkText({ text });
if (!chunks.length) {
await markIngested({ file_id: file.id, chunks: 0 });
return { status: "empty", chunks: 0 };
}
const chunk_ids = await insertChunks({ file, chunks });
let embedded = 0;
try {
/**
* A big file's chunks go in batches, so one provider refusal
* doesn't throw away the whole file.
*/
for (let start = 0; start < chunks.length; start += AppData.EmbeddingBatchSize) {
const batch = chunks.slice(start, start + AppData.EmbeddingBatchSize);
const { vectors } = await runEmbed({
scope: owner_scope,
signal,
file,
batch,
});
for (let i = 0; i < vectors.length; i++) {
await insertVector({
vector: vectors[i]!,
chunk_id: chunk_ids[start + i]!,
file_id: file.id,
scope: String(file.scope),
owner_id: Number(file.owner_id),
});
embedded++;
}
}
} catch (error) {
if (error instanceof HarnessError && error.code == "unavailable") {
/**
* No embedding model: the chunks still let a search match
* words, so the file isn't a failure.
*/
await markIngested({
file_id: file.id,
chunks: chunks.length,
error: "No embedding model is set, so this file is searchable by words only.",
});
return { status: "unavailable", chunks: chunks.length };
}
throw error;
}
await markIngested({ file_id: file.id, chunks: chunks.length });
return { status: "ingested", chunks: embedded };
} catch (error) {
/**
* Every model resting means nothing was tried: the job waits for
* them, and the file keeps whatever it said before.
*/
if (error instanceof HarnessError && error.resting) throw error;
const message = error instanceof Error ? error.message : String(error);
await BunSQLite.sql({
sql: `UPDATE bot_files SET ingest_error = ?, updated_at = ? WHERE id = ?`,
values: [message.slice(0, 500), Date.now(), file.id],
}).catch(() => {});
throw error;
}
}
async function clearChunks({ file_id }: { file_id: number }) {
await BunSQLite.sql({ sql: `DELETE FROM bot_file_vectors WHERE file_id = ?`, values: [file_id] });
await BunSQLite.sql({ sql: `DELETE FROM bot_file_chunks WHERE file_id = ?`, values: [file_id] });
}
async function insertChunks({
file,
chunks,
}: {
file: BUN_SQLITE_COBBLE_BOT_FILES;
chunks: string[];
}) {
const ids: number[] = [];
for (let index = 0; index < chunks.length; index++) {
const text = chunks[index]!;
const res = await BunSQLite.insert<
BUN_SQLITE_COBBLE_BOT_FILE_CHUNKS,
(typeof BunSQLiteTables)[number]
>({
table: "bot_file_chunks",
data: [
{
file_id: file.id,
scope: file.scope as any,
owner_id: Number(file.owner_id),
chunk_index: index,
text,
token_estimate: Math.ceil(text.length / 4),
},
],
});
const id = res.postInsertReturn?.insertId;
if (id) ids.push(id);
}
return ids;
}
async function insertVector({
vector,
chunk_id,
file_id,
scope,
owner_id,
}: {
vector: Float32Array;
chunk_id: number;
file_id: number;
scope: string;
owner_id: number;
}) {
/**
* vec0's metadata columns (its id and timestamps) are NOT NULL, so a
* raw insert has to fill them; `BunSQLite.insert` would add them but
* the vector needs `vec_f32`.
*/
const now = Date.now();
const res = await BunSQLite.sql({
sql: `INSERT INTO bot_file_vectors (created_at, updated_at, embedding, chunk_id, file_id, scope, owner_id) VALUES (?, ?, vec_f32(?), ?, ?, ?, ?)`,
values: [now, now, vector as any, chunk_id, file_id, scope, owner_id],
});
if (!res.success) {
throw new Error(`Couldn't store an embedding: ${res.error || res.msg}`);
}
}
async function markIngested({
file_id,
chunks,
error,
}: {
file_id: number;
chunks: number;
error?: string;
}) {
const now = Date.now();
await BunSQLite.sql({
sql: `UPDATE bot_files SET ingested_at = ?, ingest_error = ?, updated_at = ? WHERE id = ?`,
values: [now, error || null, now, file_id],
});
await BunSQLite.sql({
sql: `UPDATE media SET is_rag_ingested = ?, updated_at = ? WHERE id = (SELECT media_id FROM bot_files WHERE id = ?)`,
values: [chunks > 0 ? 1 : 0, now, file_id],
}).catch(() => {});
}
/**
* One batch of a file's chunks, embedded under whatever attribution owns
* it: the run that asked for the ingest, else the file's owner.
*/
async function runEmbed({
scope,
signal,
file,
batch,
}: {
scope: HarnessUsageScope | null;
signal?: AbortSignal;
file: BUN_SQLITE_COBBLE_BOT_FILES;
batch: string[];
}) {
return await runWithFallbackScope({
scope,
task: () =>
embedText({
input: batch,
bot_id: file.bot_id!,
workspace_id: file.scope == "workspace" ? file.owner_id : null,
signal,
}),
});
}
/**
* The attribution a file the queue ingests reports under: its workspace's
* creator when it shares one, else its bot's.
*/
async function resolveOwnerScope({
file,
}: {
file: BUN_SQLITE_COBBLE_BOT_FILES;
}): Promise<HarnessUsageScope | null> {
const workspace_id = file.scope == "workspace" ? file.owner_id : null;
const user_id = await grabUsageOwnerUserId({ bot_id: file.bot_id, workspace_id });
if (!user_id) return null;
return {
user_id,
bot_id: file.scope == "bot" ? file.owner_id : file.bot_id,
workspace_id,
session_id: null,
message_id: null,
job_id: null,
};
}
src/functions/backend/db/bot-files/purge-storage.ts
-16
View File
@@ -54,22 +54,6 @@ export default async function purgeStorage({ scope, owner_id, media_ids }: Param
}).catch(() => {});
}
await BunSQLite.sql({
sql: `DELETE FROM bot_file_vectors WHERE scope = ? AND owner_id = ?`,
values: [scope, owner_id],
}).catch(() => {});
/**
* Deleted here rather than left to the cascade, since SQLite has
* foreign keys off per connection unless `enableForeignKeys` ran (a
* script or CLI process never does), and a chunk left behind fails
* the next schema sync.
*/
await BunSQLite.sql({
sql: `DELETE FROM bot_file_chunks WHERE scope = ? AND owner_id = ?`,
values: [scope, owner_id],
}).catch(() => {});
/**
* Usually gone already, with the bot or workspace (the cascade).
*/
src/functions/backend/db/bot-files/record-bot-file.ts
+1 -3
View File
@@ -104,7 +104,7 @@ export default async function recordBotFile({
if (existing?.id) {
await BunSQLite.sql({
sql: `UPDATE bot_files SET media_id = ?, name = ?, folder = ?, mime_type = ?, media_type = ?, size = ?,
purpose = ?, source = ?, ingested_at = NULL, ingest_error = NULL, updated_at = ? WHERE id = ?`,
purpose = ?, source = ?, updated_at = ? WHERE id = ?`,
values: [
media_id,
resolved.name,
@@ -129,8 +129,6 @@ export default async function recordBotFile({
size: bytes.byteLength,
purpose,
source,
ingested_at: undefined,
ingest_error: undefined,
updated_at: now,
};
}
src/functions/backend/db/bot-files/search-bot-files.ts
-271
View File
@@ -1,271 +0,0 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import { AppData } from "@/src/data/app-data";
import { embedQuery } from "@/src/harness/embeddings/embed-text";
import HarnessError from "@/src/harness/harness-error";
import type { StorageScope } from "@/src/utils/storage-access";
type Params = {
scopes: StorageScope[];
query: string;
/**
* Only this file's chunks.
*/
file_id?: number | string | null;
limit?: number;
signal?: AbortSignal;
/**
* Whose embedding model to use.
*/
bot_id?: number | string;
workspace_id?: number | string | null;
};
export type BotFileSearchHit = {
file_id: number;
chunk_id: number;
chunk_index: number;
name: string;
path: string;
scope: string;
/**
* Cosine distance when the vector index answered, null for a word
* match.
*/
distance: number | null;
text: string;
};
export type BotFileSearchResult = {
hits: BotFileSearchHit[];
/**
* How the search ran: `vector`, `words` (no embedding model, or nothing
* matched) or `none` (no files at all).
*/
mode: "vector" | "words" | "none";
/**
* Said plainly for the bot's own reading of what it got back.
*/
note: string;
};
/**
* Searches a run's files: by meaning through the embedding vectors when a
* model is set, else (or when nothing matches) by words over the stored
* chunks. Never throws for an unavailable model: the bot always gets
* something it can act on.
*/
export default async function searchBotFiles({
scopes,
query,
file_id,
limit = AppData.RagTopChunks,
signal,
bot_id,
workspace_id,
}: Params): Promise<BotFileSearchResult> {
const text = query.trim();
if (!scopes.length || !text) {
return { hits: [], mode: "none", note: "No files to search." };
}
try {
const vector = await embedQuery({
text,
signal,
bot_id,
workspace_id,
});
const hits = await vectorSearch({
scopes,
query_vector: vector,
file_id,
limit,
seek: Math.min(limit * AppData.RagOverFetch, AppData.RagMaxK),
});
if (hits.length) {
return {
hits,
mode: "vector",
note: `${hits.length} passage(s), found by meaning.`,
};
}
} catch (error) {
if (!(error instanceof HarnessError)) throw error;
}
const hits = await wordSearch({ scopes, query: text, file_id, limit });
return {
hits,
mode: hits.length ? "words" : "none",
note: hits.length
? `${hits.length} passage(s), found by matching words (no embedding model, or nothing matched by meaning).`
: `Nothing in your files matched "${text}".`,
};
}
/**
* The nearest chunks by vector. sqlite-vec refuses a filter on a vec0
* side-car column in the same query as the match, so the index is asked
* for its nearest `seek` and the run's scopes are applied to that list
* here, before the chunk text is read.
*/
async function vectorSearch({
scopes,
query_vector,
file_id,
limit,
seek,
}: {
scopes: StorageScope[];
query_vector: Float32Array;
file_id?: number | string | null;
limit: number;
seek: number;
}): Promise<BotFileSearchHit[]> {
const nearest = await BunSQLite.sql<{
chunk_id: number;
file_id: number;
scope: string;
owner_id: number;
distance: number;
}>({
sql: `SELECT chunk_id, file_id, scope, owner_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?) AND k = ? ORDER BY distance`,
values: [query_vector as any, seek],
});
/**
* `BunSQLite.sql` resolves with `success: false` instead of throwing, so
* a broken vector query would otherwise read as "nothing matched" and
* silently fall back to words. A failure here is a real error.
*/
if (!nearest.success) {
throw new Error(`The file search index couldn't be read: ${nearest.error || nearest.msg}`);
}
const allowed = nearest.payload?.filter((row) =>
scopes.some((scope) => scope.scope == row.scope && Number(scope.owner_id) == Number(row.owner_id)),
);
const picked = (allowed || [])
.filter((row) => !file_id || Number(row.file_id) == Number(file_id))
.slice(0, limit);
if (!picked.length) return [];
const chunk_ids = picked.map((row) => Number(row.chunk_id));
const placeholders = chunk_ids.map(() => "?").join(", ");
const chunks = await BunSQLite.sql<{
id: number;
text: string;
chunk_index: number;
name: string;
path: string;
}>({
sql: `SELECT c.id, c.text, c.chunk_index, f.name, f.path FROM bot_file_chunks c
JOIN bot_files f ON f.id = c.file_id
WHERE c.id IN (${placeholders})`,
values: chunk_ids,
});
const by_id = new Map((chunks.payload || []).map((row) => [Number(row.id), row]));
return picked.flatMap((row) => {
const chunk = by_id.get(Number(row.chunk_id));
if (!chunk) return [];
return [
toHit({
chunk_id: row.chunk_id,
file_id: row.file_id,
scope: row.scope,
distance: Number(row.distance),
text: chunk.text,
chunk_index: chunk.chunk_index,
name: chunk.name,
path: chunk.path,
}),
];
});
}
/**
* Chunks whose text has the query's words, newest files first. Used when
* there's no embedding model, and as the fallback when the vectors
* answered nothing.
*/
async function wordSearch({
scopes,
query,
file_id,
limit,
}: {
scopes: StorageScope[];
query: string;
file_id?: number | string | null;
limit: number;
}): Promise<BotFileSearchHit[]> {
const words = [
...new Set(
query
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((word) => word.length > 2),
),
].slice(0, 6);
if (!words.length) return [];
const scope_sql = scopes.map(() => `(c.scope = ? AND c.owner_id = ?)`).join(" OR ");
const scope_values = scopes.flatMap((s) => [s.scope, s.owner_id]);
const word_sql = words.map(() => `lower(c.text) LIKE ?`).join(" OR ");
const word_values = words.map((word) => `%${word}%`);
const file_clause = file_id ? "AND c.file_id = ?" : "";
const res = await BunSQLite.sql<{
chunk_id: number;
file_id: number;
text: string;
chunk_index: number;
name: string;
path: string;
scope: string;
}>({
sql: `SELECT c.id AS chunk_id, c.file_id, c.text, c.chunk_index, c.scope, f.name, f.path
FROM bot_file_chunks c
JOIN bot_files f ON f.id = c.file_id
WHERE (${scope_sql}) ${file_clause} AND (${word_sql})
ORDER BY f.updated_at DESC, c.chunk_index
LIMIT ?`,
values: [...scope_values, ...(file_id ? [file_id] : []), ...word_values, limit],
});
return (res.payload || []).map((row) => toHit({ ...row, distance: null }));
}
function toHit(row: {
chunk_id: number;
file_id: number;
text: string;
chunk_index: number;
name: string;
path: string;
scope: string;
distance: number | null;
}): BotFileSearchHit {
return {
file_id: Number(row.file_id),
chunk_id: Number(row.chunk_id),
chunk_index: Number(row.chunk_index),
name: row.name || "",
path: row.path || "",
scope: row.scope || "bot",
distance: row.distance == null ? null : Number(row.distance),
text: row.text || "",
};
}
src/functions/backend/db/bot-files/store-chat-attachments.test.ts
+4 -6
View File
@@ -13,10 +13,8 @@ async function source() {
describe("storeChatAttachments", () => {
/**
* The same text used to be copied onto the new `bot-file` media row,
* so a chat upload kept a second copy of its extracted text (and left
* it behind when the upload was pruned). The text belongs only on the
* message's own `bot-attachment` row, where a reply reads it;
* ingestion re-reads the bytes through `extractBotFileText`.
* so a chat upload kept a second copy of its extracted text. Bot files
* keep no text at all: a bot reads the bytes when it needs them.
*/
test("never writes text_content onto the file's media row", async () => {
const code = await source();
@@ -25,10 +23,10 @@ describe("storeChatAttachments", () => {
expect(code).not.toContain("text_content = ?");
});
test("still saves the bytes and queues the ingest", async () => {
test("saves the bytes and indexes nothing", async () => {
const code = await source();
expect(code).toContain("recordBotFile(");
expect(code).toContain("enqueueFileIngest(");
expect(code).not.toContain("enqueueFileIngest(");
});
});
src/functions/backend/db/bot-files/store-chat-attachments.ts
+3 -6
View File
@@ -1,7 +1,6 @@
import mediaFilePath from "@/src/utils/media-file-path";
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
import { BotUploadFolder } from "@/src/dict/bot-file-dict";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
import { logError } from "@/src/functions/backend/logger";
import recordBotFile from "./record-bot-file";
@@ -14,7 +13,7 @@ type Params = {
/**
* Copies the files the user attached to a message into the conversation's
* folder (`uploads/<yyyy-mm>/<name>`), so they stay searchable after the
* folder (`uploads/<yyyy-mm>/<name>`), so they stay findable after the
* message scrolls away and a later run can find them. Best effort: a
* failure is logged, never shown, and the message still goes out.
*/
@@ -48,8 +47,8 @@ export default async function storeChatAttachments({
* The text stays on the message's own `bot-attachment` row,
* where a reply reads it. Copying it onto the file's row kept
* the same text twice and left the copy behind when the
* upload was pruned; ingestion re-reads the bytes once
* through `extractBotFileText` instead.
* upload was pruned; a bot reads the bytes with `read_file`
* instead.
*/
const saved = await recordBotFile({
bot_id,
@@ -63,8 +62,6 @@ export default async function storeChatAttachments({
mime_type: file.mime_type || undefined,
user_id,
});
await enqueueFileIngest({ file_id: saved.id! });
} catch (error) {
await logError({
title: `A chat attachment couldn't be saved to the bot's files`,
src/functions/backend/db/bot-files/vector-search.test.ts
-132
View File
@@ -1,132 +0,0 @@
import { afterAll, describe, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import * as sqliteVec from "sqlite-vec";
import fitVectorDimensions from "@/src/harness/embeddings/fit-vector-dimensions";
/**
* The exact insert and nearest-neighbour shape `ingestBotFile` and
* `searchBotFiles` use, on a scratch database: vec0's metadata columns are
* NOT NULL, and a side-car filter can't share the match query.
*/
function createDb() {
const db = new Database(":memory:");
sqliteVec.load(db);
db.run(`CREATE VIRTUAL TABLE bot_file_vectors USING vec0(
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at INTEGER,
updated_at INTEGER,
embedding float[8],
+chunk_id INTEGER,
+file_id INTEGER,
+scope TEXT,
+owner_id INTEGER
)`);
return db;
}
const vector = (values: number[]) => fitVectorDimensions({ vector: values, dimensions: 8 });
const insert = (db: Database, row: { chunk_id: number; file_id: number; scope: string; owner_id: number; v: number[] }) => {
const now = Date.now();
db.run(
`INSERT INTO bot_file_vectors (created_at, updated_at, embedding, chunk_id, file_id, scope, owner_id) VALUES (?, ?, vec_f32(?), ?, ?, ?, ?)`,
[now, now, vector(row.v), row.chunk_id, row.file_id, row.scope, row.owner_id],
);
};
describe("bot file vectors", () => {
test("stores a vector and matches the nearest one", () => {
const db = createDb();
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0, 0, 0] });
insert(db, { chunk_id: 2, file_id: 11, scope: "bot", owner_id: 1, v: [0, 1, 0, 0] });
const rows = db
.query(
`SELECT chunk_id, file_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 2 ORDER BY distance`,
)
.all(vector([0.9, 0.1, 0, 0])) as { chunk_id: number; distance: number }[];
expect(rows.length).toBe(2);
expect(rows[0]!.chunk_id).toBe(1);
expect(rows[0]!.distance).toBeLessThan(rows[1]!.distance);
});
test("filters scopes in the app, not in the match query", () => {
const db = createDb();
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] });
insert(db, { chunk_id: 2, file_id: 20, scope: "bot", owner_id: 2, v: [1, 0.05] });
insert(db, { chunk_id: 3, file_id: 30, scope: "workspace", owner_id: 3, v: [1, 0.1] });
const nearest = db
.query(`SELECT chunk_id, scope, owner_id, distance FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 10 ORDER BY distance`)
.all(vector([1, 0])) as { chunk_id: number; scope: string; owner_id: number }[];
expect(nearest.length).toBe(3);
const mine = nearest.filter((row) => row.scope == "bot" && row.owner_id == 1);
expect(mine.map((row) => row.chunk_id)).toEqual([1]);
});
test("a side-car filter in the match query is refused", () => {
const db = createDb();
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] });
expect(() =>
db
.query(
`SELECT chunk_id FROM bot_file_vectors WHERE embedding MATCH vec_f32(?1) AND k = 2 AND scope = 'bot'`,
)
.all(vector([1, 0])),
).toThrow(/illegal WHERE constraint/);
});
test("an insert without vec0's metadata columns is refused", () => {
const db = createDb();
expect(() =>
db.run(
`INSERT INTO bot_file_vectors (embedding, chunk_id, file_id, scope, owner_id) VALUES (vec_f32(?1), 1, 10, 'bot', 1)`,
[vector([1, 0])],
),
).toThrow();
});
test("deletes a file's vectors by its side-car column", () => {
const db = createDb();
insert(db, { chunk_id: 1, file_id: 10, scope: "bot", owner_id: 1, v: [1, 0] });
db.run(`DELETE FROM bot_file_vectors WHERE file_id = ?`, [10]);
const count = db.query(`SELECT COUNT(*) AS n FROM bot_file_vectors`).get() as { n: number };
expect(count.n).toBe(0);
});
});
describe("fitVectorDimensions", () => {
test("pads a short vector and normalizes it", () => {
const out = fitVectorDimensions({ vector: [3, 4], dimensions: 4 });
const norm = Math.sqrt([...out].reduce((sum, value) => sum + value * value, 0));
expect(out.length).toBe(4);
expect(Math.round(norm * 1000) / 1000).toBe(1);
expect(Math.round(out[0]! * 1000) / 1000).toBe(0.6);
});
test("cuts a long vector", () => {
expect(fitVectorDimensions({ vector: [1, 1, 1, 1, 1, 1], dimensions: 2 }).length).toBe(2);
});
test("leaves a zero vector as zeros", () => {
expect([...fitVectorDimensions({ vector: [0, 0], dimensions: 2 })]).toEqual([0, 0]);
});
});
afterAll(() => {});
src/functions/backend/db/bots/grab-messages-media.test.ts
+5 -11
View File
@@ -30,18 +30,16 @@ db.run(
db.run(`INSERT INTO bot_message_media (id, message_id, media_id) VALUES (1, 10, 1)`);
/**
* The query `grabMessagesMedia` builds, with the text column included or
* left out. It is the largest column in the table, so only the reply path
* that hands a document to a model asks for it.
* The query `grabMessagesMedia` builds. The text column is never read: a
* reply reads an attachment's text from the file itself.
*/
function read({ with_text_content }: { with_text_content: boolean }) {
function read() {
const columns = [
"m.id",
"m.title",
"m.media_type",
"m.media_write_path",
"m.mime_type",
...(with_text_content ? ["m.text_content"] : []),
].join(", ");
return db
@@ -55,15 +53,11 @@ function read({ with_text_content }: { with_text_content: boolean }) {
describe("grabMessagesMedia", () => {
test("a plain read never drags the extracted text along", () => {
expect(read({ with_text_content: false })).not.toHaveProperty("text_content");
});
test("the reply path can ask for it", () => {
expect(read({ with_text_content: true }).text_content).toBe("Total: 12");
expect(read()).not.toHaveProperty("text_content");
});
test("the row's name and path are always there", () => {
expect(read({ with_text_content: false })).toMatchObject({
expect(read()).toMatchObject({
title: "invoice.pdf",
media_write_path: "media/1.pdf",
message_id: 10,
src/functions/backend/db/bots/grab-messages-media.ts
+1 -8
View File
@@ -4,12 +4,6 @@ import mediaColumns from "@/src/functions/backend/db/media/media-columns";
type Params = {
message_ids: number[];
/**
* Also read `text_content`, the extracted text a reply hands a
* document to the model as. Off by default: it is the largest column
* in the table and nothing else here uses it.
*/
with_text_content?: boolean;
};
export type MessageMediaRow = BUN_SQLITE_COBBLE_MEDIA & { message_id: number };
@@ -20,14 +14,13 @@ export type MessageMediaRow = BUN_SQLITE_COBBLE_MEDIA & { message_id: number };
*/
export default async function grabMessagesMedia({
message_ids,
with_text_content,
}: Params) {
const by_message = new Map<number, MessageMediaRow[]>();
const ids = [...new Set(message_ids.filter(Boolean))];
if (!ids.length) return by_message;
const columns = mediaColumns({ alias: "m", with_text_content });
const columns = mediaColumns({ alias: "m" });
const rows =
(
src/functions/backend/db/media/media-columns.ts
+3 -5
View File
@@ -5,9 +5,8 @@
* `AppData.BotAttachmentTextStoreMaxChars` of extracted document text, and
* every caller that only needs a file's name, type or path (`SELECT m.*`
* did) pulled it into memory, into page props and into API responses.
* Read it explicitly where it is really used:
* `src/bots-queue/load-attachment-parts.ts` and
* `src/functions/backend/db/bot-files/ingest-bot-file.ts`.
* Only knowledgebase documents keep it now; read it explicitly where it
* is really used.
*/
export default function mediaColumns({
alias = "m",
@@ -31,8 +30,7 @@ export default function mediaColumns({
];
/**
* Only the two paths that really use it ask for it: the reply that
* hands a document to a model, and ingestion.
* Only a caller that really uses it asks for it.
*/
if (with_text_content) columns.push("text_content");
src/functions/backend/db/media/plan-text-content-prune.test.ts
-65
View File
@@ -1,65 +0,0 @@
import { describe, expect, test } from "bun:test";
import { AppData } from "@/src/data/app-data";
import planTextContentPrune from "./plan-text-content-prune";
describe("planTextContentPrune", () => {
test("clears the raw bytes stored for a type the extractor can't read", () => {
const changes = planTextContentPrune({
rows: [
{ id: 1, title: "logo.svg", mime_type: "image/svg+xml", text_content: "<svg><path/></svg>" },
{ id: 2, title: "bundle.zip", mime_type: "application/zip", text_content: "PK\u0003\u0004" },
{ id: 3, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", text_content: "PK\u0003\u0004" },
],
});
expect(changes.map((change) => change.media_id)).toEqual([1, 2, 3]);
expect(changes.every((change) => change.text_content === null)).toBe(true);
expect(changes.every((change) => change.after_chars === 0)).toBe(true);
});
test("leaves a document's text alone", () => {
const changes = planTextContentPrune({
rows: [
{ id: 4, title: "invoice.pdf", mime_type: "application/pdf", text_content: "Total: 12" },
{ id: 5, title: "notes.md", mime_type: "text/markdown", text_content: "# Hi" },
{ id: 6, title: "rows.csv", mime_type: "text/csv", text_content: "a,b\n1,2" },
],
});
expect(changes).toEqual([]);
});
test("cuts a document over the store ceiling", () => {
const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 100);
const changes = planTextContentPrune({
rows: [{ id: 7, title: "long.txt", mime_type: "text/plain", text_content: text }],
});
expect(changes).toHaveLength(1);
expect(changes[0]!.after_chars).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
expect(changes[0]!.text_content).toContain("Truncated");
expect(changes[0]!.before_chars).toBe(text.length);
});
test("nothing to say about a row with no text or no id", () => {
expect(
planTextContentPrune({
rows: [
{ id: 8, title: "empty.pdf", text_content: "" },
{ id: 9, title: "blank.pdf", text_content: " \n " },
{ id: 10, title: "none.pdf" },
{ title: "no id.pdf", text_content: "text" },
],
}),
).toEqual([]);
});
test("counting what each change reclaims is left to the caller", () => {
const changes = planTextContentPrune({
rows: [{ id: 11, title: "icon.svg", mime_type: "image/svg+xml", text_content: "12345" }],
});
expect(changes).toHaveLength(1);
expect(changes[0]!.before_chars - changes[0]!.after_chars).toBe(5);
});
});
src/functions/backend/db/media/plan-text-content-prune.ts
-55
View File
@@ -1,55 +0,0 @@
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
import decideTextContent from "@/src/utils/decide-text-content";
export type TextContentChange = {
media_id: number;
name: string;
before_chars: number;
after_chars: number;
/**
* What the row's `text_content` should become: `null` clears it.
*/
text_content: string | null;
};
type Params = {
/**
* The `media` rows that have any `text_content`.
*/
rows: Partial<BUN_SQLITE_COBBLE_MEDIA>[];
};
/**
* What `pruneMediaTextContent` would change, worked out without touching
* the database: one entry per row whose text belongs nowhere or is over
* the store ceiling, and nothing at all for a row that is already right.
*/
export default function planTextContentPrune({ rows }: Params): TextContentChange[] {
const changes: TextContentChange[] = [];
for (const row of rows) {
if (!row.id) continue;
const before = row.text_content || "";
if (!before.trim()) continue;
const text_content = decideTextContent({
name: row.title,
mime_type: row.mime_type,
text: before,
});
if (text_content === before) continue;
changes.push({
media_id: Number(row.id),
name: row.title || "no name",
before_chars: before.length,
after_chars: text_content?.length || 0,
text_content,
});
}
return changes;
}
src/functions/backend/db/media/prune-media-text-content.test.ts
-120
View File
@@ -1,120 +0,0 @@
import { describe, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { AppData } from "@/src/data/app-data";
import pruneMediaTextContent, { type SqlRunner } from "./prune-media-text-content";
type Row = { id: number; title: string; mime_type: string; media_type: string; text_content: string | null; updated_at?: number };
function makeDb({ rows }: { rows: Row[] }) {
const db = new Database(":memory:");
db.run(
`CREATE TABLE media (
id INTEGER PRIMARY KEY,
title TEXT,
mime_type TEXT,
media_type TEXT,
text_content TEXT,
updated_at INTEGER
)`,
);
const insert = db.prepare(
`INSERT INTO media (id, title, mime_type, media_type, text_content) VALUES (?, ?, ?, ?, ?)`,
);
for (const row of rows) insert.run(row.id, row.title, row.mime_type, row.media_type, row.text_content);
const run: SqlRunner = async ({ sql, values }) => {
const statement = db.prepare(sql);
if (sql.trim().toUpperCase().startsWith("SELECT")) {
return { payload: statement.all(...((values || []) as never[])) as unknown[] };
}
statement.run(...((values || []) as never[]));
return { payload: [] };
};
const read = (id: number) =>
(db.query(`SELECT text_content FROM media WHERE id = ?`).get(id) as { text_content: string | null }).text_content;
return { db, run, read };
}
describe("pruneMediaTextContent", () => {
test("clears raw bytes from a type the extractor can't read, and keeps a document's text", async () => {
const { run, read } = makeDb({
rows: [
{ id: 1, title: "logo.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "<svg><path d='M0 0'/></svg>" },
{ id: 2, title: "bundle.zip", mime_type: "application/zip", media_type: "document", text_content: "PK\u0003\u0004binary" },
{ id: 3, title: "invoice.pdf", mime_type: "application/pdf", media_type: "document", text_content: "Total: 12" },
],
});
const result = await pruneMediaTextContent({ run });
expect(result.checked).toBe(3);
expect(result.cleared).toBe(2);
expect(result.truncated).toBe(0);
expect(read(1)).toBeNull();
expect(read(2)).toBeNull();
expect(read(3)).toBe("Total: 12");
});
test("cuts an oversized document to the store ceiling", async () => {
const text = "a".repeat(AppData.BotAttachmentTextStoreMaxChars + 1000);
const { run, read } = makeDb({
rows: [{ id: 4, title: "huge.txt", mime_type: "text/plain", media_type: "document", text_content: text }],
});
const result = await pruneMediaTextContent({ run });
const after = read(4)!;
expect(result.truncated).toBe(1);
expect(result.cleared).toBe(0);
expect(after.length).toBeLessThanOrEqual(AppData.BotAttachmentTextStoreMaxChars);
expect(after).toContain("Truncated");
expect(result.reclaimed_bytes).toBeGreaterThan(0);
});
test("is safe to rerun: a second sweep changes nothing", async () => {
const { run, read } = makeDb({
rows: [{ id: 5, title: "icon.svg", mime_type: "image/svg+xml", media_type: "document", text_content: "raw" }],
});
await pruneMediaTextContent({ run });
const second = await pruneMediaTextContent({ run });
expect(second.checked).toBe(0);
expect(second.changes).toEqual([]);
expect(second.reclaimed_bytes).toBe(0);
expect(read(5)).toBeNull();
});
test("stamps updated_at on the rows it rewrites", async () => {
const now = 1_700_000_000_000;
const { run, db } = makeDb({
rows: [{ id: 6, title: "x.zip", mime_type: "application/zip", media_type: "document", text_content: "raw" }],
});
await pruneMediaTextContent({ run, now });
expect(
(db.query(`SELECT updated_at FROM media WHERE id = 6`).get() as { updated_at: number }).updated_at,
).toBe(now);
});
test("reports every change with its row, name and sizes", async () => {
const { run } = makeDb({
rows: [{ id: 7, title: "sheet.xlsx", mime_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", media_type: "document", text_content: "PK\u0003\u0004" }],
});
const result = await pruneMediaTextContent({ run });
expect(result.changes).toEqual([
{ media_id: 7, name: "sheet.xlsx", before_chars: 4, after_chars: 0, text_content: null },
]);
});
});
src/functions/backend/db/media/prune-media-text-content.ts
-75
View File
@@ -1,75 +0,0 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import type { BUN_SQLITE_COBBLE_MEDIA } from "@/db/types/db";
import planTextContentPrune, { type TextContentChange } from "./plan-text-content-prune";
/**
* One statement against the database. `BunSQLite.sql` is the real one; a
* test passes its own, so the whole sweep can be run without the app's
* database.
*/
export type SqlRunner = (params: {
sql: string;
values?: (string | number | null)[];
}) => Promise<{ payload?: unknown[] | null }>;
export type PruneMediaTextResult = {
checked: number;
cleared: number;
truncated: number;
reclaimed_bytes: number;
changes: TextContentChange[];
};
type Params = {
run?: SqlRunner;
/**
* When the sweep runs, for the rows it writes.
*/
now?: number;
};
/**
* Brings every `media.text_content` in line with what may be stored there
* (`planTextContentPrune`): no text on a type the extractor can't read,
* and no text longer than the store ceiling.
*
* Run once per install after the allowlist landed; safe to rerun, since a
* row that is already right is not written.
*/
export default async function pruneMediaTextContent({
run,
now = Date.now(),
}: Params = {}): Promise<PruneMediaTextResult> {
const sql: SqlRunner =
run || (({ sql, values }) => BunSQLite.sql<BUN_SQLITE_COBBLE_MEDIA>({ sql, values }));
const res = await sql({
sql: `SELECT id, title, mime_type, media_type, text_content
FROM media
WHERE text_content IS NOT NULL AND TRIM(text_content) != ''`,
});
const rows = (res.payload || []) as BUN_SQLITE_COBBLE_MEDIA[];
const changes = planTextContentPrune({ rows });
const result: PruneMediaTextResult = {
checked: rows.length,
cleared: 0,
truncated: 0,
reclaimed_bytes: 0,
changes,
};
for (const change of changes) {
await sql({
sql: `UPDATE media SET text_content = ?, updated_at = ? WHERE id = ?`,
values: [change.text_content, now, change.media_id],
});
result.reclaimed_bytes += change.before_chars - change.after_chars;
if (change.text_content === null) result.cleared += 1;
else result.truncated += 1;
}
return result;
}
src/functions/backend/db/stored-text/drop-stored-file-text-sql.test.ts
+65
View File
@@ -0,0 +1,65 @@
import { Database } from "bun:sqlite";
import { describe, expect, test } from "bun:test";
import dropStoredFileTextSql from "./drop-stored-file-text-sql";
function scratch() {
const db = new Database(":memory:");
db.run(`CREATE TABLE media (id INTEGER PRIMARY KEY, text_content TEXT)`);
db.run(`CREATE TABLE knowledgebase_documents (id INTEGER PRIMARY KEY, media_id INTEGER)`);
db.run(`CREATE TABLE file_ingest_queue (id INTEGER PRIMARY KEY, document_id INTEGER)`);
db.run(`CREATE TABLE bot_file_chunks (id INTEGER PRIMARY KEY, text TEXT)`);
db.run(`INSERT INTO media (id, text_content) VALUES (1, 'an attachment'), (2, 'a knowledgebase upload'), (3, NULL)`);
db.run(`INSERT INTO knowledgebase_documents (id, media_id) VALUES (1, 2), (2, NULL)`);
db.run(`INSERT INTO file_ingest_queue (id, document_id) VALUES (1, NULL), (2, 1)`);
db.run(`INSERT INTO bot_file_chunks (id, text) VALUES (1, 'old passage')`);
return db;
}
function run(db: Database) {
for (const sql of dropStoredFileTextSql()) db.run(sql);
}
describe("dropStoredFileTextSql", () => {
test("drops the bot file index", () => {
const db = scratch();
run(db);
const tables = db.query(`SELECT name FROM sqlite_master WHERE name LIKE 'bot_file_%'`).all();
expect(tables).toEqual([]);
});
test("clears upload text but keeps a knowledgebase document's", () => {
const db = scratch();
run(db);
const rows = db.query(`SELECT id, text_content FROM media ORDER BY id`).all();
expect(rows).toEqual([
{ id: 1, text_content: null },
{ id: 2, text_content: "a knowledgebase upload" },
{ id: 3, text_content: null },
]);
});
test("keeps only document jobs on the ingest queue", () => {
const db = scratch();
run(db);
expect(db.query(`SELECT id FROM file_ingest_queue`).all()).toEqual([{ id: 2 }]);
});
test("runs again on a clean database without failing", () => {
const db = scratch();
run(db);
expect(() => run(db)).not.toThrow();
});
});
src/functions/backend/db/stored-text/drop-stored-file-text-sql.ts
+17
View File
@@ -0,0 +1,17 @@
/**
* The statements that remove text stored outside knowledgebases: the old
* bot file index (chunks and their vectors; the vec0 drop takes its shadow
* tables with it), file jobs left on the ingest queue, and the extracted
* text uploads used to keep. A knowledgebase document's media keeps its
* text. Every statement is a no-op once it has run.
*/
export default function dropStoredFileTextSql(): string[] {
return [
`DROP TABLE IF EXISTS bot_file_vectors`,
`DROP TABLE IF EXISTS bot_file_chunks`,
`DELETE FROM file_ingest_queue WHERE document_id IS NULL`,
`UPDATE media SET text_content = NULL
WHERE text_content IS NOT NULL
AND id NOT IN (SELECT media_id FROM knowledgebase_documents WHERE media_id IS NOT NULL)`,
];
}
src/functions/backend/db/stored-text/drop-stored-file-text.ts
+46
View File
@@ -0,0 +1,46 @@
import BunSQLite from "@moduletrace/bun-sqlite";
import logEvent, { logError } from "@/src/functions/backend/logger";
import dropStoredFileTextSql from "./drop-stored-file-text-sql";
import shouldVacuum from "./should-vacuum";
/**
* Knowledgebases are the only place text is kept for search. Removes what
* bot files and uploads stored before that (`dropStoredFileTextSql`), then
* vacuums once if the database has a lot of free pages, so the space comes
* back on disk. Safe on every start: once clean, it only reads two pragmas.
*/
export default async function dropStoredFileText() {
for (const sql of dropStoredFileTextSql()) {
const res = await BunSQLite.sql({ sql });
if (!res.success) throw new Error(`${sql.split("\n")[0]}: ${res.error || res.msg}`);
}
const [page_count, freelist_count, page_size] = await Promise.all(
["page_count", "freelist_count", "page_size"].map((pragma) =>
BunSQLite.sql<Record<string, number>>({ sql: `PRAGMA ${pragma}` }).then((res) =>
Number(res.payload?.[0]?.[pragma]) || 0,
),
),
);
if (!shouldVacuum({ page_count: page_count!, freelist_count: freelist_count!, page_size: page_size! })) return;
const res = await BunSQLite.sql({ sql: `VACUUM` });
if (!res.success) {
await logError({
title: "Could not vacuum the database",
error: new Error(`${res.error || res.msg}. It needs about as much free disk as the database takes.`),
metadata: { free_bytes: freelist_count! * page_size! },
}).catch(() => {});
return;
}
await logEvent({
title: "Vacuumed the database",
category: "system",
plain_text: `Gave back ${Math.round((freelist_count! * page_size!) / (1024 * 1024))} MB of free pages to the disk.`,
metadata: { freed_bytes: freelist_count! * page_size! },
}).catch(() => {});
}
src/functions/backend/db/stored-text/should-vacuum.test.ts
+24
View File
@@ -0,0 +1,24 @@
import { describe, expect, test } from "bun:test";
import { AppData } from "@/src/data/app-data";
import shouldVacuum from "./should-vacuum";
const page_size = 4096;
const min_pages = Math.ceil(AppData.DbVacuumMinFreeBytes / page_size);
describe("shouldVacuum", () => {
test("vacuums when a large share of a big file is free", () => {
expect(shouldVacuum({ page_count: min_pages * 2, freelist_count: min_pages, page_size })).toBe(true);
});
test("leaves a database with little free space alone", () => {
expect(shouldVacuum({ page_count: min_pages * 100, freelist_count: min_pages, page_size })).toBe(false);
});
test("leaves a small database alone even when mostly free", () => {
expect(shouldVacuum({ page_count: 20, freelist_count: 15, page_size })).toBe(false);
});
test("never vacuums on missing numbers", () => {
expect(shouldVacuum({ page_count: 0, freelist_count: 0, page_size })).toBe(false);
});
});
src/functions/backend/db/stored-text/should-vacuum.ts
+22
View File
@@ -0,0 +1,22 @@
import { AppData } from "@/src/data/app-data";
type Params = {
page_count: number;
freelist_count: number;
page_size: number;
};
/**
* Whether enough of the database file sits free to be worth a `VACUUM`
* (`AppData.DbVacuumMinFreeRatio` of its pages and at least
* `AppData.DbVacuumMinFreeBytes`). Deleted rows only free pages inside the
* file; it shrinks on disk only when vacuumed.
*/
export default function shouldVacuum({ page_count, freelist_count, page_size }: Params) {
if (!(page_count > 0) || !(freelist_count > 0)) return false;
return (
freelist_count / page_count >= AppData.DbVacuumMinFreeRatio &&
freelist_count * page_size >= AppData.DbVacuumMinFreeBytes
);
}