Keep searchable text only in knowledgebases
Every bot file was chunked into bot_file_chunks and embedded into bot_file_vectors, and uploads kept their extracted text in media.text_content, which together took most of the database. Bot files are no longer indexed: bots find them by name or path with list_files and read them with read_file, and search_files, ingest_file and the files modal's re-read button are gone. Attachments keep no text; a reply reads a document from the file. Only knowledgebase documents keep text. The schema drops both tables and the ingest columns. At boot, dropStoredFileText removes what older versions stored (file jobs, upload text) and vacuums once much of the database is free. The media:prune-text script it replaces is removed. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,65 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import dropStoredFileTextSql from "./drop-stored-file-text-sql";
|
||||
|
||||
function scratch() {
|
||||
const db = new Database(":memory:");
|
||||
|
||||
db.run(`CREATE TABLE media (id INTEGER PRIMARY KEY, text_content TEXT)`);
|
||||
db.run(`CREATE TABLE knowledgebase_documents (id INTEGER PRIMARY KEY, media_id INTEGER)`);
|
||||
db.run(`CREATE TABLE file_ingest_queue (id INTEGER PRIMARY KEY, document_id INTEGER)`);
|
||||
db.run(`CREATE TABLE bot_file_chunks (id INTEGER PRIMARY KEY, text TEXT)`);
|
||||
|
||||
db.run(`INSERT INTO media (id, text_content) VALUES (1, 'an attachment'), (2, 'a knowledgebase upload'), (3, NULL)`);
|
||||
db.run(`INSERT INTO knowledgebase_documents (id, media_id) VALUES (1, 2), (2, NULL)`);
|
||||
db.run(`INSERT INTO file_ingest_queue (id, document_id) VALUES (1, NULL), (2, 1)`);
|
||||
db.run(`INSERT INTO bot_file_chunks (id, text) VALUES (1, 'old passage')`);
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
function run(db: Database) {
|
||||
for (const sql of dropStoredFileTextSql()) db.run(sql);
|
||||
}
|
||||
|
||||
describe("dropStoredFileTextSql", () => {
|
||||
test("drops the bot file index", () => {
|
||||
const db = scratch();
|
||||
|
||||
run(db);
|
||||
|
||||
const tables = db.query(`SELECT name FROM sqlite_master WHERE name LIKE 'bot_file_%'`).all();
|
||||
|
||||
expect(tables).toEqual([]);
|
||||
});
|
||||
|
||||
test("clears upload text but keeps a knowledgebase document's", () => {
|
||||
const db = scratch();
|
||||
|
||||
run(db);
|
||||
|
||||
const rows = db.query(`SELECT id, text_content FROM media ORDER BY id`).all();
|
||||
|
||||
expect(rows).toEqual([
|
||||
{ id: 1, text_content: null },
|
||||
{ id: 2, text_content: "a knowledgebase upload" },
|
||||
{ id: 3, text_content: null },
|
||||
]);
|
||||
});
|
||||
|
||||
test("keeps only document jobs on the ingest queue", () => {
|
||||
const db = scratch();
|
||||
|
||||
run(db);
|
||||
|
||||
expect(db.query(`SELECT id FROM file_ingest_queue`).all()).toEqual([{ id: 2 }]);
|
||||
});
|
||||
|
||||
test("runs again on a clean database without failing", () => {
|
||||
const db = scratch();
|
||||
|
||||
run(db);
|
||||
|
||||
expect(() => run(db)).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* The statements that remove text stored outside knowledgebases: the old
|
||||
* bot file index (chunks and their vectors; the vec0 drop takes its shadow
|
||||
* tables with it), file jobs left on the ingest queue, and the extracted
|
||||
* text uploads used to keep. A knowledgebase document's media keeps its
|
||||
* text. Every statement is a no-op once it has run.
|
||||
*/
|
||||
export default function dropStoredFileTextSql(): string[] {
|
||||
return [
|
||||
`DROP TABLE IF EXISTS bot_file_vectors`,
|
||||
`DROP TABLE IF EXISTS bot_file_chunks`,
|
||||
`DELETE FROM file_ingest_queue WHERE document_id IS NULL`,
|
||||
`UPDATE media SET text_content = NULL
|
||||
WHERE text_content IS NOT NULL
|
||||
AND id NOT IN (SELECT media_id FROM knowledgebase_documents WHERE media_id IS NOT NULL)`,
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import BunSQLite from "@moduletrace/bun-sqlite";
|
||||
import logEvent, { logError } from "@/src/functions/backend/logger";
|
||||
import dropStoredFileTextSql from "./drop-stored-file-text-sql";
|
||||
import shouldVacuum from "./should-vacuum";
|
||||
|
||||
/**
|
||||
* Knowledgebases are the only place text is kept for search. Removes what
|
||||
* bot files and uploads stored before that (`dropStoredFileTextSql`), then
|
||||
* vacuums once if the database has a lot of free pages, so the space comes
|
||||
* back on disk. Safe on every start: once clean, it only reads two pragmas.
|
||||
*/
|
||||
export default async function dropStoredFileText() {
|
||||
for (const sql of dropStoredFileTextSql()) {
|
||||
const res = await BunSQLite.sql({ sql });
|
||||
|
||||
if (!res.success) throw new Error(`${sql.split("\n")[0]}: ${res.error || res.msg}`);
|
||||
}
|
||||
|
||||
const [page_count, freelist_count, page_size] = await Promise.all(
|
||||
["page_count", "freelist_count", "page_size"].map((pragma) =>
|
||||
BunSQLite.sql<Record<string, number>>({ sql: `PRAGMA ${pragma}` }).then((res) =>
|
||||
Number(res.payload?.[0]?.[pragma]) || 0,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (!shouldVacuum({ page_count: page_count!, freelist_count: freelist_count!, page_size: page_size! })) return;
|
||||
|
||||
const res = await BunSQLite.sql({ sql: `VACUUM` });
|
||||
|
||||
if (!res.success) {
|
||||
await logError({
|
||||
title: "Could not vacuum the database",
|
||||
error: new Error(`${res.error || res.msg}. It needs about as much free disk as the database takes.`),
|
||||
metadata: { free_bytes: freelist_count! * page_size! },
|
||||
}).catch(() => {});
|
||||
return;
|
||||
}
|
||||
|
||||
await logEvent({
|
||||
title: "Vacuumed the database",
|
||||
category: "system",
|
||||
plain_text: `Gave back ${Math.round((freelist_count! * page_size!) / (1024 * 1024))} MB of free pages to the disk.`,
|
||||
metadata: { freed_bytes: freelist_count! * page_size! },
|
||||
}).catch(() => {});
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import shouldVacuum from "./should-vacuum";
|
||||
|
||||
const page_size = 4096;
|
||||
const min_pages = Math.ceil(AppData.DbVacuumMinFreeBytes / page_size);
|
||||
|
||||
describe("shouldVacuum", () => {
|
||||
test("vacuums when a large share of a big file is free", () => {
|
||||
expect(shouldVacuum({ page_count: min_pages * 2, freelist_count: min_pages, page_size })).toBe(true);
|
||||
});
|
||||
|
||||
test("leaves a database with little free space alone", () => {
|
||||
expect(shouldVacuum({ page_count: min_pages * 100, freelist_count: min_pages, page_size })).toBe(false);
|
||||
});
|
||||
|
||||
test("leaves a small database alone even when mostly free", () => {
|
||||
expect(shouldVacuum({ page_count: 20, freelist_count: 15, page_size })).toBe(false);
|
||||
});
|
||||
|
||||
test("never vacuums on missing numbers", () => {
|
||||
expect(shouldVacuum({ page_count: 0, freelist_count: 0, page_size })).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
|
||||
type Params = {
|
||||
page_count: number;
|
||||
freelist_count: number;
|
||||
page_size: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether enough of the database file sits free to be worth a `VACUUM`
|
||||
* (`AppData.DbVacuumMinFreeRatio` of its pages and at least
|
||||
* `AppData.DbVacuumMinFreeBytes`). Deleted rows only free pages inside the
|
||||
* file; it shrinks on disk only when vacuumed.
|
||||
*/
|
||||
export default function shouldVacuum({ page_count, freelist_count, page_size }: Params) {
|
||||
if (!(page_count > 0) || !(freelist_count > 0)) return false;
|
||||
|
||||
return (
|
||||
freelist_count / page_count >= AppData.DbVacuumMinFreeRatio &&
|
||||
freelist_count * page_size >= AppData.DbVacuumMinFreeBytes
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user