Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 32 Wiki Activity

Keep searchable text only in knowledgebases

Browse Source 
Every bot file was chunked into bot_file_chunks and embedded into
bot_file_vectors, and uploads kept their extracted text in
media.text_content, which together took most of the database. Bot
files are no longer indexed: bots find them by name or path with
list_files and read them with read_file, and search_files, ingest_file
and the files modal's re-read button are gone. Attachments keep no
text; a reply reads a document from the file. Only knowledgebase
documents keep text.

The schema drops both tables and the ingest columns. At boot,
dropStoredFileText removes what older versions stored (file jobs,
upload text) and vacuums once much of the database is free. The
media:prune-text script it replaces is removed.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-28 04:44:46 +01:00
co-authored by Claude Opus 5.5
parent 2896e76256
commit ac3ebc43d4
55 changed files with 271 additions and 1712 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pages/admin/bots/[bot_id]/(partials)/bot-files-modal.tsx
+5 -14
View File
@@ -29,10 +29,9 @@ type Props = {
/**
* The files a conversation works with: a bot's own folder, or the
* workspace's shared one. Each can be opened, read again so it can be
* searched, or deleted (a crew manager only). The list loads when the
* modal opens, and search goes through the same endpoint, so the page
* props stay small.
* workspace's shared one. Each can be opened or deleted (a crew manager
* only). The list loads when the modal opens, and search (by name or path)
* goes through the same endpoint, so the page props stay small.
*/
export default function BotFilesModal({ bot, workspace, menu, on_open }: Props) {
const { pageProps } = useContext(AppContext);
@@ -105,7 +104,7 @@ export default function BotFilesModal({ bot, workspace, menu, on_open }: Props)
<H2 className="text-lg! leading-6! tracking-normal! font-bold text-ink">{title}</H2>
<Span className="text-[13px] leading-[18px] text-ink-secondary">
{workspace?.id
? `Files every bot here can read and search. They're kept with the app, not on a computer.`
? `Files every bot here can find and read. They're kept with the app, not on a computer.`
: `What ${name} has saved: documents you sent, files it wrote and images it made. They're kept with the app, not on a computer.${can_manage ? " Deleting one removes it for good." : ""}`}
</Span>
{usage ? (
@@ -128,7 +127,7 @@ export default function BotFilesModal({ bot, workspace, menu, on_open }: Props)
<Stack className="w-full mt-4 gap-2! items-stretch">
<Input
placeholder="Search files"
placeholder="Search by name or path"
value={search}
onChange={(event) => setSearch(event.target.value)}
onKeyDown={(event) => {
@@ -146,14 +145,6 @@ export default function BotFilesModal({ bot, workspace, menu, on_open }: Props)
file={file}
can_manage={can_manage}
busy={busy_id == file.id}
onIngest={() =>
run({
url: `/api/admin/bots/ingest-file`,
payload: { file_id: file.id },
file_id: file.id,
done: `${file.name} will be read again shortly.`,
})
}
onDelete={() =>
run({
url: `/api/admin/bots/delete-file`,
src/pages/admin/bots/[bot_id]/(partials)/file-row.tsx
+4 -23
View File
@@ -3,7 +3,7 @@ import Row from "@moduletrace/twui/layout/Row";
import Span from "@moduletrace/twui/layout/Span";
import Stack from "@moduletrace/twui/layout/Stack";
import Button from "@moduletrace/twui/layout/Button";
import { RefreshCw, Trash2 } from "lucide-react";
import { Trash2 } from "lucide-react";
import type { MediaType, PublicBotFile } from "@/src/types";
import relativeTime from "@/src/utils/relative-time";
import AttachmentIcon from "./attachment-icon";
@@ -11,19 +11,18 @@ import AttachmentIcon from "./attachment-icon";
type Props = {
file: PublicBotFile;
/**
* A crew manager may delete and re-ingest.
* A crew manager may delete it.
*/
can_manage: boolean;
busy?: boolean;
onIngest: () => void;
onDelete: () => void;
};
/**
* One file in the files modal: its type, path, size and age, and what can
* be done with it. A file that isn't searchable says why.
* be done with it.
*/
export default function FileRow({ file, can_manage, busy, onIngest, onDelete }: Props) {
export default function FileRow({ file, can_manage, busy, onDelete }: Props) {
return (
<Row className="w-full gap-3! flex-nowrap items-start py-2 border-0 border-b border-solid border-hairline last:border-b-0">
<Row className="mt-0.5 text-ink-secondary">
@@ -44,11 +43,6 @@ export default function FileRow({ file, can_manage, busy, onIngest, onDelete }:
file.scope == "workspace" ? "Workspace" : "Bot files",
`${Math.max(1, Math.round(file.size / 1024))} KB`,
file.created_at ? relativeTime({ at: file.created_at }) : "",
file.ingest_error
? file.ingest_error
: file.ingested_at
? "searchable"
: "not searchable yet",
]
.filter(Boolean)
.join(" · ")}
@@ -56,19 +50,6 @@ export default function FileRow({ file, can_manage, busy, onIngest, onDelete }:
</Stack>
{can_manage ? (
<Row className="shrink-0 gap-0!">
<Button
title="Read it again so it can be searched"
aria-label="Reingest"
type="button"
variant="ghost"
color="gray"
size="small"
disabled={busy}
onClick={onIngest}
className="p-1.5!"
>
<RefreshCw size={14} aria-hidden />
</Button>
<Button
title="Delete it"
aria-label="Delete"
src/pages/admin/files/(partials)/files-search-input.tsx
+1 -1
View File
@@ -14,7 +14,7 @@ type Props = {
export default function FilesSearchInput({ value, loading, onChange, onSubmit }: Props) {
return (
<Input
placeholder="Search this folder"
placeholder="Search this folder by name"
value={value}
disabled={loading}
onChange={(event) => onChange(event.target.value)}
src/pages/admin/settings/database/(partials)/database-table-row.test.tsx
+3 -3
View File
@@ -4,7 +4,7 @@ import type { DatabaseTableSummary } from "@/src/types";
import DatabaseTableRow from "./database-table-row";
const table: DatabaseTableSummary = {
name: "bot_file_vectors",
name: "knowledgebase_vectors",
type: "virtual",
rows: 12,
size_bytes: 2048,
@@ -16,7 +16,7 @@ describe("DatabaseTableRow", () => {
test("names the table, its record count and its size", () => {
const html = renderToString(<DatabaseTableRow table={table} selected={false} onOpen={() => {}} />);
expect(html).toContain("bot_file_vectors");
expect(html).toContain("knowledgebase_vectors");
expect(html).toContain("12");
expect(html).toContain("2.0 KB");
});
@@ -24,7 +24,7 @@ describe("DatabaseTableRow", () => {
test("links to the table's own address", () => {
const html = renderToString(<DatabaseTableRow table={table} selected={false} onOpen={() => {}} />);
expect(html).toContain(`href="/admin/settings/database?table=bot_file_vectors"`);
expect(html).toContain(`href="/admin/settings/database?table=knowledgebase_vectors"`);
});
test("a table with no count yet shows a dash", () => {
src/pages/api/admin/bots/files.ts
-4
View File
@@ -132,8 +132,6 @@ function toPublicFile({ file }: { file: BotFileWithMediaLite }) {
purpose: file.purpose || "writing",
scope: file.scope || "bot",
owner_id: Number(file.owner_id) || 0,
ingested_at: file.ingested_at ? Number(file.ingested_at) : undefined,
ingest_error: file.ingest_error || undefined,
created_at: Number(file.created_at) || 0,
url: media_url || "",
};
@@ -149,8 +147,6 @@ type BotFileWithMediaLite = {
purpose?: string;
scope?: string;
owner_id?: number | "";
ingested_at?: number | "";
ingest_error?: string;
created_at?: number | "";
media?: Record<string, unknown> | null;
};
src/pages/api/admin/bots/ingest-file.ts
-82
View File
@@ -1,82 +0,0 @@
import userAuth from "@/src/functions/backend/auth/user-auth";
import grabBotFile from "@/src/functions/backend/db/bot-files/grab-bot-file";
import grabUserBot from "@/src/functions/backend/db/bots/grab-user-bot";
import grabUserWorkspace from "@/src/functions/backend/db/workspaces/grab-user-workspace";
import { IngestBotFileRequestSchema } from "@/src/schemas/bot-file-schema";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
import AppError from "@/src/utils/app-error";
import parseWithSchema from "@/src/utils/parse-with-schema";
import canManageCrew from "@/src/utils/rbac/can-manage-crew";
import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
import safeErrorMessage from "@/src/utils/safe-error-message";
import type { StorageScope } from "@/src/utils/storage-access";
import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
/**
* Queues a conversation's file to be read again and re-embedded, for when
* a search misses something the user knows is in it.
*/
export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({ req, body }) => {
try {
if (req.method != "POST") throw new AppError(`Method not allowed`);
const { user, user_types } = await userAuth({ req });
if (!user?.id) return { success: false, msg: `Unauthorized`, logoutUser: true };
if (isReadOnlyUser({ user_types: user_types || [] })) {
throw new AppError(`This user can only read records.`);
}
if (!canManageCrew({ user_types: user_types || [] })) {
throw new AppError(`Only a crew manager can re-read files.`);
}
const { file_id, read_media } = parseWithSchema({
schema: IngestBotFileRequestSchema,
data: body,
});
const scopes = await reachableScopes({ user_id: user.id, file_id });
const file = await grabBotFile({ scopes, file_id });
if (!file?.id) throw new AppError(`That file isn't here.`);
await enqueueFileIngest({ file_id: file.id });
return { success: true, msg: `${file.name || "The file"} will be read again shortly.` };
} catch (error: unknown) {
return { success: false, msg: safeErrorMessage({ error }) };
}
};
/**
* The directories the user reaches for this file.
*/
async function reachableScopes({
user_id,
file_id,
}: {
user_id: number;
file_id: number;
}): Promise<StorageScope[]> {
const { default: BunSQLite } = await import("@moduletrace/bun-sqlite");
const res = await BunSQLite.sql<{ scope: string; owner_id: number }>({
sql: `SELECT scope, owner_id FROM bot_files WHERE id = ? LIMIT 1`,
values: [file_id],
});
const file = res.payload?.[0];
if (!file?.owner_id) return [];
if (file.scope == "workspace") {
const workspace = await grabUserWorkspace({ workspace_id: file.owner_id, user_id });
return workspace?.id ? [{ scope: "workspace", owner_id: Number(file.owner_id) }] : [];
}
const bot = await grabUserBot({ bot_id: file.owner_id, user_id });
return bot?.id ? [{ scope: "bot", owner_id: Number(file.owner_id) }] : [];
}
src/pages/api/admin/bots/upload-attachment.ts
+2 -4
View File
@@ -8,7 +8,6 @@ import type { BotMessageAttachment } from "@/src/types";
import AppError from "@/src/utils/app-error";
import attachmentFileKind from "@/src/utils/attachment-file-kind";
import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
import canExtractText from "@/src/utils/can-extract-text";
import parseWithSchema from "@/src/utils/parse-with-schema";
import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
import safeErrorMessage from "@/src/utils/safe-error-message";
@@ -19,8 +18,8 @@ import type {
/**
* Stores one file for a chat message, private to the session user, before
* the message is sent with its id. Images are resized; documents have
* their text pulled out now, so bots can read them later.
* the message is sent with its id. Images are resized; a document keeps
* no extracted text (a reply reads it from the file).
*/
export const handler: BunextAPIRouteHandler<
APIResponseObject<BotMessageAttachment>
@@ -65,7 +64,6 @@ export const handler: BunextAPIRouteHandler<
media_mime_type: ext,
media_paradigms: ["bot-attachment"],
media: { title: name },
extract_media_text_content: canExtractText({ name, mime_type: mime }),
user: { id: Number(user.id) },
}).catch(async (error) => {
await logError({ title: `Attachment ${name} couldn't be saved`, error }).catch(() => {});
src/pages/api/admin/tasks/attach-file.ts
-2
View File
@@ -9,7 +9,6 @@ import type { TaskDetail } from "@/src/types";
import AppError from "@/src/utils/app-error";
import attachmentFileKind from "@/src/utils/attachment-file-kind";
import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
import canExtractText from "@/src/utils/can-extract-text";
import parseWithSchema from "@/src/utils/parse-with-schema";
import isReadOnlyUser from "@/src/utils/rbac/is-read-only-user";
import safeErrorMessage from "@/src/utils/safe-error-message";
@@ -66,7 +65,6 @@ export const handler: BunextAPIRouteHandler<APIResponseObject<TaskDetail>> = asy
media_mime_type: ext,
media_paradigms: ["task-media"],
media: { title: name },
extract_media_text_content: canExtractText({ name, mime_type: mime }),
user: { id: Number(user.id) },
});
src/pages/api/admin/upload-media.ts
-2
View File
@@ -44,7 +44,6 @@ export const handler: BunextAPIRouteHandler<APIResponseObject> = async (
is_media_primary,
update_on_duplicate,
reauth,
return_media_text_content,
media,
media_name,
} = body;
@@ -100,7 +99,6 @@ export const handler: BunextAPIRouteHandler<APIResponseObject> = async (
media_type,
is_primary: is_media_primary,
update_on_duplicate,
extract_media_text_content: return_media_text_content,
media,
media_mime_type: ext,
user: { id: target_user_id },