Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 32 Wiki Activity

Keep searchable text only in knowledgebases

Browse Source 
Every bot file was chunked into bot_file_chunks and embedded into
bot_file_vectors, and uploads kept their extracted text in
media.text_content, which together took most of the database. Bot
files are no longer indexed: bots find them by name or path with
list_files and read them with read_file, and search_files, ingest_file
and the files modal's re-read button are gone. Attachments keep no
text; a reply reads a document from the file. Only knowledgebase
documents keep text.

The schema drops both tables and the ingest columns. At boot,
dropStoredFileText removes what older versions stored (file jobs,
upload text) and vacuums once much of the database is free. The
media:prune-text script it replaces is removed.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-28 04:44:46 +01:00
co-authored by Claude Opus 5.5
parent 2896e76256
commit ac3ebc43d4
55 changed files with 271 additions and 1712 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/harness/embeddings/fit-vector-dimensions.ts
+1 -1
View File
@@ -4,7 +4,7 @@ type Params = {
vector: number[];
/**
* Defaults to `AppData.EmbeddingDimensions`, the width of the fixed
* `bot_file_vectors` table.
* `knowledgebase_vectors` table.
*/
dimensions?: number;
};
src/harness/tools/computer/import-file-tool.ts
+2 -7
View File
@@ -3,7 +3,6 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { z } from "zod";
import { AppData } from "@/src/data/app-data";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
import recordBotFile, { grabBotFileRow } from "@/src/functions/backend/db/bot-files/record-bot-file";
import grabComputerMcpUrl from "@/src/functions/backend/db/computers/grab-computer-mcp-url";
import computerNotReadyMessage from "@/src/utils/computer-not-ready-message";
@@ -49,7 +48,7 @@ type Args = {
* Copies what a bot made on its computer into its own files: one file, or
* a whole folder (taken as a gzipped tar and unpacked into one row per
* file). The copies are kept with the app, stay after the computer is
* rebuilt, and are searchable at once; the originals stay on the computer,
* rebuilt, and are readable at once; the originals stay on the computer,
* where the message's folder ref points at the folder holding them.
*
* The bytes come back over the sandbox's streaming download endpoint
@@ -72,7 +71,7 @@ export default function importFileTool({
name: "import_file",
description: `Copy a file or folder from this computer into your own files, so it is kept with the app and stays after the computer is rebuilt. Give its path on the computer, relative to your folder there (${dirs.cwd}), e.g. "seo/coderank/raw/crawl.json" or "seo/coderank/raw"; a folder comes in as it is, in one go. Up to ${max_mb} MB. Check with list_files and the computer's own tools what you already imported: importing the same thing twice just writes over the copy, so do it once.${
own?.scope == "workspace" ? " The copies land in the workspace's shared files." : ""
} Imports are searchable at once. Name a copy with its [file:ID] ref. Import the report you wrote before the last long steps of a run, so it is never lost to the step limit.`,
} Imports are readable at once. Name a copy with its [file:ID] ref. Import the report you wrote before the last long steps of a run, so it is never lost to the step limit.`,
schema: z.object({
path: z
.string()
@@ -190,8 +189,6 @@ async function importOneFile({
user_id: target.user_id,
});
if (saved.id) await enqueueFileIngest({ file_id: saved.id });
if (attach && saved.media_id) {
onAttach?.({ media_id: Number(saved.media_id), name: saved.name || name });
}
@@ -427,8 +424,6 @@ async function unpackArchive({
if (!row?.id) continue;
await enqueueFileIngest({ file_id: row.id }).catch(() => {});
saved.push({ id: Number(row.id), path: row.path || `${folder}/${relative}` });
}
} finally {
src/harness/tools/storage/generate-image-tool.ts
-3
View File
@@ -1,5 +1,4 @@
import { z } from "zod";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file";
import generateImage from "@/src/harness/image/generate-image";
import type { HarnessTool } from "../../types";
@@ -98,8 +97,6 @@ export default function generateImageTool(context: StorageToolContext): HarnessT
user_id: context.user_id,
});
await enqueueFileIngest({ file_id: file.id! });
if (args.attach && file.media_id) {
context.onAttach?.({ media_id: Number(file.media_id), name: file.name || "" });
}
src/harness/tools/storage/grab-storage-tools.ts
-4
View File
@@ -1,11 +1,9 @@
import type { HarnessTool } from "../../types";
import copyFileTool from "./copy-file-tool";
import deleteFileTool from "./delete-file-tool";
import ingestFileTool from "./ingest-file-tool";
import listFilesTool from "./list-files-tool";
import moveFileTool from "./move-file-tool";
import readFileTool from "./read-file-tool";
import searchFilesTool from "./search-files-tool";
import type { StorageToolContext } from "./storage-tool-context";
import writeFileTool from "./write-file-tool";
@@ -19,10 +17,8 @@ export default function grabStorageTools(context: StorageToolContext): HarnessTo
listFilesTool(context),
readFileTool(context),
writeFileTool(context),
searchFilesTool(context),
moveFileTool(context),
deleteFileTool(context),
ingestFileTool(context),
...(context.scopes.length > 1 ? [copyFileTool(context)] : []),
];
}
src/harness/tools/storage/ingest-file-tool.ts
-44
View File
@@ -1,44 +0,0 @@
import { z } from "zod";
import { AppData } from "@/src/data/app-data";
import ingestBotFile from "@/src/functions/backend/db/bot-files/ingest-bot-file";
import type { HarnessTool } from "../../types";
import type { StorageToolContext } from "./storage-tool-context";
type Args = {
file_id: number;
read_media?: boolean;
};
/**
* Ingests a file again: reads its text, chunks it and embeds it, so a
* changed or previously failed file is searchable. Images and audio are
* only read when asked for, since that costs a model call.
*/
export default function ingestFileTool(_context: StorageToolContext): HarnessTool<Args> {
return {
name: "ingest_file",
description: `Make one of your files searchable again (a course correction when search misses something you know is in it), by its ref id. Images and scans are only read when "read_media" is true.`,
schema: z.object({
file_id: z.number().int().positive(),
read_media: z.boolean().optional(),
}),
run: async ({ args, signal }) => {
const result = await ingestBotFile({
file_id: args.file_id,
read_media: args.read_media,
signal,
});
switch (result.status) {
case "ingested":
return `Ingested: ${result.chunks} passage(s), searchable now.`;
case "unavailable":
return `Its text is stored and searchable by words, but no embedding model is set, so meaning search is off.`;
case "skipped":
return `It's too large to read (over ${Math.round(AppData.IngestMaxBytes / (1024 * 1024))} MB).`;
default:
return `It has no text to read.`;
}
},
};
}
src/harness/tools/storage/list-files-tool.ts
+1 -2
View File
@@ -50,9 +50,8 @@ export default function listFilesTool(context: StorageToolContext): HarnessTool<
const lines = files.map((file) => {
const where = file.scope == "workspace" ? ` (workspace ${file.owner_id})` : "";
const ingested = file.ingested_at ? "" : " - not searchable yet";
return `- ${file.path} - ${formatBytes({ bytes: Number(file.size) || 0 })}${where}${ingested} ${fileRef({ file_id: file.id! })}`;
return `- ${file.path} - ${formatBytes({ bytes: Number(file.size) || 0 })}${where} ${fileRef({ file_id: file.id! })}`;
});
const more =
src/harness/tools/storage/make-pdf-tool.ts
-3
View File
@@ -1,6 +1,5 @@
import { z } from "zod";
import { PDFDocument, StandardFonts, rgb } from "pdf-lib";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file";
import type { HarnessTool } from "../../types";
import {
@@ -69,8 +68,6 @@ export default function makePdfTool(context: StorageToolContext): HarnessTool<Ar
user_id: context.user_id,
});
await enqueueFileIngest({ file_id: file.id! });
if (args.attach && file.media_id) {
context.onAttach?.({ media_id: Number(file.media_id), name: file.name || "" });
}
src/harness/tools/storage/search-files-tool.ts
-49
View File
@@ -1,49 +0,0 @@
import { z } from "zod";
import { AppData } from "@/src/data/app-data";
import searchBotFiles from "@/src/functions/backend/db/bot-files/search-bot-files";
import type { HarnessTool } from "../../types";
import { fileRef, type StorageToolContext } from "./storage-tool-context";
type Args = {
query: string;
file_id?: number;
limit?: number;
};
/**
* Searches the bot's saved files by meaning (through the embedding
* vectors), falling back to words, and returns the passages with their
* file's ref so the bot can read more.
*/
export default function searchFilesTool(context: StorageToolContext): HarnessTool<Args> {
return {
name: "search_files",
description: `Search the files you have saved (documents the user gave you and what you wrote) and get the passages that answer "query", with their file's ref. Search before saying you don't know something that may be in a file, and before writing a document from memory.`,
schema: z.object({
query: z.string().min(1).max(500),
file_id: z.number().int().positive().optional().describe(`Only this file's passages`),
limit: z.number().int().min(1).max(20).optional(),
}),
run: async ({ args, signal }) => {
const result = await searchBotFiles({
scopes: context.scopes,
query: args.query,
file_id: args.file_id,
limit: args.limit || AppData.RagTopChunks,
signal,
bot_id: context.bot_id,
workspace_id: context.workspace_id,
});
if (!result.hits.length) return result.note;
const passages = result.hits.map((hit, index) => {
const where = hit.scope == "workspace" ? ` (workspace ${hit.scope})` : "";
return `## ${index + 1}. ${hit.name}${where} ${fileRef({ file_id: hit.file_id })}\n${hit.text.trim()}`;
});
return `${result.note}\n\n${passages.join("\n\n")}`;
},
};
}
src/harness/tools/storage/write-file-tool.ts
+2 -5
View File
@@ -1,7 +1,6 @@
import { z } from "zod";
import { AppData } from "@/src/data/app-data";
import type { BotFilePurpose } from "@/src/dict/bot-file-dict";
import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file";
import type { HarnessTool } from "../../types";
import {
@@ -23,7 +22,7 @@ type Args = {
/**
* Writes a text file (markdown, CSV, JSON, code, plain text) into one of
* the bot's directories, making folders as needed. The file stays for
* later runs and becomes searchable.
* later runs.
*/
export default function writeFileTool(context: StorageToolContext): HarnessTool<Args> {
const own = context.scopes[0];
@@ -32,7 +31,7 @@ export default function writeFileTool(context: StorageToolContext): HarnessTool<
name: "write_file",
description: `Save a text file for later: markdown, CSV, JSON, code or plain text. Give it a path like "reports/q3.md"; folders are made as needed. Files belong to ${
own?.scope == "workspace" ? "the workspace you're working in" : "you"
} and stay between runs: write a file before telling the user something is saved, and search_files to find one you wrote before. It is readable and searchable at once.${
} and stay between runs: write a file before telling the user something is saved, and list_files (by name or path) to find one you wrote before. It is readable at once.${
context.scopes.length > 1
? ` "scope" picks a directory: ${context.scopes.map((s) => `${s.scope}:${s.owner_id}`).join(", ")}.`
: ""
@@ -74,8 +73,6 @@ export default function writeFileTool(context: StorageToolContext): HarnessTool<
user_id: context.user_id,
});
await enqueueFileIngest({ file_id: file.id! });
if (args.attach && file.media_id) {
context.onAttach?.({ media_id: Number(file.media_id), name: file.name || "" });
}