Add bots files handling logic

src/cron/index.ts

@@ -5,6 +5,7 @@ import cronSyncHarnessModels from "./sync-harness-models";
 import syncComputers from "./sync-computers";
 import syncMcpServers from "./sync-mcp-servers";
 import runCronJobs from "./run-cron-jobs";
+import ingestPendingFiles from "./ingest-pending-files";
 import registerCronJob from "./register-cron-job";
 
 export default function cron() {
@@ -54,4 +55,11 @@ export default function cron() {
         run_on_start: true,
         quiet: true,
     });
+
+    registerCronJob({
+        title: "ingest-pending-files",
+        schedule: "*/10 * * * *",
+        task: ingestPendingFiles,
+        quiet: true,
+    });
 }

src/cron/ingest-pending-files/index.ts

@@ -0,0 +1,29 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOT_FILES } from "@/db/types/db";
+import enqueueFileIngest from "@/src/file-ingest/enqueue-file-ingest";
+
+/**
+ * Queues any bot file that never got ingested (an upload that happened
+ * while the worker was down, a restart mid-ingest), so nothing is left
+ * unsearchable. Runs every ten minutes.
+ */
+export default async function ingestPendingFiles() {
+    const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_FILES>({
+        sql: `SELECT f.id FROM bot_files f
+            WHERE f.ingested_at IS NULL
+              AND f.ingest_error IS NULL
+              AND f.created_at > ?
+              AND NOT EXISTS (
+                  SELECT 1 FROM file_ingest_queue q
+                  WHERE q.file_id = f.id AND q.status IN ('pending', 'processing')
+              )
+            LIMIT 25`,
+        values: [Date.now() - 24 * 60 * 60 * 1000],
+    });
+
+    for (const file of res.payload || []) {
+        if (file.id) await enqueueFileIngest({ file_id: file.id });
+    }
+
+    return res.payload?.length || 0;
+}