Keep bunext sitemap alive while sitemap exports query the DB

Bun's SQL client (mariadb/mysql adapters, Bun 1.3.14) doesn't keep the
event loop alive for a query on an idle connection, so a second dynamic
route's sitemap export querying the DB made the command exit silently.
Hold the loop open with a timer during generation, and log what a stuck
run is waiting on.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dist/commands/sitemap/index.js

@@ -5,9 +5,10 @@ import { log } from "../../utils/log";
 import grabDirNames from "../../utils/grab-dir-names";
 import grabConfig from "../../functions/grab-config";
 import grabConstants from "../../utils/grab-constants";
-import { generateSitemapFiles } from "../../utils/generate-sitemap";
+import { generateSitemapFiles, grabPendingSitemapTask, } from "../../utils/generate-sitemap";
 import { grabPageLastmodFile } from "../../utils/page-lastmod-record";
 import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
+const SitemapStillWaitingInterval = 30_000;
 export default function () {
     return new Command("sitemap")
         .description("Generate sitemap.xml and robots.txt from the pages directory")
@@ -39,12 +40,23 @@ export default function () {
                 throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` +
                     `so the chunks are served.`);
             }
+            // Bun exits silently (code 0) when the event loop empties
+            // while a promise is pending. Bun's SQL client (mariadb /
+            // mysql adapters, Bun 1.3.14) doesn't keep the loop alive
+            // for a query on a connection that has gone idle, so a
+            // second `sitemap` export querying the DB would end the
+            // process mid-generation. Hold the loop open with a timer,
+            // which also reports what a genuinely stuck run waits on.
+            const keep_alive = setInterval(() => {
+                log.warn(`Still waiting on ` +
+                    `${grabPendingSitemapTask() || "sitemap generation"}...`);
+            }, SitemapStillWaitingInterval);
             const { sitemap, chunks, url_count } = await generateSitemapFiles({
                 ...sitemap_config,
                 origin: options.origin || sitemap_config.origin,
                 chunk_size,
                 chunks_url_path: chunks_public_path,
-            });
+            }).finally(() => clearInterval(keep_alive));
             // Remove chunks from a previous, larger sitemap
             if (existsSync(chunks_dir_path)) {
                 for (const file of readdirSync(chunks_dir_path)) {

dist/utils/generate-sitemap.d.ts

@@ -6,6 +6,7 @@ import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
 export declare const SitemapMaxUrlsPerFile = 50000;
 export declare const SitemapMaxBytesPerFile: number;
 export declare const SitemapDefaultChunkSize = 10000;
+export declare function grabPendingSitemapTask(): string | undefined;
 export type SitemapChunkFile = {
     file_name: string;
     xml: string;

dist/utils/generate-sitemap.js

@@ -18,6 +18,14 @@ export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
 export const SitemapDefaultChunkSize = 10_000;
 const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
 const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
+/**
+ * What sitemap generation is currently waiting on, so `bunext sitemap`
+ * can name it when a `sitemap` export or `additional_paths` never settles.
+ */
+let pending_sitemap_task;
+export function grabPendingSitemapTask() {
+    return pending_sitemap_task;
+}
 /**
  * # Generate a sitemap.xml string for all pages
  * Dynamic routes are only included through a `sitemap` export in their
@@ -169,9 +177,19 @@ export async function grabSitemapEntries(params) {
                     : undefined),
         });
     }
-    const additional_paths = typeof params?.additional_paths == "function"
+    let additional_paths;
-        ? await params.additional_paths()
+    if (typeof params?.additional_paths == "function") {
-        : params?.additional_paths;
+        pending_sitemap_task = "`sitemap.additional_paths` in the config";
+        try {
+            additional_paths = await params.additional_paths();
+        }
+        finally {
+            pending_sitemap_task = undefined;
+        }
+    }
+    else {
+        additional_paths = params?.additional_paths;
+    }
     entries.push(...(additional_paths || []).map(toSitemapEntry));
     const seen = new Set();
     const final_entries = [];
@@ -204,18 +222,26 @@ async function grabPageSitemapPaths({ file_path, route, }) {
     const final_server_path = server_ctx?.local_path
         ? path.join(ROOT_DIR, server_ctx.path)
         : server_file_path;
-    // Bust the import cache in development so edits to the server file
+    const relative_server_path = grabRelativePath(server_file_path);
-    // are picked up without a restart.
-    const server_module = await import(isDevelopment()
-        ? `${final_server_path}?t=${Date.now()}`
-        : final_server_path);
-    if (typeof server_module.sitemap != "function")
-        return;
     try {
-        return await server_module.sitemap({ route });
+        pending_sitemap_task = `importing ${relative_server_path}`;
+        // Bust the import cache in development so edits to the server file
+        // are picked up without a restart.
+        const server_module = await import(isDevelopment()
+            ? `${final_server_path}?t=${Date.now()}`
+            : final_server_path);
+        if (typeof server_module.sitemap != "function")
+            return;
+        pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`;
+        try {
+            return await server_module.sitemap({ route });
+        }
+        catch (error) {
+            throw new Error(`\`sitemap\` export of ${relative_server_path} failed: ${error?.message || error}`);
+        }
     }
-    catch (error) {
+    finally {
-        throw new Error(`\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${error?.message || error}`);
+        pending_sitemap_task = undefined;
     }
 }
 function toSitemapEntry(path) {

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@moduletrace/bunext",
-    "version": "1.2.1",
+    "version": "1.2.2",
     "main": "dist/index.js",
     "module": "index.ts",
     "dependencies": {

src/__tests__/commands/sitemap.test.ts

@@ -217,6 +217,35 @@ describe("bunext sitemap", () => {
         );
     });
 
+    it("waits for sitemap exports that don't keep the event loop alive", async () => {
+        // Like Bun's SQL client querying on an idle connection: the
+        // promise settles, but nothing holds the process open meanwhile.
+        const unrefServerFile = (paths: string[]) =>
+            `export const sitemap = () => new Promise((resolve) => {
+                setTimeout(() => resolve(${JSON.stringify(paths)}), 50).unref();
+            });`;
+
+        writeConfig(`{ origin: "https://example.com" }`);
+        writeProjectFile(
+            "src/pages/blog/[slug].server.ts",
+            unrefServerFile(["/blog/a"]),
+        );
+        writeProjectFile(
+            "src/pages/docs/[slug].tsx",
+            "export default () => null",
+        );
+        writeProjectFile(
+            "src/pages/docs/[slug].server.ts",
+            unrefServerFile(["/docs/a"]),
+        );
+
+        const res = await runSitemap();
+
+        expect(res.exit_code).toBe(0);
+        expect(locs(readOutput())).toContain("https://example.com/blog/a");
+        expect(locs(readOutput())).toContain("https://example.com/docs/a");
+    });
+
     it("exits with code 1 when config additional_paths throws", async () => {
         writeConfig(`{
             sitemap: {

src/commands/sitemap/index.ts

@@ -12,12 +12,17 @@ import { log } from "../../utils/log";
 import grabDirNames from "../../utils/grab-dir-names";
 import grabConfig from "../../functions/grab-config";
 import grabConstants from "../../utils/grab-constants";
-import { generateSitemapFiles } from "../../utils/generate-sitemap";
+import {
+    generateSitemapFiles,
+    grabPendingSitemapTask,
+} from "../../utils/generate-sitemap";
 import { grabPageLastmodFile } from "../../utils/page-lastmod-record";
 import generateRobotsTxt, {
     RobotsTxtGeneratedMarker,
 } from "../../utils/generate-robots-txt";
 
+const SitemapStillWaitingInterval = 30_000;
+
 type Options = {
     output?: string;
     origin?: string;
@@ -79,13 +84,27 @@ export default function () {
                     );
                 }
 
+                // Bun exits silently (code 0) when the event loop empties
+                // while a promise is pending. Bun's SQL client (mariadb /
+                // mysql adapters, Bun 1.3.14) doesn't keep the loop alive
+                // for a query on a connection that has gone idle, so a
+                // second `sitemap` export querying the DB would end the
+                // process mid-generation. Hold the loop open with a timer,
+                // which also reports what a genuinely stuck run waits on.
+                const keep_alive = setInterval(() => {
+                    log.warn(
+                        `Still waiting on ` +
+                            `${grabPendingSitemapTask() || "sitemap generation"}...`,
+                    );
+                }, SitemapStillWaitingInterval);
+
                 const { sitemap, chunks, url_count } =
                     await generateSitemapFiles({
                         ...sitemap_config,
                         origin: options.origin || sitemap_config.origin,
                         chunk_size,
                         chunks_url_path: chunks_public_path,
-                    });
+                    }).finally(() => clearInterval(keep_alive));
 
                 // Remove chunks from a previous, larger sitemap
                 if (existsSync(chunks_dir_path)) {

src/utils/generate-sitemap.ts

@@ -28,6 +28,16 @@ export const SitemapDefaultChunkSize = 10_000;
 const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
 const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
 
+/**
+ * What sitemap generation is currently waiting on, so `bunext sitemap`
+ * can name it when a `sitemap` export or `additional_paths` never settles.
+ */
+let pending_sitemap_task: string | undefined;
+
+export function grabPendingSitemapTask() {
+    return pending_sitemap_task;
+}
+
 export type SitemapChunkFile = {
     file_name: string;
     xml: string;
@@ -247,10 +257,19 @@ export async function grabSitemapEntries(
         });
     }
 
-    const additional_paths =
+    let additional_paths: BunextSitemapAdditionalPaths | undefined;
-        typeof params?.additional_paths == "function"
+
-            ? await params.additional_paths()
+    if (typeof params?.additional_paths == "function") {
-            : params?.additional_paths;
+        pending_sitemap_task = "`sitemap.additional_paths` in the config";
+
+        try {
+            additional_paths = await params.additional_paths();
+        } finally {
+            pending_sitemap_task = undefined;
+        }
+    } else {
+        additional_paths = params?.additional_paths;
+    }
 
     entries.push(...(additional_paths || []).map(toSitemapEntry));
 
@@ -295,24 +314,34 @@ async function grabPageSitemapPaths({
         ? path.join(ROOT_DIR, server_ctx.path)
         : server_file_path;
 
-    // Bust the import cache in development so edits to the server file
+    const relative_server_path = grabRelativePath(server_file_path);
-    // are picked up without a restart.
-    const server_module: BunextPageServerModule = await import(
-        isDevelopment()
-            ? `${final_server_path}?t=${Date.now()}`
-            : final_server_path
-    );
-
-    if (typeof server_module.sitemap != "function") return;
 
     try {
-        return await server_module.sitemap({ route });
+        pending_sitemap_task = `importing ${relative_server_path}`;
-    } catch (error: any) {
+
-        throw new Error(
+        // Bust the import cache in development so edits to the server file
-            `\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${
+        // are picked up without a restart.
-                error?.message || error
+        const server_module: BunextPageServerModule = await import(
-            }`,
+            isDevelopment()
+                ? `${final_server_path}?t=${Date.now()}`
+                : final_server_path
         );
+
+        if (typeof server_module.sitemap != "function") return;
+
+        pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`;
+
+        try {
+            return await server_module.sitemap({ route });
+        } catch (error: any) {
+            throw new Error(
+                `\`sitemap\` export of ${relative_server_path} failed: ${
+                    error?.message || error
+                }`,
+            );
+        }
+    } finally {
+        pending_sitemap_task = undefined;
     }
 }