Moduletrace/bunext
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep bunext sitemap alive while sitemap exports query the DB

Browse Source 
Bun's SQL client (mariadb/mysql adapters, Bun 1.3.14) doesn't keep the
event loop alive for a query on an idle connection, so a second dynamic
route's sitemap export querying the DB made the command exit silently.
Hold the loop open with a timer during generation, and log what a stuck
run is waiting on.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-15 09:00:40 +01:00
co-authored by Claude Opus 5.5
parent 23a309005a
commit 40c17f903e
7 changed files with 153 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/commands/sitemap/index.js
Vendored
+14 -2
View File
@@ -5,9 +5,10 @@ import { log } from "../../utils/log";
import grabDirNames from "../../utils/grab-dir-names";
import grabConfig from "../../functions/grab-config";
import grabConstants from "../../utils/grab-constants";
import { generateSitemapFiles } from "../../utils/generate-sitemap";
import { generateSitemapFiles, grabPendingSitemapTask, } from "../../utils/generate-sitemap";
import { grabPageLastmodFile } from "../../utils/page-lastmod-record";
import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
const SitemapStillWaitingInterval = 30_000;
export default function () {
return new Command("sitemap")
.description("Generate sitemap.xml and robots.txt from the pages directory")
@@ -39,12 +40,23 @@ export default function () {
throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` +
`so the chunks are served.`);
}
// Bun exits silently (code 0) when the event loop empties
// while a promise is pending. Bun's SQL client (mariadb /
// mysql adapters, Bun 1.3.14) doesn't keep the loop alive
// for a query on a connection that has gone idle, so a
// second `sitemap` export querying the DB would end the
// process mid-generation. Hold the loop open with a timer,
// which also reports what a genuinely stuck run waits on.
const keep_alive = setInterval(() => {
log.warn(`Still waiting on ` +
`${grabPendingSitemapTask() || "sitemap generation"}...`);
}, SitemapStillWaitingInterval);
const { sitemap, chunks, url_count } = await generateSitemapFiles({
...sitemap_config,
origin: options.origin || sitemap_config.origin,
chunk_size,
chunks_url_path: chunks_public_path,
});
}).finally(() => clearInterval(keep_alive));
// Remove chunks from a previous, larger sitemap
if (existsSync(chunks_dir_path)) {
for (const file of readdirSync(chunks_dir_path)) {
dist/utils/generate-sitemap.d.ts
Vendored
+1
View File
@@ -6,6 +6,7 @@ import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
export declare const SitemapMaxUrlsPerFile = 50000;
export declare const SitemapMaxBytesPerFile: number;
export declare const SitemapDefaultChunkSize = 10000;
export declare function grabPendingSitemapTask(): string | undefined;
export type SitemapChunkFile = {
file_name: string;
xml: string;
dist/utils/generate-sitemap.js
Vendored
+39 -13
View File
@@ -18,6 +18,14 @@ export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
export const SitemapDefaultChunkSize = 10_000;
const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
/**
* What sitemap generation is currently waiting on, so `bunext sitemap`
* can name it when a `sitemap` export or `additional_paths` never settles.
*/
let pending_sitemap_task;
export function grabPendingSitemapTask() {
return pending_sitemap_task;
}
/**
* # Generate a sitemap.xml string for all pages
* Dynamic routes are only included through a `sitemap` export in their
@@ -169,9 +177,19 @@ export async function grabSitemapEntries(params) {
: undefined),
});
}
const additional_paths = typeof params?.additional_paths == "function"
? await params.additional_paths()
: params?.additional_paths;
let additional_paths;
if (typeof params?.additional_paths == "function") {
pending_sitemap_task = "`sitemap.additional_paths` in the config";
try {
additional_paths = await params.additional_paths();
}
finally {
pending_sitemap_task = undefined;
}
}
else {
additional_paths = params?.additional_paths;
}
entries.push(...(additional_paths || []).map(toSitemapEntry));
const seen = new Set();
const final_entries = [];
@@ -204,18 +222,26 @@ async function grabPageSitemapPaths({ file_path, route, }) {
const final_server_path = server_ctx?.local_path
? path.join(ROOT_DIR, server_ctx.path)
: server_file_path;
// Bust the import cache in development so edits to the server file
// are picked up without a restart.
const server_module = await import(isDevelopment()
? `${final_server_path}?t=${Date.now()}`
: final_server_path);
if (typeof server_module.sitemap != "function")
return;
const relative_server_path = grabRelativePath(server_file_path);
try {
return await server_module.sitemap({ route });
pending_sitemap_task = `importing ${relative_server_path}`;
// Bust the import cache in development so edits to the server file
// are picked up without a restart.
const server_module = await import(isDevelopment()
? `${final_server_path}?t=${Date.now()}`
: final_server_path);
if (typeof server_module.sitemap != "function")
return;
pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`;
try {
return await server_module.sitemap({ route });
}
catch (error) {
throw new Error(`\`sitemap\` export of ${relative_server_path} failed: ${error?.message || error}`);
}
}
catch (error) {
throw new Error(`\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${error?.message || error}`);
finally {
pending_sitemap_task = undefined;
}
}
function toSitemapEntry(path) {
package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@moduletrace/bunext",
"version": "1.2.1",
"version": "1.2.2",
"main": "dist/index.js",
"module": "index.ts",
"dependencies": {
src/__tests__/commands/sitemap.test.ts
+29
View File
@@ -217,6 +217,35 @@ describe("bunext sitemap", () => {
);
});
it("waits for sitemap exports that don't keep the event loop alive", async () => {
// Like Bun's SQL client querying on an idle connection: the
// promise settles, but nothing holds the process open meanwhile.
const unrefServerFile = (paths: string[]) =>
`export const sitemap = () => new Promise((resolve) => {
setTimeout(() => resolve(${JSON.stringify(paths)}), 50).unref();
});`;
writeConfig(`{ origin: "https://example.com" }`);
writeProjectFile(
"src/pages/blog/[slug].server.ts",
unrefServerFile(["/blog/a"]),
);
writeProjectFile(
"src/pages/docs/[slug].tsx",
"export default () => null",
);
writeProjectFile(
"src/pages/docs/[slug].server.ts",
unrefServerFile(["/docs/a"]),
);
const res = await runSitemap();
expect(res.exit_code).toBe(0);
expect(locs(readOutput())).toContain("https://example.com/blog/a");
expect(locs(readOutput())).toContain("https://example.com/docs/a");
});
it("exits with code 1 when config additional_paths throws", async () => {
writeConfig(`{
sitemap: {
src/commands/sitemap/index.ts
+21 -2
View File
@@ -12,12 +12,17 @@ import { log } from "../../utils/log";
import grabDirNames from "../../utils/grab-dir-names";
import grabConfig from "../../functions/grab-config";
import grabConstants from "../../utils/grab-constants";
import { generateSitemapFiles } from "../../utils/generate-sitemap";
import {
generateSitemapFiles,
grabPendingSitemapTask,
} from "../../utils/generate-sitemap";
import { grabPageLastmodFile } from "../../utils/page-lastmod-record";
import generateRobotsTxt, {
RobotsTxtGeneratedMarker,
} from "../../utils/generate-robots-txt";
const SitemapStillWaitingInterval = 30_000;
type Options = {
output?: string;
origin?: string;
@@ -79,13 +84,27 @@ export default function () {
);
}
// Bun exits silently (code 0) when the event loop empties
// while a promise is pending. Bun's SQL client (mariadb /
// mysql adapters, Bun 1.3.14) doesn't keep the loop alive
// for a query on a connection that has gone idle, so a
// second `sitemap` export querying the DB would end the
// process mid-generation. Hold the loop open with a timer,
// which also reports what a genuinely stuck run waits on.
const keep_alive = setInterval(() => {
log.warn(
`Still waiting on ` +
`${grabPendingSitemapTask() || "sitemap generation"}...`,
);
}, SitemapStillWaitingInterval);
const { sitemap, chunks, url_count } =
await generateSitemapFiles({
...sitemap_config,
origin: options.origin || sitemap_config.origin,
chunk_size,
chunks_url_path: chunks_public_path,
});
}).finally(() => clearInterval(keep_alive));
// Remove chunks from a previous, larger sitemap
if (existsSync(chunks_dir_path)) {
src/utils/generate-sitemap.ts
+48 -19
View File
@@ -28,6 +28,16 @@ export const SitemapDefaultChunkSize = 10_000;
const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
/**
* What sitemap generation is currently waiting on, so `bunext sitemap`
* can name it when a `sitemap` export or `additional_paths` never settles.
*/
let pending_sitemap_task: string | undefined;
export function grabPendingSitemapTask() {
return pending_sitemap_task;
}
export type SitemapChunkFile = {
file_name: string;
xml: string;
@@ -247,10 +257,19 @@ export async function grabSitemapEntries(
});
}
const additional_paths =
typeof params?.additional_paths == "function"
? await params.additional_paths()
: params?.additional_paths;
let additional_paths: BunextSitemapAdditionalPaths | undefined;
if (typeof params?.additional_paths == "function") {
pending_sitemap_task = "`sitemap.additional_paths` in the config";
try {
additional_paths = await params.additional_paths();
} finally {
pending_sitemap_task = undefined;
}
} else {
additional_paths = params?.additional_paths;
}
entries.push(...(additional_paths || []).map(toSitemapEntry));
@@ -295,24 +314,34 @@ async function grabPageSitemapPaths({
? path.join(ROOT_DIR, server_ctx.path)
: server_file_path;
// Bust the import cache in development so edits to the server file
// are picked up without a restart.
const server_module: BunextPageServerModule = await import(
isDevelopment()
? `${final_server_path}?t=${Date.now()}`
: final_server_path
);
if (typeof server_module.sitemap != "function") return;
const relative_server_path = grabRelativePath(server_file_path);
try {
return await server_module.sitemap({ route });
} catch (error: any) {
throw new Error(
`\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${
error?.message || error
}`,
pending_sitemap_task = `importing ${relative_server_path}`;
// Bust the import cache in development so edits to the server file
// are picked up without a restart.
const server_module: BunextPageServerModule = await import(
isDevelopment()
? `${final_server_path}?t=${Date.now()}`
: final_server_path
);
if (typeof server_module.sitemap != "function") return;
pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`;
try {
return await server_module.sitemap({ route });
} catch (error: any) {
throw new Error(
`\`sitemap\` export of ${relative_server_path} failed: ${
error?.message || error
}`,
);
}
} finally {
pending_sitemap_task = undefined;
}
}