diff --git a/dist/commands/sitemap/index.js b/dist/commands/sitemap/index.js index a4e9277..af13644 100644 --- a/dist/commands/sitemap/index.js +++ b/dist/commands/sitemap/index.js @@ -5,9 +5,10 @@ import { log } from "../../utils/log"; import grabDirNames from "../../utils/grab-dir-names"; import grabConfig from "../../functions/grab-config"; import grabConstants from "../../utils/grab-constants"; -import { generateSitemapFiles } from "../../utils/generate-sitemap"; +import { generateSitemapFiles, grabPendingSitemapTask, } from "../../utils/generate-sitemap"; import { grabPageLastmodFile } from "../../utils/page-lastmod-record"; import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt"; +const SitemapStillWaitingInterval = 30_000; export default function () { return new Command("sitemap") .description("Generate sitemap.xml and robots.txt from the pages directory") @@ -39,12 +40,23 @@ export default function () { throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` + `so the chunks are served.`); } + // Bun exits silently (code 0) when the event loop empties + // while a promise is pending. Bun's SQL client (mariadb / + // mysql adapters, Bun 1.3.14) doesn't keep the loop alive + // for a query on a connection that has gone idle, so a + // second `sitemap` export querying the DB would end the + // process mid-generation. Hold the loop open with a timer, + // which also reports what a genuinely stuck run waits on. + const keep_alive = setInterval(() => { + log.warn(`Still waiting on ` + + `${grabPendingSitemapTask() || "sitemap generation"}...`); + }, SitemapStillWaitingInterval); const { sitemap, chunks, url_count } = await generateSitemapFiles({ ...sitemap_config, origin: options.origin || sitemap_config.origin, chunk_size, chunks_url_path: chunks_public_path, - }); + }).finally(() => clearInterval(keep_alive)); // Remove chunks from a previous, larger sitemap if (existsSync(chunks_dir_path)) { for (const file of readdirSync(chunks_dir_path)) { diff --git a/dist/utils/generate-sitemap.d.ts b/dist/utils/generate-sitemap.d.ts index 41d5749..00b106c 100644 --- a/dist/utils/generate-sitemap.d.ts +++ b/dist/utils/generate-sitemap.d.ts @@ -6,6 +6,7 @@ import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types"; export declare const SitemapMaxUrlsPerFile = 50000; export declare const SitemapMaxBytesPerFile: number; export declare const SitemapDefaultChunkSize = 10000; +export declare function grabPendingSitemapTask(): string | undefined; export type SitemapChunkFile = { file_name: string; xml: string; diff --git a/dist/utils/generate-sitemap.js b/dist/utils/generate-sitemap.js index 3b3281e..584b347 100644 --- a/dist/utils/generate-sitemap.js +++ b/dist/utils/generate-sitemap.js @@ -18,6 +18,14 @@ export const SitemapMaxBytesPerFile = 50 * 1024 * 1024; export const SitemapDefaultChunkSize = 10_000; const SitemapXmlHeader = ``; const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`; +/** + * What sitemap generation is currently waiting on, so `bunext sitemap` + * can name it when a `sitemap` export or `additional_paths` never settles. + */ +let pending_sitemap_task; +export function grabPendingSitemapTask() { + return pending_sitemap_task; +} /** * # Generate a sitemap.xml string for all pages * Dynamic routes are only included through a `sitemap` export in their @@ -169,9 +177,19 @@ export async function grabSitemapEntries(params) { : undefined), }); } - const additional_paths = typeof params?.additional_paths == "function" - ? await params.additional_paths() - : params?.additional_paths; + let additional_paths; + if (typeof params?.additional_paths == "function") { + pending_sitemap_task = "`sitemap.additional_paths` in the config"; + try { + additional_paths = await params.additional_paths(); + } + finally { + pending_sitemap_task = undefined; + } + } + else { + additional_paths = params?.additional_paths; + } entries.push(...(additional_paths || []).map(toSitemapEntry)); const seen = new Set(); const final_entries = []; @@ -204,18 +222,26 @@ async function grabPageSitemapPaths({ file_path, route, }) { const final_server_path = server_ctx?.local_path ? path.join(ROOT_DIR, server_ctx.path) : server_file_path; - // Bust the import cache in development so edits to the server file - // are picked up without a restart. - const server_module = await import(isDevelopment() - ? `${final_server_path}?t=${Date.now()}` - : final_server_path); - if (typeof server_module.sitemap != "function") - return; + const relative_server_path = grabRelativePath(server_file_path); try { - return await server_module.sitemap({ route }); + pending_sitemap_task = `importing ${relative_server_path}`; + // Bust the import cache in development so edits to the server file + // are picked up without a restart. + const server_module = await import(isDevelopment() + ? `${final_server_path}?t=${Date.now()}` + : final_server_path); + if (typeof server_module.sitemap != "function") + return; + pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`; + try { + return await server_module.sitemap({ route }); + } + catch (error) { + throw new Error(`\`sitemap\` export of ${relative_server_path} failed: ${error?.message || error}`); + } } - catch (error) { - throw new Error(`\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${error?.message || error}`); + finally { + pending_sitemap_task = undefined; } } function toSitemapEntry(path) { diff --git a/package.json b/package.json index 702e1a2..48179d6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@moduletrace/bunext", - "version": "1.2.1", + "version": "1.2.2", "main": "dist/index.js", "module": "index.ts", "dependencies": { diff --git a/src/__tests__/commands/sitemap.test.ts b/src/__tests__/commands/sitemap.test.ts index 1ef8455..7d409c5 100644 --- a/src/__tests__/commands/sitemap.test.ts +++ b/src/__tests__/commands/sitemap.test.ts @@ -217,6 +217,35 @@ describe("bunext sitemap", () => { ); }); + it("waits for sitemap exports that don't keep the event loop alive", async () => { + // Like Bun's SQL client querying on an idle connection: the + // promise settles, but nothing holds the process open meanwhile. + const unrefServerFile = (paths: string[]) => + `export const sitemap = () => new Promise((resolve) => { + setTimeout(() => resolve(${JSON.stringify(paths)}), 50).unref(); + });`; + + writeConfig(`{ origin: "https://example.com" }`); + writeProjectFile( + "src/pages/blog/[slug].server.ts", + unrefServerFile(["/blog/a"]), + ); + writeProjectFile( + "src/pages/docs/[slug].tsx", + "export default () => null", + ); + writeProjectFile( + "src/pages/docs/[slug].server.ts", + unrefServerFile(["/docs/a"]), + ); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(locs(readOutput())).toContain("https://example.com/blog/a"); + expect(locs(readOutput())).toContain("https://example.com/docs/a"); + }); + it("exits with code 1 when config additional_paths throws", async () => { writeConfig(`{ sitemap: { diff --git a/src/commands/sitemap/index.ts b/src/commands/sitemap/index.ts index aac0da5..163b91d 100644 --- a/src/commands/sitemap/index.ts +++ b/src/commands/sitemap/index.ts @@ -12,12 +12,17 @@ import { log } from "../../utils/log"; import grabDirNames from "../../utils/grab-dir-names"; import grabConfig from "../../functions/grab-config"; import grabConstants from "../../utils/grab-constants"; -import { generateSitemapFiles } from "../../utils/generate-sitemap"; +import { + generateSitemapFiles, + grabPendingSitemapTask, +} from "../../utils/generate-sitemap"; import { grabPageLastmodFile } from "../../utils/page-lastmod-record"; import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt"; +const SitemapStillWaitingInterval = 30_000; + type Options = { output?: string; origin?: string; @@ -79,13 +84,27 @@ export default function () { ); } + // Bun exits silently (code 0) when the event loop empties + // while a promise is pending. Bun's SQL client (mariadb / + // mysql adapters, Bun 1.3.14) doesn't keep the loop alive + // for a query on a connection that has gone idle, so a + // second `sitemap` export querying the DB would end the + // process mid-generation. Hold the loop open with a timer, + // which also reports what a genuinely stuck run waits on. + const keep_alive = setInterval(() => { + log.warn( + `Still waiting on ` + + `${grabPendingSitemapTask() || "sitemap generation"}...`, + ); + }, SitemapStillWaitingInterval); + const { sitemap, chunks, url_count } = await generateSitemapFiles({ ...sitemap_config, origin: options.origin || sitemap_config.origin, chunk_size, chunks_url_path: chunks_public_path, - }); + }).finally(() => clearInterval(keep_alive)); // Remove chunks from a previous, larger sitemap if (existsSync(chunks_dir_path)) { diff --git a/src/utils/generate-sitemap.ts b/src/utils/generate-sitemap.ts index cf0bc4b..6c0413f 100644 --- a/src/utils/generate-sitemap.ts +++ b/src/utils/generate-sitemap.ts @@ -28,6 +28,16 @@ export const SitemapDefaultChunkSize = 10_000; const SitemapXmlHeader = ``; const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`; +/** + * What sitemap generation is currently waiting on, so `bunext sitemap` + * can name it when a `sitemap` export or `additional_paths` never settles. + */ +let pending_sitemap_task: string | undefined; + +export function grabPendingSitemapTask() { + return pending_sitemap_task; +} + export type SitemapChunkFile = { file_name: string; xml: string; @@ -247,10 +257,19 @@ export async function grabSitemapEntries( }); } - const additional_paths = - typeof params?.additional_paths == "function" - ? await params.additional_paths() - : params?.additional_paths; + let additional_paths: BunextSitemapAdditionalPaths | undefined; + + if (typeof params?.additional_paths == "function") { + pending_sitemap_task = "`sitemap.additional_paths` in the config"; + + try { + additional_paths = await params.additional_paths(); + } finally { + pending_sitemap_task = undefined; + } + } else { + additional_paths = params?.additional_paths; + } entries.push(...(additional_paths || []).map(toSitemapEntry)); @@ -295,24 +314,34 @@ async function grabPageSitemapPaths({ ? path.join(ROOT_DIR, server_ctx.path) : server_file_path; - // Bust the import cache in development so edits to the server file - // are picked up without a restart. - const server_module: BunextPageServerModule = await import( - isDevelopment() - ? `${final_server_path}?t=${Date.now()}` - : final_server_path - ); - - if (typeof server_module.sitemap != "function") return; + const relative_server_path = grabRelativePath(server_file_path); try { - return await server_module.sitemap({ route }); - } catch (error: any) { - throw new Error( - `\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${ - error?.message || error - }`, + pending_sitemap_task = `importing ${relative_server_path}`; + + // Bust the import cache in development so edits to the server file + // are picked up without a restart. + const server_module: BunextPageServerModule = await import( + isDevelopment() + ? `${final_server_path}?t=${Date.now()}` + : final_server_path ); + + if (typeof server_module.sitemap != "function") return; + + pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`; + + try { + return await server_module.sitemap({ route }); + } catch (error: any) { + throw new Error( + `\`sitemap\` export of ${relative_server_path} failed: ${ + error?.message || error + }`, + ); + } + } finally { + pending_sitemap_task = undefined; } }