tbenandClaude Opus 5.5
40c17f903e
Bun's SQL client (mariadb/mysql adapters, Bun 1.3.14) doesn't keep the event loop alive for a query on an idle connection, so a second dynamic route's sitemap export querying the DB made the command exit silently. Hold the loop open with a timer during generation, and log what a stuck run is waiting on. Co-Authored-By: Claude Opus 5.5 <[email protected]>
291 lines
11 KiB
JavaScript
291 lines
11 KiB
JavaScript
import { statSync } from "fs";
|
|
import path from "path";
|
|
import grabPageServerPath from "../functions/server/web-pages/grab-page-server-path";
|
|
import grabAllPages from "./grab-all-pages";
|
|
import grabDirNames from "./grab-dir-names";
|
|
import grabOrigin from "./grab-origin";
|
|
import grabRelativePath from "./grab-relative-path";
|
|
import isDevelopment from "./is-development";
|
|
import { log } from "./log";
|
|
import { readPageLastmodRecord } from "./page-lastmod-record";
|
|
const ErrorPagePaths = ["/404", "/500"];
|
|
/**
|
|
* Protocol limits for a single sitemap file
|
|
* https://www.sitemaps.org/protocol.html#index
|
|
*/
|
|
export const SitemapMaxUrlsPerFile = 50_000;
|
|
export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
|
|
export const SitemapDefaultChunkSize = 10_000;
|
|
const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
|
|
const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
|
|
/**
|
|
* What sitemap generation is currently waiting on, so `bunext sitemap`
|
|
* can name it when a `sitemap` export or `additional_paths` never settles.
|
|
*/
|
|
let pending_sitemap_task;
|
|
export function grabPendingSitemapTask() {
|
|
return pending_sitemap_task;
|
|
}
|
|
/**
|
|
* # Generate a sitemap.xml string for all pages
|
|
* Dynamic routes are only included through a `sitemap` export in their
|
|
* `.server.ts` file, or through `additional_paths`. Always returns a
|
|
* single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
|
|
*/
|
|
export default async function generateSitemap(params) {
|
|
const entries = await grabSitemapEntries(params);
|
|
const origin = grabSitemapOrigin(params?.origin);
|
|
if (entries.length > SitemapMaxUrlsPerFile) {
|
|
log.warn(`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
|
|
`per-file limit. Use \`bunext sitemap\` to split it into chunks.`);
|
|
}
|
|
return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
|
|
}
|
|
/**
|
|
* # Generate sitemap.xml, split into chunk files when needed
|
|
* Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
|
|
* is the public URL path the chunk files are served from.
|
|
*/
|
|
export async function generateSitemapFiles({ chunk_size = SitemapDefaultChunkSize, chunks_url_path, ...params }) {
|
|
const entries = await grabSitemapEntries(params);
|
|
const origin = grabSitemapOrigin(params.origin);
|
|
const max_urls = Math.min(Math.max(Math.floor(chunk_size) || 1, 1), SitemapMaxUrlsPerFile);
|
|
const urls = entries.map((entry) => renderUrl(entry, origin));
|
|
const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
|
|
const chunks = [];
|
|
let chunk;
|
|
for (let i = 0; i < urls.length; i++) {
|
|
const url_bytes = Buffer.byteLength(urls[i]) + 1;
|
|
if (!chunk ||
|
|
chunk.urls.length >= max_urls ||
|
|
chunk.bytes + url_bytes > SitemapMaxBytesPerFile) {
|
|
chunk = { urls: [], bytes: empty_urlset_bytes };
|
|
chunks.push(chunk);
|
|
}
|
|
chunk.urls.push(urls[i]);
|
|
chunk.bytes += url_bytes;
|
|
const lastmod = parseLastmod(entries[i].lastmod);
|
|
if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
|
|
chunk.lastmod = lastmod;
|
|
}
|
|
}
|
|
if (chunks.length <= 1) {
|
|
return {
|
|
sitemap: renderUrlset(chunks[0]?.urls || []),
|
|
chunks: [],
|
|
url_count: urls.length,
|
|
};
|
|
}
|
|
const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
|
|
const chunk_base_url = chunks_dir_path
|
|
? `${origin}/${chunks_dir_path}`
|
|
: origin;
|
|
const chunk_files = chunks.map((c, i) => ({
|
|
file_name: `sitemap-${i + 1}.xml`,
|
|
xml: renderUrlset(c.urls),
|
|
url_count: c.urls.length,
|
|
}));
|
|
const sitemap = [
|
|
SitemapXmlHeader,
|
|
`<sitemapindex ${SitemapXmlns}>`,
|
|
...chunk_files.map((file, i) => {
|
|
const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
|
|
const lines = [` <loc>${loc}</loc>`];
|
|
const lastmod = chunks[i].lastmod?.toISOString();
|
|
if (lastmod)
|
|
lines.push(` <lastmod>${lastmod}</lastmod>`);
|
|
return ` <sitemap>\n${lines.join("\n")}\n </sitemap>`;
|
|
}),
|
|
`</sitemapindex>`,
|
|
``,
|
|
].join("\n");
|
|
return { sitemap, chunks: chunk_files, url_count: urls.length };
|
|
}
|
|
function grabSitemapOrigin(origin) {
|
|
return (origin || grabOrigin()).replace(/\/+$/, "");
|
|
}
|
|
function renderUrlset(urls) {
|
|
return [
|
|
SitemapXmlHeader,
|
|
`<urlset ${SitemapXmlns}>`,
|
|
...urls,
|
|
`</urlset>`,
|
|
``,
|
|
].join("\n");
|
|
}
|
|
function renderUrl(entry, origin) {
|
|
const loc = entry.path.match(/^https?:\/\//)
|
|
? entry.path
|
|
: `${origin}${entry.path}`;
|
|
const lines = [` <loc>${escapeXml(loc)}</loc>`];
|
|
if (entry.lastmod) {
|
|
const lastmod = entry.lastmod instanceof Date
|
|
? entry.lastmod.toISOString()
|
|
: entry.lastmod;
|
|
lines.push(` <lastmod>${escapeXml(lastmod)}</lastmod>`);
|
|
}
|
|
if (entry.changefreq) {
|
|
lines.push(` <changefreq>${entry.changefreq}</changefreq>`);
|
|
}
|
|
if (typeof entry.priority == "number") {
|
|
const priority = Math.min(Math.max(entry.priority, 0), 1);
|
|
lines.push(` <priority>${priority.toFixed(1)}</priority>`);
|
|
}
|
|
return ` <url>\n${lines.join("\n")}\n </url>`;
|
|
}
|
|
function parseLastmod(lastmod) {
|
|
if (!lastmod)
|
|
return;
|
|
const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
|
|
return isNaN(date.getTime()) ? undefined : date;
|
|
}
|
|
export async function grabSitemapEntries(params) {
|
|
const use_file_lastmod = params?.use_file_lastmod ?? false;
|
|
const lastmod_record = readPageLastmodRecord(params?.lastmod_file);
|
|
const entries = [];
|
|
const pages = grabAllPages({ exclude_api: true });
|
|
for (let i = 0; i < pages.length; i++) {
|
|
const page = pages[i];
|
|
if (!page.local_path)
|
|
continue;
|
|
const page_path = page.url_path.replace(/\/index$/, "") || "/";
|
|
if (ErrorPagePaths.includes(page_path))
|
|
continue;
|
|
const page_sitemap_paths = await grabPageSitemapPaths({
|
|
file_path: page.local_path,
|
|
route: page_path,
|
|
});
|
|
// When the page's client bundle last changed, per `bunext build`
|
|
const build_lastmod = lastmod_record?.pages[grabRelativePath(page.local_path)]?.lastmod;
|
|
if (page_sitemap_paths) {
|
|
for (const page_sitemap_path of page_sitemap_paths) {
|
|
const entry = toSitemapEntry(page_sitemap_path);
|
|
entries.push({
|
|
...entry,
|
|
lastmod: entry.lastmod || build_lastmod,
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
if (page.url_path.match(/\[/))
|
|
continue;
|
|
entries.push({
|
|
path: page_path,
|
|
lastmod: build_lastmod ||
|
|
(use_file_lastmod
|
|
? statSync(page.local_path).mtime
|
|
: undefined),
|
|
});
|
|
}
|
|
let additional_paths;
|
|
if (typeof params?.additional_paths == "function") {
|
|
pending_sitemap_task = "`sitemap.additional_paths` in the config";
|
|
try {
|
|
additional_paths = await params.additional_paths();
|
|
}
|
|
finally {
|
|
pending_sitemap_task = undefined;
|
|
}
|
|
}
|
|
else {
|
|
additional_paths = params?.additional_paths;
|
|
}
|
|
entries.push(...(additional_paths || []).map(toSitemapEntry));
|
|
const seen = new Set();
|
|
const final_entries = [];
|
|
for (const entry of entries) {
|
|
const path = normalizePath(entry.path, params?.trailing_slash);
|
|
if (seen.has(path))
|
|
continue;
|
|
if (isExcluded(path, params?.exclude))
|
|
continue;
|
|
seen.add(path);
|
|
final_entries.push({
|
|
...entry,
|
|
path,
|
|
lastmod: entry.lastmod || params?.default_lastmod,
|
|
changefreq: entry.changefreq || params?.default_changefreq,
|
|
priority: entry.priority ?? params?.default_priority,
|
|
});
|
|
}
|
|
return final_entries;
|
|
}
|
|
/**
|
|
* # Run the `sitemap` export of a page's server file, if any
|
|
*/
|
|
async function grabPageSitemapPaths({ file_path, route, }) {
|
|
const { server_file_path } = grabPageServerPath({ file_path });
|
|
if (!server_file_path)
|
|
return;
|
|
const { ROOT_DIR } = grabDirNames();
|
|
const server_ctx = global.BUNEXT_SSR_BUNDLER_CTX_MAP?.[grabRelativePath(server_file_path)];
|
|
const final_server_path = server_ctx?.local_path
|
|
? path.join(ROOT_DIR, server_ctx.path)
|
|
: server_file_path;
|
|
const relative_server_path = grabRelativePath(server_file_path);
|
|
try {
|
|
pending_sitemap_task = `importing ${relative_server_path}`;
|
|
// Bust the import cache in development so edits to the server file
|
|
// are picked up without a restart.
|
|
const server_module = await import(isDevelopment()
|
|
? `${final_server_path}?t=${Date.now()}`
|
|
: final_server_path);
|
|
if (typeof server_module.sitemap != "function")
|
|
return;
|
|
pending_sitemap_task = `the \`sitemap\` export of ${relative_server_path}`;
|
|
try {
|
|
return await server_module.sitemap({ route });
|
|
}
|
|
catch (error) {
|
|
throw new Error(`\`sitemap\` export of ${relative_server_path} failed: ${error?.message || error}`);
|
|
}
|
|
}
|
|
finally {
|
|
pending_sitemap_task = undefined;
|
|
}
|
|
}
|
|
function toSitemapEntry(path) {
|
|
return typeof path == "string" ? { path } : path;
|
|
}
|
|
function normalizePath(path, trailing_slash) {
|
|
if (path.match(/^https?:\/\//))
|
|
return path;
|
|
let new_path = `/${path}`.replace(/\/{2,}/g, "/");
|
|
if (new_path != "/")
|
|
new_path = new_path.replace(/\/$/, "");
|
|
if (trailing_slash && new_path != "/")
|
|
new_path += "/";
|
|
return new_path;
|
|
}
|
|
function isExcluded(path, exclude) {
|
|
if (!exclude?.length)
|
|
return false;
|
|
const bare_path = path.replace(/^https?:\/\/[^/]+/, "").replace(/(.)\/$/, "$1") || "/";
|
|
for (const pattern of exclude) {
|
|
if (pattern instanceof RegExp) {
|
|
if (pattern.test(path) || pattern.test(bare_path))
|
|
return true;
|
|
continue;
|
|
}
|
|
const normalized = normalizePath(pattern.replace(/\/\*$/, ""));
|
|
if (pattern.endsWith("/*")) {
|
|
if (bare_path == normalized ||
|
|
bare_path.startsWith(`${normalized.replace(/\/$/, "")}/`)) {
|
|
return true;
|
|
}
|
|
}
|
|
else if (bare_path == normalized) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
function escapeXml(value) {
|
|
return value
|
|
.replace(/&/g, "&")
|
|
.replace(/</g, "<")
|
|
.replace(/>/g, ">")
|
|
.replace(/"/g, """)
|
|
.replace(/'/g, "'");
|
|
}
|