Add robots.txt generation and sitemap chunking

- bunext sitemap now writes public/robots.txt (configurable via the robots config key; robots: false or --no-robots to skip). Hand-written robots.txt files are never overwritten without a robots config - Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with sitemap.xml as a sitemap index; stale chunks are cleaned up - Export bunextGenerateRobotsTxt - Run tests in publish.sh Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-14 18:02:52 +01:00
parent 1169729b06
commit be4bebaa22
20 changed files with 1406 additions and 89 deletions
@@ -6,43 +6,130 @@ import grabDirNames from "./grab-dir-names";
 import grabOrigin from "./grab-origin";
 import grabRelativePath from "./grab-relative-path";
 import isDevelopment from "./is-development";
+import { log } from "./log";
 const ErrorPagePaths = ["/404", "/500"];
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export const SitemapMaxUrlsPerFile = 50_000;
+export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
+export const SitemapDefaultChunkSize = 10_000;
+const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
+const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
 /**
 * # Generate a sitemap.xml string for all pages
 * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
 */
 export default async function generateSitemap(params) {
    const entries = await grabSitemapEntries(params);
-    const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
-    const urls = entries.map((entry) => {
-        const loc = entry.path.match(/^https?:\/\//)
-            ? entry.path
-            : `${origin}${entry.path}`;
-        const lines = [`        <loc>${escapeXml(loc)}</loc>`];
-        if (entry.lastmod) {
-            const lastmod = entry.lastmod instanceof Date
-                ? entry.lastmod.toISOString()
-                : entry.lastmod;
-            lines.push(`        <lastmod>${escapeXml(lastmod)}</lastmod>`);
+    const origin = grabSitemapOrigin(params?.origin);
+    if (entries.length > SitemapMaxUrlsPerFile) {
+        log.warn(`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
+            `per-file limit. Use \`bunext sitemap\` to split it into chunks.`);
+    }
+    return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
+}
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export async function generateSitemapFiles({ chunk_size = SitemapDefaultChunkSize, chunks_url_path, ...params }) {
+    const entries = await grabSitemapEntries(params);
+    const origin = grabSitemapOrigin(params.origin);
+    const max_urls = Math.min(Math.max(Math.floor(chunk_size) || 1, 1), SitemapMaxUrlsPerFile);
+    const urls = entries.map((entry) => renderUrl(entry, origin));
+    const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
+    const chunks = [];
+    let chunk;
+    for (let i = 0; i < urls.length; i++) {
+        const url_bytes = Buffer.byteLength(urls[i]) + 1;
+        if (!chunk ||
+            chunk.urls.length >= max_urls ||
+            chunk.bytes + url_bytes > SitemapMaxBytesPerFile) {
+            chunk = { urls: [], bytes: empty_urlset_bytes };
+            chunks.push(chunk);
        }
-        if (entry.changefreq) {
-            lines.push(`        <changefreq>${entry.changefreq}</changefreq>`);
+        chunk.urls.push(urls[i]);
+        chunk.bytes += url_bytes;
+        const lastmod = parseLastmod(entries[i].lastmod);
+        if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
+            chunk.lastmod = lastmod;
        }
-        if (typeof entry.priority == "number") {
-            const priority = Math.min(Math.max(entry.priority, 0), 1);
-            lines.push(`        <priority>${priority.toFixed(1)}</priority>`);
-        }
-        return `    <url>\n${lines.join("\n")}\n    </url>`;
-    });
+    }
+    if (chunks.length <= 1) {
+        return {
+            sitemap: renderUrlset(chunks[0]?.urls || []),
+            chunks: [],
+            url_count: urls.length,
+        };
+    }
+    const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
+    const chunk_base_url = chunks_dir_path
+        ? `${origin}/${chunks_dir_path}`
+        : origin;
+    const chunk_files = chunks.map((c, i) => ({
+        file_name: `sitemap-${i + 1}.xml`,
+        xml: renderUrlset(c.urls),
+        url_count: c.urls.length,
+    }));
+    const sitemap = [
+        SitemapXmlHeader,
+        `<sitemapindex ${SitemapXmlns}>`,
+        ...chunk_files.map((file, i) => {
+            const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
+            const lines = [`        <loc>${loc}</loc>`];
+            const lastmod = chunks[i].lastmod?.toISOString();
+            if (lastmod)
+                lines.push(`        <lastmod>${lastmod}</lastmod>`);
+            return `    <sitemap>\n${lines.join("\n")}\n    </sitemap>`;
+        }),
+        `</sitemapindex>`,
+        ``,
+    ].join("\n");
+    return { sitemap, chunks: chunk_files, url_count: urls.length };
+}
+function grabSitemapOrigin(origin) {
+    return (origin || grabOrigin()).replace(/\/+$/, "");
+}
+function renderUrlset(urls) {
    return [
-        `<?xml version="1.0" encoding="UTF-8"?>`,
-        `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+        SitemapXmlHeader,
+        `<urlset ${SitemapXmlns}>`,
        ...urls,
        `</urlset>`,
        ``,
    ].join("\n");
 }
+function renderUrl(entry, origin) {
+    const loc = entry.path.match(/^https?:\/\//)
+        ? entry.path
+        : `${origin}${entry.path}`;
+    const lines = [`        <loc>${escapeXml(loc)}</loc>`];
+    if (entry.lastmod) {
+        const lastmod = entry.lastmod instanceof Date
+            ? entry.lastmod.toISOString()
+            : entry.lastmod;
+        lines.push(`        <lastmod>${escapeXml(lastmod)}</lastmod>`);
+    }
+    if (entry.changefreq) {
+        lines.push(`        <changefreq>${entry.changefreq}</changefreq>`);
+    }
+    if (typeof entry.priority == "number") {
+        const priority = Math.min(Math.max(entry.priority, 0), 1);
+        lines.push(`        <priority>${priority.toFixed(1)}</priority>`);
+    }
+    return `    <url>\n${lines.join("\n")}\n    </url>`;
+}
+function parseLastmod(lastmod) {
+    if (!lastmod)
+        return;
+    const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
+    return isNaN(date.getTime()) ? undefined : date;
+}
 export async function grabSitemapEntries(params) {
    const use_file_lastmod = params?.use_file_lastmod ?? true;
    const entries = [];