Add robots.txt generation and sitemap chunking

- bunext sitemap now writes public/robots.txt (configurable via the robots config key; robots: false or --no-robots to skip). Hand-written robots.txt files are never overwritten without a robots config - Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with sitemap.xml as a sitemap index; stale chunks are cleaned up - Export bunextGenerateRobotsTxt - Run tests in publish.sh Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-14 18:02:52 +01:00
parent 1169729b06
commit be4bebaa22
20 changed files with 1406 additions and 89 deletions
@@ -0,0 +1,11 @@
+import type { BunextGenerateRobotsTxtParams } from "../types";
+/**
+ * First line of every generated robots.txt. `bunext sitemap` only
+ * overwrites an existing robots.txt that starts with it, unless robots
+ * is configured explicitly.
+ */
+export declare const RobotsTxtGeneratedMarker = "# Generated by bunext";
+/**
+ * # Generate a robots.txt string
+ */
+export default function generateRobotsTxt(params?: BunextGenerateRobotsTxtParams): string;
@@ -0,0 +1,57 @@
+import grabOrigin from "./grab-origin";
+/**
+ * First line of every generated robots.txt. `bunext sitemap` only
+ * overwrites an existing robots.txt that starts with it, unless robots
+ * is configured explicitly.
+ */
+export const RobotsTxtGeneratedMarker = "# Generated by bunext";
+/**
+ * # Generate a robots.txt string
+ */
+export default function generateRobotsTxt(params) {
+    const lines = [RobotsTxtGeneratedMarker, ""];
+    if (params?.disallow_all) {
+        lines.push("User-agent: *", "Disallow: /", "");
+        return lines.join("\n");
+    }
+    const rules = params?.rules?.length
+        ? params.rules
+        : [{ user_agent: "*", allow: ["/"] }];
+    for (const rule of rules) {
+        const user_agents = [rule.user_agent].flat();
+        for (const user_agent of user_agents) {
+            lines.push(`User-agent: ${sanitizeValue(user_agent)}`);
+        }
+        for (const allow of rule.allow || []) {
+            lines.push(`Allow: ${sanitizeValue(allow)}`);
+        }
+        for (const disallow of rule.disallow || []) {
+            lines.push(`Disallow: ${sanitizeValue(disallow)}`);
+        }
+        // A group needs at least one rule; an empty Disallow allows all.
+        if (!rule.allow?.length && !rule.disallow?.length) {
+            lines.push("Disallow:");
+        }
+        if (typeof rule.crawl_delay == "number") {
+            lines.push(`Crawl-delay: ${rule.crawl_delay}`);
+        }
+        lines.push("");
+    }
+    if (params?.sitemaps?.length) {
+        const origin = (params.origin || grabOrigin()).replace(/\/+$/, "");
+        for (const sitemap of new Set(params.sitemaps)) {
+            const url = sitemap.match(/^https?:\/\//)
+                ? sitemap
+                : `${origin}/${sitemap.replace(/^\/+/, "")}`;
+            lines.push(`Sitemap: ${sanitizeValue(url)}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Line breaks would let a value inject extra directives.
+ */
+function sanitizeValue(value) {
+    return value.replace(/[\r\n]+/g, " ").trim();
+}
@@ -1,8 +1,39 @@
 import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export declare const SitemapMaxUrlsPerFile = 50000;
+export declare const SitemapMaxBytesPerFile: number;
+export declare const SitemapDefaultChunkSize = 10000;
+export type SitemapChunkFile = {
+    file_name: string;
+    xml: string;
+    url_count: number;
+};
+export type SitemapFiles = {
+    /**
+     * A `<urlset>` when everything fits in one file, otherwise a
+     * `<sitemapindex>` referencing `chunks`.
+     */
+    sitemap: string;
+    chunks: SitemapChunkFile[];
+    url_count: number;
+};
 /**
 * # Generate a sitemap.xml string for all pages
 * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
 */
 export default function generateSitemap(params?: BunextGenerateSitemapParams): Promise<string>;
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export declare function generateSitemapFiles({ chunk_size, chunks_url_path, ...params }: BunextGenerateSitemapParams & {
+    chunk_size?: number;
+    chunks_url_path: string;
+}): Promise<SitemapFiles>;
 export declare function grabSitemapEntries(params?: BunextGenerateSitemapParams): Promise<BunextSitemapEntry[]>;
@@ -6,43 +6,130 @@ import grabDirNames from "./grab-dir-names";
 import grabOrigin from "./grab-origin";
 import grabRelativePath from "./grab-relative-path";
 import isDevelopment from "./is-development";
+import { log } from "./log";
 const ErrorPagePaths = ["/404", "/500"];
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export const SitemapMaxUrlsPerFile = 50_000;
+export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
+export const SitemapDefaultChunkSize = 10_000;
+const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
+const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
 /**
 * # Generate a sitemap.xml string for all pages
 * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
 */
 export default async function generateSitemap(params) {
    const entries = await grabSitemapEntries(params);
-    const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
-    const urls = entries.map((entry) => {
-        const loc = entry.path.match(/^https?:\/\//)
-            ? entry.path
-            : `${origin}${entry.path}`;
-        const lines = [`        <loc>${escapeXml(loc)}</loc>`];
-        if (entry.lastmod) {
-            const lastmod = entry.lastmod instanceof Date
-                ? entry.lastmod.toISOString()
-                : entry.lastmod;
-            lines.push(`        <lastmod>${escapeXml(lastmod)}</lastmod>`);
+    const origin = grabSitemapOrigin(params?.origin);
+    if (entries.length > SitemapMaxUrlsPerFile) {
+        log.warn(`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
+            `per-file limit. Use \`bunext sitemap\` to split it into chunks.`);
+    }
+    return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
+}
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export async function generateSitemapFiles({ chunk_size = SitemapDefaultChunkSize, chunks_url_path, ...params }) {
+    const entries = await grabSitemapEntries(params);
+    const origin = grabSitemapOrigin(params.origin);
+    const max_urls = Math.min(Math.max(Math.floor(chunk_size) || 1, 1), SitemapMaxUrlsPerFile);
+    const urls = entries.map((entry) => renderUrl(entry, origin));
+    const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
+    const chunks = [];
+    let chunk;
+    for (let i = 0; i < urls.length; i++) {
+        const url_bytes = Buffer.byteLength(urls[i]) + 1;
+        if (!chunk ||
+            chunk.urls.length >= max_urls ||
+            chunk.bytes + url_bytes > SitemapMaxBytesPerFile) {
+            chunk = { urls: [], bytes: empty_urlset_bytes };
+            chunks.push(chunk);
        }
-        if (entry.changefreq) {
-            lines.push(`        <changefreq>${entry.changefreq}</changefreq>`);
+        chunk.urls.push(urls[i]);
+        chunk.bytes += url_bytes;
+        const lastmod = parseLastmod(entries[i].lastmod);
+        if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
+            chunk.lastmod = lastmod;
        }
-        if (typeof entry.priority == "number") {
-            const priority = Math.min(Math.max(entry.priority, 0), 1);
-            lines.push(`        <priority>${priority.toFixed(1)}</priority>`);
-        }
-        return `    <url>\n${lines.join("\n")}\n    </url>`;
-    });
+    }
+    if (chunks.length <= 1) {
+        return {
+            sitemap: renderUrlset(chunks[0]?.urls || []),
+            chunks: [],
+            url_count: urls.length,
+        };
+    }
+    const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
+    const chunk_base_url = chunks_dir_path
+        ? `${origin}/${chunks_dir_path}`
+        : origin;
+    const chunk_files = chunks.map((c, i) => ({
+        file_name: `sitemap-${i + 1}.xml`,
+        xml: renderUrlset(c.urls),
+        url_count: c.urls.length,
+    }));
+    const sitemap = [
+        SitemapXmlHeader,
+        `<sitemapindex ${SitemapXmlns}>`,
+        ...chunk_files.map((file, i) => {
+            const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
+            const lines = [`        <loc>${loc}</loc>`];
+            const lastmod = chunks[i].lastmod?.toISOString();
+            if (lastmod)
+                lines.push(`        <lastmod>${lastmod}</lastmod>`);
+            return `    <sitemap>\n${lines.join("\n")}\n    </sitemap>`;
+        }),
+        `</sitemapindex>`,
+        ``,
+    ].join("\n");
+    return { sitemap, chunks: chunk_files, url_count: urls.length };
+}
+function grabSitemapOrigin(origin) {
+    return (origin || grabOrigin()).replace(/\/+$/, "");
+}
+function renderUrlset(urls) {
    return [
-        `<?xml version="1.0" encoding="UTF-8"?>`,
-        `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+        SitemapXmlHeader,
+        `<urlset ${SitemapXmlns}>`,
        ...urls,
        `</urlset>`,
        ``,
    ].join("\n");
 }
+function renderUrl(entry, origin) {
+    const loc = entry.path.match(/^https?:\/\//)
+        ? entry.path
+        : `${origin}${entry.path}`;
+    const lines = [`        <loc>${escapeXml(loc)}</loc>`];
+    if (entry.lastmod) {
+        const lastmod = entry.lastmod instanceof Date
+            ? entry.lastmod.toISOString()
+            : entry.lastmod;
+        lines.push(`        <lastmod>${escapeXml(lastmod)}</lastmod>`);
+    }
+    if (entry.changefreq) {
+        lines.push(`        <changefreq>${entry.changefreq}</changefreq>`);
+    }
+    if (typeof entry.priority == "number") {
+        const priority = Math.min(Math.max(entry.priority, 0), 1);
+        lines.push(`        <priority>${priority.toFixed(1)}</priority>`);
+    }
+    return `    <url>\n${lines.join("\n")}\n    </url>`;
+}
+function parseLastmod(lastmod) {
+    if (!lastmod)
+        return;
+    const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
+    return isNaN(date.getTime()) ? undefined : date;
+}
 export async function grabSitemapEntries(params) {
    const use_file_lastmod = params?.use_file_lastmod ?? true;
    const entries = [];