Add robots.txt generation and sitemap chunking

- bunext sitemap now writes public/robots.txt (configurable via the
  robots config key; robots: false or --no-robots to skip). Hand-written
  robots.txt files are never overwritten without a robots config
- Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml
  (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with
  sitemap.xml as a sitemap index; stale chunks are cleaned up
- Export bunextGenerateRobotsTxt
- Run tests in publish.sh

Co-Authored-By: Claude Opus 5.5 <[email protected]>

CLAUDE.md

@@ -19,7 +19,7 @@ bun run dev         # tsc --watch
 bunext dev          # Start dev server with HMR and file watcher
 bunext build        # Bundle all pages for production
 bunext start        # Start production server from pre-built artifacts
-bunext sitemap      # Write public/sitemap.xml using config.sitemap
+bunext sitemap      # Write public/sitemap.xml and public/robots.txt using config.sitemap / config.robots
 ```
 
 ## Architecture

README.md

@@ -39,7 +39,9 @@ The goal is a framework that is:
 - [Sitemap](#sitemap)
     - [Page `sitemap` Export](#page-sitemap-export)
     - [`bunext sitemap` Command](#bunext-sitemap-command)
+    - [Large Sitemaps](#large-sitemaps)
     - [Generating at Runtime](#generating-at-runtime)
+    - [robots.txt](#robotstxt)
 - [Static Files](#static-files)
 - [Caching](#caching)
     - [Enabling Cache Per Page](#enabling-cache-per-page)
@@ -159,7 +161,7 @@ bun run dev
 | `bunext dev`   | Start the development server with HMR and file watching.                       |
 | `bunext build` | Bundle all pages for production. Outputs artifacts to `.bunext/public/pages/` using paths relative to the project root. |
 | `bunext start` | Start the production server using pre-built artifacts.                         |
-| `bunext sitemap` | Generate `public/sitemap.xml` from your pages, using the `sitemap` options in `bunext.config.ts`. See [Sitemap](#sitemap). |
+| `bunext sitemap` | Generate `public/sitemap.xml` and `public/robots.txt` from your pages, using the `sitemap` and `robots` options in `bunext.config.ts`. See [Sitemap](#sitemap). |
 
 ### Running the CLI
 
@@ -694,16 +696,37 @@ export default config;
 ```
 
 ```bash
-bunext sitemap                                   # writes public/sitemap.xml
+bunext sitemap                                   # writes public/sitemap.xml and public/robots.txt
-bunext sitemap -o public/sitemaps/main.xml      # custom output path
+bunext sitemap -o public/sitemaps/main.xml      # custom sitemap output path
 bunext sitemap --origin https://staging.example.com
+bunext sitemap --no-robots                       # sitemap only
 ```
 
-`--output` and `--origin` override `sitemap.output` and `sitemap.origin` from the config.
+#### Large Sitemaps
+
+A sitemap file may hold at most 50,000 URLs and 50MB. When the sitemap has more than `chunk_size` URLs (default 10,000), `bunext sitemap` splits it into chunk files and makes `sitemap.xml` a sitemap index pointing to them:
+
+```
+public/
+├── sitemap.xml                 ← <sitemapindex> referencing the chunks
+└── sitemap-chunks/
+    ├── sitemap-1.xml           ← up to chunk_size URLs each
+    ├── sitemap-2.xml
+    └── sitemap-3.xml
+```
+
+Each index entry's `lastmod` is the newest `lastmod` in its chunk. Chunk files left over from a previous, larger run are removed; other files in the chunks directory are left alone. If the sitemap shrinks back under `chunk_size`, `sitemap.xml` returns to a plain `<urlset>`.
+
+| Config option | Type     | Description                                                                                         |
+| ------------- | -------- | --------------------------------------------------------------------------------------------------- |
+| `chunk_size`  | `number` | Maximum URLs per file. Default `10000`, capped at `50000`. Chunks are also split at 50MB.           |
+| `chunks_dir`  | `string` | Chunk directory relative to the project root. Must be inside `public/`. Default `public/sitemap-chunks`. |
+
+`--output` and `--origin` override `sitemap.output` and `sitemap.origin` from the config. `--origin` also applies to robots.txt. The command also writes a `robots.txt` — see [robots.txt](#robotstxt).
 
 ### Generating at Runtime
 
-`bunextGenerateSitemap()` resolves to the same XML as a string, so you can serve it from a route instead of a file:
+`bunextGenerateSitemap()` resolves to the same XML as a single `<urlset>` (it does not chunk, and logs a warning above 50,000 URLs — use `bunext sitemap` for large sites), so you can serve it from a route instead of a file:
 
 ```ts
 // src/pages/api/sitemap.ts
@@ -738,7 +761,51 @@ export default async function handler(req: Request) {
 | `default_changefreq` | `BunextSitemapChangeFreq`                  | `changefreq` for entries that do not set one.                                                               |
 | `default_priority`   | `number`                                   | `priority` (0.0 to 1.0) for entries that do not set one.                                                    |
 
-The `sitemap` config key takes the same options, plus `output`: the file path relative to the project root (default `public/sitemap.xml`).
+The `sitemap` config key takes the same options, plus `output` (the file path relative to the project root, default `public/sitemap.xml`) and the [chunking options](#large-sitemaps).
+
+### robots.txt
+
+`bunext sitemap` also writes `public/robots.txt`, served at `/robots.txt`. With no `robots` config it allows every crawler and points to the generated sitemap:
+
+```
+# Generated by bunext
+
+User-agent: *
+Allow: /
+
+Sitemap: https://example.com/sitemap.xml
+```
+
+Configure it with the `robots` key:
+
+```ts
+// bunext.config.ts
+const config: BunextConfig = {
+    origin: "https://example.com",
+    robots: {
+        rules: [
+            { user_agent: "*", disallow: ["/admin", "/api/"] },
+            { user_agent: ["GPTBot", "CCBot"], disallow: ["/"] },
+        ],
+        // Block everything on staging
+        disallow_all: process.env.STAGING === "true",
+    },
+};
+```
+
+| Option         | Type                 | Description                                                                                                                       |
+| -------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
+| `rules`        | `BunextRobotsRule[]` | Groups of `user_agent` (string or array), `allow`, `disallow` and `crawl_delay`. Default: allow everything for `*`.               |
+| `sitemaps`     | `string[]`           | Extra `Sitemap:` URLs or paths. The generated sitemap is added automatically when it is inside `public/`.                         |
+| `disallow_all` | `boolean`            | Block every crawler from the whole site. `rules` and `sitemaps` are ignored.                                                      |
+| `origin`       | `string`             | Base URL for relative `sitemaps` paths. Defaults to `--origin`, then `sitemap.origin`, then `config.origin`.                       |
+| `output`       | `string`             | Output file relative to the project root. Default `public/robots.txt`.                                                             |
+
+- **Existing files are protected.** Generated files start with `# Generated by bunext`. Without a `robots` config, a `robots.txt` that lacks this line (one you wrote yourself) is left alone and a warning is shown. Setting `robots` in the config overwrites it.
+- **Opting out.** Set `robots: false`, or pass `--no-robots`, to skip robots.txt entirely.
+- **Sitemap `exclude` does not add `Disallow` lines.** Leaving a page out of the sitemap and blocking crawlers from it are different things; add `disallow` rules explicitly. Remember robots.txt is public, so `Disallow` paths are visible to anyone.
+
+To serve it from a route instead, `bunext.bunextGenerateRobotsTxt()` takes the same options except `output` and returns the text.
 
 ---
 
@@ -883,6 +950,7 @@ export default config;
 | `websocket`          | `WebSocketHandler<any>`                                                           | —                | Bun WebSocket handler — see [WebSocket](#websocket)                                               |
 | `serverOptions`      | `ServeOptions`                                                                    | —                | Extra options passed to `Bun.serve()` (excluding `fetch`) — see [Server Options](#server-options) |
 | `sitemap`            | `BunextSitemapConfig`                                                             | —                | Options for `bunext sitemap` — see [Sitemap](#sitemap)                                            |
+| `robots`             | `BunextRobotsConfig \| false`                                                     | —                | robots.txt options for `bunext sitemap`, or `false` to skip it — see [robots.txt](#robotstxt)     |
 
 ### Middleware
 

dist/commands/sitemap/index.js

@@ -1,39 +1,128 @@
 import { Command } from "commander";
-import { mkdirSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "fs";
 import path from "path";
 import { log } from "../../utils/log";
 import grabDirNames from "../../utils/grab-dir-names";
 import grabConfig from "../../functions/grab-config";
 import grabConstants from "../../utils/grab-constants";
-import generateSitemap from "../../utils/generate-sitemap";
+import { generateSitemapFiles } from "../../utils/generate-sitemap";
+import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
 export default function () {
     return new Command("sitemap")
-        .description("Generate sitemap.xml from the pages directory")
+        .description("Generate sitemap.xml and robots.txt from the pages directory")
         .option("-o, --output <path>", "Output file relative to the project root (default: public/sitemap.xml)")
         .option("--origin <url>", "Base URL for sitemap links")
+        .option("--no-robots", "Skip generating robots.txt")
         .action(async (options) => {
         const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
         global.BUNEXT_CONFIG = (await grabConfig()) || {};
         global.BUNEXT_CONSTANTS = grabConstants();
-        const { output, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
+        const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
+        let sitemap_file;
         try {
-            const xml = await generateSitemap({
-                ...sitemap_config,
-                origin: options.origin || sitemap_config.origin,
-            });
             const output_path = options.output || output;
-            const output_file = output_path
+            sitemap_file = output_path
                 ? path.resolve(ROOT_DIR, output_path)
                 : path.join(PUBLIC_DIR, "sitemap.xml");
-            mkdirSync(path.dirname(output_file), { recursive: true });
+            const chunks_dir_path = path.resolve(ROOT_DIR, chunks_dir || "public/sitemap-chunks");
-            writeFileSync(output_file, xml, "utf-8");
+            const chunks_public_path = grabPublicUrlPath(chunks_dir_path);
-            const url_count = xml.match(/<url>/g)?.length || 0;
+            if (chunks_public_path === undefined) {
-            log.success(`Sitemap with ${url_count} URLs written to ${path.relative(ROOT_DIR, output_file)}`);
+                throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` +
+                    `so the chunks are served.`);
+            }
+            const { sitemap, chunks, url_count } = await generateSitemapFiles({
+                ...sitemap_config,
+                origin: options.origin || sitemap_config.origin,
+                chunk_size,
+                chunks_url_path: chunks_public_path,
+            });
+            // Remove chunks from a previous, larger sitemap
+            if (existsSync(chunks_dir_path)) {
+                for (const file of readdirSync(chunks_dir_path)) {
+                    if (file.match(/^sitemap-\d+\.xml$/)) {
+                        rmSync(path.join(chunks_dir_path, file));
+                    }
+                }
+            }
+            if (chunks.length) {
+                mkdirSync(chunks_dir_path, { recursive: true });
+            }
+            for (const chunk of chunks) {
+                writeFileSync(path.join(chunks_dir_path, chunk.file_name), chunk.xml, "utf-8");
+            }
+            mkdirSync(path.dirname(sitemap_file), { recursive: true });
+            writeFileSync(sitemap_file, sitemap, "utf-8");
+            const sitemap_relative_path = path.relative(ROOT_DIR, sitemap_file);
+            if (chunks.length) {
+                log.success(`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` +
+                    `written to ${sitemap_relative_path} ` +
+                    `(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`);
+            }
+            else {
+                log.success(`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`);
+            }
         }
         catch (error) {
             log.error(`Failed to generate sitemap: ${error?.message || error}`);
             process.exit(1);
         }
+        const robots_config = global.BUNEXT_CONFIG.robots;
+        if (options.robots && robots_config !== false) {
+            writeRobotsTxt({
+                sitemap_file,
+                cli_origin: options.origin,
+                sitemap_origin: sitemap_config.origin,
+            });
+        }
         process.exit();
     });
 }
+function writeRobotsTxt({ sitemap_file, cli_origin, sitemap_origin, }) {
+    const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
+    const robots_config = global.BUNEXT_CONFIG.robots || undefined;
+    const { output, ...robots_params } = robots_config || {};
+    const robots_file = output
+        ? path.resolve(ROOT_DIR, output)
+        : path.join(PUBLIC_DIR, "robots.txt");
+    const robots_relative_path = path.relative(ROOT_DIR, robots_file);
+    // Without a robots config, never replace a hand-written robots.txt.
+    if (!robots_config &&
+        existsSync(robots_file) &&
+        !readFileSync(robots_file, "utf-8").startsWith(RobotsTxtGeneratedMarker)) {
+        log.warn(`Skipped ${robots_relative_path}: it was not generated by bunext. ` +
+            `Set \`robots\` in bunext.config.ts to overwrite it, ` +
+            `or \`robots: false\` to silence this.`);
+        return;
+    }
+    const sitemap_public_path = grabPublicUrlPath(sitemap_file);
+    if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) {
+        log.warn(`Sitemap is outside public/, so robots.txt has no Sitemap line. ` +
+            `Add its URL to \`robots.sitemaps\`.`);
+    }
+    const robots_txt = generateRobotsTxt({
+        ...robots_params,
+        origin: cli_origin || robots_params.origin || sitemap_origin,
+        sitemaps: [
+            ...(sitemap_public_path !== undefined
+                ? [sitemap_public_path]
+                : []),
+            ...(robots_params.sitemaps || []),
+        ],
+    });
+    mkdirSync(path.dirname(robots_file), { recursive: true });
+    writeFileSync(robots_file, robots_txt, "utf-8");
+    log.success(`robots.txt written to ${robots_relative_path}`);
+}
+/**
+ * # URL path a file in public/ is served at
+ * Public files are served from the site root, e.g. public/sitemap.xml
+ * at /sitemap.xml. Returns `undefined` for paths outside public/.
+ */
+function grabPublicUrlPath(file_path) {
+    const { PUBLIC_DIR } = grabDirNames();
+    const relative_path = path.relative(PUBLIC_DIR, file_path);
+    if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) {
+        return;
+    }
+    return `/${relative_path.split(path.sep).join("/")}`;
+}

dist/index.d.ts

@@ -1,5 +1,6 @@
 import bunextInit from "./functions/bunext-init";
 import bunextRequestHandler from "./functions/server/bunext-req-handler";
+import generateRobotsTxt from "./utils/generate-robots-txt";
 import generateSitemap from "./utils/generate-sitemap";
 declare const bunext: {
     bunextRequestHandler: typeof bunextRequestHandler;
@@ -15,5 +16,6 @@ declare const bunext: {
     };
     bunextInit: typeof bunextInit;
     bunextGenerateSitemap: typeof generateSitemap;
+    bunextGenerateRobotsTxt: typeof generateRobotsTxt;
 };
 export default bunext;

dist/index.js

@@ -1,5 +1,6 @@
 import bunextInit from "./functions/bunext-init";
 import bunextRequestHandler from "./functions/server/bunext-req-handler";
+import generateRobotsTxt from "./utils/generate-robots-txt";
 import generateSitemap from "./utils/generate-sitemap";
 import { log } from "./utils/log";
 const bunext = {
@@ -7,5 +8,6 @@ const bunext = {
     bunextLog: log,
     bunextInit,
     bunextGenerateSitemap: generateSitemap,
+    bunextGenerateRobotsTxt: generateRobotsTxt,
 };
 export default bunext;

dist/types/index.d.ts

@@ -97,6 +97,11 @@ export type BunextConfig = {
      * Options for the `bunext sitemap` command
      */
     sitemap?: BunextSitemapConfig;
+    /**
+     * Options for the robots.txt written by `bunext sitemap`. Set to
+     * `false` to skip generating it.
+     */
+    robots?: BunextRobotsConfig | false;
 };
 export type BunextConfigMiddlewareParams = {
     req: Request;
@@ -445,9 +450,61 @@ export type BunextGenerateSitemapParams = {
     default_priority?: number;
 };
 export type BunextSitemapConfig = BunextGenerateSitemapParams & {
+    /**
+     * Maximum URLs per sitemap file. Above this, `bunext sitemap` writes
+     * chunk files to `chunks_dir` and makes sitemap.xml a sitemap index
+     * referencing them. Defaults to 10,000; capped at 50,000, the protocol
+     * limit. Chunks are also split at 50MB.
+     */
+    chunk_size?: number;
+    /**
+     * Directory for sitemap chunk files, relative to the project root.
+     * Must be inside `public/` so the files are served. Defaults to
+     * `public/sitemap-chunks`.
+     */
+    chunks_dir?: string;
     /**
      * Output file, relative to the project root. Defaults to
      * `public/sitemap.xml`, which is served at `/sitemap.xml`.
      */
     output?: string;
 };
+export type BunextRobotsRule = {
+    /**
+     * One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]`
+     */
+    user_agent: string | string[];
+    allow?: string[];
+    disallow?: string[];
+    /**
+     * Seconds between requests. Not supported by every crawler.
+     */
+    crawl_delay?: number;
+};
+export type BunextGenerateRobotsTxtParams = {
+    /**
+     * Base URL used to turn relative `sitemaps` paths into absolute URLs.
+     * Defaults to `config.origin`, falling back to `http://localhost:<port>`.
+     */
+    origin?: string;
+    /**
+     * Defaults to allowing every crawler everywhere.
+     */
+    rules?: BunextRobotsRule[];
+    /**
+     * Sitemap URLs or paths, each written as a `Sitemap:` line.
+     */
+    sitemaps?: string[];
+    /**
+     * Block every crawler from the whole site, ignoring `rules` and
+     * `sitemaps`. Useful for staging deployments.
+     */
+    disallow_all?: boolean;
+};
+export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & {
+    /**
+     * Output file, relative to the project root. Defaults to
+     * `public/robots.txt`, which is served at `/robots.txt`.
+     */
+    output?: string;
+};

dist/utils/generate-robots-txt.d.ts

@@ -0,0 +1,11 @@
+import type { BunextGenerateRobotsTxtParams } from "../types";
+/**
+ * First line of every generated robots.txt. `bunext sitemap` only
+ * overwrites an existing robots.txt that starts with it, unless robots
+ * is configured explicitly.
+ */
+export declare const RobotsTxtGeneratedMarker = "# Generated by bunext";
+/**
+ * # Generate a robots.txt string
+ */
+export default function generateRobotsTxt(params?: BunextGenerateRobotsTxtParams): string;

dist/utils/generate-robots-txt.js

@@ -0,0 +1,57 @@
+import grabOrigin from "./grab-origin";
+/**
+ * First line of every generated robots.txt. `bunext sitemap` only
+ * overwrites an existing robots.txt that starts with it, unless robots
+ * is configured explicitly.
+ */
+export const RobotsTxtGeneratedMarker = "# Generated by bunext";
+/**
+ * # Generate a robots.txt string
+ */
+export default function generateRobotsTxt(params) {
+    const lines = [RobotsTxtGeneratedMarker, ""];
+    if (params?.disallow_all) {
+        lines.push("User-agent: *", "Disallow: /", "");
+        return lines.join("\n");
+    }
+    const rules = params?.rules?.length
+        ? params.rules
+        : [{ user_agent: "*", allow: ["/"] }];
+    for (const rule of rules) {
+        const user_agents = [rule.user_agent].flat();
+        for (const user_agent of user_agents) {
+            lines.push(`User-agent: ${sanitizeValue(user_agent)}`);
+        }
+        for (const allow of rule.allow || []) {
+            lines.push(`Allow: ${sanitizeValue(allow)}`);
+        }
+        for (const disallow of rule.disallow || []) {
+            lines.push(`Disallow: ${sanitizeValue(disallow)}`);
+        }
+        // A group needs at least one rule; an empty Disallow allows all.
+        if (!rule.allow?.length && !rule.disallow?.length) {
+            lines.push("Disallow:");
+        }
+        if (typeof rule.crawl_delay == "number") {
+            lines.push(`Crawl-delay: ${rule.crawl_delay}`);
+        }
+        lines.push("");
+    }
+    if (params?.sitemaps?.length) {
+        const origin = (params.origin || grabOrigin()).replace(/\/+$/, "");
+        for (const sitemap of new Set(params.sitemaps)) {
+            const url = sitemap.match(/^https?:\/\//)
+                ? sitemap
+                : `${origin}/${sitemap.replace(/^\/+/, "")}`;
+            lines.push(`Sitemap: ${sanitizeValue(url)}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Line breaks would let a value inject extra directives.
+ */
+function sanitizeValue(value) {
+    return value.replace(/[\r\n]+/g, " ").trim();
+}

dist/utils/generate-sitemap.d.ts

@@ -1,8 +1,39 @@
 import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export declare const SitemapMaxUrlsPerFile = 50000;
+export declare const SitemapMaxBytesPerFile: number;
+export declare const SitemapDefaultChunkSize = 10000;
+export type SitemapChunkFile = {
+    file_name: string;
+    xml: string;
+    url_count: number;
+};
+export type SitemapFiles = {
+    /**
+     * A `<urlset>` when everything fits in one file, otherwise a
+     * `<sitemapindex>` referencing `chunks`.
+     */
+    sitemap: string;
+    chunks: SitemapChunkFile[];
+    url_count: number;
+};
 /**
  * # Generate a sitemap.xml string for all pages
  * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
  */
 export default function generateSitemap(params?: BunextGenerateSitemapParams): Promise<string>;
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export declare function generateSitemapFiles({ chunk_size, chunks_url_path, ...params }: BunextGenerateSitemapParams & {
+    chunk_size?: number;
+    chunks_url_path: string;
+}): Promise<SitemapFiles>;
 export declare function grabSitemapEntries(params?: BunextGenerateSitemapParams): Promise<BunextSitemapEntry[]>;

dist/utils/generate-sitemap.js

@@ -6,16 +6,105 @@ import grabDirNames from "./grab-dir-names";
 import grabOrigin from "./grab-origin";
 import grabRelativePath from "./grab-relative-path";
 import isDevelopment from "./is-development";
+import { log } from "./log";
 const ErrorPagePaths = ["/404", "/500"];
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export const SitemapMaxUrlsPerFile = 50_000;
+export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
+export const SitemapDefaultChunkSize = 10_000;
+const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
+const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
 /**
  * # Generate a sitemap.xml string for all pages
  * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
  */
 export default async function generateSitemap(params) {
     const entries = await grabSitemapEntries(params);
-    const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
+    const origin = grabSitemapOrigin(params?.origin);
-    const urls = entries.map((entry) => {
+    if (entries.length > SitemapMaxUrlsPerFile) {
+        log.warn(`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
+            `per-file limit. Use \`bunext sitemap\` to split it into chunks.`);
+    }
+    return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
+}
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export async function generateSitemapFiles({ chunk_size = SitemapDefaultChunkSize, chunks_url_path, ...params }) {
+    const entries = await grabSitemapEntries(params);
+    const origin = grabSitemapOrigin(params.origin);
+    const max_urls = Math.min(Math.max(Math.floor(chunk_size) || 1, 1), SitemapMaxUrlsPerFile);
+    const urls = entries.map((entry) => renderUrl(entry, origin));
+    const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
+    const chunks = [];
+    let chunk;
+    for (let i = 0; i < urls.length; i++) {
+        const url_bytes = Buffer.byteLength(urls[i]) + 1;
+        if (!chunk ||
+            chunk.urls.length >= max_urls ||
+            chunk.bytes + url_bytes > SitemapMaxBytesPerFile) {
+            chunk = { urls: [], bytes: empty_urlset_bytes };
+            chunks.push(chunk);
+        }
+        chunk.urls.push(urls[i]);
+        chunk.bytes += url_bytes;
+        const lastmod = parseLastmod(entries[i].lastmod);
+        if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
+            chunk.lastmod = lastmod;
+        }
+    }
+    if (chunks.length <= 1) {
+        return {
+            sitemap: renderUrlset(chunks[0]?.urls || []),
+            chunks: [],
+            url_count: urls.length,
+        };
+    }
+    const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
+    const chunk_base_url = chunks_dir_path
+        ? `${origin}/${chunks_dir_path}`
+        : origin;
+    const chunk_files = chunks.map((c, i) => ({
+        file_name: `sitemap-${i + 1}.xml`,
+        xml: renderUrlset(c.urls),
+        url_count: c.urls.length,
+    }));
+    const sitemap = [
+        SitemapXmlHeader,
+        `<sitemapindex ${SitemapXmlns}>`,
+        ...chunk_files.map((file, i) => {
+            const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
+            const lines = [`        <loc>${loc}</loc>`];
+            const lastmod = chunks[i].lastmod?.toISOString();
+            if (lastmod)
+                lines.push(`        <lastmod>${lastmod}</lastmod>`);
+            return `    <sitemap>\n${lines.join("\n")}\n    </sitemap>`;
+        }),
+        `</sitemapindex>`,
+        ``,
+    ].join("\n");
+    return { sitemap, chunks: chunk_files, url_count: urls.length };
+}
+function grabSitemapOrigin(origin) {
+    return (origin || grabOrigin()).replace(/\/+$/, "");
+}
+function renderUrlset(urls) {
+    return [
+        SitemapXmlHeader,
+        `<urlset ${SitemapXmlns}>`,
+        ...urls,
+        `</urlset>`,
+        ``,
+    ].join("\n");
+}
+function renderUrl(entry, origin) {
     const loc = entry.path.match(/^https?:\/\//)
         ? entry.path
         : `${origin}${entry.path}`;
@@ -34,14 +123,12 @@ export default async function generateSitemap(params) {
         lines.push(`        <priority>${priority.toFixed(1)}</priority>`);
     }
     return `    <url>\n${lines.join("\n")}\n    </url>`;
-    });
+}
-    return [
+function parseLastmod(lastmod) {
-        `<?xml version="1.0" encoding="UTF-8"?>`,
+    if (!lastmod)
-        `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+        return;
-        ...urls,
+    const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
-        `</urlset>`,
+    return isNaN(date.getTime()) ? undefined : date;
-        ``,
-    ].join("\n");
 }
 export async function grabSitemapEntries(params) {
     const use_file_lastmod = params?.use_file_lastmod ?? true;

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@moduletrace/bunext",
-    "version": "1.1.21",
+    "version": "1.1.22",
     "main": "dist/index.js",
     "module": "index.ts",
     "dependencies": {

publish.sh

@@ -2,14 +2,15 @@
 
 set -e
 
+tsc --noEmit
+bun run test
+
 if [ -z "$1" ]; then
     msg="Updates"
 else
     msg="$1"
 fi
 
-tsc --noEmit
-
 rm -rf dist
 
 tsc

src/__tests__/commands/sitemap.test.ts

@@ -41,6 +41,14 @@ function readOutput(file = "public/sitemap.xml") {
     return fs.readFileSync(path.join(projectDir, file), "utf-8");
 }
 
+function readRobots(file = "public/robots.txt") {
+    return fs.readFileSync(path.join(projectDir, file), "utf-8");
+}
+
+function exists(file: string) {
+    return fs.existsSync(path.join(projectDir, file));
+}
+
 function locs(xml: string) {
     return [...xml.matchAll(/<loc>(.*?)<\/loc>/g)].map((m) => m[1]).sort();
 }
@@ -221,4 +229,300 @@ describe("bunext sitemap", () => {
         expect(res.exit_code).toBe(1);
         expect(res.output).toContain("api down");
     });
+
+    describe("robots.txt", () => {
+        it("writes an allow-all robots.txt pointing at the sitemap by default", async () => {
+            writeConfig(`{ origin: "https://example.com" }`);
+
+            const res = await runSitemap();
+
+            expect(res.exit_code).toBe(0);
+            expect(res.output).toContain("robots.txt written to public/robots.txt");
+            expect(readRobots()).toBe(
+                [
+                    "# Generated by bunext",
+                    "",
+                    "User-agent: *",
+                    "Allow: /",
+                    "",
+                    "Sitemap: https://example.com/sitemap.xml",
+                    "",
+                ].join("\n"),
+            );
+        });
+
+        it("applies the robots config and writes to robots.output", async () => {
+            writeConfig(`{
+                origin: "https://example.com",
+                robots: {
+                    output: "out/robots.txt",
+                    rules: [
+                        { user_agent: "*", disallow: ["/admin"] },
+                        { user_agent: "GPTBot", disallow: ["/"] },
+                    ],
+                    sitemaps: ["/sitemaps/extra.xml"],
+                },
+            }`);
+
+            const res = await runSitemap();
+            const robots = readRobots("out/robots.txt");
+
+            expect(res.exit_code).toBe(0);
+            expect(exists("public/robots.txt")).toBe(false);
+            expect(robots).toContain("User-agent: *\nDisallow: /admin\n");
+            expect(robots).toContain("User-agent: GPTBot\nDisallow: /\n");
+            expect(robots).toContain("Sitemap: https://example.com/sitemap.xml");
+            expect(robots).toContain(
+                "Sitemap: https://example.com/sitemaps/extra.xml",
+            );
+        });
+
+        it("points at a custom sitemap output inside public/", async () => {
+            writeConfig(`{
+                origin: "https://example.com",
+                sitemap: { output: "public/maps/site.xml" },
+            }`);
+
+            await runSitemap();
+
+            expect(readRobots()).toContain(
+                "Sitemap: https://example.com/maps/site.xml",
+            );
+        });
+
+        it("warns and omits the Sitemap line when the sitemap is outside public/", async () => {
+            writeConfig(`{ origin: "https://example.com" }`);
+
+            const res = await runSitemap("-o", "dist/sitemap.xml");
+
+            expect(res.exit_code).toBe(0);
+            expect(res.output).toContain("outside public/");
+            expect(readRobots()).not.toContain("Sitemap:");
+        });
+
+        it("uses --origin over robots.origin and sitemap.origin", async () => {
+            writeConfig(`{
+                sitemap: { origin: "https://sitemap.dev" },
+                robots: { origin: "https://robots.dev" },
+            }`);
+
+            await runSitemap("--origin", "https://cli.dev");
+
+            expect(readRobots()).toContain("Sitemap: https://cli.dev/sitemap.xml");
+        });
+
+        it("uses robots.origin over sitemap.origin", async () => {
+            writeConfig(`{
+                sitemap: { origin: "https://sitemap.dev" },
+                robots: { origin: "https://robots.dev" },
+            }`);
+
+            await runSitemap();
+
+            expect(readRobots()).toContain(
+                "Sitemap: https://robots.dev/sitemap.xml",
+            );
+        });
+
+        it("blocks all crawlers with disallow_all", async () => {
+            writeConfig(`{ robots: { disallow_all: true } }`);
+
+            await runSitemap();
+
+            expect(readRobots()).toBe(
+                "# Generated by bunext\n\nUser-agent: *\nDisallow: /\n",
+            );
+        });
+
+        it("keeps a hand-written robots.txt when robots is not configured", async () => {
+            writeConfig(`{ origin: "https://example.com" }`);
+            writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n");
+
+            const res = await runSitemap();
+
+            expect(res.exit_code).toBe(0);
+            expect(res.output).toContain("not generated by bunext");
+            expect(readRobots()).toBe("User-agent: *\nDisallow: /x\n");
+        });
+
+        it("overwrites a hand-written robots.txt when robots is configured", async () => {
+            writeConfig(`{ robots: { rules: [{ user_agent: "*", allow: ["/"] }] } }`);
+            writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n");
+
+            await runSitemap();
+
+            expect(readRobots()).toStartWith("# Generated by bunext");
+        });
+
+        it("regenerates its own robots.txt without a robots config", async () => {
+            writeConfig(`{ origin: "https://example.com" }`);
+            writeProjectFile(
+                "public/robots.txt",
+                "# Generated by bunext\n\nUser-agent: *\nDisallow: /old\n",
+            );
+
+            await runSitemap();
+
+            expect(readRobots()).not.toContain("/old");
+            expect(readRobots()).toContain("Allow: /");
+        });
+
+        it("skips robots.txt with robots: false", async () => {
+            writeConfig(`{ robots: false }`);
+
+            const res = await runSitemap();
+
+            expect(res.exit_code).toBe(0);
+            expect(exists("public/sitemap.xml")).toBe(true);
+            expect(exists("public/robots.txt")).toBe(false);
+            expect(res.output).not.toContain("robots.txt");
+        });
+
+        it("skips robots.txt with --no-robots", async () => {
+            writeConfig(`{ robots: { disallow_all: true } }`);
+
+            await runSitemap("--no-robots");
+
+            expect(exists("public/robots.txt")).toBe(false);
+        });
+
+        it("does not write robots.txt when sitemap generation fails", async () => {
+            writeConfig(`{
+                sitemap: {
+                    additional_paths: async () => { throw new Error("api down"); },
+                },
+            }`);
+
+            const res = await runSitemap();
+
+            expect(res.exit_code).toBe(1);
+            expect(exists("public/robots.txt")).toBe(false);
+        });
+    });
+
+    describe("chunking", () => {
+        const itemsServerFile = (count: number) =>
+            `export const sitemap = () => Array.from({ length: ${count} }, (_, i) => "/items/" + i);`;
+
+        const chunkFiles = (dir = "public/sitemap-chunks") =>
+            exists(dir)
+                ? fs
+                      .readdirSync(path.join(projectDir, dir))
+                      .sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
+                : [];
+
+        beforeEach(() => {
+            writeProjectFile("src/pages/items/[id].tsx", "export default () => null");
+        });
+
+        it("writes chunks and a sitemap index above chunk_size", async () => {
+            writeConfig(`{
+                origin: "https://example.com",
+                sitemap: { chunk_size: 1000 },
+            }`);
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500));
+
+            const res = await runSitemap();
+            const index = readOutput();
+
+            expect(res.exit_code).toBe(0);
+            expect(res.output).toContain("2503 URLs in 3 chunks");
+            expect(chunkFiles()).toEqual([
+                "sitemap-1.xml",
+                "sitemap-2.xml",
+                "sitemap-3.xml",
+            ]);
+            expect(index).toContain("<sitemapindex");
+            expect(locs(index)).toEqual([
+                "https://example.com/sitemap-chunks/sitemap-1.xml",
+                "https://example.com/sitemap-chunks/sitemap-2.xml",
+                "https://example.com/sitemap-chunks/sitemap-3.xml",
+            ]);
+
+            const all_locs = chunkFiles().flatMap((f) =>
+                locs(readOutput(`public/sitemap-chunks/${f}`)),
+            );
+            expect(all_locs).toHaveLength(2503);
+            expect(new Set(all_locs).size).toBe(2503);
+            expect(readRobots()).toContain(
+                "Sitemap: https://example.com/sitemap.xml",
+            );
+        });
+
+        it("chunks 20,000+ URLs with the default chunk size", async () => {
+            writeConfig(`{ origin: "https://example.com" }`);
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(25_000));
+
+            const res = await runSitemap();
+
+            expect(res.exit_code).toBe(0);
+            expect(chunkFiles()).toEqual([
+                "sitemap-1.xml",
+                "sitemap-2.xml",
+                "sitemap-3.xml",
+            ]);
+            expect(
+                locs(readOutput("public/sitemap-chunks/sitemap-3.xml")),
+            ).toHaveLength(5003);
+        });
+
+        it("removes stale chunks but keeps other files in the chunks dir", async () => {
+            writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`);
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(3500));
+            writeProjectFile("public/sitemap-chunks/keep-me.txt", "mine");
+
+            await runSitemap();
+            expect(chunkFiles()).toHaveLength(5);
+
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500));
+            await runSitemap();
+
+            expect(chunkFiles()).toEqual([
+                "keep-me.txt",
+                "sitemap-1.xml",
+                "sitemap-2.xml",
+            ]);
+        });
+
+        it("goes back to a single sitemap and removes chunks when it shrinks", async () => {
+            writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`);
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500));
+            await runSitemap();
+
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(10));
+            const res = await runSitemap();
+
+            expect(res.output).toContain("Sitemap with 13 URLs");
+            expect(readOutput()).toContain("<urlset");
+            expect(chunkFiles()).toEqual([]);
+        });
+
+        it("writes chunks to a custom chunks_dir inside public/", async () => {
+            writeConfig(`{
+                origin: "https://example.com",
+                sitemap: { chunk_size: 1000, chunks_dir: "public/seo/maps" },
+            }`);
+            writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500));
+
+            await runSitemap();
+
+            expect(chunkFiles("public/seo/maps")).toEqual([
+                "sitemap-1.xml",
+                "sitemap-2.xml",
+            ]);
+            expect(locs(readOutput())).toContain(
+                "https://example.com/seo/maps/sitemap-1.xml",
+            );
+        });
+
+        it("fails when chunks_dir is outside public/", async () => {
+            writeConfig(`{ sitemap: { chunks_dir: "dist/chunks" } }`);
+
+            const res = await runSitemap();
+
+            expect(res.exit_code).toBe(1);
+            expect(res.output).toContain("must be inside public/");
+            expect(exists("public/sitemap.xml")).toBe(false);
+        });
+    });
 });

src/__tests__/utils/generate-robots-txt.test.ts

@@ -0,0 +1,86 @@
+import { describe, it, expect, beforeEach } from "bun:test";
+import generateRobotsTxt, {
+    RobotsTxtGeneratedMarker,
+} from "../../utils/generate-robots-txt";
+
+beforeEach(() => {
+    (global as any).BUNEXT_CONFIG = { origin: "https://example.com" };
+});
+
+describe("generateRobotsTxt", () => {
+    it("allows everything by default, starting with the marker", () => {
+        expect(generateRobotsTxt()).toBe(
+            `${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nAllow: /\n`,
+        );
+    });
+
+    it("renders rule groups with multiple user agents and crawl delay", () => {
+        const txt = generateRobotsTxt({
+            rules: [
+                {
+                    user_agent: "*",
+                    allow: ["/"],
+                    disallow: ["/admin", "/api/"],
+                    crawl_delay: 5,
+                },
+                { user_agent: ["GPTBot", "CCBot"], disallow: ["/"] },
+            ],
+        });
+
+        expect(txt).toContain(
+            "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /api/\nCrawl-delay: 5\n",
+        );
+        expect(txt).toContain("User-agent: GPTBot\nUser-agent: CCBot\nDisallow: /\n");
+    });
+
+    it("writes an empty Disallow for a rule with no paths", () => {
+        const txt = generateRobotsTxt({ rules: [{ user_agent: "Bingbot" }] });
+        expect(txt).toContain("User-agent: Bingbot\nDisallow:\n");
+    });
+
+    it("resolves sitemap paths against the origin and dedupes them", () => {
+        const txt = generateRobotsTxt({
+            sitemaps: [
+                "/sitemap.xml",
+                "sitemaps/blog.xml",
+                "https://cdn.example.com/sitemap.xml",
+                "/sitemap.xml",
+            ],
+        });
+
+        expect(txt.match(/^Sitemap: .*$/gm)).toEqual([
+            "Sitemap: https://example.com/sitemap.xml",
+            "Sitemap: https://example.com/sitemaps/blog.xml",
+            "Sitemap: https://cdn.example.com/sitemap.xml",
+        ]);
+    });
+
+    it("uses the origin param over config and strips its trailing slash", () => {
+        const txt = generateRobotsTxt({
+            origin: "https://other.dev/",
+            sitemaps: ["/sitemap.xml"],
+        });
+        expect(txt).toContain("Sitemap: https://other.dev/sitemap.xml");
+    });
+
+    it("blocks everything with disallow_all, ignoring rules and sitemaps", () => {
+        const txt = generateRobotsTxt({
+            disallow_all: true,
+            rules: [{ user_agent: "*", allow: ["/"] }],
+            sitemaps: ["/sitemap.xml"],
+        });
+
+        expect(txt).toBe(
+            `${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nDisallow: /\n`,
+        );
+    });
+
+    it("strips line breaks so values cannot inject directives", () => {
+        const txt = generateRobotsTxt({
+            rules: [{ user_agent: "*", disallow: ["/a\nAllow: /secret"] }],
+        });
+
+        expect(txt).not.toContain("\nAllow: /secret");
+        expect(txt).toContain("Disallow: /a Allow: /secret");
+    });
+});

src/__tests__/utils/generate-sitemap.test.ts

@@ -3,7 +3,9 @@ import path from "path";
 import fs from "fs";
 import os from "os";
 import generateSitemap, {
+    generateSitemapFiles,
     grabSitemapEntries,
+    SitemapMaxUrlsPerFile,
 } from "../../utils/generate-sitemap";
 
 // grabAllPages skips anything under __tests__, so fixtures live in the OS tmp dir
@@ -178,3 +180,101 @@ describe("generateSitemap", () => {
         expect(xml).toContain("<loc>https://other.dev/about</loc>");
     });
 });
+
+describe("generateSitemapFiles", () => {
+    const manyPaths = (count: number) =>
+        Array.from({ length: count }, (_, i) => `/items/${i}`);
+
+    const countUrls = (xml: string) => xml.match(/<url>/g)?.length || 0;
+
+    it("returns a single urlset with no chunks when within chunk_size", async () => {
+        const { sitemap, chunks, url_count } = await generateSitemapFiles({
+            chunks_url_path: "/sitemap-chunks",
+            use_file_lastmod: false,
+        });
+
+        expect(chunks).toEqual([]);
+        expect(sitemap).toContain("<urlset");
+        expect(countUrls(sitemap)).toBe(url_count);
+    });
+
+    it("splits into chunks referenced by a sitemap index", async () => {
+        const { sitemap, chunks, url_count } = await generateSitemapFiles({
+            chunks_url_path: "/sitemap-chunks",
+            chunk_size: 1000,
+            exclude: [/^(?!\/items\/)/],
+            additional_paths: manyPaths(2500),
+            use_file_lastmod: false,
+        });
+
+        expect(url_count).toBe(2500);
+        expect(chunks.map((c) => c.file_name)).toEqual([
+            "sitemap-1.xml",
+            "sitemap-2.xml",
+            "sitemap-3.xml",
+        ]);
+        expect(chunks.map((c) => c.url_count)).toEqual([1000, 1000, 500]);
+        expect(chunks.map((c) => countUrls(c.xml))).toEqual([1000, 1000, 500]);
+        expect(chunks[0].xml).toContain("<loc>https://example.com/items/0</loc>");
+        expect(chunks[2].xml).toContain(
+            "<loc>https://example.com/items/2499</loc>",
+        );
+
+        expect(sitemap).toContain("<sitemapindex");
+        expect(sitemap).not.toContain("<url>");
+        expect(sitemap.match(/<loc>.*?<\/loc>/g)).toEqual([
+            "<loc>https://example.com/sitemap-chunks/sitemap-1.xml</loc>",
+            "<loc>https://example.com/sitemap-chunks/sitemap-2.xml</loc>",
+            "<loc>https://example.com/sitemap-chunks/sitemap-3.xml</loc>",
+        ]);
+    });
+
+    it("sets each index entry's lastmod to the newest in its chunk", async () => {
+        const { sitemap } = await generateSitemapFiles({
+            chunks_url_path: "/maps/",
+            origin: "https://other.dev/",
+            chunk_size: 2,
+            exclude: [/^(?!\/items\/)/],
+            use_file_lastmod: false,
+            additional_paths: [
+                { path: "/items/a", lastmod: "2026-01-01" },
+                { path: "/items/b", lastmod: new Date("2026-03-01") },
+                { path: "/items/c" },
+                { path: "/items/d", lastmod: "not a date" },
+            ],
+        });
+
+        expect(sitemap).toContain(
+            "<loc>https://other.dev/maps/sitemap-1.xml</loc>\n        <lastmod>2026-03-01T00:00:00.000Z</lastmod>",
+        );
+        expect(sitemap).toContain(
+            "<loc>https://other.dev/maps/sitemap-2.xml</loc>\n    </sitemap>",
+        );
+    });
+
+    it("caps chunk_size at the protocol limit", async () => {
+        const { chunks } = await generateSitemapFiles({
+            chunks_url_path: "/sitemap-chunks",
+            chunk_size: 1_000_000,
+            exclude: [/^(?!\/items\/)/],
+            additional_paths: manyPaths(SitemapMaxUrlsPerFile + 1),
+            use_file_lastmod: false,
+        });
+
+        expect(chunks.map((c) => c.url_count)).toEqual([
+            SitemapMaxUrlsPerFile,
+            1,
+        ]);
+    });
+
+    it("defaults to 10,000 URLs per chunk", async () => {
+        const { chunks } = await generateSitemapFiles({
+            chunks_url_path: "/sitemap-chunks",
+            exclude: [/^(?!\/items\/)/],
+            additional_paths: manyPaths(20_001),
+            use_file_lastmod: false,
+        });
+
+        expect(chunks.map((c) => c.url_count)).toEqual([10_000, 10_000, 1]);
+    });
+});

src/commands/sitemap/index.ts

@@ -1,56 +1,117 @@
 import { Command } from "commander";
-import { mkdirSync, writeFileSync } from "fs";
+import {
+    existsSync,
+    mkdirSync,
+    readdirSync,
+    readFileSync,
+    rmSync,
+    writeFileSync,
+} from "fs";
 import path from "path";
 import { log } from "../../utils/log";
 import grabDirNames from "../../utils/grab-dir-names";
 import grabConfig from "../../functions/grab-config";
 import grabConstants from "../../utils/grab-constants";
-import generateSitemap from "../../utils/generate-sitemap";
+import { generateSitemapFiles } from "../../utils/generate-sitemap";
+import generateRobotsTxt, {
+    RobotsTxtGeneratedMarker,
+} from "../../utils/generate-robots-txt";
 
 type Options = {
     output?: string;
     origin?: string;
+    robots: boolean;
 };
 
 export default function () {
     return new Command("sitemap")
-        .description("Generate sitemap.xml from the pages directory")
+        .description(
+            "Generate sitemap.xml and robots.txt from the pages directory",
+        )
         .option(
             "-o, --output <path>",
             "Output file relative to the project root (default: public/sitemap.xml)",
         )
         .option("--origin <url>", "Base URL for sitemap links")
+        .option("--no-robots", "Skip generating robots.txt")
         .action(async (options: Options) => {
             const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
 
             global.BUNEXT_CONFIG = (await grabConfig()) || {};
             global.BUNEXT_CONSTANTS = grabConstants();
 
-            const { output, ...sitemap_config } =
+            const { output, chunk_size, chunks_dir, ...sitemap_config } =
                 global.BUNEXT_CONFIG.sitemap || {};
 
-            try {
+            let sitemap_file: string;
-                const xml = await generateSitemap({
-                    ...sitemap_config,
-                    origin: options.origin || sitemap_config.origin,
-                });
 
+            try {
                 const output_path = options.output || output;
-                const output_file = output_path
+                sitemap_file = output_path
                     ? path.resolve(ROOT_DIR, output_path)
                     : path.join(PUBLIC_DIR, "sitemap.xml");
 
-                mkdirSync(path.dirname(output_file), { recursive: true });
+                const chunks_dir_path = path.resolve(
-                writeFileSync(output_file, xml, "utf-8");
-
-                const url_count = xml.match(/<url>/g)?.length || 0;
-
-                log.success(
-                    `Sitemap with ${url_count} URLs written to ${path.relative(
                     ROOT_DIR,
-                        output_file,
+                    chunks_dir || "public/sitemap-chunks",
-                    )}`,
                 );
+                const chunks_public_path = grabPublicUrlPath(chunks_dir_path);
+
+                if (chunks_public_path === undefined) {
+                    throw new Error(
+                        `\`sitemap.chunks_dir\` must be inside public/ ` +
+                            `so the chunks are served.`,
+                    );
+                }
+
+                const { sitemap, chunks, url_count } =
+                    await generateSitemapFiles({
+                        ...sitemap_config,
+                        origin: options.origin || sitemap_config.origin,
+                        chunk_size,
+                        chunks_url_path: chunks_public_path,
+                    });
+
+                // Remove chunks from a previous, larger sitemap
+                if (existsSync(chunks_dir_path)) {
+                    for (const file of readdirSync(chunks_dir_path)) {
+                        if (file.match(/^sitemap-\d+\.xml$/)) {
+                            rmSync(path.join(chunks_dir_path, file));
+                        }
+                    }
+                }
+
+                if (chunks.length) {
+                    mkdirSync(chunks_dir_path, { recursive: true });
+                }
+
+                for (const chunk of chunks) {
+                    writeFileSync(
+                        path.join(chunks_dir_path, chunk.file_name),
+                        chunk.xml,
+                        "utf-8",
+                    );
+                }
+
+                mkdirSync(path.dirname(sitemap_file), { recursive: true });
+                writeFileSync(sitemap_file, sitemap, "utf-8");
+
+                const sitemap_relative_path = path.relative(
+                    ROOT_DIR,
+                    sitemap_file,
+                );
+
+                if (chunks.length) {
+                    log.success(
+                        `Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` +
+                            `written to ${sitemap_relative_path} ` +
+                            `(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`,
+                    );
+                } else {
+                    log.success(
+                        `Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`,
+                    );
+                }
             } catch (error: any) {
                 log.error(
                     `Failed to generate sitemap: ${error?.message || error}`,
@@ -58,6 +119,92 @@ export default function () {
                 process.exit(1);
             }
 
+            const robots_config = global.BUNEXT_CONFIG.robots;
+
+            if (options.robots && robots_config !== false) {
+                writeRobotsTxt({
+                    sitemap_file,
+                    cli_origin: options.origin,
+                    sitemap_origin: sitemap_config.origin,
+                });
+            }
+
             process.exit();
         });
 }
+
+function writeRobotsTxt({
+    sitemap_file,
+    cli_origin,
+    sitemap_origin,
+}: {
+    sitemap_file: string;
+    cli_origin?: string;
+    sitemap_origin?: string;
+}) {
+    const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
+    const robots_config = global.BUNEXT_CONFIG.robots || undefined;
+    const { output, ...robots_params } = robots_config || {};
+
+    const robots_file = output
+        ? path.resolve(ROOT_DIR, output)
+        : path.join(PUBLIC_DIR, "robots.txt");
+    const robots_relative_path = path.relative(ROOT_DIR, robots_file);
+
+    // Without a robots config, never replace a hand-written robots.txt.
+    if (
+        !robots_config &&
+        existsSync(robots_file) &&
+        !readFileSync(robots_file, "utf-8").startsWith(
+            RobotsTxtGeneratedMarker,
+        )
+    ) {
+        log.warn(
+            `Skipped ${robots_relative_path}: it was not generated by bunext. ` +
+                `Set \`robots\` in bunext.config.ts to overwrite it, ` +
+                `or \`robots: false\` to silence this.`,
+        );
+        return;
+    }
+
+    const sitemap_public_path = grabPublicUrlPath(sitemap_file);
+
+    if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) {
+        log.warn(
+            `Sitemap is outside public/, so robots.txt has no Sitemap line. ` +
+                `Add its URL to \`robots.sitemaps\`.`,
+        );
+    }
+
+    const robots_txt = generateRobotsTxt({
+        ...robots_params,
+        origin: cli_origin || robots_params.origin || sitemap_origin,
+        sitemaps: [
+            ...(sitemap_public_path !== undefined
+                ? [sitemap_public_path]
+                : []),
+            ...(robots_params.sitemaps || []),
+        ],
+    });
+
+    mkdirSync(path.dirname(robots_file), { recursive: true });
+    writeFileSync(robots_file, robots_txt, "utf-8");
+
+    log.success(`robots.txt written to ${robots_relative_path}`);
+}
+
+/**
+ * # URL path a file in public/ is served at
+ * Public files are served from the site root, e.g. public/sitemap.xml
+ * at /sitemap.xml. Returns `undefined` for paths outside public/.
+ */
+function grabPublicUrlPath(file_path: string) {
+    const { PUBLIC_DIR } = grabDirNames();
+    const relative_path = path.relative(PUBLIC_DIR, file_path);
+
+    if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) {
+        return;
+    }
+
+    return `/${relative_path.split(path.sep).join("/")}`;
+}

src/index.ts

@@ -1,5 +1,6 @@
 import bunextInit from "./functions/bunext-init";
 import bunextRequestHandler from "./functions/server/bunext-req-handler";
+import generateRobotsTxt from "./utils/generate-robots-txt";
 import generateSitemap from "./utils/generate-sitemap";
 import { log } from "./utils/log";
 
@@ -8,5 +9,6 @@ const bunext = {
     bunextLog: log,
     bunextInit,
     bunextGenerateSitemap: generateSitemap,
+    bunextGenerateRobotsTxt: generateRobotsTxt,
 };
 export default bunext;

src/types/index.ts

@@ -114,6 +114,11 @@ export type BunextConfig = {
      * Options for the `bunext sitemap` command
      */
     sitemap?: BunextSitemapConfig;
+    /**
+     * Options for the robots.txt written by `bunext sitemap`. Set to
+     * `false` to skip generating it.
+     */
+    robots?: BunextRobotsConfig | false;
 };
 
 export type BunextConfigMiddlewareParams = {
@@ -531,9 +536,64 @@ export type BunextGenerateSitemapParams = {
 };
 
 export type BunextSitemapConfig = BunextGenerateSitemapParams & {
+    /**
+     * Maximum URLs per sitemap file. Above this, `bunext sitemap` writes
+     * chunk files to `chunks_dir` and makes sitemap.xml a sitemap index
+     * referencing them. Defaults to 10,000; capped at 50,000, the protocol
+     * limit. Chunks are also split at 50MB.
+     */
+    chunk_size?: number;
+    /**
+     * Directory for sitemap chunk files, relative to the project root.
+     * Must be inside `public/` so the files are served. Defaults to
+     * `public/sitemap-chunks`.
+     */
+    chunks_dir?: string;
     /**
      * Output file, relative to the project root. Defaults to
      * `public/sitemap.xml`, which is served at `/sitemap.xml`.
      */
     output?: string;
 };
+
+export type BunextRobotsRule = {
+    /**
+     * One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]`
+     */
+    user_agent: string | string[];
+    allow?: string[];
+    disallow?: string[];
+    /**
+     * Seconds between requests. Not supported by every crawler.
+     */
+    crawl_delay?: number;
+};
+
+export type BunextGenerateRobotsTxtParams = {
+    /**
+     * Base URL used to turn relative `sitemaps` paths into absolute URLs.
+     * Defaults to `config.origin`, falling back to `http://localhost:<port>`.
+     */
+    origin?: string;
+    /**
+     * Defaults to allowing every crawler everywhere.
+     */
+    rules?: BunextRobotsRule[];
+    /**
+     * Sitemap URLs or paths, each written as a `Sitemap:` line.
+     */
+    sitemaps?: string[];
+    /**
+     * Block every crawler from the whole site, ignoring `rules` and
+     * `sitemaps`. Useful for staging deployments.
+     */
+    disallow_all?: boolean;
+};
+
+export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & {
+    /**
+     * Output file, relative to the project root. Defaults to
+     * `public/robots.txt`, which is served at `/robots.txt`.
+     */
+    output?: string;
+};

src/utils/generate-robots-txt.ts

@@ -0,0 +1,77 @@
+import type { BunextGenerateRobotsTxtParams } from "../types";
+import grabOrigin from "./grab-origin";
+
+/**
+ * First line of every generated robots.txt. `bunext sitemap` only
+ * overwrites an existing robots.txt that starts with it, unless robots
+ * is configured explicitly.
+ */
+export const RobotsTxtGeneratedMarker = "# Generated by bunext";
+
+/**
+ * # Generate a robots.txt string
+ */
+export default function generateRobotsTxt(
+    params?: BunextGenerateRobotsTxtParams,
+): string {
+    const lines = [RobotsTxtGeneratedMarker, ""];
+
+    if (params?.disallow_all) {
+        lines.push("User-agent: *", "Disallow: /", "");
+        return lines.join("\n");
+    }
+
+    const rules = params?.rules?.length
+        ? params.rules
+        : [{ user_agent: "*", allow: ["/"] }];
+
+    for (const rule of rules) {
+        const user_agents = [rule.user_agent].flat();
+
+        for (const user_agent of user_agents) {
+            lines.push(`User-agent: ${sanitizeValue(user_agent)}`);
+        }
+
+        for (const allow of rule.allow || []) {
+            lines.push(`Allow: ${sanitizeValue(allow)}`);
+        }
+
+        for (const disallow of rule.disallow || []) {
+            lines.push(`Disallow: ${sanitizeValue(disallow)}`);
+        }
+
+        // A group needs at least one rule; an empty Disallow allows all.
+        if (!rule.allow?.length && !rule.disallow?.length) {
+            lines.push("Disallow:");
+        }
+
+        if (typeof rule.crawl_delay == "number") {
+            lines.push(`Crawl-delay: ${rule.crawl_delay}`);
+        }
+
+        lines.push("");
+    }
+
+    if (params?.sitemaps?.length) {
+        const origin = (params.origin || grabOrigin()).replace(/\/+$/, "");
+
+        for (const sitemap of new Set(params.sitemaps)) {
+            const url = sitemap.match(/^https?:\/\//)
+                ? sitemap
+                : `${origin}/${sitemap.replace(/^\/+/, "")}`;
+
+            lines.push(`Sitemap: ${sanitizeValue(url)}`);
+        }
+
+        lines.push("");
+    }
+
+    return lines.join("\n");
+}
+
+/**
+ * Line breaks would let a value inject extra directives.
+ */
+function sanitizeValue(value: string) {
+    return value.replace(/[\r\n]+/g, " ").trim();
+}

src/utils/generate-sitemap.ts

@@ -12,21 +12,160 @@ import grabDirNames from "./grab-dir-names";
 import grabOrigin from "./grab-origin";
 import grabRelativePath from "./grab-relative-path";
 import isDevelopment from "./is-development";
+import { log } from "./log";
 
 const ErrorPagePaths = ["/404", "/500"];
 
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export const SitemapMaxUrlsPerFile = 50_000;
+export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
+export const SitemapDefaultChunkSize = 10_000;
+
+const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
+const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
+
+export type SitemapChunkFile = {
+    file_name: string;
+    xml: string;
+    url_count: number;
+};
+
+export type SitemapFiles = {
+    /**
+     * A `<urlset>` when everything fits in one file, otherwise a
+     * `<sitemapindex>` referencing `chunks`.
+     */
+    sitemap: string;
+    chunks: SitemapChunkFile[];
+    url_count: number;
+};
+
 /**
  * # Generate a sitemap.xml string for all pages
  * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
  */
 export default async function generateSitemap(
     params?: BunextGenerateSitemapParams,
 ): Promise<string> {
     const entries = await grabSitemapEntries(params);
-    const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
+    const origin = grabSitemapOrigin(params?.origin);
 
-    const urls = entries.map((entry) => {
+    if (entries.length > SitemapMaxUrlsPerFile) {
+        log.warn(
+            `Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
+                `per-file limit. Use \`bunext sitemap\` to split it into chunks.`,
+        );
+    }
+
+    return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
+}
+
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export async function generateSitemapFiles({
+    chunk_size = SitemapDefaultChunkSize,
+    chunks_url_path,
+    ...params
+}: BunextGenerateSitemapParams & {
+    chunk_size?: number;
+    chunks_url_path: string;
+}): Promise<SitemapFiles> {
+    const entries = await grabSitemapEntries(params);
+    const origin = grabSitemapOrigin(params.origin);
+    const max_urls = Math.min(
+        Math.max(Math.floor(chunk_size) || 1, 1),
+        SitemapMaxUrlsPerFile,
+    );
+
+    const urls = entries.map((entry) => renderUrl(entry, origin));
+    const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
+
+    type Chunk = { urls: string[]; bytes: number; lastmod?: Date };
+    const chunks: Chunk[] = [];
+    let chunk: Chunk | undefined;
+
+    for (let i = 0; i < urls.length; i++) {
+        const url_bytes = Buffer.byteLength(urls[i]) + 1;
+
+        if (
+            !chunk ||
+            chunk.urls.length >= max_urls ||
+            chunk.bytes + url_bytes > SitemapMaxBytesPerFile
+        ) {
+            chunk = { urls: [], bytes: empty_urlset_bytes };
+            chunks.push(chunk);
+        }
+
+        chunk.urls.push(urls[i]);
+        chunk.bytes += url_bytes;
+
+        const lastmod = parseLastmod(entries[i].lastmod);
+        if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
+            chunk.lastmod = lastmod;
+        }
+    }
+
+    if (chunks.length <= 1) {
+        return {
+            sitemap: renderUrlset(chunks[0]?.urls || []),
+            chunks: [],
+            url_count: urls.length,
+        };
+    }
+
+    const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
+    const chunk_base_url = chunks_dir_path
+        ? `${origin}/${chunks_dir_path}`
+        : origin;
+
+    const chunk_files: SitemapChunkFile[] = chunks.map((c, i) => ({
+        file_name: `sitemap-${i + 1}.xml`,
+        xml: renderUrlset(c.urls),
+        url_count: c.urls.length,
+    }));
+
+    const sitemap = [
+        SitemapXmlHeader,
+        `<sitemapindex ${SitemapXmlns}>`,
+        ...chunk_files.map((file, i) => {
+            const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
+            const lines = [`        <loc>${loc}</loc>`];
+
+            const lastmod = chunks[i].lastmod?.toISOString();
+            if (lastmod) lines.push(`        <lastmod>${lastmod}</lastmod>`);
+
+            return `    <sitemap>\n${lines.join("\n")}\n    </sitemap>`;
+        }),
+        `</sitemapindex>`,
+        ``,
+    ].join("\n");
+
+    return { sitemap, chunks: chunk_files, url_count: urls.length };
+}
+
+function grabSitemapOrigin(origin?: string) {
+    return (origin || grabOrigin()).replace(/\/+$/, "");
+}
+
+function renderUrlset(urls: string[]) {
+    return [
+        SitemapXmlHeader,
+        `<urlset ${SitemapXmlns}>`,
+        ...urls,
+        `</urlset>`,
+        ``,
+    ].join("\n");
+}
+
+function renderUrl(entry: BunextSitemapEntry, origin: string) {
     const loc = entry.path.match(/^https?:\/\//)
         ? entry.path
         : `${origin}${entry.path}`;
@@ -51,15 +190,12 @@ export default async function generateSitemap(
     }
 
     return `    <url>\n${lines.join("\n")}\n    </url>`;
-    });
+}
 
-    return [
+function parseLastmod(lastmod?: Date | string) {
-        `<?xml version="1.0" encoding="UTF-8"?>`,
+    if (!lastmod) return;
-        `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+    const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
-        ...urls,
+    return isNaN(date.getTime()) ? undefined : date;
-        `</urlset>`,
-        ``,
-    ].join("\n");
 }
 
 export async function grabSitemapEntries(