Add robots.txt generation and sitemap chunking

- bunext sitemap now writes public/robots.txt (configurable via the
  robots config key; robots: false or --no-robots to skip). Hand-written
  robots.txt files are never overwritten without a robots config
- Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml
  (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with
  sitemap.xml as a sitemap index; stale chunks are cleaned up
- Export bunextGenerateRobotsTxt
- Run tests in publish.sh

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/types/index.ts

@@ -114,6 +114,11 @@ export type BunextConfig = {
      * Options for the `bunext sitemap` command
      */
     sitemap?: BunextSitemapConfig;
+    /**
+     * Options for the robots.txt written by `bunext sitemap`. Set to
+     * `false` to skip generating it.
+     */
+    robots?: BunextRobotsConfig | false;
 };
 
 export type BunextConfigMiddlewareParams = {
@@ -531,9 +536,64 @@ export type BunextGenerateSitemapParams = {
 };
 
 export type BunextSitemapConfig = BunextGenerateSitemapParams & {
+    /**
+     * Maximum URLs per sitemap file. Above this, `bunext sitemap` writes
+     * chunk files to `chunks_dir` and makes sitemap.xml a sitemap index
+     * referencing them. Defaults to 10,000; capped at 50,000, the protocol
+     * limit. Chunks are also split at 50MB.
+     */
+    chunk_size?: number;
+    /**
+     * Directory for sitemap chunk files, relative to the project root.
+     * Must be inside `public/` so the files are served. Defaults to
+     * `public/sitemap-chunks`.
+     */
+    chunks_dir?: string;
     /**
      * Output file, relative to the project root. Defaults to
      * `public/sitemap.xml`, which is served at `/sitemap.xml`.
      */
     output?: string;
 };
+
+export type BunextRobotsRule = {
+    /**
+     * One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]`
+     */
+    user_agent: string | string[];
+    allow?: string[];
+    disallow?: string[];
+    /**
+     * Seconds between requests. Not supported by every crawler.
+     */
+    crawl_delay?: number;
+};
+
+export type BunextGenerateRobotsTxtParams = {
+    /**
+     * Base URL used to turn relative `sitemaps` paths into absolute URLs.
+     * Defaults to `config.origin`, falling back to `http://localhost:<port>`.
+     */
+    origin?: string;
+    /**
+     * Defaults to allowing every crawler everywhere.
+     */
+    rules?: BunextRobotsRule[];
+    /**
+     * Sitemap URLs or paths, each written as a `Sitemap:` line.
+     */
+    sitemaps?: string[];
+    /**
+     * Block every crawler from the whole site, ignoring `rules` and
+     * `sitemaps`. Useful for staging deployments.
+     */
+    disallow_all?: boolean;
+};
+
+export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & {
+    /**
+     * Output file, relative to the project root. Defaults to
+     * `public/robots.txt`, which is served at `/robots.txt`.
+     */
+    output?: string;
+};