Add robots.txt generation and sitemap chunking

- bunext sitemap now writes public/robots.txt (configurable via the
  robots config key; robots: false or --no-robots to skip). Hand-written
  robots.txt files are never overwritten without a robots config
- Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml
  (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with
  sitemap.xml as a sitemap index; stale chunks are cleaned up
- Export bunextGenerateRobotsTxt
- Run tests in publish.sh

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dist/utils/generate-sitemap.d.ts

@@ -1,8 +1,39 @@
 import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
+/**
+ * Protocol limits for a single sitemap file
+ * https://www.sitemaps.org/protocol.html#index
+ */
+export declare const SitemapMaxUrlsPerFile = 50000;
+export declare const SitemapMaxBytesPerFile: number;
+export declare const SitemapDefaultChunkSize = 10000;
+export type SitemapChunkFile = {
+    file_name: string;
+    xml: string;
+    url_count: number;
+};
+export type SitemapFiles = {
+    /**
+     * A `<urlset>` when everything fits in one file, otherwise a
+     * `<sitemapindex>` referencing `chunks`.
+     */
+    sitemap: string;
+    chunks: SitemapChunkFile[];
+    url_count: number;
+};
 /**
  * # Generate a sitemap.xml string for all pages
  * Dynamic routes are only included through a `sitemap` export in their
- * `.server.ts` file, or through `additional_paths`.
+ * `.server.ts` file, or through `additional_paths`. Always returns a
+ * single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
  */
 export default function generateSitemap(params?: BunextGenerateSitemapParams): Promise<string>;
+/**
+ * # Generate sitemap.xml, split into chunk files when needed
+ * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
+ * is the public URL path the chunk files are served from.
+ */
+export declare function generateSitemapFiles({ chunk_size, chunks_url_path, ...params }: BunextGenerateSitemapParams & {
+    chunk_size?: number;
+    chunks_url_path: string;
+}): Promise<SitemapFiles>;
 export declare function grabSitemapEntries(params?: BunextGenerateSitemapParams): Promise<BunextSitemapEntry[]>;