diff --git a/CLAUDE.md b/CLAUDE.md index 5355e7b..c573899 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ bun run dev # tsc --watch bunext dev # Start dev server with HMR and file watcher bunext build # Bundle all pages for production bunext start # Start production server from pre-built artifacts -bunext sitemap # Write public/sitemap.xml using config.sitemap +bunext sitemap # Write public/sitemap.xml and public/robots.txt using config.sitemap / config.robots ``` ## Architecture diff --git a/README.md b/README.md index 686af10..6e07be9 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,9 @@ The goal is a framework that is: - [Sitemap](#sitemap) - [Page `sitemap` Export](#page-sitemap-export) - [`bunext sitemap` Command](#bunext-sitemap-command) + - [Large Sitemaps](#large-sitemaps) - [Generating at Runtime](#generating-at-runtime) + - [robots.txt](#robotstxt) - [Static Files](#static-files) - [Caching](#caching) - [Enabling Cache Per Page](#enabling-cache-per-page) @@ -159,7 +161,7 @@ bun run dev | `bunext dev` | Start the development server with HMR and file watching. | | `bunext build` | Bundle all pages for production. Outputs artifacts to `.bunext/public/pages/` using paths relative to the project root. | | `bunext start` | Start the production server using pre-built artifacts. | -| `bunext sitemap` | Generate `public/sitemap.xml` from your pages, using the `sitemap` options in `bunext.config.ts`. See [Sitemap](#sitemap). | +| `bunext sitemap` | Generate `public/sitemap.xml` and `public/robots.txt` from your pages, using the `sitemap` and `robots` options in `bunext.config.ts`. See [Sitemap](#sitemap). | ### Running the CLI @@ -694,16 +696,37 @@ export default config; ``` ```bash -bunext sitemap # writes public/sitemap.xml -bunext sitemap -o public/sitemaps/main.xml # custom output path +bunext sitemap # writes public/sitemap.xml and public/robots.txt +bunext sitemap -o public/sitemaps/main.xml # custom sitemap output path bunext sitemap --origin https://staging.example.com +bunext sitemap --no-robots # sitemap only ``` -`--output` and `--origin` override `sitemap.output` and `sitemap.origin` from the config. +#### Large Sitemaps + +A sitemap file may hold at most 50,000 URLs and 50MB. When the sitemap has more than `chunk_size` URLs (default 10,000), `bunext sitemap` splits it into chunk files and makes `sitemap.xml` a sitemap index pointing to them: + +``` +public/ +├── sitemap.xml ← referencing the chunks +└── sitemap-chunks/ + ├── sitemap-1.xml ← up to chunk_size URLs each + ├── sitemap-2.xml + └── sitemap-3.xml +``` + +Each index entry's `lastmod` is the newest `lastmod` in its chunk. Chunk files left over from a previous, larger run are removed; other files in the chunks directory are left alone. If the sitemap shrinks back under `chunk_size`, `sitemap.xml` returns to a plain ``. + +| Config option | Type | Description | +| ------------- | -------- | --------------------------------------------------------------------------------------------------- | +| `chunk_size` | `number` | Maximum URLs per file. Default `10000`, capped at `50000`. Chunks are also split at 50MB. | +| `chunks_dir` | `string` | Chunk directory relative to the project root. Must be inside `public/`. Default `public/sitemap-chunks`. | + +`--output` and `--origin` override `sitemap.output` and `sitemap.origin` from the config. `--origin` also applies to robots.txt. The command also writes a `robots.txt` — see [robots.txt](#robotstxt). ### Generating at Runtime -`bunextGenerateSitemap()` resolves to the same XML as a string, so you can serve it from a route instead of a file: +`bunextGenerateSitemap()` resolves to the same XML as a single `` (it does not chunk, and logs a warning above 50,000 URLs — use `bunext sitemap` for large sites), so you can serve it from a route instead of a file: ```ts // src/pages/api/sitemap.ts @@ -738,7 +761,51 @@ export default async function handler(req: Request) { | `default_changefreq` | `BunextSitemapChangeFreq` | `changefreq` for entries that do not set one. | | `default_priority` | `number` | `priority` (0.0 to 1.0) for entries that do not set one. | -The `sitemap` config key takes the same options, plus `output`: the file path relative to the project root (default `public/sitemap.xml`). +The `sitemap` config key takes the same options, plus `output` (the file path relative to the project root, default `public/sitemap.xml`) and the [chunking options](#large-sitemaps). + +### robots.txt + +`bunext sitemap` also writes `public/robots.txt`, served at `/robots.txt`. With no `robots` config it allows every crawler and points to the generated sitemap: + +``` +# Generated by bunext + +User-agent: * +Allow: / + +Sitemap: https://example.com/sitemap.xml +``` + +Configure it with the `robots` key: + +```ts +// bunext.config.ts +const config: BunextConfig = { + origin: "https://example.com", + robots: { + rules: [ + { user_agent: "*", disallow: ["/admin", "/api/"] }, + { user_agent: ["GPTBot", "CCBot"], disallow: ["/"] }, + ], + // Block everything on staging + disallow_all: process.env.STAGING === "true", + }, +}; +``` + +| Option | Type | Description | +| -------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `rules` | `BunextRobotsRule[]` | Groups of `user_agent` (string or array), `allow`, `disallow` and `crawl_delay`. Default: allow everything for `*`. | +| `sitemaps` | `string[]` | Extra `Sitemap:` URLs or paths. The generated sitemap is added automatically when it is inside `public/`. | +| `disallow_all` | `boolean` | Block every crawler from the whole site. `rules` and `sitemaps` are ignored. | +| `origin` | `string` | Base URL for relative `sitemaps` paths. Defaults to `--origin`, then `sitemap.origin`, then `config.origin`. | +| `output` | `string` | Output file relative to the project root. Default `public/robots.txt`. | + +- **Existing files are protected.** Generated files start with `# Generated by bunext`. Without a `robots` config, a `robots.txt` that lacks this line (one you wrote yourself) is left alone and a warning is shown. Setting `robots` in the config overwrites it. +- **Opting out.** Set `robots: false`, or pass `--no-robots`, to skip robots.txt entirely. +- **Sitemap `exclude` does not add `Disallow` lines.** Leaving a page out of the sitemap and blocking crawlers from it are different things; add `disallow` rules explicitly. Remember robots.txt is public, so `Disallow` paths are visible to anyone. + +To serve it from a route instead, `bunext.bunextGenerateRobotsTxt()` takes the same options except `output` and returns the text. --- @@ -883,6 +950,7 @@ export default config; | `websocket` | `WebSocketHandler` | — | Bun WebSocket handler — see [WebSocket](#websocket) | | `serverOptions` | `ServeOptions` | — | Extra options passed to `Bun.serve()` (excluding `fetch`) — see [Server Options](#server-options) | | `sitemap` | `BunextSitemapConfig` | — | Options for `bunext sitemap` — see [Sitemap](#sitemap) | +| `robots` | `BunextRobotsConfig \| false` | — | robots.txt options for `bunext sitemap`, or `false` to skip it — see [robots.txt](#robotstxt) | ### Middleware diff --git a/dist/commands/sitemap/index.js b/dist/commands/sitemap/index.js index 03fad2c..602f16f 100644 --- a/dist/commands/sitemap/index.js +++ b/dist/commands/sitemap/index.js @@ -1,39 +1,128 @@ import { Command } from "commander"; -import { mkdirSync, writeFileSync } from "fs"; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "fs"; import path from "path"; import { log } from "../../utils/log"; import grabDirNames from "../../utils/grab-dir-names"; import grabConfig from "../../functions/grab-config"; import grabConstants from "../../utils/grab-constants"; -import generateSitemap from "../../utils/generate-sitemap"; +import { generateSitemapFiles } from "../../utils/generate-sitemap"; +import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt"; export default function () { return new Command("sitemap") - .description("Generate sitemap.xml from the pages directory") + .description("Generate sitemap.xml and robots.txt from the pages directory") .option("-o, --output ", "Output file relative to the project root (default: public/sitemap.xml)") .option("--origin ", "Base URL for sitemap links") + .option("--no-robots", "Skip generating robots.txt") .action(async (options) => { const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); global.BUNEXT_CONFIG = (await grabConfig()) || {}; global.BUNEXT_CONSTANTS = grabConstants(); - const { output, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {}; + const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {}; + let sitemap_file; try { - const xml = await generateSitemap({ - ...sitemap_config, - origin: options.origin || sitemap_config.origin, - }); const output_path = options.output || output; - const output_file = output_path + sitemap_file = output_path ? path.resolve(ROOT_DIR, output_path) : path.join(PUBLIC_DIR, "sitemap.xml"); - mkdirSync(path.dirname(output_file), { recursive: true }); - writeFileSync(output_file, xml, "utf-8"); - const url_count = xml.match(//g)?.length || 0; - log.success(`Sitemap with ${url_count} URLs written to ${path.relative(ROOT_DIR, output_file)}`); + const chunks_dir_path = path.resolve(ROOT_DIR, chunks_dir || "public/sitemap-chunks"); + const chunks_public_path = grabPublicUrlPath(chunks_dir_path); + if (chunks_public_path === undefined) { + throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` + + `so the chunks are served.`); + } + const { sitemap, chunks, url_count } = await generateSitemapFiles({ + ...sitemap_config, + origin: options.origin || sitemap_config.origin, + chunk_size, + chunks_url_path: chunks_public_path, + }); + // Remove chunks from a previous, larger sitemap + if (existsSync(chunks_dir_path)) { + for (const file of readdirSync(chunks_dir_path)) { + if (file.match(/^sitemap-\d+\.xml$/)) { + rmSync(path.join(chunks_dir_path, file)); + } + } + } + if (chunks.length) { + mkdirSync(chunks_dir_path, { recursive: true }); + } + for (const chunk of chunks) { + writeFileSync(path.join(chunks_dir_path, chunk.file_name), chunk.xml, "utf-8"); + } + mkdirSync(path.dirname(sitemap_file), { recursive: true }); + writeFileSync(sitemap_file, sitemap, "utf-8"); + const sitemap_relative_path = path.relative(ROOT_DIR, sitemap_file); + if (chunks.length) { + log.success(`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` + + `written to ${sitemap_relative_path} ` + + `(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`); + } + else { + log.success(`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`); + } } catch (error) { log.error(`Failed to generate sitemap: ${error?.message || error}`); process.exit(1); } + const robots_config = global.BUNEXT_CONFIG.robots; + if (options.robots && robots_config !== false) { + writeRobotsTxt({ + sitemap_file, + cli_origin: options.origin, + sitemap_origin: sitemap_config.origin, + }); + } process.exit(); }); } +function writeRobotsTxt({ sitemap_file, cli_origin, sitemap_origin, }) { + const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); + const robots_config = global.BUNEXT_CONFIG.robots || undefined; + const { output, ...robots_params } = robots_config || {}; + const robots_file = output + ? path.resolve(ROOT_DIR, output) + : path.join(PUBLIC_DIR, "robots.txt"); + const robots_relative_path = path.relative(ROOT_DIR, robots_file); + // Without a robots config, never replace a hand-written robots.txt. + if (!robots_config && + existsSync(robots_file) && + !readFileSync(robots_file, "utf-8").startsWith(RobotsTxtGeneratedMarker)) { + log.warn(`Skipped ${robots_relative_path}: it was not generated by bunext. ` + + `Set \`robots\` in bunext.config.ts to overwrite it, ` + + `or \`robots: false\` to silence this.`); + return; + } + const sitemap_public_path = grabPublicUrlPath(sitemap_file); + if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) { + log.warn(`Sitemap is outside public/, so robots.txt has no Sitemap line. ` + + `Add its URL to \`robots.sitemaps\`.`); + } + const robots_txt = generateRobotsTxt({ + ...robots_params, + origin: cli_origin || robots_params.origin || sitemap_origin, + sitemaps: [ + ...(sitemap_public_path !== undefined + ? [sitemap_public_path] + : []), + ...(robots_params.sitemaps || []), + ], + }); + mkdirSync(path.dirname(robots_file), { recursive: true }); + writeFileSync(robots_file, robots_txt, "utf-8"); + log.success(`robots.txt written to ${robots_relative_path}`); +} +/** + * # URL path a file in public/ is served at + * Public files are served from the site root, e.g. public/sitemap.xml + * at /sitemap.xml. Returns `undefined` for paths outside public/. + */ +function grabPublicUrlPath(file_path) { + const { PUBLIC_DIR } = grabDirNames(); + const relative_path = path.relative(PUBLIC_DIR, file_path); + if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) { + return; + } + return `/${relative_path.split(path.sep).join("/")}`; +} diff --git a/dist/index.d.ts b/dist/index.d.ts index 013eb82..f5b663d 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,5 +1,6 @@ import bunextInit from "./functions/bunext-init"; import bunextRequestHandler from "./functions/server/bunext-req-handler"; +import generateRobotsTxt from "./utils/generate-robots-txt"; import generateSitemap from "./utils/generate-sitemap"; declare const bunext: { bunextRequestHandler: typeof bunextRequestHandler; @@ -15,5 +16,6 @@ declare const bunext: { }; bunextInit: typeof bunextInit; bunextGenerateSitemap: typeof generateSitemap; + bunextGenerateRobotsTxt: typeof generateRobotsTxt; }; export default bunext; diff --git a/dist/index.js b/dist/index.js index a0ffe6f..ee4efbf 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,5 +1,6 @@ import bunextInit from "./functions/bunext-init"; import bunextRequestHandler from "./functions/server/bunext-req-handler"; +import generateRobotsTxt from "./utils/generate-robots-txt"; import generateSitemap from "./utils/generate-sitemap"; import { log } from "./utils/log"; const bunext = { @@ -7,5 +8,6 @@ const bunext = { bunextLog: log, bunextInit, bunextGenerateSitemap: generateSitemap, + bunextGenerateRobotsTxt: generateRobotsTxt, }; export default bunext; diff --git a/dist/types/index.d.ts b/dist/types/index.d.ts index 0d501a2..2216995 100644 --- a/dist/types/index.d.ts +++ b/dist/types/index.d.ts @@ -97,6 +97,11 @@ export type BunextConfig = { * Options for the `bunext sitemap` command */ sitemap?: BunextSitemapConfig; + /** + * Options for the robots.txt written by `bunext sitemap`. Set to + * `false` to skip generating it. + */ + robots?: BunextRobotsConfig | false; }; export type BunextConfigMiddlewareParams = { req: Request; @@ -445,9 +450,61 @@ export type BunextGenerateSitemapParams = { default_priority?: number; }; export type BunextSitemapConfig = BunextGenerateSitemapParams & { + /** + * Maximum URLs per sitemap file. Above this, `bunext sitemap` writes + * chunk files to `chunks_dir` and makes sitemap.xml a sitemap index + * referencing them. Defaults to 10,000; capped at 50,000, the protocol + * limit. Chunks are also split at 50MB. + */ + chunk_size?: number; + /** + * Directory for sitemap chunk files, relative to the project root. + * Must be inside `public/` so the files are served. Defaults to + * `public/sitemap-chunks`. + */ + chunks_dir?: string; /** * Output file, relative to the project root. Defaults to * `public/sitemap.xml`, which is served at `/sitemap.xml`. */ output?: string; }; +export type BunextRobotsRule = { + /** + * One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]` + */ + user_agent: string | string[]; + allow?: string[]; + disallow?: string[]; + /** + * Seconds between requests. Not supported by every crawler. + */ + crawl_delay?: number; +}; +export type BunextGenerateRobotsTxtParams = { + /** + * Base URL used to turn relative `sitemaps` paths into absolute URLs. + * Defaults to `config.origin`, falling back to `http://localhost:`. + */ + origin?: string; + /** + * Defaults to allowing every crawler everywhere. + */ + rules?: BunextRobotsRule[]; + /** + * Sitemap URLs or paths, each written as a `Sitemap:` line. + */ + sitemaps?: string[]; + /** + * Block every crawler from the whole site, ignoring `rules` and + * `sitemaps`. Useful for staging deployments. + */ + disallow_all?: boolean; +}; +export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & { + /** + * Output file, relative to the project root. Defaults to + * `public/robots.txt`, which is served at `/robots.txt`. + */ + output?: string; +}; diff --git a/dist/utils/generate-robots-txt.d.ts b/dist/utils/generate-robots-txt.d.ts new file mode 100644 index 0000000..e20d342 --- /dev/null +++ b/dist/utils/generate-robots-txt.d.ts @@ -0,0 +1,11 @@ +import type { BunextGenerateRobotsTxtParams } from "../types"; +/** + * First line of every generated robots.txt. `bunext sitemap` only + * overwrites an existing robots.txt that starts with it, unless robots + * is configured explicitly. + */ +export declare const RobotsTxtGeneratedMarker = "# Generated by bunext"; +/** + * # Generate a robots.txt string + */ +export default function generateRobotsTxt(params?: BunextGenerateRobotsTxtParams): string; diff --git a/dist/utils/generate-robots-txt.js b/dist/utils/generate-robots-txt.js new file mode 100644 index 0000000..c413065 --- /dev/null +++ b/dist/utils/generate-robots-txt.js @@ -0,0 +1,57 @@ +import grabOrigin from "./grab-origin"; +/** + * First line of every generated robots.txt. `bunext sitemap` only + * overwrites an existing robots.txt that starts with it, unless robots + * is configured explicitly. + */ +export const RobotsTxtGeneratedMarker = "# Generated by bunext"; +/** + * # Generate a robots.txt string + */ +export default function generateRobotsTxt(params) { + const lines = [RobotsTxtGeneratedMarker, ""]; + if (params?.disallow_all) { + lines.push("User-agent: *", "Disallow: /", ""); + return lines.join("\n"); + } + const rules = params?.rules?.length + ? params.rules + : [{ user_agent: "*", allow: ["/"] }]; + for (const rule of rules) { + const user_agents = [rule.user_agent].flat(); + for (const user_agent of user_agents) { + lines.push(`User-agent: ${sanitizeValue(user_agent)}`); + } + for (const allow of rule.allow || []) { + lines.push(`Allow: ${sanitizeValue(allow)}`); + } + for (const disallow of rule.disallow || []) { + lines.push(`Disallow: ${sanitizeValue(disallow)}`); + } + // A group needs at least one rule; an empty Disallow allows all. + if (!rule.allow?.length && !rule.disallow?.length) { + lines.push("Disallow:"); + } + if (typeof rule.crawl_delay == "number") { + lines.push(`Crawl-delay: ${rule.crawl_delay}`); + } + lines.push(""); + } + if (params?.sitemaps?.length) { + const origin = (params.origin || grabOrigin()).replace(/\/+$/, ""); + for (const sitemap of new Set(params.sitemaps)) { + const url = sitemap.match(/^https?:\/\//) + ? sitemap + : `${origin}/${sitemap.replace(/^\/+/, "")}`; + lines.push(`Sitemap: ${sanitizeValue(url)}`); + } + lines.push(""); + } + return lines.join("\n"); +} +/** + * Line breaks would let a value inject extra directives. + */ +function sanitizeValue(value) { + return value.replace(/[\r\n]+/g, " ").trim(); +} diff --git a/dist/utils/generate-sitemap.d.ts b/dist/utils/generate-sitemap.d.ts index 59ec60d..41d5749 100644 --- a/dist/utils/generate-sitemap.d.ts +++ b/dist/utils/generate-sitemap.d.ts @@ -1,8 +1,39 @@ import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types"; +/** + * Protocol limits for a single sitemap file + * https://www.sitemaps.org/protocol.html#index + */ +export declare const SitemapMaxUrlsPerFile = 50000; +export declare const SitemapMaxBytesPerFile: number; +export declare const SitemapDefaultChunkSize = 10000; +export type SitemapChunkFile = { + file_name: string; + xml: string; + url_count: number; +}; +export type SitemapFiles = { + /** + * A `` when everything fits in one file, otherwise a + * `` referencing `chunks`. + */ + sitemap: string; + chunks: SitemapChunkFile[]; + url_count: number; +}; /** * # Generate a sitemap.xml string for all pages * Dynamic routes are only included through a `sitemap` export in their - * `.server.ts` file, or through `additional_paths`. + * `.server.ts` file, or through `additional_paths`. Always returns a + * single ``; `bunext sitemap` splits large sitemaps into chunks. */ export default function generateSitemap(params?: BunextGenerateSitemapParams): Promise; +/** + * # Generate sitemap.xml, split into chunk files when needed + * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path` + * is the public URL path the chunk files are served from. + */ +export declare function generateSitemapFiles({ chunk_size, chunks_url_path, ...params }: BunextGenerateSitemapParams & { + chunk_size?: number; + chunks_url_path: string; +}): Promise; export declare function grabSitemapEntries(params?: BunextGenerateSitemapParams): Promise; diff --git a/dist/utils/generate-sitemap.js b/dist/utils/generate-sitemap.js index 210a291..20f92ee 100644 --- a/dist/utils/generate-sitemap.js +++ b/dist/utils/generate-sitemap.js @@ -6,43 +6,130 @@ import grabDirNames from "./grab-dir-names"; import grabOrigin from "./grab-origin"; import grabRelativePath from "./grab-relative-path"; import isDevelopment from "./is-development"; +import { log } from "./log"; const ErrorPagePaths = ["/404", "/500"]; +/** + * Protocol limits for a single sitemap file + * https://www.sitemaps.org/protocol.html#index + */ +export const SitemapMaxUrlsPerFile = 50_000; +export const SitemapMaxBytesPerFile = 50 * 1024 * 1024; +export const SitemapDefaultChunkSize = 10_000; +const SitemapXmlHeader = ``; +const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`; /** * # Generate a sitemap.xml string for all pages * Dynamic routes are only included through a `sitemap` export in their - * `.server.ts` file, or through `additional_paths`. + * `.server.ts` file, or through `additional_paths`. Always returns a + * single ``; `bunext sitemap` splits large sitemaps into chunks. */ export default async function generateSitemap(params) { const entries = await grabSitemapEntries(params); - const origin = (params?.origin || grabOrigin()).replace(/\/+$/, ""); - const urls = entries.map((entry) => { - const loc = entry.path.match(/^https?:\/\//) - ? entry.path - : `${origin}${entry.path}`; - const lines = [` ${escapeXml(loc)}`]; - if (entry.lastmod) { - const lastmod = entry.lastmod instanceof Date - ? entry.lastmod.toISOString() - : entry.lastmod; - lines.push(` ${escapeXml(lastmod)}`); + const origin = grabSitemapOrigin(params?.origin); + if (entries.length > SitemapMaxUrlsPerFile) { + log.warn(`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` + + `per-file limit. Use \`bunext sitemap\` to split it into chunks.`); + } + return renderUrlset(entries.map((entry) => renderUrl(entry, origin))); +} +/** + * # Generate sitemap.xml, split into chunk files when needed + * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path` + * is the public URL path the chunk files are served from. + */ +export async function generateSitemapFiles({ chunk_size = SitemapDefaultChunkSize, chunks_url_path, ...params }) { + const entries = await grabSitemapEntries(params); + const origin = grabSitemapOrigin(params.origin); + const max_urls = Math.min(Math.max(Math.floor(chunk_size) || 1, 1), SitemapMaxUrlsPerFile); + const urls = entries.map((entry) => renderUrl(entry, origin)); + const empty_urlset_bytes = Buffer.byteLength(renderUrlset([])); + const chunks = []; + let chunk; + for (let i = 0; i < urls.length; i++) { + const url_bytes = Buffer.byteLength(urls[i]) + 1; + if (!chunk || + chunk.urls.length >= max_urls || + chunk.bytes + url_bytes > SitemapMaxBytesPerFile) { + chunk = { urls: [], bytes: empty_urlset_bytes }; + chunks.push(chunk); } - if (entry.changefreq) { - lines.push(` ${entry.changefreq}`); + chunk.urls.push(urls[i]); + chunk.bytes += url_bytes; + const lastmod = parseLastmod(entries[i].lastmod); + if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) { + chunk.lastmod = lastmod; } - if (typeof entry.priority == "number") { - const priority = Math.min(Math.max(entry.priority, 0), 1); - lines.push(` ${priority.toFixed(1)}`); - } - return ` \n${lines.join("\n")}\n `; - }); + } + if (chunks.length <= 1) { + return { + sitemap: renderUrlset(chunks[0]?.urls || []), + chunks: [], + url_count: urls.length, + }; + } + const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, ""); + const chunk_base_url = chunks_dir_path + ? `${origin}/${chunks_dir_path}` + : origin; + const chunk_files = chunks.map((c, i) => ({ + file_name: `sitemap-${i + 1}.xml`, + xml: renderUrlset(c.urls), + url_count: c.urls.length, + })); + const sitemap = [ + SitemapXmlHeader, + ``, + ...chunk_files.map((file, i) => { + const loc = escapeXml(`${chunk_base_url}/${file.file_name}`); + const lines = [` ${loc}`]; + const lastmod = chunks[i].lastmod?.toISOString(); + if (lastmod) + lines.push(` ${lastmod}`); + return ` \n${lines.join("\n")}\n `; + }), + ``, + ``, + ].join("\n"); + return { sitemap, chunks: chunk_files, url_count: urls.length }; +} +function grabSitemapOrigin(origin) { + return (origin || grabOrigin()).replace(/\/+$/, ""); +} +function renderUrlset(urls) { return [ - ``, - ``, + SitemapXmlHeader, + ``, ...urls, ``, ``, ].join("\n"); } +function renderUrl(entry, origin) { + const loc = entry.path.match(/^https?:\/\//) + ? entry.path + : `${origin}${entry.path}`; + const lines = [` ${escapeXml(loc)}`]; + if (entry.lastmod) { + const lastmod = entry.lastmod instanceof Date + ? entry.lastmod.toISOString() + : entry.lastmod; + lines.push(` ${escapeXml(lastmod)}`); + } + if (entry.changefreq) { + lines.push(` ${entry.changefreq}`); + } + if (typeof entry.priority == "number") { + const priority = Math.min(Math.max(entry.priority, 0), 1); + lines.push(` ${priority.toFixed(1)}`); + } + return ` \n${lines.join("\n")}\n `; +} +function parseLastmod(lastmod) { + if (!lastmod) + return; + const date = lastmod instanceof Date ? lastmod : new Date(lastmod); + return isNaN(date.getTime()) ? undefined : date; +} export async function grabSitemapEntries(params) { const use_file_lastmod = params?.use_file_lastmod ?? true; const entries = []; diff --git a/package.json b/package.json index 9cbbd96..2859ea5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@moduletrace/bunext", - "version": "1.1.21", + "version": "1.1.22", "main": "dist/index.js", "module": "index.ts", "dependencies": { diff --git a/publish.sh b/publish.sh index 95cc39a..20c9eae 100755 --- a/publish.sh +++ b/publish.sh @@ -2,14 +2,15 @@ set -e +tsc --noEmit +bun run test + if [ -z "$1" ]; then msg="Updates" else msg="$1" fi -tsc --noEmit - rm -rf dist tsc diff --git a/src/__tests__/commands/sitemap.test.ts b/src/__tests__/commands/sitemap.test.ts index d20e914..f64b540 100644 --- a/src/__tests__/commands/sitemap.test.ts +++ b/src/__tests__/commands/sitemap.test.ts @@ -41,6 +41,14 @@ function readOutput(file = "public/sitemap.xml") { return fs.readFileSync(path.join(projectDir, file), "utf-8"); } +function readRobots(file = "public/robots.txt") { + return fs.readFileSync(path.join(projectDir, file), "utf-8"); +} + +function exists(file: string) { + return fs.existsSync(path.join(projectDir, file)); +} + function locs(xml: string) { return [...xml.matchAll(/(.*?)<\/loc>/g)].map((m) => m[1]).sort(); } @@ -221,4 +229,300 @@ describe("bunext sitemap", () => { expect(res.exit_code).toBe(1); expect(res.output).toContain("api down"); }); + + describe("robots.txt", () => { + it("writes an allow-all robots.txt pointing at the sitemap by default", async () => { + writeConfig(`{ origin: "https://example.com" }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(res.output).toContain("robots.txt written to public/robots.txt"); + expect(readRobots()).toBe( + [ + "# Generated by bunext", + "", + "User-agent: *", + "Allow: /", + "", + "Sitemap: https://example.com/sitemap.xml", + "", + ].join("\n"), + ); + }); + + it("applies the robots config and writes to robots.output", async () => { + writeConfig(`{ + origin: "https://example.com", + robots: { + output: "out/robots.txt", + rules: [ + { user_agent: "*", disallow: ["/admin"] }, + { user_agent: "GPTBot", disallow: ["/"] }, + ], + sitemaps: ["/sitemaps/extra.xml"], + }, + }`); + + const res = await runSitemap(); + const robots = readRobots("out/robots.txt"); + + expect(res.exit_code).toBe(0); + expect(exists("public/robots.txt")).toBe(false); + expect(robots).toContain("User-agent: *\nDisallow: /admin\n"); + expect(robots).toContain("User-agent: GPTBot\nDisallow: /\n"); + expect(robots).toContain("Sitemap: https://example.com/sitemap.xml"); + expect(robots).toContain( + "Sitemap: https://example.com/sitemaps/extra.xml", + ); + }); + + it("points at a custom sitemap output inside public/", async () => { + writeConfig(`{ + origin: "https://example.com", + sitemap: { output: "public/maps/site.xml" }, + }`); + + await runSitemap(); + + expect(readRobots()).toContain( + "Sitemap: https://example.com/maps/site.xml", + ); + }); + + it("warns and omits the Sitemap line when the sitemap is outside public/", async () => { + writeConfig(`{ origin: "https://example.com" }`); + + const res = await runSitemap("-o", "dist/sitemap.xml"); + + expect(res.exit_code).toBe(0); + expect(res.output).toContain("outside public/"); + expect(readRobots()).not.toContain("Sitemap:"); + }); + + it("uses --origin over robots.origin and sitemap.origin", async () => { + writeConfig(`{ + sitemap: { origin: "https://sitemap.dev" }, + robots: { origin: "https://robots.dev" }, + }`); + + await runSitemap("--origin", "https://cli.dev"); + + expect(readRobots()).toContain("Sitemap: https://cli.dev/sitemap.xml"); + }); + + it("uses robots.origin over sitemap.origin", async () => { + writeConfig(`{ + sitemap: { origin: "https://sitemap.dev" }, + robots: { origin: "https://robots.dev" }, + }`); + + await runSitemap(); + + expect(readRobots()).toContain( + "Sitemap: https://robots.dev/sitemap.xml", + ); + }); + + it("blocks all crawlers with disallow_all", async () => { + writeConfig(`{ robots: { disallow_all: true } }`); + + await runSitemap(); + + expect(readRobots()).toBe( + "# Generated by bunext\n\nUser-agent: *\nDisallow: /\n", + ); + }); + + it("keeps a hand-written robots.txt when robots is not configured", async () => { + writeConfig(`{ origin: "https://example.com" }`); + writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n"); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(res.output).toContain("not generated by bunext"); + expect(readRobots()).toBe("User-agent: *\nDisallow: /x\n"); + }); + + it("overwrites a hand-written robots.txt when robots is configured", async () => { + writeConfig(`{ robots: { rules: [{ user_agent: "*", allow: ["/"] }] } }`); + writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n"); + + await runSitemap(); + + expect(readRobots()).toStartWith("# Generated by bunext"); + }); + + it("regenerates its own robots.txt without a robots config", async () => { + writeConfig(`{ origin: "https://example.com" }`); + writeProjectFile( + "public/robots.txt", + "# Generated by bunext\n\nUser-agent: *\nDisallow: /old\n", + ); + + await runSitemap(); + + expect(readRobots()).not.toContain("/old"); + expect(readRobots()).toContain("Allow: /"); + }); + + it("skips robots.txt with robots: false", async () => { + writeConfig(`{ robots: false }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(exists("public/sitemap.xml")).toBe(true); + expect(exists("public/robots.txt")).toBe(false); + expect(res.output).not.toContain("robots.txt"); + }); + + it("skips robots.txt with --no-robots", async () => { + writeConfig(`{ robots: { disallow_all: true } }`); + + await runSitemap("--no-robots"); + + expect(exists("public/robots.txt")).toBe(false); + }); + + it("does not write robots.txt when sitemap generation fails", async () => { + writeConfig(`{ + sitemap: { + additional_paths: async () => { throw new Error("api down"); }, + }, + }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(1); + expect(exists("public/robots.txt")).toBe(false); + }); + }); + + describe("chunking", () => { + const itemsServerFile = (count: number) => + `export const sitemap = () => Array.from({ length: ${count} }, (_, i) => "/items/" + i);`; + + const chunkFiles = (dir = "public/sitemap-chunks") => + exists(dir) + ? fs + .readdirSync(path.join(projectDir, dir)) + .sort((a, b) => a.localeCompare(b, undefined, { numeric: true })) + : []; + + beforeEach(() => { + writeProjectFile("src/pages/items/[id].tsx", "export default () => null"); + }); + + it("writes chunks and a sitemap index above chunk_size", async () => { + writeConfig(`{ + origin: "https://example.com", + sitemap: { chunk_size: 1000 }, + }`); + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500)); + + const res = await runSitemap(); + const index = readOutput(); + + expect(res.exit_code).toBe(0); + expect(res.output).toContain("2503 URLs in 3 chunks"); + expect(chunkFiles()).toEqual([ + "sitemap-1.xml", + "sitemap-2.xml", + "sitemap-3.xml", + ]); + expect(index).toContain(" + locs(readOutput(`public/sitemap-chunks/${f}`)), + ); + expect(all_locs).toHaveLength(2503); + expect(new Set(all_locs).size).toBe(2503); + expect(readRobots()).toContain( + "Sitemap: https://example.com/sitemap.xml", + ); + }); + + it("chunks 20,000+ URLs with the default chunk size", async () => { + writeConfig(`{ origin: "https://example.com" }`); + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(25_000)); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(chunkFiles()).toEqual([ + "sitemap-1.xml", + "sitemap-2.xml", + "sitemap-3.xml", + ]); + expect( + locs(readOutput("public/sitemap-chunks/sitemap-3.xml")), + ).toHaveLength(5003); + }); + + it("removes stale chunks but keeps other files in the chunks dir", async () => { + writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`); + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(3500)); + writeProjectFile("public/sitemap-chunks/keep-me.txt", "mine"); + + await runSitemap(); + expect(chunkFiles()).toHaveLength(5); + + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500)); + await runSitemap(); + + expect(chunkFiles()).toEqual([ + "keep-me.txt", + "sitemap-1.xml", + "sitemap-2.xml", + ]); + }); + + it("goes back to a single sitemap and removes chunks when it shrinks", async () => { + writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`); + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500)); + await runSitemap(); + + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(10)); + const res = await runSitemap(); + + expect(res.output).toContain("Sitemap with 13 URLs"); + expect(readOutput()).toContain(" { + writeConfig(`{ + origin: "https://example.com", + sitemap: { chunk_size: 1000, chunks_dir: "public/seo/maps" }, + }`); + writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500)); + + await runSitemap(); + + expect(chunkFiles("public/seo/maps")).toEqual([ + "sitemap-1.xml", + "sitemap-2.xml", + ]); + expect(locs(readOutput())).toContain( + "https://example.com/seo/maps/sitemap-1.xml", + ); + }); + + it("fails when chunks_dir is outside public/", async () => { + writeConfig(`{ sitemap: { chunks_dir: "dist/chunks" } }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(1); + expect(res.output).toContain("must be inside public/"); + expect(exists("public/sitemap.xml")).toBe(false); + }); + }); }); diff --git a/src/__tests__/utils/generate-robots-txt.test.ts b/src/__tests__/utils/generate-robots-txt.test.ts new file mode 100644 index 0000000..6b114ba --- /dev/null +++ b/src/__tests__/utils/generate-robots-txt.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, beforeEach } from "bun:test"; +import generateRobotsTxt, { + RobotsTxtGeneratedMarker, +} from "../../utils/generate-robots-txt"; + +beforeEach(() => { + (global as any).BUNEXT_CONFIG = { origin: "https://example.com" }; +}); + +describe("generateRobotsTxt", () => { + it("allows everything by default, starting with the marker", () => { + expect(generateRobotsTxt()).toBe( + `${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nAllow: /\n`, + ); + }); + + it("renders rule groups with multiple user agents and crawl delay", () => { + const txt = generateRobotsTxt({ + rules: [ + { + user_agent: "*", + allow: ["/"], + disallow: ["/admin", "/api/"], + crawl_delay: 5, + }, + { user_agent: ["GPTBot", "CCBot"], disallow: ["/"] }, + ], + }); + + expect(txt).toContain( + "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /api/\nCrawl-delay: 5\n", + ); + expect(txt).toContain("User-agent: GPTBot\nUser-agent: CCBot\nDisallow: /\n"); + }); + + it("writes an empty Disallow for a rule with no paths", () => { + const txt = generateRobotsTxt({ rules: [{ user_agent: "Bingbot" }] }); + expect(txt).toContain("User-agent: Bingbot\nDisallow:\n"); + }); + + it("resolves sitemap paths against the origin and dedupes them", () => { + const txt = generateRobotsTxt({ + sitemaps: [ + "/sitemap.xml", + "sitemaps/blog.xml", + "https://cdn.example.com/sitemap.xml", + "/sitemap.xml", + ], + }); + + expect(txt.match(/^Sitemap: .*$/gm)).toEqual([ + "Sitemap: https://example.com/sitemap.xml", + "Sitemap: https://example.com/sitemaps/blog.xml", + "Sitemap: https://cdn.example.com/sitemap.xml", + ]); + }); + + it("uses the origin param over config and strips its trailing slash", () => { + const txt = generateRobotsTxt({ + origin: "https://other.dev/", + sitemaps: ["/sitemap.xml"], + }); + expect(txt).toContain("Sitemap: https://other.dev/sitemap.xml"); + }); + + it("blocks everything with disallow_all, ignoring rules and sitemaps", () => { + const txt = generateRobotsTxt({ + disallow_all: true, + rules: [{ user_agent: "*", allow: ["/"] }], + sitemaps: ["/sitemap.xml"], + }); + + expect(txt).toBe( + `${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nDisallow: /\n`, + ); + }); + + it("strips line breaks so values cannot inject directives", () => { + const txt = generateRobotsTxt({ + rules: [{ user_agent: "*", disallow: ["/a\nAllow: /secret"] }], + }); + + expect(txt).not.toContain("\nAllow: /secret"); + expect(txt).toContain("Disallow: /a Allow: /secret"); + }); +}); diff --git a/src/__tests__/utils/generate-sitemap.test.ts b/src/__tests__/utils/generate-sitemap.test.ts index 3bcc4d0..9057e4f 100644 --- a/src/__tests__/utils/generate-sitemap.test.ts +++ b/src/__tests__/utils/generate-sitemap.test.ts @@ -3,7 +3,9 @@ import path from "path"; import fs from "fs"; import os from "os"; import generateSitemap, { + generateSitemapFiles, grabSitemapEntries, + SitemapMaxUrlsPerFile, } from "../../utils/generate-sitemap"; // grabAllPages skips anything under __tests__, so fixtures live in the OS tmp dir @@ -178,3 +180,101 @@ describe("generateSitemap", () => { expect(xml).toContain("https://other.dev/about"); }); }); + +describe("generateSitemapFiles", () => { + const manyPaths = (count: number) => + Array.from({ length: count }, (_, i) => `/items/${i}`); + + const countUrls = (xml: string) => xml.match(//g)?.length || 0; + + it("returns a single urlset with no chunks when within chunk_size", async () => { + const { sitemap, chunks, url_count } = await generateSitemapFiles({ + chunks_url_path: "/sitemap-chunks", + use_file_lastmod: false, + }); + + expect(chunks).toEqual([]); + expect(sitemap).toContain(" { + const { sitemap, chunks, url_count } = await generateSitemapFiles({ + chunks_url_path: "/sitemap-chunks", + chunk_size: 1000, + exclude: [/^(?!\/items\/)/], + additional_paths: manyPaths(2500), + use_file_lastmod: false, + }); + + expect(url_count).toBe(2500); + expect(chunks.map((c) => c.file_name)).toEqual([ + "sitemap-1.xml", + "sitemap-2.xml", + "sitemap-3.xml", + ]); + expect(chunks.map((c) => c.url_count)).toEqual([1000, 1000, 500]); + expect(chunks.map((c) => countUrls(c.xml))).toEqual([1000, 1000, 500]); + expect(chunks[0].xml).toContain("https://example.com/items/0"); + expect(chunks[2].xml).toContain( + "https://example.com/items/2499", + ); + + expect(sitemap).toContain(""); + expect(sitemap.match(/.*?<\/loc>/g)).toEqual([ + "https://example.com/sitemap-chunks/sitemap-1.xml", + "https://example.com/sitemap-chunks/sitemap-2.xml", + "https://example.com/sitemap-chunks/sitemap-3.xml", + ]); + }); + + it("sets each index entry's lastmod to the newest in its chunk", async () => { + const { sitemap } = await generateSitemapFiles({ + chunks_url_path: "/maps/", + origin: "https://other.dev/", + chunk_size: 2, + exclude: [/^(?!\/items\/)/], + use_file_lastmod: false, + additional_paths: [ + { path: "/items/a", lastmod: "2026-01-01" }, + { path: "/items/b", lastmod: new Date("2026-03-01") }, + { path: "/items/c" }, + { path: "/items/d", lastmod: "not a date" }, + ], + }); + + expect(sitemap).toContain( + "https://other.dev/maps/sitemap-1.xml\n 2026-03-01T00:00:00.000Z", + ); + expect(sitemap).toContain( + "https://other.dev/maps/sitemap-2.xml\n ", + ); + }); + + it("caps chunk_size at the protocol limit", async () => { + const { chunks } = await generateSitemapFiles({ + chunks_url_path: "/sitemap-chunks", + chunk_size: 1_000_000, + exclude: [/^(?!\/items\/)/], + additional_paths: manyPaths(SitemapMaxUrlsPerFile + 1), + use_file_lastmod: false, + }); + + expect(chunks.map((c) => c.url_count)).toEqual([ + SitemapMaxUrlsPerFile, + 1, + ]); + }); + + it("defaults to 10,000 URLs per chunk", async () => { + const { chunks } = await generateSitemapFiles({ + chunks_url_path: "/sitemap-chunks", + exclude: [/^(?!\/items\/)/], + additional_paths: manyPaths(20_001), + use_file_lastmod: false, + }); + + expect(chunks.map((c) => c.url_count)).toEqual([10_000, 10_000, 1]); + }); +}); diff --git a/src/commands/sitemap/index.ts b/src/commands/sitemap/index.ts index 3bcdd3e..033e358 100644 --- a/src/commands/sitemap/index.ts +++ b/src/commands/sitemap/index.ts @@ -1,56 +1,117 @@ import { Command } from "commander"; -import { mkdirSync, writeFileSync } from "fs"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; import path from "path"; import { log } from "../../utils/log"; import grabDirNames from "../../utils/grab-dir-names"; import grabConfig from "../../functions/grab-config"; import grabConstants from "../../utils/grab-constants"; -import generateSitemap from "../../utils/generate-sitemap"; +import { generateSitemapFiles } from "../../utils/generate-sitemap"; +import generateRobotsTxt, { + RobotsTxtGeneratedMarker, +} from "../../utils/generate-robots-txt"; type Options = { output?: string; origin?: string; + robots: boolean; }; export default function () { return new Command("sitemap") - .description("Generate sitemap.xml from the pages directory") + .description( + "Generate sitemap.xml and robots.txt from the pages directory", + ) .option( "-o, --output ", "Output file relative to the project root (default: public/sitemap.xml)", ) .option("--origin ", "Base URL for sitemap links") + .option("--no-robots", "Skip generating robots.txt") .action(async (options: Options) => { const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); global.BUNEXT_CONFIG = (await grabConfig()) || {}; global.BUNEXT_CONSTANTS = grabConstants(); - const { output, ...sitemap_config } = + const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {}; - try { - const xml = await generateSitemap({ - ...sitemap_config, - origin: options.origin || sitemap_config.origin, - }); + let sitemap_file: string; + try { const output_path = options.output || output; - const output_file = output_path + sitemap_file = output_path ? path.resolve(ROOT_DIR, output_path) : path.join(PUBLIC_DIR, "sitemap.xml"); - mkdirSync(path.dirname(output_file), { recursive: true }); - writeFileSync(output_file, xml, "utf-8"); - - const url_count = xml.match(//g)?.length || 0; - - log.success( - `Sitemap with ${url_count} URLs written to ${path.relative( - ROOT_DIR, - output_file, - )}`, + const chunks_dir_path = path.resolve( + ROOT_DIR, + chunks_dir || "public/sitemap-chunks", ); + const chunks_public_path = grabPublicUrlPath(chunks_dir_path); + + if (chunks_public_path === undefined) { + throw new Error( + `\`sitemap.chunks_dir\` must be inside public/ ` + + `so the chunks are served.`, + ); + } + + const { sitemap, chunks, url_count } = + await generateSitemapFiles({ + ...sitemap_config, + origin: options.origin || sitemap_config.origin, + chunk_size, + chunks_url_path: chunks_public_path, + }); + + // Remove chunks from a previous, larger sitemap + if (existsSync(chunks_dir_path)) { + for (const file of readdirSync(chunks_dir_path)) { + if (file.match(/^sitemap-\d+\.xml$/)) { + rmSync(path.join(chunks_dir_path, file)); + } + } + } + + if (chunks.length) { + mkdirSync(chunks_dir_path, { recursive: true }); + } + + for (const chunk of chunks) { + writeFileSync( + path.join(chunks_dir_path, chunk.file_name), + chunk.xml, + "utf-8", + ); + } + + mkdirSync(path.dirname(sitemap_file), { recursive: true }); + writeFileSync(sitemap_file, sitemap, "utf-8"); + + const sitemap_relative_path = path.relative( + ROOT_DIR, + sitemap_file, + ); + + if (chunks.length) { + log.success( + `Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` + + `written to ${sitemap_relative_path} ` + + `(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`, + ); + } else { + log.success( + `Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`, + ); + } } catch (error: any) { log.error( `Failed to generate sitemap: ${error?.message || error}`, @@ -58,6 +119,92 @@ export default function () { process.exit(1); } + const robots_config = global.BUNEXT_CONFIG.robots; + + if (options.robots && robots_config !== false) { + writeRobotsTxt({ + sitemap_file, + cli_origin: options.origin, + sitemap_origin: sitemap_config.origin, + }); + } + process.exit(); }); } + +function writeRobotsTxt({ + sitemap_file, + cli_origin, + sitemap_origin, +}: { + sitemap_file: string; + cli_origin?: string; + sitemap_origin?: string; +}) { + const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); + const robots_config = global.BUNEXT_CONFIG.robots || undefined; + const { output, ...robots_params } = robots_config || {}; + + const robots_file = output + ? path.resolve(ROOT_DIR, output) + : path.join(PUBLIC_DIR, "robots.txt"); + const robots_relative_path = path.relative(ROOT_DIR, robots_file); + + // Without a robots config, never replace a hand-written robots.txt. + if ( + !robots_config && + existsSync(robots_file) && + !readFileSync(robots_file, "utf-8").startsWith( + RobotsTxtGeneratedMarker, + ) + ) { + log.warn( + `Skipped ${robots_relative_path}: it was not generated by bunext. ` + + `Set \`robots\` in bunext.config.ts to overwrite it, ` + + `or \`robots: false\` to silence this.`, + ); + return; + } + + const sitemap_public_path = grabPublicUrlPath(sitemap_file); + + if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) { + log.warn( + `Sitemap is outside public/, so robots.txt has no Sitemap line. ` + + `Add its URL to \`robots.sitemaps\`.`, + ); + } + + const robots_txt = generateRobotsTxt({ + ...robots_params, + origin: cli_origin || robots_params.origin || sitemap_origin, + sitemaps: [ + ...(sitemap_public_path !== undefined + ? [sitemap_public_path] + : []), + ...(robots_params.sitemaps || []), + ], + }); + + mkdirSync(path.dirname(robots_file), { recursive: true }); + writeFileSync(robots_file, robots_txt, "utf-8"); + + log.success(`robots.txt written to ${robots_relative_path}`); +} + +/** + * # URL path a file in public/ is served at + * Public files are served from the site root, e.g. public/sitemap.xml + * at /sitemap.xml. Returns `undefined` for paths outside public/. + */ +function grabPublicUrlPath(file_path: string) { + const { PUBLIC_DIR } = grabDirNames(); + const relative_path = path.relative(PUBLIC_DIR, file_path); + + if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) { + return; + } + + return `/${relative_path.split(path.sep).join("/")}`; +} diff --git a/src/index.ts b/src/index.ts index 8df9995..8a21888 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ import bunextInit from "./functions/bunext-init"; import bunextRequestHandler from "./functions/server/bunext-req-handler"; +import generateRobotsTxt from "./utils/generate-robots-txt"; import generateSitemap from "./utils/generate-sitemap"; import { log } from "./utils/log"; @@ -8,5 +9,6 @@ const bunext = { bunextLog: log, bunextInit, bunextGenerateSitemap: generateSitemap, + bunextGenerateRobotsTxt: generateRobotsTxt, }; export default bunext; diff --git a/src/types/index.ts b/src/types/index.ts index d86a43c..8f89972 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -114,6 +114,11 @@ export type BunextConfig = { * Options for the `bunext sitemap` command */ sitemap?: BunextSitemapConfig; + /** + * Options for the robots.txt written by `bunext sitemap`. Set to + * `false` to skip generating it. + */ + robots?: BunextRobotsConfig | false; }; export type BunextConfigMiddlewareParams = { @@ -531,9 +536,64 @@ export type BunextGenerateSitemapParams = { }; export type BunextSitemapConfig = BunextGenerateSitemapParams & { + /** + * Maximum URLs per sitemap file. Above this, `bunext sitemap` writes + * chunk files to `chunks_dir` and makes sitemap.xml a sitemap index + * referencing them. Defaults to 10,000; capped at 50,000, the protocol + * limit. Chunks are also split at 50MB. + */ + chunk_size?: number; + /** + * Directory for sitemap chunk files, relative to the project root. + * Must be inside `public/` so the files are served. Defaults to + * `public/sitemap-chunks`. + */ + chunks_dir?: string; /** * Output file, relative to the project root. Defaults to * `public/sitemap.xml`, which is served at `/sitemap.xml`. */ output?: string; }; + +export type BunextRobotsRule = { + /** + * One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]` + */ + user_agent: string | string[]; + allow?: string[]; + disallow?: string[]; + /** + * Seconds between requests. Not supported by every crawler. + */ + crawl_delay?: number; +}; + +export type BunextGenerateRobotsTxtParams = { + /** + * Base URL used to turn relative `sitemaps` paths into absolute URLs. + * Defaults to `config.origin`, falling back to `http://localhost:`. + */ + origin?: string; + /** + * Defaults to allowing every crawler everywhere. + */ + rules?: BunextRobotsRule[]; + /** + * Sitemap URLs or paths, each written as a `Sitemap:` line. + */ + sitemaps?: string[]; + /** + * Block every crawler from the whole site, ignoring `rules` and + * `sitemaps`. Useful for staging deployments. + */ + disallow_all?: boolean; +}; + +export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & { + /** + * Output file, relative to the project root. Defaults to + * `public/robots.txt`, which is served at `/robots.txt`. + */ + output?: string; +}; diff --git a/src/utils/generate-robots-txt.ts b/src/utils/generate-robots-txt.ts new file mode 100644 index 0000000..9d0d2b9 --- /dev/null +++ b/src/utils/generate-robots-txt.ts @@ -0,0 +1,77 @@ +import type { BunextGenerateRobotsTxtParams } from "../types"; +import grabOrigin from "./grab-origin"; + +/** + * First line of every generated robots.txt. `bunext sitemap` only + * overwrites an existing robots.txt that starts with it, unless robots + * is configured explicitly. + */ +export const RobotsTxtGeneratedMarker = "# Generated by bunext"; + +/** + * # Generate a robots.txt string + */ +export default function generateRobotsTxt( + params?: BunextGenerateRobotsTxtParams, +): string { + const lines = [RobotsTxtGeneratedMarker, ""]; + + if (params?.disallow_all) { + lines.push("User-agent: *", "Disallow: /", ""); + return lines.join("\n"); + } + + const rules = params?.rules?.length + ? params.rules + : [{ user_agent: "*", allow: ["/"] }]; + + for (const rule of rules) { + const user_agents = [rule.user_agent].flat(); + + for (const user_agent of user_agents) { + lines.push(`User-agent: ${sanitizeValue(user_agent)}`); + } + + for (const allow of rule.allow || []) { + lines.push(`Allow: ${sanitizeValue(allow)}`); + } + + for (const disallow of rule.disallow || []) { + lines.push(`Disallow: ${sanitizeValue(disallow)}`); + } + + // A group needs at least one rule; an empty Disallow allows all. + if (!rule.allow?.length && !rule.disallow?.length) { + lines.push("Disallow:"); + } + + if (typeof rule.crawl_delay == "number") { + lines.push(`Crawl-delay: ${rule.crawl_delay}`); + } + + lines.push(""); + } + + if (params?.sitemaps?.length) { + const origin = (params.origin || grabOrigin()).replace(/\/+$/, ""); + + for (const sitemap of new Set(params.sitemaps)) { + const url = sitemap.match(/^https?:\/\//) + ? sitemap + : `${origin}/${sitemap.replace(/^\/+/, "")}`; + + lines.push(`Sitemap: ${sanitizeValue(url)}`); + } + + lines.push(""); + } + + return lines.join("\n"); +} + +/** + * Line breaks would let a value inject extra directives. + */ +function sanitizeValue(value: string) { + return value.replace(/[\r\n]+/g, " ").trim(); +} diff --git a/src/utils/generate-sitemap.ts b/src/utils/generate-sitemap.ts index 1d808bf..970c869 100644 --- a/src/utils/generate-sitemap.ts +++ b/src/utils/generate-sitemap.ts @@ -12,56 +12,192 @@ import grabDirNames from "./grab-dir-names"; import grabOrigin from "./grab-origin"; import grabRelativePath from "./grab-relative-path"; import isDevelopment from "./is-development"; +import { log } from "./log"; const ErrorPagePaths = ["/404", "/500"]; +/** + * Protocol limits for a single sitemap file + * https://www.sitemaps.org/protocol.html#index + */ +export const SitemapMaxUrlsPerFile = 50_000; +export const SitemapMaxBytesPerFile = 50 * 1024 * 1024; +export const SitemapDefaultChunkSize = 10_000; + +const SitemapXmlHeader = ``; +const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`; + +export type SitemapChunkFile = { + file_name: string; + xml: string; + url_count: number; +}; + +export type SitemapFiles = { + /** + * A `` when everything fits in one file, otherwise a + * `` referencing `chunks`. + */ + sitemap: string; + chunks: SitemapChunkFile[]; + url_count: number; +}; + /** * # Generate a sitemap.xml string for all pages * Dynamic routes are only included through a `sitemap` export in their - * `.server.ts` file, or through `additional_paths`. + * `.server.ts` file, or through `additional_paths`. Always returns a + * single ``; `bunext sitemap` splits large sitemaps into chunks. */ export default async function generateSitemap( params?: BunextGenerateSitemapParams, ): Promise { const entries = await grabSitemapEntries(params); - const origin = (params?.origin || grabOrigin()).replace(/\/+$/, ""); + const origin = grabSitemapOrigin(params?.origin); - const urls = entries.map((entry) => { - const loc = entry.path.match(/^https?:\/\//) - ? entry.path - : `${origin}${entry.path}`; + if (entries.length > SitemapMaxUrlsPerFile) { + log.warn( + `Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` + + `per-file limit. Use \`bunext sitemap\` to split it into chunks.`, + ); + } - const lines = [` ${escapeXml(loc)}`]; + return renderUrlset(entries.map((entry) => renderUrl(entry, origin))); +} - if (entry.lastmod) { - const lastmod = - entry.lastmod instanceof Date - ? entry.lastmod.toISOString() - : entry.lastmod; - lines.push(` ${escapeXml(lastmod)}`); +/** + * # Generate sitemap.xml, split into chunk files when needed + * Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path` + * is the public URL path the chunk files are served from. + */ +export async function generateSitemapFiles({ + chunk_size = SitemapDefaultChunkSize, + chunks_url_path, + ...params +}: BunextGenerateSitemapParams & { + chunk_size?: number; + chunks_url_path: string; +}): Promise { + const entries = await grabSitemapEntries(params); + const origin = grabSitemapOrigin(params.origin); + const max_urls = Math.min( + Math.max(Math.floor(chunk_size) || 1, 1), + SitemapMaxUrlsPerFile, + ); + + const urls = entries.map((entry) => renderUrl(entry, origin)); + const empty_urlset_bytes = Buffer.byteLength(renderUrlset([])); + + type Chunk = { urls: string[]; bytes: number; lastmod?: Date }; + const chunks: Chunk[] = []; + let chunk: Chunk | undefined; + + for (let i = 0; i < urls.length; i++) { + const url_bytes = Buffer.byteLength(urls[i]) + 1; + + if ( + !chunk || + chunk.urls.length >= max_urls || + chunk.bytes + url_bytes > SitemapMaxBytesPerFile + ) { + chunk = { urls: [], bytes: empty_urlset_bytes }; + chunks.push(chunk); } - if (entry.changefreq) { - lines.push(` ${entry.changefreq}`); + chunk.urls.push(urls[i]); + chunk.bytes += url_bytes; + + const lastmod = parseLastmod(entries[i].lastmod); + if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) { + chunk.lastmod = lastmod; } + } - if (typeof entry.priority == "number") { - const priority = Math.min(Math.max(entry.priority, 0), 1); - lines.push(` ${priority.toFixed(1)}`); - } + if (chunks.length <= 1) { + return { + sitemap: renderUrlset(chunks[0]?.urls || []), + chunks: [], + url_count: urls.length, + }; + } - return ` \n${lines.join("\n")}\n `; - }); + const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, ""); + const chunk_base_url = chunks_dir_path + ? `${origin}/${chunks_dir_path}` + : origin; + const chunk_files: SitemapChunkFile[] = chunks.map((c, i) => ({ + file_name: `sitemap-${i + 1}.xml`, + xml: renderUrlset(c.urls), + url_count: c.urls.length, + })); + + const sitemap = [ + SitemapXmlHeader, + ``, + ...chunk_files.map((file, i) => { + const loc = escapeXml(`${chunk_base_url}/${file.file_name}`); + const lines = [` ${loc}`]; + + const lastmod = chunks[i].lastmod?.toISOString(); + if (lastmod) lines.push(` ${lastmod}`); + + return ` \n${lines.join("\n")}\n `; + }), + ``, + ``, + ].join("\n"); + + return { sitemap, chunks: chunk_files, url_count: urls.length }; +} + +function grabSitemapOrigin(origin?: string) { + return (origin || grabOrigin()).replace(/\/+$/, ""); +} + +function renderUrlset(urls: string[]) { return [ - ``, - ``, + SitemapXmlHeader, + ``, ...urls, ``, ``, ].join("\n"); } +function renderUrl(entry: BunextSitemapEntry, origin: string) { + const loc = entry.path.match(/^https?:\/\//) + ? entry.path + : `${origin}${entry.path}`; + + const lines = [` ${escapeXml(loc)}`]; + + if (entry.lastmod) { + const lastmod = + entry.lastmod instanceof Date + ? entry.lastmod.toISOString() + : entry.lastmod; + lines.push(` ${escapeXml(lastmod)}`); + } + + if (entry.changefreq) { + lines.push(` ${entry.changefreq}`); + } + + if (typeof entry.priority == "number") { + const priority = Math.min(Math.max(entry.priority, 0), 1); + lines.push(` ${priority.toFixed(1)}`); + } + + return ` \n${lines.join("\n")}\n `; +} + +function parseLastmod(lastmod?: Date | string) { + if (!lastmod) return; + const date = lastmod instanceof Date ? lastmod : new Date(lastmod); + return isNaN(date.getTime()) ? undefined : date; +} + export async function grabSitemapEntries( params?: BunextGenerateSitemapParams, ): Promise {