Moduletrace/bunext
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add robots.txt generation and sitemap chunking

Browse Source 
- bunext sitemap now writes public/robots.txt (configurable via the
  robots config key; robots: false or --no-robots to skip). Hand-written
  robots.txt files are never overwritten without a robots config
- Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml
  (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with
  sitemap.xml as a sitemap index; stale chunks are cleaned up
- Export bunextGenerateRobotsTxt
- Run tests in publish.sh

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-14 18:02:52 +01:00
co-authored by Claude Opus 5.5
parent 1169729b06
commit be4bebaa22
20 changed files with 1406 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/commands/sitemap/index.js
Vendored
+102 -13
View File
@@ -1,39 +1,128 @@
import { Command } from "commander";
import { mkdirSync, writeFileSync } from "fs";
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "fs";
import path from "path";
import { log } from "../../utils/log";
import grabDirNames from "../../utils/grab-dir-names";
import grabConfig from "../../functions/grab-config";
import grabConstants from "../../utils/grab-constants";
import generateSitemap from "../../utils/generate-sitemap";
import { generateSitemapFiles } from "../../utils/generate-sitemap";
import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
export default function () {
return new Command("sitemap")
.description("Generate sitemap.xml from the pages directory")
.description("Generate sitemap.xml and robots.txt from the pages directory")
.option("-o, --output <path>", "Output file relative to the project root (default: public/sitemap.xml)")
.option("--origin <url>", "Base URL for sitemap links")
.option("--no-robots", "Skip generating robots.txt")
.action(async (options) => {
const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
global.BUNEXT_CONFIG = (await grabConfig()) || {};
global.BUNEXT_CONSTANTS = grabConstants();
const { output, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
let sitemap_file;
try {
const xml = await generateSitemap({
...sitemap_config,
origin: options.origin || sitemap_config.origin,
});
const output_path = options.output || output;
const output_file = output_path
sitemap_file = output_path
? path.resolve(ROOT_DIR, output_path)
: path.join(PUBLIC_DIR, "sitemap.xml");
mkdirSync(path.dirname(output_file), { recursive: true });
writeFileSync(output_file, xml, "utf-8");
const url_count = xml.match(/<url>/g)?.length || 0;
log.success(`Sitemap with ${url_count} URLs written to ${path.relative(ROOT_DIR, output_file)}`);
const chunks_dir_path = path.resolve(ROOT_DIR, chunks_dir || "public/sitemap-chunks");
const chunks_public_path = grabPublicUrlPath(chunks_dir_path);
if (chunks_public_path === undefined) {
throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` +
`so the chunks are served.`);
}
const { sitemap, chunks, url_count } = await generateSitemapFiles({
...sitemap_config,
origin: options.origin || sitemap_config.origin,
chunk_size,
chunks_url_path: chunks_public_path,
});
// Remove chunks from a previous, larger sitemap
if (existsSync(chunks_dir_path)) {
for (const file of readdirSync(chunks_dir_path)) {
if (file.match(/^sitemap-\d+\.xml$/)) {
rmSync(path.join(chunks_dir_path, file));
}
}
}
if (chunks.length) {
mkdirSync(chunks_dir_path, { recursive: true });
}
for (const chunk of chunks) {
writeFileSync(path.join(chunks_dir_path, chunk.file_name), chunk.xml, "utf-8");
}
mkdirSync(path.dirname(sitemap_file), { recursive: true });
writeFileSync(sitemap_file, sitemap, "utf-8");
const sitemap_relative_path = path.relative(ROOT_DIR, sitemap_file);
if (chunks.length) {
log.success(`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` +
`written to ${sitemap_relative_path} ` +
`(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`);
}
else {
log.success(`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`);
}
}
catch (error) {
log.error(`Failed to generate sitemap: ${error?.message || error}`);
process.exit(1);
}
const robots_config = global.BUNEXT_CONFIG.robots;
if (options.robots && robots_config !== false) {
writeRobotsTxt({
sitemap_file,
cli_origin: options.origin,
sitemap_origin: sitemap_config.origin,
});
}
process.exit();
});
}
function writeRobotsTxt({ sitemap_file, cli_origin, sitemap_origin, }) {
const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
const robots_config = global.BUNEXT_CONFIG.robots || undefined;
const { output, ...robots_params } = robots_config || {};
const robots_file = output
? path.resolve(ROOT_DIR, output)
: path.join(PUBLIC_DIR, "robots.txt");
const robots_relative_path = path.relative(ROOT_DIR, robots_file);
// Without a robots config, never replace a hand-written robots.txt.
if (!robots_config &&
existsSync(robots_file) &&
!readFileSync(robots_file, "utf-8").startsWith(RobotsTxtGeneratedMarker)) {
log.warn(`Skipped ${robots_relative_path}: it was not generated by bunext. ` +
`Set \`robots\` in bunext.config.ts to overwrite it, ` +
`or \`robots: false\` to silence this.`);
return;
}
const sitemap_public_path = grabPublicUrlPath(sitemap_file);
if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) {
log.warn(`Sitemap is outside public/, so robots.txt has no Sitemap line. ` +
`Add its URL to \`robots.sitemaps\`.`);
}
const robots_txt = generateRobotsTxt({
...robots_params,
origin: cli_origin || robots_params.origin || sitemap_origin,
sitemaps: [
...(sitemap_public_path !== undefined
? [sitemap_public_path]
: []),
...(robots_params.sitemaps || []),
],
});
mkdirSync(path.dirname(robots_file), { recursive: true });
writeFileSync(robots_file, robots_txt, "utf-8");
log.success(`robots.txt written to ${robots_relative_path}`);
}
/**
* # URL path a file in public/ is served at
* Public files are served from the site root, e.g. public/sitemap.xml
* at /sitemap.xml. Returns `undefined` for paths outside public/.
*/
function grabPublicUrlPath(file_path) {
const { PUBLIC_DIR } = grabDirNames();
const relative_path = path.relative(PUBLIC_DIR, file_path);
if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) {
return;
}
return `/${relative_path.split(path.sep).join("/")}`;
}
dist/index.d.ts
Vendored
+2
View File
@@ -1,5 +1,6 @@
import bunextInit from "./functions/bunext-init";
import bunextRequestHandler from "./functions/server/bunext-req-handler";
import generateRobotsTxt from "./utils/generate-robots-txt";
import generateSitemap from "./utils/generate-sitemap";
declare const bunext: {
bunextRequestHandler: typeof bunextRequestHandler;
@@ -15,5 +16,6 @@ declare const bunext: {
};
bunextInit: typeof bunextInit;
bunextGenerateSitemap: typeof generateSitemap;
bunextGenerateRobotsTxt: typeof generateRobotsTxt;
};
export default bunext;
dist/index.js
Vendored
+2
View File
@@ -1,5 +1,6 @@
import bunextInit from "./functions/bunext-init";
import bunextRequestHandler from "./functions/server/bunext-req-handler";
import generateRobotsTxt from "./utils/generate-robots-txt";
import generateSitemap from "./utils/generate-sitemap";
import { log } from "./utils/log";
const bunext = {
@@ -7,5 +8,6 @@ const bunext = {
bunextLog: log,
bunextInit,
bunextGenerateSitemap: generateSitemap,
bunextGenerateRobotsTxt: generateRobotsTxt,
};
export default bunext;
dist/types/index.d.ts
Vendored
+57
View File
@@ -97,6 +97,11 @@ export type BunextConfig = {
* Options for the `bunext sitemap` command
*/
sitemap?: BunextSitemapConfig;
/**
* Options for the robots.txt written by `bunext sitemap`. Set to
* `false` to skip generating it.
*/
robots?: BunextRobotsConfig | false;
};
export type BunextConfigMiddlewareParams = {
req: Request;
@@ -445,9 +450,61 @@ export type BunextGenerateSitemapParams = {
default_priority?: number;
};
export type BunextSitemapConfig = BunextGenerateSitemapParams & {
/**
* Maximum URLs per sitemap file. Above this, `bunext sitemap` writes
* chunk files to `chunks_dir` and makes sitemap.xml a sitemap index
* referencing them. Defaults to 10,000; capped at 50,000, the protocol
* limit. Chunks are also split at 50MB.
*/
chunk_size?: number;
/**
* Directory for sitemap chunk files, relative to the project root.
* Must be inside `public/` so the files are served. Defaults to
* `public/sitemap-chunks`.
*/
chunks_dir?: string;
/**
* Output file, relative to the project root. Defaults to
* `public/sitemap.xml`, which is served at `/sitemap.xml`.
*/
output?: string;
};
export type BunextRobotsRule = {
/**
* One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]`
*/
user_agent: string | string[];
allow?: string[];
disallow?: string[];
/**
* Seconds between requests. Not supported by every crawler.
*/
crawl_delay?: number;
};
export type BunextGenerateRobotsTxtParams = {
/**
* Base URL used to turn relative `sitemaps` paths into absolute URLs.
* Defaults to `config.origin`, falling back to `http://localhost:<port>`.
*/
origin?: string;
/**
* Defaults to allowing every crawler everywhere.
*/
rules?: BunextRobotsRule[];
/**
* Sitemap URLs or paths, each written as a `Sitemap:` line.
*/
sitemaps?: string[];
/**
* Block every crawler from the whole site, ignoring `rules` and
* `sitemaps`. Useful for staging deployments.
*/
disallow_all?: boolean;
};
export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & {
/**
* Output file, relative to the project root. Defaults to
* `public/robots.txt`, which is served at `/robots.txt`.
*/
output?: string;
};
dist/utils/generate-robots-txt.d.ts
Vendored
+11
View File
@@ -0,0 +1,11 @@
import type { BunextGenerateRobotsTxtParams } from "../types";
/**
* First line of every generated robots.txt. `bunext sitemap` only
* overwrites an existing robots.txt that starts with it, unless robots
* is configured explicitly.
*/
export declare const RobotsTxtGeneratedMarker = "# Generated by bunext";
/**
* # Generate a robots.txt string
*/
export default function generateRobotsTxt(params?: BunextGenerateRobotsTxtParams): string;
dist/utils/generate-robots-txt.js
Vendored
+57
View File
@@ -0,0 +1,57 @@
import grabOrigin from "./grab-origin";
/**
* First line of every generated robots.txt. `bunext sitemap` only
* overwrites an existing robots.txt that starts with it, unless robots
* is configured explicitly.
*/
export const RobotsTxtGeneratedMarker = "# Generated by bunext";
/**
* # Generate a robots.txt string
*/
export default function generateRobotsTxt(params) {
const lines = [RobotsTxtGeneratedMarker, ""];
if (params?.disallow_all) {
lines.push("User-agent: *", "Disallow: /", "");
return lines.join("\n");
}
const rules = params?.rules?.length
? params.rules
: [{ user_agent: "*", allow: ["/"] }];
for (const rule of rules) {
const user_agents = [rule.user_agent].flat();
for (const user_agent of user_agents) {
lines.push(`User-agent: ${sanitizeValue(user_agent)}`);
}
for (const allow of rule.allow || []) {
lines.push(`Allow: ${sanitizeValue(allow)}`);
}
for (const disallow of rule.disallow || []) {
lines.push(`Disallow: ${sanitizeValue(disallow)}`);
}
// A group needs at least one rule; an empty Disallow allows all.
if (!rule.allow?.length && !rule.disallow?.length) {
lines.push("Disallow:");
}
if (typeof rule.crawl_delay == "number") {
lines.push(`Crawl-delay: ${rule.crawl_delay}`);
}
lines.push("");
}
if (params?.sitemaps?.length) {
const origin = (params.origin || grabOrigin()).replace(/\/+$/, "");
for (const sitemap of new Set(params.sitemaps)) {
const url = sitemap.match(/^https?:\/\//)
? sitemap
: `${origin}/${sitemap.replace(/^\/+/, "")}`;
lines.push(`Sitemap: ${sanitizeValue(url)}`);
}
lines.push("");
}
return lines.join("\n");
}
/**
* Line breaks would let a value inject extra directives.
*/
function sanitizeValue(value) {
return value.replace(/[\r\n]+/g, " ").trim();
}
dist/utils/generate-sitemap.d.ts
Vendored
+32 -1
View File
@@ -1,8 +1,39 @@
import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
/**
* Protocol limits for a single sitemap file
* https://www.sitemaps.org/protocol.html#index
*/
export declare const SitemapMaxUrlsPerFile = 50000;
export declare const SitemapMaxBytesPerFile: number;
export declare const SitemapDefaultChunkSize = 10000;
export type SitemapChunkFile = {
file_name: string;
xml: string;
url_count: number;
};
export type SitemapFiles = {
/**
* A `<urlset>` when everything fits in one file, otherwise a
* `<sitemapindex>` referencing `chunks`.
*/
sitemap: string;
chunks: SitemapChunkFile[];
url_count: number;
};
/**
* # Generate a sitemap.xml string for all pages
* Dynamic routes are only included through a `sitemap` export in their
* `.server.ts` file, or through `additional_paths`.
* `.server.ts` file, or through `additional_paths`. Always returns a
* single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
*/
export default function generateSitemap(params?: BunextGenerateSitemapParams): Promise<string>;
/**
* # Generate sitemap.xml, split into chunk files when needed
* Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
* is the public URL path the chunk files are served from.
*/
export declare function generateSitemapFiles({ chunk_size, chunks_url_path, ...params }: BunextGenerateSitemapParams & {
chunk_size?: number;
chunks_url_path: string;
}): Promise<SitemapFiles>;
export declare function grabSitemapEntries(params?: BunextGenerateSitemapParams): Promise<BunextSitemapEntry[]>;
dist/utils/generate-sitemap.js
Vendored
+109 -22
View File
@@ -6,43 +6,130 @@ import grabDirNames from "./grab-dir-names";
import grabOrigin from "./grab-origin";
import grabRelativePath from "./grab-relative-path";
import isDevelopment from "./is-development";
import { log } from "./log";
const ErrorPagePaths = ["/404", "/500"];
/**
* Protocol limits for a single sitemap file
* https://www.sitemaps.org/protocol.html#index
*/
export const SitemapMaxUrlsPerFile = 50_000;
export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
export const SitemapDefaultChunkSize = 10_000;
const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
/**
* # Generate a sitemap.xml string for all pages
* Dynamic routes are only included through a `sitemap` export in their
* `.server.ts` file, or through `additional_paths`.
* `.server.ts` file, or through `additional_paths`. Always returns a
* single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
*/
export default async function generateSitemap(params) {
const entries = await grabSitemapEntries(params);
const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
const urls = entries.map((entry) => {
const loc = entry.path.match(/^https?:\/\//)
? entry.path
: `${origin}${entry.path}`;
const lines = [` <loc>${escapeXml(loc)}</loc>`];
if (entry.lastmod) {
const lastmod = entry.lastmod instanceof Date
? entry.lastmod.toISOString()
: entry.lastmod;
lines.push(` <lastmod>${escapeXml(lastmod)}</lastmod>`);
const origin = grabSitemapOrigin(params?.origin);
if (entries.length > SitemapMaxUrlsPerFile) {
log.warn(`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
`per-file limit. Use \`bunext sitemap\` to split it into chunks.`);
}
return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
}
/**
* # Generate sitemap.xml, split into chunk files when needed
* Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
* is the public URL path the chunk files are served from.
*/
export async function generateSitemapFiles({ chunk_size = SitemapDefaultChunkSize, chunks_url_path, ...params }) {
const entries = await grabSitemapEntries(params);
const origin = grabSitemapOrigin(params.origin);
const max_urls = Math.min(Math.max(Math.floor(chunk_size) || 1, 1), SitemapMaxUrlsPerFile);
const urls = entries.map((entry) => renderUrl(entry, origin));
const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
const chunks = [];
let chunk;
for (let i = 0; i < urls.length; i++) {
const url_bytes = Buffer.byteLength(urls[i]) + 1;
if (!chunk ||
chunk.urls.length >= max_urls ||
chunk.bytes + url_bytes > SitemapMaxBytesPerFile) {
chunk = { urls: [], bytes: empty_urlset_bytes };
chunks.push(chunk);
}
if (entry.changefreq) {
lines.push(` <changefreq>${entry.changefreq}</changefreq>`);
chunk.urls.push(urls[i]);
chunk.bytes += url_bytes;
const lastmod = parseLastmod(entries[i].lastmod);
if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
chunk.lastmod = lastmod;
}
if (typeof entry.priority == "number") {
const priority = Math.min(Math.max(entry.priority, 0), 1);
lines.push(` <priority>${priority.toFixed(1)}</priority>`);
}
return ` <url>\n${lines.join("\n")}\n </url>`;
});
}
if (chunks.length <= 1) {
return {
sitemap: renderUrlset(chunks[0]?.urls || []),
chunks: [],
url_count: urls.length,
};
}
const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
const chunk_base_url = chunks_dir_path
? `${origin}/${chunks_dir_path}`
: origin;
const chunk_files = chunks.map((c, i) => ({
file_name: `sitemap-${i + 1}.xml`,
xml: renderUrlset(c.urls),
url_count: c.urls.length,
}));
const sitemap = [
SitemapXmlHeader,
`<sitemapindex ${SitemapXmlns}>`,
...chunk_files.map((file, i) => {
const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
const lines = [` <loc>${loc}</loc>`];
const lastmod = chunks[i].lastmod?.toISOString();
if (lastmod)
lines.push(` <lastmod>${lastmod}</lastmod>`);
return ` <sitemap>\n${lines.join("\n")}\n </sitemap>`;
}),
`</sitemapindex>`,
``,
].join("\n");
return { sitemap, chunks: chunk_files, url_count: urls.length };
}
function grabSitemapOrigin(origin) {
return (origin || grabOrigin()).replace(/\/+$/, "");
}
function renderUrlset(urls) {
return [
`<?xml version="1.0" encoding="UTF-8"?>`,
`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
SitemapXmlHeader,
`<urlset ${SitemapXmlns}>`,
...urls,
`</urlset>`,
``,
].join("\n");
}
function renderUrl(entry, origin) {
const loc = entry.path.match(/^https?:\/\//)
? entry.path
: `${origin}${entry.path}`;
const lines = [` <loc>${escapeXml(loc)}</loc>`];
if (entry.lastmod) {
const lastmod = entry.lastmod instanceof Date
? entry.lastmod.toISOString()
: entry.lastmod;
lines.push(` <lastmod>${escapeXml(lastmod)}</lastmod>`);
}
if (entry.changefreq) {
lines.push(` <changefreq>${entry.changefreq}</changefreq>`);
}
if (typeof entry.priority == "number") {
const priority = Math.min(Math.max(entry.priority, 0), 1);
lines.push(` <priority>${priority.toFixed(1)}</priority>`);
}
return ` <url>\n${lines.join("\n")}\n </url>`;
}
function parseLastmod(lastmod) {
if (!lastmod)
return;
const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
return isNaN(date.getTime()) ? undefined : date;
}
export async function grabSitemapEntries(params) {
const use_file_lastmod = params?.use_file_lastmod ?? true;
const entries = [];