Add robots.txt generation and sitemap chunking
- bunext sitemap now writes public/robots.txt (configurable via the robots config key; robots: false or --no-robots to skip). Hand-written robots.txt files are never overwritten without a robots config - Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with sitemap.xml as a sitemap index; stale chunks are cleaned up - Export bunextGenerateRobotsTxt - Run tests in publish.sh Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
Vendored
+102
-13
@@ -1,39 +1,128 @@
|
||||
import { Command } from "commander";
|
||||
import { mkdirSync, writeFileSync } from "fs";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "fs";
|
||||
import path from "path";
|
||||
import { log } from "../../utils/log";
|
||||
import grabDirNames from "../../utils/grab-dir-names";
|
||||
import grabConfig from "../../functions/grab-config";
|
||||
import grabConstants from "../../utils/grab-constants";
|
||||
import generateSitemap from "../../utils/generate-sitemap";
|
||||
import { generateSitemapFiles } from "../../utils/generate-sitemap";
|
||||
import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
|
||||
export default function () {
|
||||
return new Command("sitemap")
|
||||
.description("Generate sitemap.xml from the pages directory")
|
||||
.description("Generate sitemap.xml and robots.txt from the pages directory")
|
||||
.option("-o, --output <path>", "Output file relative to the project root (default: public/sitemap.xml)")
|
||||
.option("--origin <url>", "Base URL for sitemap links")
|
||||
.option("--no-robots", "Skip generating robots.txt")
|
||||
.action(async (options) => {
|
||||
const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
|
||||
global.BUNEXT_CONFIG = (await grabConfig()) || {};
|
||||
global.BUNEXT_CONSTANTS = grabConstants();
|
||||
const { output, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
|
||||
const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
|
||||
let sitemap_file;
|
||||
try {
|
||||
const xml = await generateSitemap({
|
||||
...sitemap_config,
|
||||
origin: options.origin || sitemap_config.origin,
|
||||
});
|
||||
const output_path = options.output || output;
|
||||
const output_file = output_path
|
||||
sitemap_file = output_path
|
||||
? path.resolve(ROOT_DIR, output_path)
|
||||
: path.join(PUBLIC_DIR, "sitemap.xml");
|
||||
mkdirSync(path.dirname(output_file), { recursive: true });
|
||||
writeFileSync(output_file, xml, "utf-8");
|
||||
const url_count = xml.match(/<url>/g)?.length || 0;
|
||||
log.success(`Sitemap with ${url_count} URLs written to ${path.relative(ROOT_DIR, output_file)}`);
|
||||
const chunks_dir_path = path.resolve(ROOT_DIR, chunks_dir || "public/sitemap-chunks");
|
||||
const chunks_public_path = grabPublicUrlPath(chunks_dir_path);
|
||||
if (chunks_public_path === undefined) {
|
||||
throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` +
|
||||
`so the chunks are served.`);
|
||||
}
|
||||
const { sitemap, chunks, url_count } = await generateSitemapFiles({
|
||||
...sitemap_config,
|
||||
origin: options.origin || sitemap_config.origin,
|
||||
chunk_size,
|
||||
chunks_url_path: chunks_public_path,
|
||||
});
|
||||
// Remove chunks from a previous, larger sitemap
|
||||
if (existsSync(chunks_dir_path)) {
|
||||
for (const file of readdirSync(chunks_dir_path)) {
|
||||
if (file.match(/^sitemap-\d+\.xml$/)) {
|
||||
rmSync(path.join(chunks_dir_path, file));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (chunks.length) {
|
||||
mkdirSync(chunks_dir_path, { recursive: true });
|
||||
}
|
||||
for (const chunk of chunks) {
|
||||
writeFileSync(path.join(chunks_dir_path, chunk.file_name), chunk.xml, "utf-8");
|
||||
}
|
||||
mkdirSync(path.dirname(sitemap_file), { recursive: true });
|
||||
writeFileSync(sitemap_file, sitemap, "utf-8");
|
||||
const sitemap_relative_path = path.relative(ROOT_DIR, sitemap_file);
|
||||
if (chunks.length) {
|
||||
log.success(`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` +
|
||||
`written to ${sitemap_relative_path} ` +
|
||||
`(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`);
|
||||
}
|
||||
else {
|
||||
log.success(`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`);
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
log.error(`Failed to generate sitemap: ${error?.message || error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const robots_config = global.BUNEXT_CONFIG.robots;
|
||||
if (options.robots && robots_config !== false) {
|
||||
writeRobotsTxt({
|
||||
sitemap_file,
|
||||
cli_origin: options.origin,
|
||||
sitemap_origin: sitemap_config.origin,
|
||||
});
|
||||
}
|
||||
process.exit();
|
||||
});
|
||||
}
|
||||
function writeRobotsTxt({ sitemap_file, cli_origin, sitemap_origin, }) {
|
||||
const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
|
||||
const robots_config = global.BUNEXT_CONFIG.robots || undefined;
|
||||
const { output, ...robots_params } = robots_config || {};
|
||||
const robots_file = output
|
||||
? path.resolve(ROOT_DIR, output)
|
||||
: path.join(PUBLIC_DIR, "robots.txt");
|
||||
const robots_relative_path = path.relative(ROOT_DIR, robots_file);
|
||||
// Without a robots config, never replace a hand-written robots.txt.
|
||||
if (!robots_config &&
|
||||
existsSync(robots_file) &&
|
||||
!readFileSync(robots_file, "utf-8").startsWith(RobotsTxtGeneratedMarker)) {
|
||||
log.warn(`Skipped ${robots_relative_path}: it was not generated by bunext. ` +
|
||||
`Set \`robots\` in bunext.config.ts to overwrite it, ` +
|
||||
`or \`robots: false\` to silence this.`);
|
||||
return;
|
||||
}
|
||||
const sitemap_public_path = grabPublicUrlPath(sitemap_file);
|
||||
if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) {
|
||||
log.warn(`Sitemap is outside public/, so robots.txt has no Sitemap line. ` +
|
||||
`Add its URL to \`robots.sitemaps\`.`);
|
||||
}
|
||||
const robots_txt = generateRobotsTxt({
|
||||
...robots_params,
|
||||
origin: cli_origin || robots_params.origin || sitemap_origin,
|
||||
sitemaps: [
|
||||
...(sitemap_public_path !== undefined
|
||||
? [sitemap_public_path]
|
||||
: []),
|
||||
...(robots_params.sitemaps || []),
|
||||
],
|
||||
});
|
||||
mkdirSync(path.dirname(robots_file), { recursive: true });
|
||||
writeFileSync(robots_file, robots_txt, "utf-8");
|
||||
log.success(`robots.txt written to ${robots_relative_path}`);
|
||||
}
|
||||
/**
|
||||
* # URL path a file in public/ is served at
|
||||
* Public files are served from the site root, e.g. public/sitemap.xml
|
||||
* at /sitemap.xml. Returns `undefined` for paths outside public/.
|
||||
*/
|
||||
function grabPublicUrlPath(file_path) {
|
||||
const { PUBLIC_DIR } = grabDirNames();
|
||||
const relative_path = path.relative(PUBLIC_DIR, file_path);
|
||||
if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) {
|
||||
return;
|
||||
}
|
||||
return `/${relative_path.split(path.sep).join("/")}`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user