import { Command } from "commander"; import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "fs"; import path from "path"; import { log } from "../../utils/log"; import grabDirNames from "../../utils/grab-dir-names"; import grabConfig from "../../functions/grab-config"; import grabConstants from "../../utils/grab-constants"; import { generateSitemapFiles } from "../../utils/generate-sitemap"; import { grabPageLastmodFile } from "../../utils/page-lastmod-record"; import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt"; export default function () { return new Command("sitemap") .description("Generate sitemap.xml and robots.txt from the pages directory") .option("-o, --output ", "Output file relative to the project root (default: public/sitemap.xml)") .option("--origin ", "Base URL for sitemap links") .option("--no-robots", "Skip generating robots.txt") .action(async (options) => { const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); global.BUNEXT_CONFIG = (await grabConfig()) || {}; global.BUNEXT_CONSTANTS = grabConstants(); const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {}; if (!existsSync(grabPageLastmodFile(sitemap_config.lastmod_file))) { log.warn(`No page lastmod record found. Run \`bunext build\` first ` + `so pages get lastmod dates.`); } let sitemap_file; try { const output_path = options.output || output; sitemap_file = output_path ? path.resolve(ROOT_DIR, output_path) : path.join(PUBLIC_DIR, "sitemap.xml"); const chunks_dir_path = path.resolve(ROOT_DIR, chunks_dir || "public/sitemap-chunks"); const chunks_public_path = grabPublicUrlPath(chunks_dir_path); if (chunks_public_path === undefined) { throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` + `so the chunks are served.`); } const { sitemap, chunks, url_count } = await generateSitemapFiles({ ...sitemap_config, origin: options.origin || sitemap_config.origin, chunk_size, chunks_url_path: chunks_public_path, }); // Remove chunks from a previous, larger sitemap if (existsSync(chunks_dir_path)) { for (const file of readdirSync(chunks_dir_path)) { if (file.match(/^sitemap-\d+\.xml$/)) { rmSync(path.join(chunks_dir_path, file)); } } } if (chunks.length) { mkdirSync(chunks_dir_path, { recursive: true }); } for (const chunk of chunks) { writeFileSync(path.join(chunks_dir_path, chunk.file_name), chunk.xml, "utf-8"); } mkdirSync(path.dirname(sitemap_file), { recursive: true }); writeFileSync(sitemap_file, sitemap, "utf-8"); const sitemap_relative_path = path.relative(ROOT_DIR, sitemap_file); if (chunks.length) { log.success(`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` + `written to ${sitemap_relative_path} ` + `(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`); } else { log.success(`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`); } } catch (error) { log.error(`Failed to generate sitemap: ${error?.message || error}`); process.exit(1); } const robots_config = global.BUNEXT_CONFIG.robots; if (options.robots && robots_config !== false) { writeRobotsTxt({ sitemap_file, cli_origin: options.origin, sitemap_origin: sitemap_config.origin, }); } process.exit(); }); } function writeRobotsTxt({ sitemap_file, cli_origin, sitemap_origin, }) { const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); const robots_config = global.BUNEXT_CONFIG.robots || undefined; const { output, ...robots_params } = robots_config || {}; const robots_file = output ? path.resolve(ROOT_DIR, output) : path.join(PUBLIC_DIR, "robots.txt"); const robots_relative_path = path.relative(ROOT_DIR, robots_file); // Without a robots config, never replace a hand-written robots.txt. if (!robots_config && existsSync(robots_file) && !readFileSync(robots_file, "utf-8").startsWith(RobotsTxtGeneratedMarker)) { log.warn(`Skipped ${robots_relative_path}: it was not generated by bunext. ` + `Set \`robots\` in bunext.config.ts to overwrite it, ` + `or \`robots: false\` to silence this.`); return; } const sitemap_public_path = grabPublicUrlPath(sitemap_file); if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) { log.warn(`Sitemap is outside public/, so robots.txt has no Sitemap line. ` + `Add its URL to \`robots.sitemaps\`.`); } const robots_txt = generateRobotsTxt({ ...robots_params, origin: cli_origin || robots_params.origin || sitemap_origin, sitemaps: [ ...(sitemap_public_path !== undefined ? [sitemap_public_path] : []), ...(robots_params.sitemaps || []), ], }); mkdirSync(path.dirname(robots_file), { recursive: true }); writeFileSync(robots_file, robots_txt, "utf-8"); log.success(`robots.txt written to ${robots_relative_path}`); } /** * # URL path a file in public/ is served at * Public files are served from the site root, e.g. public/sitemap.xml * at /sitemap.xml. Returns `undefined` for paths outside public/. */ function grabPublicUrlPath(file_path) { const { PUBLIC_DIR } = grabDirNames(); const relative_path = path.relative(PUBLIC_DIR, file_path); if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) { return; } return `/${relative_path.split(path.sep).join("/")}`; }