bunext/dist/commands/sitemap/index.js

import { Command } from "commander";
import { DistDirOptionDescription, DistDirOptionFlags, } from "../dist-dir-option";
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "fs";
import path from "path";
import { log } from "../../utils/log";
import grabDirNames from "../../utils/grab-dir-names";
import grabConfig from "../../functions/grab-config";
import grabConstants from "../../utils/grab-constants";
import { generateSitemapFiles, grabPendingSitemapTask, } from "../../utils/generate-sitemap";
import { grabPageLastmodFile } from "../../utils/page-lastmod-record";
import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
const SitemapStillWaitingInterval = 30_000;
export default function () {
    return new Command("sitemap")
        .description("Generate sitemap.xml and robots.txt from the pages directory")
        .option("-o, --output <path>", "Output file relative to the project root (default: public/sitemap.xml)")
        .option("--origin <url>", "Base URL for sitemap links")
        .option("--no-robots", "Skip generating robots.txt")
        .option(DistDirOptionFlags, DistDirOptionDescription)
        .action(async (options) => {
        const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
        global.BUNEXT_CONFIG = (await grabConfig()) || {};
        global.BUNEXT_CONSTANTS = grabConstants();
        if (global.BUNEXT_CONFIG.disable_seo) {
            log.warn(`\`disable_seo\` is set: the server blocks all crawlers at ` +
                `/robots.txt, so the generated robots.txt is not served.`);
        }
        const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
        if (!existsSync(grabPageLastmodFile(sitemap_config.lastmod_file))) {
            log.warn(`No page lastmod record found. Run \`bunext build\` first ` +
                `so pages get lastmod dates.`);
        }
        let sitemap_file;
        try {
            const output_path = options.output || output;
            sitemap_file = output_path
                ? path.resolve(ROOT_DIR, output_path)
                : path.join(PUBLIC_DIR, "sitemap.xml");
            const chunks_dir_path = path.resolve(ROOT_DIR, chunks_dir || "public/sitemap-chunks");
            const chunks_public_path = grabPublicUrlPath(chunks_dir_path);
            if (chunks_public_path === undefined) {
                throw new Error(`\`sitemap.chunks_dir\` must be inside public/ ` +
                    `so the chunks are served.`);
            }
            // Bun exits silently (code 0) when the event loop empties
            // while a promise is pending. Bun's SQL client (mariadb /
            // mysql adapters, Bun 1.3.14) doesn't keep the loop alive
            // for a query on a connection that has gone idle, so a
            // second `sitemap` export querying the DB would end the
            // process mid-generation. Hold the loop open with a timer,
            // which also reports what a genuinely stuck run waits on.
            const keep_alive = setInterval(() => {
                log.warn(`Still waiting on ` +
                    `${grabPendingSitemapTask() || "sitemap generation"}...`);
            }, SitemapStillWaitingInterval);
            const { sitemap, chunks, url_count } = await generateSitemapFiles({
                ...sitemap_config,
                origin: options.origin || sitemap_config.origin,
                chunk_size,
                chunks_url_path: chunks_public_path,
            }).finally(() => clearInterval(keep_alive));
            // Remove chunks from a previous, larger sitemap
            if (existsSync(chunks_dir_path)) {
                for (const file of readdirSync(chunks_dir_path)) {
                    if (file.match(/^sitemap-\d+\.xml$/)) {
                        rmSync(path.join(chunks_dir_path, file));
                    }
                }
            }
            if (chunks.length) {
                mkdirSync(chunks_dir_path, { recursive: true });
            }
            for (const chunk of chunks) {
                writeFileSync(path.join(chunks_dir_path, chunk.file_name), chunk.xml, "utf-8");
            }
            mkdirSync(path.dirname(sitemap_file), { recursive: true });
            writeFileSync(sitemap_file, sitemap, "utf-8");
            const sitemap_relative_path = path.relative(ROOT_DIR, sitemap_file);
            if (chunks.length) {
                log.success(`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` +
                    `written to ${sitemap_relative_path} ` +
                    `(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`);
            }
            else {
                log.success(`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`);
            }
        }
        catch (error) {
            log.error(`Failed to generate sitemap: ${error?.message || error}`);
            process.exit(1);
        }
        const robots_config = global.BUNEXT_CONFIG.robots;
        if (options.robots && robots_config !== false) {
            writeRobotsTxt({
                sitemap_file,
                cli_origin: options.origin,
                sitemap_origin: sitemap_config.origin,
            });
        }
        process.exit();
    });
}
function writeRobotsTxt({ sitemap_file, cli_origin, sitemap_origin, }) {
    const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
    const robots_config = global.BUNEXT_CONFIG.robots || undefined;
    const { output, ...robots_params } = robots_config || {};
    const robots_file = output
        ? path.resolve(ROOT_DIR, output)
        : path.join(PUBLIC_DIR, "robots.txt");
    const robots_relative_path = path.relative(ROOT_DIR, robots_file);
    // Without a robots config, never replace a hand-written robots.txt.
    if (!robots_config &&
        existsSync(robots_file) &&
        !readFileSync(robots_file, "utf-8").startsWith(RobotsTxtGeneratedMarker)) {
        log.warn(`Skipped ${robots_relative_path}: it was not generated by bunext. ` +
            `Set \`robots\` in bunext.config.ts to overwrite it, ` +
            `or \`robots: false\` to silence this.`);
        return;
    }
    const sitemap_public_path = grabPublicUrlPath(sitemap_file);
    if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) {
        log.warn(`Sitemap is outside public/, so robots.txt has no Sitemap line. ` +
            `Add its URL to \`robots.sitemaps\`.`);
    }
    const robots_txt = generateRobotsTxt({
        ...robots_params,
        origin: cli_origin || robots_params.origin || sitemap_origin,
        sitemaps: [
            ...(sitemap_public_path !== undefined
                ? [sitemap_public_path]
                : []),
            ...(robots_params.sitemaps || []),
        ],
    });
    mkdirSync(path.dirname(robots_file), { recursive: true });
    writeFileSync(robots_file, robots_txt, "utf-8");
    log.success(`robots.txt written to ${robots_relative_path}`);
}
/**
 * # URL path a file in public/ is served at
 * Public files are served from the site root, e.g. public/sitemap.xml
 * at /sitemap.xml. Returns `undefined` for paths outside public/.
 */
function grabPublicUrlPath(file_path) {
    const { PUBLIC_DIR } = grabDirNames();
    const relative_path = path.relative(PUBLIC_DIR, file_path);
    if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) {
        return;
    }
    return `/${relative_path.split(path.sep).join("/")}`;
}