Add sitemap generation utility

dist/commands/index.js

@@ -3,6 +3,7 @@ import { program } from "commander";
 import start from "./start";
 import dev from "./dev";
 import build from "./build";
+import sitemap from "./sitemap";
 import { log } from "../utils/log";
 /**
  * # Describe Program
@@ -17,6 +18,7 @@ program
 program.addCommand(dev());
 program.addCommand(start());
 program.addCommand(build());
+program.addCommand(sitemap());
 /**
  * # Handle Unavailable Commands
  */

dist/commands/sitemap/index.d.ts

@@ -0,0 +1,2 @@
+import { Command } from "commander";
+export default function (): Command;

dist/commands/sitemap/index.js

@@ -0,0 +1,39 @@
+import { Command } from "commander";
+import { mkdirSync, writeFileSync } from "fs";
+import path from "path";
+import { log } from "../../utils/log";
+import grabDirNames from "../../utils/grab-dir-names";
+import grabConfig from "../../functions/grab-config";
+import grabConstants from "../../utils/grab-constants";
+import generateSitemap from "../../utils/generate-sitemap";
+export default function () {
+    return new Command("sitemap")
+        .description("Generate sitemap.xml from the pages directory")
+        .option("-o, --output <path>", "Output file relative to the project root (default: public/sitemap.xml)")
+        .option("--origin <url>", "Base URL for sitemap links")
+        .action(async (options) => {
+        const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
+        global.BUNEXT_CONFIG = (await grabConfig()) || {};
+        global.BUNEXT_CONSTANTS = grabConstants();
+        const { output, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
+        try {
+            const xml = await generateSitemap({
+                ...sitemap_config,
+                origin: options.origin || sitemap_config.origin,
+            });
+            const output_path = options.output || output;
+            const output_file = output_path
+                ? path.resolve(ROOT_DIR, output_path)
+                : path.join(PUBLIC_DIR, "sitemap.xml");
+            mkdirSync(path.dirname(output_file), { recursive: true });
+            writeFileSync(output_file, xml, "utf-8");
+            const url_count = xml.match(/<url>/g)?.length || 0;
+            log.success(`Sitemap with ${url_count} URLs written to ${path.relative(ROOT_DIR, output_file)}`);
+        }
+        catch (error) {
+            log.error(`Failed to generate sitemap: ${error?.message || error}`);
+            process.exit(1);
+        }
+        process.exit();
+    });
+}

dist/index.d.ts

@@ -1,5 +1,6 @@
 import bunextInit from "./functions/bunext-init";
 import bunextRequestHandler from "./functions/server/bunext-req-handler";
+import generateSitemap from "./utils/generate-sitemap";
 declare const bunext: {
     bunextRequestHandler: typeof bunextRequestHandler;
     bunextLog: {
@@ -13,5 +14,6 @@ declare const bunext: {
         banner: () => void;
     };
     bunextInit: typeof bunextInit;
+    bunextGenerateSitemap: typeof generateSitemap;
 };
 export default bunext;

dist/index.js

@@ -1,9 +1,11 @@
 import bunextInit from "./functions/bunext-init";
 import bunextRequestHandler from "./functions/server/bunext-req-handler";
+import generateSitemap from "./utils/generate-sitemap";
 import { log } from "./utils/log";
 const bunext = {
     bunextRequestHandler,
     bunextLog: log,
     bunextInit,
+    bunextGenerateSitemap: generateSitemap,
 };
 export default bunext;

dist/types/index.d.ts

@@ -93,6 +93,10 @@ export type BunextConfig = {
      * Public environment variables to be passed to the client
      */
     public_envs?: Record<string, string>;
+    /**
+     * Options for the `bunext sitemap` command
+     */
+    sitemap?: BunextSitemapConfig;
 };
 export type BunextConfigMiddlewareParams = {
     req: Request;
@@ -191,7 +195,19 @@ export type BunextPageModule = {
 export type BunextPageServerModule = {
     default?: BunextPageServerFn;
     server?: BunextPageServerFn;
+    sitemap?: BunextPageSitemapFn;
 };
+/**
+ * Returns the sitemap entries for a page. Required for dynamic routes
+ * (`[slug]`) to appear in the sitemap. On a static page, the returned
+ * entries replace the page's default entry.
+ */
+export type BunextPageSitemapFn = (params: {
+    /**
+     * The page's route pattern, e.g. `/blog/[slug]`
+     */
+    route: string;
+}) => BunextSitemapAdditionalPaths | Promise<BunextSitemapAdditionalPaths>;
 export type BunextHTMLProps = DetailedHTMLProps<HtmlHTMLAttributes<HTMLHtmlElement>, HTMLHtmlElement>;
 export type BunextPageModuleMetaFn = (params: {
     ctx: BunxRouteParams;
@@ -383,3 +399,55 @@ export type BuildDataType = {
     build_date_unix: number;
     build_date: string;
 };
+export type BunextSitemapChangeFreq = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
+export type BunextSitemapEntry = {
+    /**
+     * URL path (e.g. `/blog/my-post`) or absolute URL.
+     */
+    path: string;
+    lastmod?: Date | string;
+    changefreq?: BunextSitemapChangeFreq;
+    /**
+     * Between `0.0` and `1.0`.
+     */
+    priority?: number;
+};
+export type BunextSitemapAdditionalPaths = (string | BunextSitemapEntry)[];
+export type BunextGenerateSitemapParams = {
+    /**
+     * Base URL prepended to every path. Defaults to `config.origin`,
+     * falling back to `http://localhost:<port>`.
+     */
+    origin?: string;
+    /**
+     * Paths to leave out. A string matches a path exactly, or a whole
+     * subtree when it ends with `/*` (e.g. `/admin/*`). A RegExp is
+     * tested against the path.
+     */
+    exclude?: (string | RegExp)[];
+    /**
+     * Extra entries, typically concrete URLs for dynamic routes
+     * (`[slug]`), which cannot be discovered from the file system. Can
+     * also be a function (sync or async) returning them, e.g. to load
+     * them from a database.
+     */
+    additional_paths?: BunextSitemapAdditionalPaths | (() => BunextSitemapAdditionalPaths | Promise<BunextSitemapAdditionalPaths>);
+    /**
+     * Use each page file's modification time as `lastmod`. Defaults to
+     * `true`.
+     */
+    use_file_lastmod?: boolean;
+    /**
+     * Append a trailing slash to every non-root path. Defaults to `false`.
+     */
+    trailing_slash?: boolean;
+    default_changefreq?: BunextSitemapChangeFreq;
+    default_priority?: number;
+};
+export type BunextSitemapConfig = BunextGenerateSitemapParams & {
+    /**
+     * Output file, relative to the project root. Defaults to
+     * `public/sitemap.xml`, which is served at `/sitemap.xml`.
+     */
+    output?: string;
+};

dist/utils/generate-sitemap.d.ts

@@ -0,0 +1,8 @@
+import type { BunextGenerateSitemapParams, BunextSitemapEntry } from "../types";
+/**
+ * # Generate a sitemap.xml string for all pages
+ * Dynamic routes are only included through a `sitemap` export in their
+ * `.server.ts` file, or through `additional_paths`.
+ */
+export default function generateSitemap(params?: BunextGenerateSitemapParams): Promise<string>;
+export declare function grabSitemapEntries(params?: BunextGenerateSitemapParams): Promise<BunextSitemapEntry[]>;

dist/utils/generate-sitemap.js

@@ -0,0 +1,165 @@
+import { statSync } from "fs";
+import path from "path";
+import grabPageServerPath from "../functions/server/web-pages/grab-page-server-path";
+import grabAllPages from "./grab-all-pages";
+import grabDirNames from "./grab-dir-names";
+import grabOrigin from "./grab-origin";
+import grabRelativePath from "./grab-relative-path";
+import isDevelopment from "./is-development";
+const ErrorPagePaths = ["/404", "/500"];
+/**
+ * # Generate a sitemap.xml string for all pages
+ * Dynamic routes are only included through a `sitemap` export in their
+ * `.server.ts` file, or through `additional_paths`.
+ */
+export default async function generateSitemap(params) {
+    const entries = await grabSitemapEntries(params);
+    const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
+    const urls = entries.map((entry) => {
+        const loc = entry.path.match(/^https?:\/\//)
+            ? entry.path
+            : `${origin}${entry.path}`;
+        const lines = [`        <loc>${escapeXml(loc)}</loc>`];
+        if (entry.lastmod) {
+            const lastmod = entry.lastmod instanceof Date
+                ? entry.lastmod.toISOString()
+                : entry.lastmod;
+            lines.push(`        <lastmod>${escapeXml(lastmod)}</lastmod>`);
+        }
+        if (entry.changefreq) {
+            lines.push(`        <changefreq>${entry.changefreq}</changefreq>`);
+        }
+        if (typeof entry.priority == "number") {
+            const priority = Math.min(Math.max(entry.priority, 0), 1);
+            lines.push(`        <priority>${priority.toFixed(1)}</priority>`);
+        }
+        return `    <url>\n${lines.join("\n")}\n    </url>`;
+    });
+    return [
+        `<?xml version="1.0" encoding="UTF-8"?>`,
+        `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+        ...urls,
+        `</urlset>`,
+        ``,
+    ].join("\n");
+}
+export async function grabSitemapEntries(params) {
+    const use_file_lastmod = params?.use_file_lastmod ?? true;
+    const entries = [];
+    const pages = grabAllPages({ exclude_api: true });
+    for (let i = 0; i < pages.length; i++) {
+        const page = pages[i];
+        if (!page.local_path)
+            continue;
+        const page_path = page.url_path.replace(/\/index$/, "") || "/";
+        if (ErrorPagePaths.includes(page_path))
+            continue;
+        const page_sitemap_paths = await grabPageSitemapPaths({
+            file_path: page.local_path,
+            route: page_path,
+        });
+        if (page_sitemap_paths) {
+            entries.push(...page_sitemap_paths.map(toSitemapEntry));
+            continue;
+        }
+        if (page.url_path.match(/\[/))
+            continue;
+        entries.push({
+            path: page_path,
+            lastmod: use_file_lastmod
+                ? statSync(page.local_path).mtime
+                : undefined,
+        });
+    }
+    const additional_paths = typeof params?.additional_paths == "function"
+        ? await params.additional_paths()
+        : params?.additional_paths;
+    entries.push(...(additional_paths || []).map(toSitemapEntry));
+    const seen = new Set();
+    const final_entries = [];
+    for (const entry of entries) {
+        const path = normalizePath(entry.path, params?.trailing_slash);
+        if (seen.has(path))
+            continue;
+        if (isExcluded(path, params?.exclude))
+            continue;
+        seen.add(path);
+        final_entries.push({
+            changefreq: params?.default_changefreq,
+            priority: params?.default_priority,
+            ...entry,
+            path,
+        });
+    }
+    return final_entries;
+}
+/**
+ * # Run the `sitemap` export of a page's server file, if any
+ */
+async function grabPageSitemapPaths({ file_path, route, }) {
+    const { server_file_path } = grabPageServerPath({ file_path });
+    if (!server_file_path)
+        return;
+    const { ROOT_DIR } = grabDirNames();
+    const server_ctx = global.BUNEXT_SSR_BUNDLER_CTX_MAP?.[grabRelativePath(server_file_path)];
+    const final_server_path = server_ctx?.local_path
+        ? path.join(ROOT_DIR, server_ctx.path)
+        : server_file_path;
+    // Bust the import cache in development so edits to the server file
+    // are picked up without a restart.
+    const server_module = await import(isDevelopment()
+        ? `${final_server_path}?t=${Date.now()}`
+        : final_server_path);
+    if (typeof server_module.sitemap != "function")
+        return;
+    try {
+        return await server_module.sitemap({ route });
+    }
+    catch (error) {
+        throw new Error(`\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${error?.message || error}`);
+    }
+}
+function toSitemapEntry(path) {
+    return typeof path == "string" ? { path } : path;
+}
+function normalizePath(path, trailing_slash) {
+    if (path.match(/^https?:\/\//))
+        return path;
+    let new_path = `/${path}`.replace(/\/{2,}/g, "/");
+    if (new_path != "/")
+        new_path = new_path.replace(/\/$/, "");
+    if (trailing_slash && new_path != "/")
+        new_path += "/";
+    return new_path;
+}
+function isExcluded(path, exclude) {
+    if (!exclude?.length)
+        return false;
+    const bare_path = path.replace(/^https?:\/\/[^/]+/, "").replace(/(.)\/$/, "$1") || "/";
+    for (const pattern of exclude) {
+        if (pattern instanceof RegExp) {
+            if (pattern.test(path) || pattern.test(bare_path))
+                return true;
+            continue;
+        }
+        const normalized = normalizePath(pattern.replace(/\/\*$/, ""));
+        if (pattern.endsWith("/*")) {
+            if (bare_path == normalized ||
+                bare_path.startsWith(`${normalized.replace(/\/$/, "")}/`)) {
+                return true;
+            }
+        }
+        else if (bare_path == normalized) {
+            return true;
+        }
+    }
+    return false;
+}
+function escapeXml(value) {
+    return value
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;")
+        .replace(/'/g, "&apos;");
+}

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@moduletrace/bunext",
-    "version": "1.1.19",
+    "version": "1.1.20",
     "main": "dist/index.js",
     "module": "index.ts",
     "dependencies": {