Date sitemap lastmod from the build pipeline; warn on skipped pages

- bunext build records when each page's client bundle hash last changed
  in .bunext/page-lastmod.json (sitemap.lastmod_file to relocate). The
  hash covers the page's whole import graph, so component edits re-date
  every page that uses them. The sitemap uses these dates as lastmod
- Add sitemap.default_lastmod
- BREAKING: use_file_lastmod now defaults to false (file mtimes are reset
  by checkouts); it is a fallback after the build record
- Warn when a page or __root.tsx is skipped because its default export
  is not at the start of a line (e.g. export { Page as default })
- Replace leaking module mocks in tests (log, grab-root-file-path) with
  spies and fixtures

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dist/commands/build/index.js

@@ -4,6 +4,7 @@ import grabDirNames from "../../utils/grab-dir-names";
 import { rmSync, writeFileSync } from "fs";
 import bunextInit from "../../functions/bunext-init";
 import { v4 } from "uuid";
+import { updatePageLastmodRecord } from "../../utils/page-lastmod-record";
 const { HYDRATION_DST_DIR, BUNX_CWD_PAGES_REWRITE_DIR, BUNDLER_CTX_MAP_JSON, SSR_BUNDLER_CTX_MAP_JSON, BUNX_CWD_MODULE_CACHE_DIR, BUILD_INFO_JSON, } = grabDirNames();
 export default function () {
     return new Command("build")
@@ -39,6 +40,18 @@ export default function () {
             build_date_unix: Date.now(),
             build_date: Date(),
         };
+        if (global.BUNEXT_BUNDLER_CTX_MAP) {
+            try {
+                updatePageLastmodRecord({
+                    ctx_map: global.BUNEXT_BUNDLER_CTX_MAP,
+                    build_date: new Date(build_data.build_date_unix),
+                });
+            }
+            catch (error) {
+                // Only sitemap dates depend on this; never fail the build.
+                log.warn(`Could not update page lastmod record: ${error?.message || error}`);
+            }
+        }
         writeFileSync(BUILD_INFO_JSON, JSON.stringify(build_data, null, 4), "utf-8");
         log.success("Modules Built Successfully!");
         process.exit();

dist/commands/sitemap/index.js

@@ -6,6 +6,7 @@ import grabDirNames from "../../utils/grab-dir-names";
 import grabConfig from "../../functions/grab-config";
 import grabConstants from "../../utils/grab-constants";
 import { generateSitemapFiles } from "../../utils/generate-sitemap";
+import { grabPageLastmodFile } from "../../utils/page-lastmod-record";
 import generateRobotsTxt, { RobotsTxtGeneratedMarker, } from "../../utils/generate-robots-txt";
 export default function () {
     return new Command("sitemap")
@@ -18,6 +19,10 @@ export default function () {
         global.BUNEXT_CONFIG = (await grabConfig()) || {};
         global.BUNEXT_CONSTANTS = grabConstants();
         const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
+        if (!existsSync(grabPageLastmodFile(sitemap_config.lastmod_file))) {
+            log.warn(`No page lastmod record found. Run \`bunext build\` first ` +
+                `so pages get lastmod dates.`);
+        }
         let sitemap_file;
         try {
             const output_path = options.output || output;

dist/functions/bundler/grab-client-hydration-script.js

@@ -2,6 +2,8 @@ import { existsSync } from "fs";
 import grabDirNames from "../../utils/grab-dir-names";
 import grabConstants from "../../utils/grab-constants";
 import grabRootFilePath from "../server/web-pages/grab-root-file-path";
+import grabRelativePath from "../../utils/grab-relative-path";
+import { log } from "../../utils/log";
 const { ROOT_DIR } = grabDirNames();
 export default async function grabClientHydrationScript({ page_local_path, }) {
     const { ClientRootElementIDName, ClientRootComponentWindowName, ClientWindowPagePropsName, } = grabConstants();
@@ -18,12 +20,19 @@ export default async function grabClientHydrationScript({ page_local_path, }) {
             return undefined;
         }
         const root_content = await Bun.file(root_file_path).text();
-        if (!root_content.match(/^export default/m)) {
+        if (!hasBundlableDefaultExport({
+            file_path: root_file_path,
+            content: root_content,
+            is_root: true,
+        })) {
             return undefined;
         }
     }
     const page_content = await Bun.file(page_local_path).text();
-    if (!page_content.match(/^export default/m)) {
+    if (!hasBundlableDefaultExport({
+        file_path: page_local_path,
+        content: page_content,
+    })) {
         return undefined;
     }
     let txt = ``;
@@ -54,3 +63,36 @@ export default async function grabClientHydrationScript({ page_local_path, }) {
     txt += `}\n`;
     return txt;
 }
+// File path -> content hash of the last warning, so rebuilds don't repeat
+// a warning until the file changes.
+const warned_files = new Map();
+/**
+ * # Check a page or root file can be bundled
+ * Files are only bundled when `export default` starts a line. Files with
+ * no default export at all (e.g. a page still being written) are skipped
+ * silently; this warns about the confusing case where one exists
+ * elsewhere, like `import X from "x"; export default X` on one line, or
+ * `export { Page as default }`.
+ */
+function hasBundlableDefaultExport({ file_path, content, is_root, }) {
+    if (/^export default/m.test(content)) {
+        warned_files.delete(file_path);
+        return true;
+    }
+    const has_default_export = /\bexport\s+default\b/.test(content) ||
+        /\bas\s+default\b/.test(content);
+    if (!has_default_export) {
+        warned_files.delete(file_path);
+        return false;
+    }
+    const content_hash = Bun.hash(content);
+    if (warned_files.get(file_path) === content_hash)
+        return false;
+    warned_files.set(file_path, content_hash);
+    log.warn(`${grabRelativePath(file_path)} was skipped: ` +
+        `\`export default\` must be at the start of a line. ` +
+        (is_root
+            ? `No pages can be bundled until the root layout is fixed.`
+            : `This page will not be bundled.`));
+    return false;
+}

dist/types/index.d.ts

@@ -438,10 +438,21 @@ export type BunextGenerateSitemapParams = {
      */
     additional_paths?: BunextSitemapAdditionalPaths | (() => BunextSitemapAdditionalPaths | Promise<BunextSitemapAdditionalPaths>);
     /**
-     * Use each page file's modification time as `lastmod`. Defaults to
-     * `true`.
+     * Fall back to each page file's modification time as `lastmod` when
+     * the build record has no date for it. Defaults to `false`: checkouts
+     * reset modification times, so they are rarely meaningful.
      */
     use_file_lastmod?: boolean;
+    /**
+     * `lastmod` for entries that have no other date
+     */
+    default_lastmod?: Date | string;
+    /**
+     * Page last-modified record written by `bunext build`, relative to the
+     * project root. Defaults to `.bunext/page-lastmod.json`. Point it at a
+     * cached or committed location when builds start from a fresh checkout.
+     */
+    lastmod_file?: string;
     /**
      * Append a trailing slash to every non-root path. Defaults to `false`.
      */
@@ -508,3 +519,15 @@ export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & {
      */
     output?: string;
 };
+/**
+ * Written by `bunext build`. Keyed by page path relative to the project
+ * root. `lastmod` is the build date at which `fingerprint` last changed.
+ */
+export type BunextPageLastmodRecord = {
+    pages: {
+        [local_path: string]: {
+            fingerprint: string;
+            lastmod: string;
+        };
+    };
+};

dist/utils/generate-sitemap.js

@@ -7,6 +7,7 @@ import grabOrigin from "./grab-origin";
 import grabRelativePath from "./grab-relative-path";
 import isDevelopment from "./is-development";
 import { log } from "./log";
+import { readPageLastmodRecord } from "./page-lastmod-record";
 const ErrorPagePaths = ["/404", "/500"];
 /**
  * Protocol limits for a single sitemap file
@@ -131,7 +132,8 @@ function parseLastmod(lastmod) {
     return isNaN(date.getTime()) ? undefined : date;
 }
 export async function grabSitemapEntries(params) {
-    const use_file_lastmod = params?.use_file_lastmod ?? true;
+    const use_file_lastmod = params?.use_file_lastmod ?? false;
+    const lastmod_record = readPageLastmodRecord(params?.lastmod_file);
     const entries = [];
     const pages = grabAllPages({ exclude_api: true });
     for (let i = 0; i < pages.length; i++) {
@@ -145,17 +147,26 @@ export async function grabSitemapEntries(params) {
             file_path: page.local_path,
             route: page_path,
         });
+        // When the page's client bundle last changed, per `bunext build`
+        const build_lastmod = lastmod_record?.pages[grabRelativePath(page.local_path)]?.lastmod;
         if (page_sitemap_paths) {
-            entries.push(...page_sitemap_paths.map(toSitemapEntry));
+            for (const page_sitemap_path of page_sitemap_paths) {
+                const entry = toSitemapEntry(page_sitemap_path);
+                entries.push({
+                    ...entry,
+                    lastmod: entry.lastmod || build_lastmod,
+                });
+            }
             continue;
         }
         if (page.url_path.match(/\[/))
             continue;
         entries.push({
             path: page_path,
-            lastmod: use_file_lastmod
-                ? statSync(page.local_path).mtime
-                : undefined,
+            lastmod: build_lastmod ||
+                (use_file_lastmod
+                    ? statSync(page.local_path).mtime
+                    : undefined),
         });
     }
     const additional_paths = typeof params?.additional_paths == "function"
@@ -172,10 +183,11 @@ export async function grabSitemapEntries(params) {
             continue;
         seen.add(path);
         final_entries.push({
-            changefreq: params?.default_changefreq,
-            priority: params?.default_priority,
             ...entry,
             path,
+            lastmod: entry.lastmod || params?.default_lastmod,
+            changefreq: entry.changefreq || params?.default_changefreq,
+            priority: entry.priority ?? params?.default_priority,
         });
     }
     return final_entries;

dist/utils/grab-dir-names.d.ts

@@ -29,5 +29,6 @@ export type DirNames = {
     BUNDLER_CTX_MAP_JSON: string;
     SSR_BUNDLER_CTX_MAP_JSON: string;
     BUILD_INFO_JSON: string;
+    PAGE_LASTMOD_JSON: string;
 };
 export default function grabDirNames(): DirNames;

dist/utils/grab-dir-names.js

@@ -33,6 +33,7 @@ export default function grabDirNames() {
     const BUNDLER_CTX_MAP_JSON = path.join(BUNX_CWD_DIR, "bundler-ctx-map.json");
     const SSR_BUNDLER_CTX_MAP_JSON = path.join(BUNX_CWD_DIR, "ssr-bundler-ctx-map.json");
     const BUILD_INFO_JSON = path.join(BUNX_CWD_DIR, "build-info.json");
+    const PAGE_LASTMOD_JSON = path.join(BUNX_CWD_DIR, "page-lastmod.json");
     return {
         ROOT_DIR,
         SRC_DIR,
@@ -64,6 +65,7 @@ export default function grabDirNames() {
         BUNDLER_CTX_MAP_JSON,
         SSR_BUNDLER_CTX_MAP_JSON,
         BUILD_INFO_JSON,
+        PAGE_LASTMOD_JSON,
     };
 }
 // const NODE_MODULES_DIR = path.resolve(

dist/utils/page-lastmod-record.d.ts

@@ -0,0 +1,19 @@
+import type { BundlerCTXMap, BunextPageLastmodRecord } from "../types";
+/**
+ * # Path of the page last-modified record
+ */
+export declare function grabPageLastmodFile(lastmod_file?: string): string;
+export declare function readPageLastmodRecord(lastmod_file?: string): BunextPageLastmodRecord | undefined;
+/**
+ * # Record when each page's client bundle last changed
+ * A page's bundle hash covers its whole client import graph (including
+ * the root layout), so a component edit re-dates every page that uses it.
+ * Unchanged pages keep their previous date; removed pages are dropped.
+ */
+export declare function updatePageLastmodRecord({ ctx_map, build_date, lastmod_file, }: {
+    ctx_map: {
+        [local_path: string]: BundlerCTXMap;
+    };
+    build_date?: Date;
+    lastmod_file?: string;
+}): BunextPageLastmodRecord;

dist/utils/page-lastmod-record.js

@@ -0,0 +1,51 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import path from "path";
+import grabDirNames from "./grab-dir-names";
+import { log } from "./log";
+/**
+ * # Path of the page last-modified record
+ */
+export function grabPageLastmodFile(lastmod_file) {
+    const { ROOT_DIR, PAGE_LASTMOD_JSON } = grabDirNames();
+    const target = lastmod_file || global.BUNEXT_CONFIG?.sitemap?.lastmod_file;
+    return target ? path.resolve(ROOT_DIR, target) : PAGE_LASTMOD_JSON;
+}
+export function readPageLastmodRecord(lastmod_file) {
+    const file = grabPageLastmodFile(lastmod_file);
+    if (!existsSync(file))
+        return;
+    try {
+        const record = JSON.parse(readFileSync(file, "utf-8"));
+        if (record?.pages && typeof record.pages == "object")
+            return record;
+    }
+    catch { }
+    log.warn(`Ignoring invalid page lastmod record ${file}`);
+}
+/**
+ * # Record when each page's client bundle last changed
+ * A page's bundle hash covers its whole client import graph (including
+ * the root layout), so a component edit re-dates every page that uses it.
+ * Unchanged pages keep their previous date; removed pages are dropped.
+ */
+export function updatePageLastmodRecord({ ctx_map, build_date = new Date(), lastmod_file, }) {
+    const prior = readPageLastmodRecord(lastmod_file);
+    const record = { pages: {} };
+    // Sorted so a committed record produces minimal diffs
+    const local_paths = Object.keys(ctx_map).sort();
+    for (const local_path of local_paths) {
+        const artifact = ctx_map[local_path];
+        if (!artifact?.hash)
+            continue;
+        const fingerprint = `${artifact.hash}:${artifact.css_path || ""}`;
+        const prior_page = prior?.pages[local_path];
+        record.pages[local_path] =
+            prior_page?.fingerprint == fingerprint
+                ? prior_page
+                : { fingerprint, lastmod: build_date.toISOString() };
+    }
+    const file = grabPageLastmodFile(lastmod_file);
+    mkdirSync(path.dirname(file), { recursive: true });
+    writeFileSync(file, JSON.stringify(record, null, 4), "utf-8");
+    return record;
+}