diff --git a/.gitignore b/.gitignore index deac8f3..a33d132 100644 --- a/.gitignore +++ b/.gitignore @@ -182,4 +182,5 @@ __fixtures__ /.dump /.vscode /source.md -SECURITY.md \ No newline at end of file +SECURITY.md +/.tmp \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 03dee9c..5355e7b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,7 @@ bun run dev # tsc --watch bunext dev # Start dev server with HMR and file watcher bunext build # Bundle all pages for production bunext start # Start production server from pre-built artifacts +bunext sitemap # Write public/sitemap.xml using config.sitemap ``` ## Architecture @@ -51,6 +52,7 @@ Pages live in `src/pages/`. Server logic is **separated from page files** into c **Server file** (`page.server.ts` or `page.server.tsx`) — server-only, never sent to the browser: - Default export or `export const server`: `BunextPageServerFn` — runs server-side before rendering, return value becomes props +- `export const sitemap`: `BunextPageSitemapFn` — optional; returns the page's sitemap entries (how dynamic routes get into `bunext sitemap`) The framework resolves the companion by replacing the page extension with `.server.ts` or `.server.tsx`. If neither exists, no server function runs and only the default `url` prop is injected. diff --git a/README.md b/README.md index a36c8f0..686af10 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,10 @@ The goal is a framework that is: - [API Routes](#api-routes) - [Route Config (Body Size Limit)](#route-config-body-size-limit) - [Error Pages](#error-pages) +- [Sitemap](#sitemap) + - [Page `sitemap` Export](#page-sitemap-export) + - [`bunext sitemap` Command](#bunext-sitemap-command) + - [Generating at Runtime](#generating-at-runtime) - [Static Files](#static-files) - [Caching](#caching) - [Enabling Cache Per Page](#enabling-cache-per-page) @@ -155,6 +159,7 @@ bun run dev | `bunext dev` | Start the development server with HMR and file watching. | | `bunext build` | Bundle all pages for production. Outputs artifacts to `.bunext/public/pages/` using paths relative to the project root. | | `bunext start` | Start the production server using pre-built artifacts. | +| `bunext sitemap` | Generate `public/sitemap.xml` from your pages, using the `sitemap` options in `bunext.config.ts`. See [Sitemap](#sitemap). | ### Running the CLI @@ -325,6 +330,8 @@ export default function ProfilePage({ props, url }: Props) { } ``` +> Server files can also export a `sitemap` function to list a page's URLs in the sitemap — see [Page `sitemap` Export](#page-sitemap-export). + > **Why separate files?** Bundling server code (Bun APIs, database clients, `fs`, secrets) into the same file as a React component causes TypeScript compilation errors because the bundler processes the page file for the browser. The `.server.ts` companion file is loaded only by the server at request time and is never included in the client bundle. The server function receives a `ctx` object (type `BunxRouteParams`) with: @@ -624,6 +631,117 @@ If no custom error pages exist, Bunext uses built-in preset components. --- +## Sitemap + +Bunext builds a sitemap from every static page in `src/pages/`. API routes, `404`/`500` pages and ignored directories are left out. Dynamic routes (`[slug]`) cannot be discovered from the file system, so they are only included when their server file exports a `sitemap` function (see below), or when you pass their URLs through `additional_paths`. + +### Page `sitemap` Export + +A page's `.server.ts` file can export a `sitemap` function returning that page's entries. This is the way to list URLs for a dynamic route: + +```ts +// src/pages/blog/[slug].server.ts +import type { BunextPageServerFn, BunextPageSitemapFn } from "@moduletrace/bunext/types"; + +const server: BunextPageServerFn = async (ctx) => { + // ... +}; + +export default server; + +export const sitemap: BunextPageSitemapFn = async ({ route }) => { + // route === "/blog/[slug]" + const posts = await getPublishedPosts(); + + return posts.map((post) => ({ + path: `/blog/${post.slug}`, + lastmod: post.updated_at, + priority: 0.7, + })); +}; +``` + +- It can be sync or async, and returns strings or `BunextSitemapEntry` objects, like `additional_paths`. +- On a static page, the returned entries **replace** the page's default entry, so you can set its `priority`/`changefreq`, or return `[]` to leave the page out. +- `exclude` and the `default_*` options still apply to the returned entries. +- Generating a sitemap imports every page's server file, so their top-level code runs. If a `sitemap` function throws, generation fails with the file name in the error. + +### `bunext sitemap` Command + +Add a `sitemap` key to `bunext.config.ts`, then run `bunext sitemap`. By default the file is written to `public/sitemap.xml`, which is served at `/sitemap.xml`. Run it after `bunext build` in your deploy step to keep it current. + +```ts +// bunext.config.ts +import type { BunextConfig } from "@moduletrace/bunext/types"; + +const config: BunextConfig = { + origin: "https://example.com", + sitemap: { + exclude: ["/login", "/admin/*"], + default_changefreq: "weekly", + // A function (sync or async) lets you load dynamic routes + additional_paths: async () => { + const posts = await getPublishedPosts(); + return posts.map((post) => ({ + path: `/blog/${post.slug}`, + lastmod: post.updated_at, + })); + }, + }, +}; + +export default config; +``` + +```bash +bunext sitemap # writes public/sitemap.xml +bunext sitemap -o public/sitemaps/main.xml # custom output path +bunext sitemap --origin https://staging.example.com +``` + +`--output` and `--origin` override `sitemap.output` and `sitemap.origin` from the config. + +### Generating at Runtime + +`bunextGenerateSitemap()` resolves to the same XML as a string, so you can serve it from a route instead of a file: + +```ts +// src/pages/api/sitemap.ts +import bunext from "@moduletrace/bunext"; + +export default async function handler(req: Request) { + const xml = await bunext.bunextGenerateSitemap({ + exclude: ["/login", "/admin/*", /^\/drafts/], + additional_paths: async () => { + const posts = await getPublishedPosts(); + return posts.map((post) => ({ + path: `/blog/${post.slug}`, + lastmod: post.updated_at, + })); + }, + default_changefreq: "weekly", + }); + + return new Response(xml, { + headers: { "Content-Type": "application/xml" }, + }); +} +``` + +| Option | Type | Description | +| -------------------- | ------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| `origin` | `string` | Base URL for each ``. Defaults to `config.origin`, then `http://localhost:`. | +| `exclude` | `(string \| RegExp)[]` | Exact paths, subtrees ending in `/*` (`/admin/*` also excludes `/admin`), or regexes tested on the path. | +| `additional_paths` | `entries \| () => entries \| Promise` | Extra paths or entries (`path`, `lastmod`, `changefreq`, `priority`), or a sync/async function returning them, e.g. for dynamic routes. Each entry is a `string` or `BunextSitemapEntry`. | +| `use_file_lastmod` | `boolean` | Use each page file's modification time as `lastmod`. Default `true`. | +| `trailing_slash` | `boolean` | Append `/` to every non-root path. Default `false`. | +| `default_changefreq` | `BunextSitemapChangeFreq` | `changefreq` for entries that do not set one. | +| `default_priority` | `number` | `priority` (0.0 to 1.0) for entries that do not set one. | + +The `sitemap` config key takes the same options, plus `output`: the file path relative to the project root (default `public/sitemap.xml`). + +--- + ## Static Files Place static files in the `public/` directory. They are served at the `/public/` URL path: @@ -764,6 +882,7 @@ export default config; | `middleware` | `(params: BunextConfigMiddlewareParams) => Response \| undefined \| Promise<...>` | — | Global middleware — see [Middleware](#middleware) | | `websocket` | `WebSocketHandler` | — | Bun WebSocket handler — see [WebSocket](#websocket) | | `serverOptions` | `ServeOptions` | — | Extra options passed to `Bun.serve()` (excluding `fetch`) — see [Server Options](#server-options) | +| `sitemap` | `BunextSitemapConfig` | — | Options for `bunext sitemap` — see [Sitemap](#sitemap) | ### Middleware diff --git a/src/__tests__/commands/sitemap.test.ts b/src/__tests__/commands/sitemap.test.ts new file mode 100644 index 0000000..d20e914 --- /dev/null +++ b/src/__tests__/commands/sitemap.test.ts @@ -0,0 +1,224 @@ +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import path from "path"; +import fs from "fs"; +import os from "os"; + +const cliPath = path.resolve(import.meta.dir, "../../commands/index.ts"); + +let projectDir: string; + +function writeProjectFile(file: string, content: string) { + const file_path = path.join(projectDir, file); + fs.mkdirSync(path.dirname(file_path), { recursive: true }); + fs.writeFileSync(file_path, content); +} + +function writeConfig(config: string) { + writeProjectFile( + "bunext.config.ts", + `const config = ${config};\nexport default config;\n`, + ); +} + +async function runSitemap(...args: string[]) { + const proc = Bun.spawn(["bun", cliPath, "sitemap", ...args], { + cwd: projectDir, + stdout: "pipe", + stderr: "pipe", + env: { ...process.env, NODE_ENV: "production" }, + }); + + const [stdout, stderr, exit_code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + return { stdout, stderr, output: stdout + stderr, exit_code }; +} + +function readOutput(file = "public/sitemap.xml") { + return fs.readFileSync(path.join(projectDir, file), "utf-8"); +} + +function locs(xml: string) { + return [...xml.matchAll(/(.*?)<\/loc>/g)].map((m) => m[1]).sort(); +} + +beforeEach(() => { + // Outside __tests__, which the page scanner skips + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "bunext-sitemap-cmd-")); + + for (const page of ["index", "about", "blog/index", "blog/[slug]", "404"]) { + writeProjectFile(`src/pages/${page}.tsx`, "export default () => null"); + } + + writeProjectFile( + "src/pages/api/hello.ts", + `export default () => new Response("hi")`, + ); +}); + +afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); +}); + +describe("bunext sitemap", () => { + it("writes public/sitemap.xml using config.origin", async () => { + writeConfig(`{ origin: "https://example.com" }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(res.output).toContain( + "Sitemap with 3 URLs written to public/sitemap.xml", + ); + expect(locs(readOutput())).toEqual([ + "https://example.com/", + "https://example.com/about", + "https://example.com/blog", + ]); + }); + + it("applies the sitemap config options", async () => { + writeConfig(`{ + origin: "https://example.com", + sitemap: { + exclude: ["/about"], + default_changefreq: "daily", + use_file_lastmod: false, + trailing_slash: true, + additional_paths: async () => [ + "/blog/from-config", + { path: "/blog/priority", priority: 0.9 }, + ], + }, + }`); + + const res = await runSitemap(); + const xml = readOutput(); + + expect(res.exit_code).toBe(0); + expect(locs(xml)).toEqual([ + "https://example.com/", + "https://example.com/blog/", + "https://example.com/blog/from-config/", + "https://example.com/blog/priority/", + ]); + expect(xml).toContain("daily"); + expect(xml).toContain("0.9"); + expect(xml).not.toContain(""); + }); + + it("includes dynamic route URLs from a page server sitemap export", async () => { + writeConfig(`{ origin: "https://example.com" }`); + writeProjectFile( + "src/pages/blog/[slug].server.ts", + `export default async () => ({ props: {} }); + export const sitemap = async ({ route }) => [ + "/blog/first", + { path: "/blog/" + (route == "/blog/[slug]" ? "second" : "bad") }, + ];`, + ); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(locs(readOutput())).toEqual([ + "https://example.com/", + "https://example.com/about", + "https://example.com/blog", + "https://example.com/blog/first", + "https://example.com/blog/second", + ]); + }); + + it("writes to config.sitemap.output", async () => { + writeConfig(`{ + origin: "https://example.com", + sitemap: { output: "out/site/sitemap.xml" }, + }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(fs.existsSync(path.join(projectDir, "public/sitemap.xml"))).toBe( + false, + ); + expect(locs(readOutput("out/site/sitemap.xml"))).toContain( + "https://example.com/about", + ); + }); + + it("lets --output and --origin override the config", async () => { + writeConfig(`{ + origin: "https://example.com", + sitemap: { origin: "https://config.dev", output: "out/config.xml" }, + }`); + + const res = await runSitemap( + "-o", + "cli/map.xml", + "--origin", + "https://cli.dev", + ); + + expect(res.exit_code).toBe(0); + expect(fs.existsSync(path.join(projectDir, "out/config.xml"))).toBe( + false, + ); + expect(locs(readOutput("cli/map.xml"))).toContain( + "https://cli.dev/about", + ); + }); + + it("falls back to sitemap.origin over config.origin", async () => { + writeConfig(`{ + origin: "https://example.com", + sitemap: { origin: "https://sitemap.dev" }, + }`); + + await runSitemap(); + + expect(locs(readOutput())).toContain("https://sitemap.dev/about"); + }); + + it("works without a sitemap config key, defaulting to localhost", async () => { + writeConfig(`{}`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(0); + expect(locs(readOutput())).toContain("http://localhost:7000/about"); + }); + + it("exits with code 1 when a page sitemap export throws", async () => { + writeConfig(`{ origin: "https://example.com" }`); + writeProjectFile( + "src/pages/about.server.ts", + `export const sitemap = () => { throw new Error("db down"); };`, + ); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(1); + expect(res.output).toContain("src/pages/about.server.ts"); + expect(res.output).toContain("db down"); + expect(fs.existsSync(path.join(projectDir, "public/sitemap.xml"))).toBe( + false, + ); + }); + + it("exits with code 1 when config additional_paths throws", async () => { + writeConfig(`{ + sitemap: { + additional_paths: async () => { throw new Error("api down"); }, + }, + }`); + + const res = await runSitemap(); + + expect(res.exit_code).toBe(1); + expect(res.output).toContain("api down"); + }); +}); diff --git a/src/__tests__/functions/server/handle-routes.test.ts b/src/__tests__/functions/server/handle-routes.test.ts index 01f3ba7..7b9dd41 100644 --- a/src/__tests__/functions/server/handle-routes.test.ts +++ b/src/__tests__/functions/server/handle-routes.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test, mock, afterAll } from "bun:test"; +import { describe, expect, test, mock, afterAll, beforeAll } from "bun:test"; import handleRoutes from "../../../../src/functions/server/handle-routes"; mock.module("../../../../src/utils/is-development", () => ({ @@ -9,15 +9,6 @@ mock.module("../../../../src/utils/grab-constants", () => ({ default: () => ({ MBInBytes: 1048576, ServerDefaultRequestBodyLimitBytes: 5242880 }) })); -mock.module("../../../../src/utils/grab-router", () => ({ - default: () => ({ - match: (path: string) => { - if (path === "/api/test") return { filePath: "/test-path" }; - if (path === "/api/large") return { filePath: "/large-path" }; - return null; - } - }) -})); mock.module("../../../../src/utils/grab-route-params", () => ({ default: async () => ({ params: {}, searchParams: {} }) @@ -36,7 +27,24 @@ mock.module("/large-path", () => ({ * Tests for routing logic within `handle-routes`. */ describe("handle-routes", () => { + let originalRouter: typeof global.BUNEXT_ROUTER; + + // grabRouter() reads this global. Mocking the grab-router module + // instead leaks into other test files (mock.restore() does not undo + // mock.module), breaking route matching in bunext-req-handler tests. + beforeAll(() => { + originalRouter = global.BUNEXT_ROUTER; + global.BUNEXT_ROUTER = { + match: (path: string) => { + if (path === "/api/test") return { filePath: "/test-path" }; + if (path === "/api/large") return { filePath: "/large-path" }; + return null; + }, + } as any; + }); + afterAll(() => { + global.BUNEXT_ROUTER = originalRouter; mock.restore(); }); diff --git a/src/__tests__/utils/generate-sitemap.test.ts b/src/__tests__/utils/generate-sitemap.test.ts new file mode 100644 index 0000000..3bcc4d0 --- /dev/null +++ b/src/__tests__/utils/generate-sitemap.test.ts @@ -0,0 +1,180 @@ +import { describe, it, expect, beforeAll, afterAll } from "bun:test"; +import path from "path"; +import fs from "fs"; +import os from "os"; +import generateSitemap, { + grabSitemapEntries, +} from "../../utils/generate-sitemap"; + +// grabAllPages skips anything under __tests__, so fixtures live in the OS tmp dir +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "bunext-sitemap-")); +const pagesDir = path.join(tmpDir, "pages"); + +const pageFiles = [ + "index.tsx", + "about.tsx", + "404.tsx", + "__root.tsx", + "blog/index.tsx", + "blog/[slug].tsx", + "admin/index.tsx", + "admin/users.tsx", + "docs/a&b.tsx", + "api/hello.ts", + "(components)/button.tsx", +]; + +const serverFiles: Record = { + // Dynamic route listing its own URLs + "products/[id].server.ts": ` + export default async () => ({ props: {} }); + export const sitemap = async ({ route }: { route: string }) => [ + "/products/1", + { path: "/products/2", priority: 0.8 }, + route == "/products/[id]" ? "/products/route-ok" : "/products/route-bad", + ]; + `, + // Static page overriding its own entry + "contact.server.ts": ` + export const sitemap = () => [{ path: "/contact", priority: 0.3 }]; + `, + // Server file without a sitemap export keeps the default entry + "about.server.ts": "export default async () => ({ props: {} });", +}; + +beforeAll(() => { + for (const file of [...pageFiles, "products/[id].tsx", "contact.tsx"]) { + const file_path = path.join(pagesDir, file); + fs.mkdirSync(path.dirname(file_path), { recursive: true }); + fs.writeFileSync(file_path, "export default () => null"); + } + + for (const [file, content] of Object.entries(serverFiles)) { + fs.writeFileSync(path.join(pagesDir, file), content); + } + + (global as any).BUNEXT_DIR_NAMES = { PAGES_DIR: pagesDir }; + (global as any).BUNEXT_CONSTANTS = { RouteIgnorePatterns: [/\/\(/] }; + (global as any).BUNEXT_CONFIG = { origin: "https://example.com" }; +}); + +afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + (global as any).BUNEXT_DIR_NAMES = undefined; + (global as any).BUNEXT_CONSTANTS = undefined; + (global as any).BUNEXT_CONFIG = undefined; +}); + +const paths = async (params?: Parameters[0]) => + (await grabSitemapEntries(params)).map((e) => e.path).sort(); + +describe("grabSitemapEntries", () => { + it("lists static pages, skipping api, dynamic, error and ignored routes", async () => { + expect(await paths()).toEqual( + [ + "/", + "/about", + "/admin", + "/admin/users", + "/blog", + "/contact", + "/docs/a&b", + "/products/1", + "/products/2", + "/products/route-ok", + ].sort(), + ); + }); + + it("excludes exact paths, subtrees and regex matches", async () => { + expect( + await paths({ + exclude: ["/about", "/admin/*", /^\/docs/, "/products/*"], + }), + ).toEqual(["/", "/blog", "/contact"]); + }); + + it("adds and dedupes additional paths, applying exclusions", async () => { + const result = await paths({ + additional_paths: [ + "/blog/first-post", + { path: "blog/second-post/", priority: 0.4 }, + "/about", + "/admin/secret", + ], + exclude: ["/admin/*"], + }); + + expect(result).toContain("/blog/first-post"); + expect(result).toContain("/blog/second-post"); + expect(result.filter((p) => p == "/about")).toHaveLength(1); + expect(result).not.toContain("/admin/secret"); + }); + + it("uses a page's server sitemap export for its entries", async () => { + const entries = await grabSitemapEntries(); + const find = (p: string) => entries.find((e) => e.path == p); + + expect(find("/products/2")?.priority).toBe(0.8); + expect(find("/contact")?.priority).toBe(0.3); + expect(find("/contact")?.lastmod).toBeUndefined(); + expect(find("/about")?.lastmod).toBeInstanceOf(Date); + }); + + it("accepts a sync or async function for additional paths", async () => { + expect( + await paths({ additional_paths: () => ["/blog/sync-post"] }), + ).toContain("/blog/sync-post"); + + const result = await paths({ + additional_paths: async () => [ + "/blog/async-post", + { path: "/admin/hidden" }, + ], + exclude: ["/admin/*"], + }); + + expect(result).toContain("/blog/async-post"); + expect(result).not.toContain("/admin/hidden"); + }); + + it("applies trailing slashes except on root", async () => { + expect(await paths({ trailing_slash: true })).toContain("/about/"); + expect(await paths({ trailing_slash: true })).toContain("/"); + }); + + it("uses file mtime as lastmod unless disabled", async () => { + const [entry] = await grabSitemapEntries(); + expect(entry.lastmod).toBeInstanceOf(Date); + + const [no_lastmod] = await grabSitemapEntries({ use_file_lastmod: false }); + expect(no_lastmod.lastmod).toBeUndefined(); + }); +}); + +describe("generateSitemap", () => { + it("renders valid sitemap xml with escaped absolute urls", async () => { + const xml = await generateSitemap({ + default_changefreq: "weekly", + default_priority: 0.5, + use_file_lastmod: false, + additional_paths: [{ path: "/blog/post", priority: 2 }], + }); + + expect(xml).toStartWith(``); + expect(xml).toContain( + ``, + ); + expect(xml).toContain("https://example.com/"); + expect(xml).toContain("https://example.com/docs/a&b"); + expect(xml).toContain("weekly"); + expect(xml).toContain("0.5"); + expect(xml).toContain("1.0"); + expect(xml).not.toContain(""); + }); + + it("uses the origin param over config and strips its trailing slash", async () => { + const xml = await generateSitemap({ origin: "https://other.dev/" }); + expect(xml).toContain("https://other.dev/about"); + }); +}); diff --git a/src/commands/index.ts b/src/commands/index.ts index f96d67b..35cef33 100755 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -4,6 +4,7 @@ import { program } from "commander"; import start from "./start"; import dev from "./dev"; import build from "./build"; +import sitemap from "./sitemap"; import { log } from "../utils/log"; /** @@ -20,6 +21,7 @@ program program.addCommand(dev()); program.addCommand(start()); program.addCommand(build()); +program.addCommand(sitemap()); /** * # Handle Unavailable Commands diff --git a/src/commands/sitemap/index.ts b/src/commands/sitemap/index.ts new file mode 100644 index 0000000..3bcdd3e --- /dev/null +++ b/src/commands/sitemap/index.ts @@ -0,0 +1,63 @@ +import { Command } from "commander"; +import { mkdirSync, writeFileSync } from "fs"; +import path from "path"; +import { log } from "../../utils/log"; +import grabDirNames from "../../utils/grab-dir-names"; +import grabConfig from "../../functions/grab-config"; +import grabConstants from "../../utils/grab-constants"; +import generateSitemap from "../../utils/generate-sitemap"; + +type Options = { + output?: string; + origin?: string; +}; + +export default function () { + return new Command("sitemap") + .description("Generate sitemap.xml from the pages directory") + .option( + "-o, --output ", + "Output file relative to the project root (default: public/sitemap.xml)", + ) + .option("--origin ", "Base URL for sitemap links") + .action(async (options: Options) => { + const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); + + global.BUNEXT_CONFIG = (await grabConfig()) || {}; + global.BUNEXT_CONSTANTS = grabConstants(); + + const { output, ...sitemap_config } = + global.BUNEXT_CONFIG.sitemap || {}; + + try { + const xml = await generateSitemap({ + ...sitemap_config, + origin: options.origin || sitemap_config.origin, + }); + + const output_path = options.output || output; + const output_file = output_path + ? path.resolve(ROOT_DIR, output_path) + : path.join(PUBLIC_DIR, "sitemap.xml"); + + mkdirSync(path.dirname(output_file), { recursive: true }); + writeFileSync(output_file, xml, "utf-8"); + + const url_count = xml.match(//g)?.length || 0; + + log.success( + `Sitemap with ${url_count} URLs written to ${path.relative( + ROOT_DIR, + output_file, + )}`, + ); + } catch (error: any) { + log.error( + `Failed to generate sitemap: ${error?.message || error}`, + ); + process.exit(1); + } + + process.exit(); + }); +} diff --git a/src/index.ts b/src/index.ts index 2232f9c..8df9995 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,12 @@ import bunextInit from "./functions/bunext-init"; import bunextRequestHandler from "./functions/server/bunext-req-handler"; +import generateSitemap from "./utils/generate-sitemap"; import { log } from "./utils/log"; const bunext = { bunextRequestHandler, bunextLog: log, bunextInit, + bunextGenerateSitemap: generateSitemap, }; export default bunext; diff --git a/src/types/index.ts b/src/types/index.ts index 2ffffbe..d86a43c 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -110,6 +110,10 @@ export type BunextConfig = { * Public environment variables to be passed to the client */ public_envs?: Record; + /** + * Options for the `bunext sitemap` command + */ + sitemap?: BunextSitemapConfig; }; export type BunextConfigMiddlewareParams = { @@ -216,8 +220,21 @@ export type BunextPageModule = { export type BunextPageServerModule = { default?: BunextPageServerFn; server?: BunextPageServerFn; + sitemap?: BunextPageSitemapFn; }; +/** + * Returns the sitemap entries for a page. Required for dynamic routes + * (`[slug]`) to appear in the sitemap. On a static page, the returned + * entries replace the page's default entry. + */ +export type BunextPageSitemapFn = (params: { + /** + * The page's route pattern, e.g. `/blog/[slug]` + */ + route: string; +}) => BunextSitemapAdditionalPaths | Promise; + export type BunextHTMLProps = DetailedHTMLProps< HtmlHTMLAttributes, HTMLHtmlElement @@ -452,3 +469,71 @@ export type BuildDataType = { build_date_unix: number; build_date: string; }; + +export type BunextSitemapChangeFreq = + | "always" + | "hourly" + | "daily" + | "weekly" + | "monthly" + | "yearly" + | "never"; + +export type BunextSitemapEntry = { + /** + * URL path (e.g. `/blog/my-post`) or absolute URL. + */ + path: string; + lastmod?: Date | string; + changefreq?: BunextSitemapChangeFreq; + /** + * Between `0.0` and `1.0`. + */ + priority?: number; +}; + +export type BunextSitemapAdditionalPaths = (string | BunextSitemapEntry)[]; + +export type BunextGenerateSitemapParams = { + /** + * Base URL prepended to every path. Defaults to `config.origin`, + * falling back to `http://localhost:`. + */ + origin?: string; + /** + * Paths to leave out. A string matches a path exactly, or a whole + * subtree when it ends with `/*` (e.g. `/admin/*`). A RegExp is + * tested against the path. + */ + exclude?: (string | RegExp)[]; + /** + * Extra entries, typically concrete URLs for dynamic routes + * (`[slug]`), which cannot be discovered from the file system. Can + * also be a function (sync or async) returning them, e.g. to load + * them from a database. + */ + additional_paths?: + | BunextSitemapAdditionalPaths + | (() => + | BunextSitemapAdditionalPaths + | Promise); + /** + * Use each page file's modification time as `lastmod`. Defaults to + * `true`. + */ + use_file_lastmod?: boolean; + /** + * Append a trailing slash to every non-root path. Defaults to `false`. + */ + trailing_slash?: boolean; + default_changefreq?: BunextSitemapChangeFreq; + default_priority?: number; +}; + +export type BunextSitemapConfig = BunextGenerateSitemapParams & { + /** + * Output file, relative to the project root. Defaults to + * `public/sitemap.xml`, which is served at `/sitemap.xml`. + */ + output?: string; +}; diff --git a/src/utils/generate-sitemap.ts b/src/utils/generate-sitemap.ts new file mode 100644 index 0000000..1d808bf --- /dev/null +++ b/src/utils/generate-sitemap.ts @@ -0,0 +1,220 @@ +import { statSync } from "fs"; +import path from "path"; +import type { + BunextGenerateSitemapParams, + BunextPageServerModule, + BunextSitemapAdditionalPaths, + BunextSitemapEntry, +} from "../types"; +import grabPageServerPath from "../functions/server/web-pages/grab-page-server-path"; +import grabAllPages from "./grab-all-pages"; +import grabDirNames from "./grab-dir-names"; +import grabOrigin from "./grab-origin"; +import grabRelativePath from "./grab-relative-path"; +import isDevelopment from "./is-development"; + +const ErrorPagePaths = ["/404", "/500"]; + +/** + * # Generate a sitemap.xml string for all pages + * Dynamic routes are only included through a `sitemap` export in their + * `.server.ts` file, or through `additional_paths`. + */ +export default async function generateSitemap( + params?: BunextGenerateSitemapParams, +): Promise { + const entries = await grabSitemapEntries(params); + const origin = (params?.origin || grabOrigin()).replace(/\/+$/, ""); + + const urls = entries.map((entry) => { + const loc = entry.path.match(/^https?:\/\//) + ? entry.path + : `${origin}${entry.path}`; + + const lines = [` ${escapeXml(loc)}`]; + + if (entry.lastmod) { + const lastmod = + entry.lastmod instanceof Date + ? entry.lastmod.toISOString() + : entry.lastmod; + lines.push(` ${escapeXml(lastmod)}`); + } + + if (entry.changefreq) { + lines.push(` ${entry.changefreq}`); + } + + if (typeof entry.priority == "number") { + const priority = Math.min(Math.max(entry.priority, 0), 1); + lines.push(` ${priority.toFixed(1)}`); + } + + return ` \n${lines.join("\n")}\n `; + }); + + return [ + ``, + ``, + ...urls, + ``, + ``, + ].join("\n"); +} + +export async function grabSitemapEntries( + params?: BunextGenerateSitemapParams, +): Promise { + const use_file_lastmod = params?.use_file_lastmod ?? true; + const entries: BunextSitemapEntry[] = []; + + const pages = grabAllPages({ exclude_api: true }); + + for (let i = 0; i < pages.length; i++) { + const page = pages[i]; + if (!page.local_path) continue; + + const page_path = page.url_path.replace(/\/index$/, "") || "/"; + if (ErrorPagePaths.includes(page_path)) continue; + + const page_sitemap_paths = await grabPageSitemapPaths({ + file_path: page.local_path, + route: page_path, + }); + + if (page_sitemap_paths) { + entries.push(...page_sitemap_paths.map(toSitemapEntry)); + continue; + } + + if (page.url_path.match(/\[/)) continue; + + entries.push({ + path: page_path, + lastmod: use_file_lastmod + ? statSync(page.local_path).mtime + : undefined, + }); + } + + const additional_paths = + typeof params?.additional_paths == "function" + ? await params.additional_paths() + : params?.additional_paths; + + entries.push(...(additional_paths || []).map(toSitemapEntry)); + + const seen = new Set(); + const final_entries: BunextSitemapEntry[] = []; + + for (const entry of entries) { + const path = normalizePath(entry.path, params?.trailing_slash); + if (seen.has(path)) continue; + if (isExcluded(path, params?.exclude)) continue; + seen.add(path); + + final_entries.push({ + changefreq: params?.default_changefreq, + priority: params?.default_priority, + ...entry, + path, + }); + } + + return final_entries; +} + +/** + * # Run the `sitemap` export of a page's server file, if any + */ +async function grabPageSitemapPaths({ + file_path, + route, +}: { + file_path: string; + route: string; +}): Promise { + const { server_file_path } = grabPageServerPath({ file_path }); + if (!server_file_path) return; + + const { ROOT_DIR } = grabDirNames(); + const server_ctx = + global.BUNEXT_SSR_BUNDLER_CTX_MAP?.[grabRelativePath(server_file_path)]; + const final_server_path = server_ctx?.local_path + ? path.join(ROOT_DIR, server_ctx.path) + : server_file_path; + + // Bust the import cache in development so edits to the server file + // are picked up without a restart. + const server_module: BunextPageServerModule = await import( + isDevelopment() + ? `${final_server_path}?t=${Date.now()}` + : final_server_path + ); + + if (typeof server_module.sitemap != "function") return; + + try { + return await server_module.sitemap({ route }); + } catch (error: any) { + throw new Error( + `\`sitemap\` export of ${grabRelativePath(server_file_path)} failed: ${ + error?.message || error + }`, + ); + } +} + +function toSitemapEntry( + path: BunextSitemapAdditionalPaths[number], +): BunextSitemapEntry { + return typeof path == "string" ? { path } : path; +} + +function normalizePath(path: string, trailing_slash?: boolean) { + if (path.match(/^https?:\/\//)) return path; + + let new_path = `/${path}`.replace(/\/{2,}/g, "/"); + if (new_path != "/") new_path = new_path.replace(/\/$/, ""); + if (trailing_slash && new_path != "/") new_path += "/"; + + return new_path; +} + +function isExcluded(path: string, exclude?: (string | RegExp)[]) { + if (!exclude?.length) return false; + + const bare_path = + path.replace(/^https?:\/\/[^/]+/, "").replace(/(.)\/$/, "$1") || "/"; + + for (const pattern of exclude) { + if (pattern instanceof RegExp) { + if (pattern.test(path) || pattern.test(bare_path)) return true; + continue; + } + + const normalized = normalizePath(pattern.replace(/\/\*$/, "")); + + if (pattern.endsWith("/*")) { + if ( + bare_path == normalized || + bare_path.startsWith(`${normalized.replace(/\/$/, "")}/`) + ) { + return true; + } + } else if (bare_path == normalized) { + return true; + } + } + + return false; +} + +function escapeXml(value: string) { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +}