Add robots.txt generation and sitemap chunking
- bunext sitemap now writes public/robots.txt (configurable via the robots config key; robots: false or --no-robots to skip). Hand-written robots.txt files are never overwritten without a robots config - Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with sitemap.xml as a sitemap index; stale chunks are cleaned up - Export bunextGenerateRobotsTxt - Run tests in publish.sh Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -41,6 +41,14 @@ function readOutput(file = "public/sitemap.xml") {
|
||||
return fs.readFileSync(path.join(projectDir, file), "utf-8");
|
||||
}
|
||||
|
||||
function readRobots(file = "public/robots.txt") {
|
||||
return fs.readFileSync(path.join(projectDir, file), "utf-8");
|
||||
}
|
||||
|
||||
function exists(file: string) {
|
||||
return fs.existsSync(path.join(projectDir, file));
|
||||
}
|
||||
|
||||
function locs(xml: string) {
|
||||
return [...xml.matchAll(/<loc>(.*?)<\/loc>/g)].map((m) => m[1]).sort();
|
||||
}
|
||||
@@ -221,4 +229,300 @@ describe("bunext sitemap", () => {
|
||||
expect(res.exit_code).toBe(1);
|
||||
expect(res.output).toContain("api down");
|
||||
});
|
||||
|
||||
describe("robots.txt", () => {
|
||||
it("writes an allow-all robots.txt pointing at the sitemap by default", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("robots.txt written to public/robots.txt");
|
||||
expect(readRobots()).toBe(
|
||||
[
|
||||
"# Generated by bunext",
|
||||
"",
|
||||
"User-agent: *",
|
||||
"Allow: /",
|
||||
"",
|
||||
"Sitemap: https://example.com/sitemap.xml",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("applies the robots config and writes to robots.output", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
robots: {
|
||||
output: "out/robots.txt",
|
||||
rules: [
|
||||
{ user_agent: "*", disallow: ["/admin"] },
|
||||
{ user_agent: "GPTBot", disallow: ["/"] },
|
||||
],
|
||||
sitemaps: ["/sitemaps/extra.xml"],
|
||||
},
|
||||
}`);
|
||||
|
||||
const res = await runSitemap();
|
||||
const robots = readRobots("out/robots.txt");
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
expect(robots).toContain("User-agent: *\nDisallow: /admin\n");
|
||||
expect(robots).toContain("User-agent: GPTBot\nDisallow: /\n");
|
||||
expect(robots).toContain("Sitemap: https://example.com/sitemap.xml");
|
||||
expect(robots).toContain(
|
||||
"Sitemap: https://example.com/sitemaps/extra.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("points at a custom sitemap output inside public/", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
sitemap: { output: "public/maps/site.xml" },
|
||||
}`);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toContain(
|
||||
"Sitemap: https://example.com/maps/site.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns and omits the Sitemap line when the sitemap is outside public/", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
|
||||
const res = await runSitemap("-o", "dist/sitemap.xml");
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("outside public/");
|
||||
expect(readRobots()).not.toContain("Sitemap:");
|
||||
});
|
||||
|
||||
it("uses --origin over robots.origin and sitemap.origin", async () => {
|
||||
writeConfig(`{
|
||||
sitemap: { origin: "https://sitemap.dev" },
|
||||
robots: { origin: "https://robots.dev" },
|
||||
}`);
|
||||
|
||||
await runSitemap("--origin", "https://cli.dev");
|
||||
|
||||
expect(readRobots()).toContain("Sitemap: https://cli.dev/sitemap.xml");
|
||||
});
|
||||
|
||||
it("uses robots.origin over sitemap.origin", async () => {
|
||||
writeConfig(`{
|
||||
sitemap: { origin: "https://sitemap.dev" },
|
||||
robots: { origin: "https://robots.dev" },
|
||||
}`);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toContain(
|
||||
"Sitemap: https://robots.dev/sitemap.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks all crawlers with disallow_all", async () => {
|
||||
writeConfig(`{ robots: { disallow_all: true } }`);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toBe(
|
||||
"# Generated by bunext\n\nUser-agent: *\nDisallow: /\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a hand-written robots.txt when robots is not configured", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n");
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("not generated by bunext");
|
||||
expect(readRobots()).toBe("User-agent: *\nDisallow: /x\n");
|
||||
});
|
||||
|
||||
it("overwrites a hand-written robots.txt when robots is configured", async () => {
|
||||
writeConfig(`{ robots: { rules: [{ user_agent: "*", allow: ["/"] }] } }`);
|
||||
writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n");
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toStartWith("# Generated by bunext");
|
||||
});
|
||||
|
||||
it("regenerates its own robots.txt without a robots config", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
writeProjectFile(
|
||||
"public/robots.txt",
|
||||
"# Generated by bunext\n\nUser-agent: *\nDisallow: /old\n",
|
||||
);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).not.toContain("/old");
|
||||
expect(readRobots()).toContain("Allow: /");
|
||||
});
|
||||
|
||||
it("skips robots.txt with robots: false", async () => {
|
||||
writeConfig(`{ robots: false }`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(exists("public/sitemap.xml")).toBe(true);
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
expect(res.output).not.toContain("robots.txt");
|
||||
});
|
||||
|
||||
it("skips robots.txt with --no-robots", async () => {
|
||||
writeConfig(`{ robots: { disallow_all: true } }`);
|
||||
|
||||
await runSitemap("--no-robots");
|
||||
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
});
|
||||
|
||||
it("does not write robots.txt when sitemap generation fails", async () => {
|
||||
writeConfig(`{
|
||||
sitemap: {
|
||||
additional_paths: async () => { throw new Error("api down"); },
|
||||
},
|
||||
}`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(1);
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunking", () => {
|
||||
const itemsServerFile = (count: number) =>
|
||||
`export const sitemap = () => Array.from({ length: ${count} }, (_, i) => "/items/" + i);`;
|
||||
|
||||
const chunkFiles = (dir = "public/sitemap-chunks") =>
|
||||
exists(dir)
|
||||
? fs
|
||||
.readdirSync(path.join(projectDir, dir))
|
||||
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
|
||||
: [];
|
||||
|
||||
beforeEach(() => {
|
||||
writeProjectFile("src/pages/items/[id].tsx", "export default () => null");
|
||||
});
|
||||
|
||||
it("writes chunks and a sitemap index above chunk_size", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
sitemap: { chunk_size: 1000 },
|
||||
}`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500));
|
||||
|
||||
const res = await runSitemap();
|
||||
const index = readOutput();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("2503 URLs in 3 chunks");
|
||||
expect(chunkFiles()).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
"sitemap-3.xml",
|
||||
]);
|
||||
expect(index).toContain("<sitemapindex");
|
||||
expect(locs(index)).toEqual([
|
||||
"https://example.com/sitemap-chunks/sitemap-1.xml",
|
||||
"https://example.com/sitemap-chunks/sitemap-2.xml",
|
||||
"https://example.com/sitemap-chunks/sitemap-3.xml",
|
||||
]);
|
||||
|
||||
const all_locs = chunkFiles().flatMap((f) =>
|
||||
locs(readOutput(`public/sitemap-chunks/${f}`)),
|
||||
);
|
||||
expect(all_locs).toHaveLength(2503);
|
||||
expect(new Set(all_locs).size).toBe(2503);
|
||||
expect(readRobots()).toContain(
|
||||
"Sitemap: https://example.com/sitemap.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("chunks 20,000+ URLs with the default chunk size", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(25_000));
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(chunkFiles()).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
"sitemap-3.xml",
|
||||
]);
|
||||
expect(
|
||||
locs(readOutput("public/sitemap-chunks/sitemap-3.xml")),
|
||||
).toHaveLength(5003);
|
||||
});
|
||||
|
||||
it("removes stale chunks but keeps other files in the chunks dir", async () => {
|
||||
writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(3500));
|
||||
writeProjectFile("public/sitemap-chunks/keep-me.txt", "mine");
|
||||
|
||||
await runSitemap();
|
||||
expect(chunkFiles()).toHaveLength(5);
|
||||
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500));
|
||||
await runSitemap();
|
||||
|
||||
expect(chunkFiles()).toEqual([
|
||||
"keep-me.txt",
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
]);
|
||||
});
|
||||
|
||||
it("goes back to a single sitemap and removes chunks when it shrinks", async () => {
|
||||
writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500));
|
||||
await runSitemap();
|
||||
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(10));
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.output).toContain("Sitemap with 13 URLs");
|
||||
expect(readOutput()).toContain("<urlset");
|
||||
expect(chunkFiles()).toEqual([]);
|
||||
});
|
||||
|
||||
it("writes chunks to a custom chunks_dir inside public/", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
sitemap: { chunk_size: 1000, chunks_dir: "public/seo/maps" },
|
||||
}`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500));
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(chunkFiles("public/seo/maps")).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
]);
|
||||
expect(locs(readOutput())).toContain(
|
||||
"https://example.com/seo/maps/sitemap-1.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("fails when chunks_dir is outside public/", async () => {
|
||||
writeConfig(`{ sitemap: { chunks_dir: "dist/chunks" } }`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(1);
|
||||
expect(res.output).toContain("must be inside public/");
|
||||
expect(exists("public/sitemap.xml")).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, it, expect, beforeEach } from "bun:test";
|
||||
import generateRobotsTxt, {
|
||||
RobotsTxtGeneratedMarker,
|
||||
} from "../../utils/generate-robots-txt";
|
||||
|
||||
beforeEach(() => {
|
||||
(global as any).BUNEXT_CONFIG = { origin: "https://example.com" };
|
||||
});
|
||||
|
||||
describe("generateRobotsTxt", () => {
|
||||
it("allows everything by default, starting with the marker", () => {
|
||||
expect(generateRobotsTxt()).toBe(
|
||||
`${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nAllow: /\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("renders rule groups with multiple user agents and crawl delay", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
rules: [
|
||||
{
|
||||
user_agent: "*",
|
||||
allow: ["/"],
|
||||
disallow: ["/admin", "/api/"],
|
||||
crawl_delay: 5,
|
||||
},
|
||||
{ user_agent: ["GPTBot", "CCBot"], disallow: ["/"] },
|
||||
],
|
||||
});
|
||||
|
||||
expect(txt).toContain(
|
||||
"User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /api/\nCrawl-delay: 5\n",
|
||||
);
|
||||
expect(txt).toContain("User-agent: GPTBot\nUser-agent: CCBot\nDisallow: /\n");
|
||||
});
|
||||
|
||||
it("writes an empty Disallow for a rule with no paths", () => {
|
||||
const txt = generateRobotsTxt({ rules: [{ user_agent: "Bingbot" }] });
|
||||
expect(txt).toContain("User-agent: Bingbot\nDisallow:\n");
|
||||
});
|
||||
|
||||
it("resolves sitemap paths against the origin and dedupes them", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
sitemaps: [
|
||||
"/sitemap.xml",
|
||||
"sitemaps/blog.xml",
|
||||
"https://cdn.example.com/sitemap.xml",
|
||||
"/sitemap.xml",
|
||||
],
|
||||
});
|
||||
|
||||
expect(txt.match(/^Sitemap: .*$/gm)).toEqual([
|
||||
"Sitemap: https://example.com/sitemap.xml",
|
||||
"Sitemap: https://example.com/sitemaps/blog.xml",
|
||||
"Sitemap: https://cdn.example.com/sitemap.xml",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the origin param over config and strips its trailing slash", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
origin: "https://other.dev/",
|
||||
sitemaps: ["/sitemap.xml"],
|
||||
});
|
||||
expect(txt).toContain("Sitemap: https://other.dev/sitemap.xml");
|
||||
});
|
||||
|
||||
it("blocks everything with disallow_all, ignoring rules and sitemaps", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
disallow_all: true,
|
||||
rules: [{ user_agent: "*", allow: ["/"] }],
|
||||
sitemaps: ["/sitemap.xml"],
|
||||
});
|
||||
|
||||
expect(txt).toBe(
|
||||
`${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nDisallow: /\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("strips line breaks so values cannot inject directives", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
rules: [{ user_agent: "*", disallow: ["/a\nAllow: /secret"] }],
|
||||
});
|
||||
|
||||
expect(txt).not.toContain("\nAllow: /secret");
|
||||
expect(txt).toContain("Disallow: /a Allow: /secret");
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,9 @@ import path from "path";
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import generateSitemap, {
|
||||
generateSitemapFiles,
|
||||
grabSitemapEntries,
|
||||
SitemapMaxUrlsPerFile,
|
||||
} from "../../utils/generate-sitemap";
|
||||
|
||||
// grabAllPages skips anything under __tests__, so fixtures live in the OS tmp dir
|
||||
@@ -178,3 +180,101 @@ describe("generateSitemap", () => {
|
||||
expect(xml).toContain("<loc>https://other.dev/about</loc>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateSitemapFiles", () => {
|
||||
const manyPaths = (count: number) =>
|
||||
Array.from({ length: count }, (_, i) => `/items/${i}`);
|
||||
|
||||
const countUrls = (xml: string) => xml.match(/<url>/g)?.length || 0;
|
||||
|
||||
it("returns a single urlset with no chunks when within chunk_size", async () => {
|
||||
const { sitemap, chunks, url_count } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(chunks).toEqual([]);
|
||||
expect(sitemap).toContain("<urlset");
|
||||
expect(countUrls(sitemap)).toBe(url_count);
|
||||
});
|
||||
|
||||
it("splits into chunks referenced by a sitemap index", async () => {
|
||||
const { sitemap, chunks, url_count } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
chunk_size: 1000,
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
additional_paths: manyPaths(2500),
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(url_count).toBe(2500);
|
||||
expect(chunks.map((c) => c.file_name)).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
"sitemap-3.xml",
|
||||
]);
|
||||
expect(chunks.map((c) => c.url_count)).toEqual([1000, 1000, 500]);
|
||||
expect(chunks.map((c) => countUrls(c.xml))).toEqual([1000, 1000, 500]);
|
||||
expect(chunks[0].xml).toContain("<loc>https://example.com/items/0</loc>");
|
||||
expect(chunks[2].xml).toContain(
|
||||
"<loc>https://example.com/items/2499</loc>",
|
||||
);
|
||||
|
||||
expect(sitemap).toContain("<sitemapindex");
|
||||
expect(sitemap).not.toContain("<url>");
|
||||
expect(sitemap.match(/<loc>.*?<\/loc>/g)).toEqual([
|
||||
"<loc>https://example.com/sitemap-chunks/sitemap-1.xml</loc>",
|
||||
"<loc>https://example.com/sitemap-chunks/sitemap-2.xml</loc>",
|
||||
"<loc>https://example.com/sitemap-chunks/sitemap-3.xml</loc>",
|
||||
]);
|
||||
});
|
||||
|
||||
it("sets each index entry's lastmod to the newest in its chunk", async () => {
|
||||
const { sitemap } = await generateSitemapFiles({
|
||||
chunks_url_path: "/maps/",
|
||||
origin: "https://other.dev/",
|
||||
chunk_size: 2,
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
use_file_lastmod: false,
|
||||
additional_paths: [
|
||||
{ path: "/items/a", lastmod: "2026-01-01" },
|
||||
{ path: "/items/b", lastmod: new Date("2026-03-01") },
|
||||
{ path: "/items/c" },
|
||||
{ path: "/items/d", lastmod: "not a date" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(sitemap).toContain(
|
||||
"<loc>https://other.dev/maps/sitemap-1.xml</loc>\n <lastmod>2026-03-01T00:00:00.000Z</lastmod>",
|
||||
);
|
||||
expect(sitemap).toContain(
|
||||
"<loc>https://other.dev/maps/sitemap-2.xml</loc>\n </sitemap>",
|
||||
);
|
||||
});
|
||||
|
||||
it("caps chunk_size at the protocol limit", async () => {
|
||||
const { chunks } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
chunk_size: 1_000_000,
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
additional_paths: manyPaths(SitemapMaxUrlsPerFile + 1),
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(chunks.map((c) => c.url_count)).toEqual([
|
||||
SitemapMaxUrlsPerFile,
|
||||
1,
|
||||
]);
|
||||
});
|
||||
|
||||
it("defaults to 10,000 URLs per chunk", async () => {
|
||||
const { chunks } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
additional_paths: manyPaths(20_001),
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(chunks.map((c) => c.url_count)).toEqual([10_000, 10_000, 1]);
|
||||
});
|
||||
});
|
||||
|
||||
+167
-20
@@ -1,56 +1,117 @@
|
||||
import { Command } from "commander";
|
||||
import { mkdirSync, writeFileSync } from "fs";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "fs";
|
||||
import path from "path";
|
||||
import { log } from "../../utils/log";
|
||||
import grabDirNames from "../../utils/grab-dir-names";
|
||||
import grabConfig from "../../functions/grab-config";
|
||||
import grabConstants from "../../utils/grab-constants";
|
||||
import generateSitemap from "../../utils/generate-sitemap";
|
||||
import { generateSitemapFiles } from "../../utils/generate-sitemap";
|
||||
import generateRobotsTxt, {
|
||||
RobotsTxtGeneratedMarker,
|
||||
} from "../../utils/generate-robots-txt";
|
||||
|
||||
type Options = {
|
||||
output?: string;
|
||||
origin?: string;
|
||||
robots: boolean;
|
||||
};
|
||||
|
||||
export default function () {
|
||||
return new Command("sitemap")
|
||||
.description("Generate sitemap.xml from the pages directory")
|
||||
.description(
|
||||
"Generate sitemap.xml and robots.txt from the pages directory",
|
||||
)
|
||||
.option(
|
||||
"-o, --output <path>",
|
||||
"Output file relative to the project root (default: public/sitemap.xml)",
|
||||
)
|
||||
.option("--origin <url>", "Base URL for sitemap links")
|
||||
.option("--no-robots", "Skip generating robots.txt")
|
||||
.action(async (options: Options) => {
|
||||
const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
|
||||
|
||||
global.BUNEXT_CONFIG = (await grabConfig()) || {};
|
||||
global.BUNEXT_CONSTANTS = grabConstants();
|
||||
|
||||
const { output, ...sitemap_config } =
|
||||
const { output, chunk_size, chunks_dir, ...sitemap_config } =
|
||||
global.BUNEXT_CONFIG.sitemap || {};
|
||||
|
||||
try {
|
||||
const xml = await generateSitemap({
|
||||
...sitemap_config,
|
||||
origin: options.origin || sitemap_config.origin,
|
||||
});
|
||||
let sitemap_file: string;
|
||||
|
||||
try {
|
||||
const output_path = options.output || output;
|
||||
const output_file = output_path
|
||||
sitemap_file = output_path
|
||||
? path.resolve(ROOT_DIR, output_path)
|
||||
: path.join(PUBLIC_DIR, "sitemap.xml");
|
||||
|
||||
mkdirSync(path.dirname(output_file), { recursive: true });
|
||||
writeFileSync(output_file, xml, "utf-8");
|
||||
|
||||
const url_count = xml.match(/<url>/g)?.length || 0;
|
||||
|
||||
log.success(
|
||||
`Sitemap with ${url_count} URLs written to ${path.relative(
|
||||
ROOT_DIR,
|
||||
output_file,
|
||||
)}`,
|
||||
const chunks_dir_path = path.resolve(
|
||||
ROOT_DIR,
|
||||
chunks_dir || "public/sitemap-chunks",
|
||||
);
|
||||
const chunks_public_path = grabPublicUrlPath(chunks_dir_path);
|
||||
|
||||
if (chunks_public_path === undefined) {
|
||||
throw new Error(
|
||||
`\`sitemap.chunks_dir\` must be inside public/ ` +
|
||||
`so the chunks are served.`,
|
||||
);
|
||||
}
|
||||
|
||||
const { sitemap, chunks, url_count } =
|
||||
await generateSitemapFiles({
|
||||
...sitemap_config,
|
||||
origin: options.origin || sitemap_config.origin,
|
||||
chunk_size,
|
||||
chunks_url_path: chunks_public_path,
|
||||
});
|
||||
|
||||
// Remove chunks from a previous, larger sitemap
|
||||
if (existsSync(chunks_dir_path)) {
|
||||
for (const file of readdirSync(chunks_dir_path)) {
|
||||
if (file.match(/^sitemap-\d+\.xml$/)) {
|
||||
rmSync(path.join(chunks_dir_path, file));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (chunks.length) {
|
||||
mkdirSync(chunks_dir_path, { recursive: true });
|
||||
}
|
||||
|
||||
for (const chunk of chunks) {
|
||||
writeFileSync(
|
||||
path.join(chunks_dir_path, chunk.file_name),
|
||||
chunk.xml,
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
|
||||
mkdirSync(path.dirname(sitemap_file), { recursive: true });
|
||||
writeFileSync(sitemap_file, sitemap, "utf-8");
|
||||
|
||||
const sitemap_relative_path = path.relative(
|
||||
ROOT_DIR,
|
||||
sitemap_file,
|
||||
);
|
||||
|
||||
if (chunks.length) {
|
||||
log.success(
|
||||
`Sitemap index with ${url_count} URLs in ${chunks.length} chunks ` +
|
||||
`written to ${sitemap_relative_path} ` +
|
||||
`(chunks in ${path.relative(ROOT_DIR, chunks_dir_path)})`,
|
||||
);
|
||||
} else {
|
||||
log.success(
|
||||
`Sitemap with ${url_count} URLs written to ${sitemap_relative_path}`,
|
||||
);
|
||||
}
|
||||
} catch (error: any) {
|
||||
log.error(
|
||||
`Failed to generate sitemap: ${error?.message || error}`,
|
||||
@@ -58,6 +119,92 @@ export default function () {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const robots_config = global.BUNEXT_CONFIG.robots;
|
||||
|
||||
if (options.robots && robots_config !== false) {
|
||||
writeRobotsTxt({
|
||||
sitemap_file,
|
||||
cli_origin: options.origin,
|
||||
sitemap_origin: sitemap_config.origin,
|
||||
});
|
||||
}
|
||||
|
||||
process.exit();
|
||||
});
|
||||
}
|
||||
|
||||
function writeRobotsTxt({
|
||||
sitemap_file,
|
||||
cli_origin,
|
||||
sitemap_origin,
|
||||
}: {
|
||||
sitemap_file: string;
|
||||
cli_origin?: string;
|
||||
sitemap_origin?: string;
|
||||
}) {
|
||||
const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
|
||||
const robots_config = global.BUNEXT_CONFIG.robots || undefined;
|
||||
const { output, ...robots_params } = robots_config || {};
|
||||
|
||||
const robots_file = output
|
||||
? path.resolve(ROOT_DIR, output)
|
||||
: path.join(PUBLIC_DIR, "robots.txt");
|
||||
const robots_relative_path = path.relative(ROOT_DIR, robots_file);
|
||||
|
||||
// Without a robots config, never replace a hand-written robots.txt.
|
||||
if (
|
||||
!robots_config &&
|
||||
existsSync(robots_file) &&
|
||||
!readFileSync(robots_file, "utf-8").startsWith(
|
||||
RobotsTxtGeneratedMarker,
|
||||
)
|
||||
) {
|
||||
log.warn(
|
||||
`Skipped ${robots_relative_path}: it was not generated by bunext. ` +
|
||||
`Set \`robots\` in bunext.config.ts to overwrite it, ` +
|
||||
`or \`robots: false\` to silence this.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const sitemap_public_path = grabPublicUrlPath(sitemap_file);
|
||||
|
||||
if (sitemap_public_path === undefined && !robots_params.sitemaps?.length) {
|
||||
log.warn(
|
||||
`Sitemap is outside public/, so robots.txt has no Sitemap line. ` +
|
||||
`Add its URL to \`robots.sitemaps\`.`,
|
||||
);
|
||||
}
|
||||
|
||||
const robots_txt = generateRobotsTxt({
|
||||
...robots_params,
|
||||
origin: cli_origin || robots_params.origin || sitemap_origin,
|
||||
sitemaps: [
|
||||
...(sitemap_public_path !== undefined
|
||||
? [sitemap_public_path]
|
||||
: []),
|
||||
...(robots_params.sitemaps || []),
|
||||
],
|
||||
});
|
||||
|
||||
mkdirSync(path.dirname(robots_file), { recursive: true });
|
||||
writeFileSync(robots_file, robots_txt, "utf-8");
|
||||
|
||||
log.success(`robots.txt written to ${robots_relative_path}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* # URL path a file in public/ is served at
|
||||
* Public files are served from the site root, e.g. public/sitemap.xml
|
||||
* at /sitemap.xml. Returns `undefined` for paths outside public/.
|
||||
*/
|
||||
function grabPublicUrlPath(file_path: string) {
|
||||
const { PUBLIC_DIR } = grabDirNames();
|
||||
const relative_path = path.relative(PUBLIC_DIR, file_path);
|
||||
|
||||
if (relative_path.startsWith("..") || path.isAbsolute(relative_path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
return `/${relative_path.split(path.sep).join("/")}`;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import bunextInit from "./functions/bunext-init";
|
||||
import bunextRequestHandler from "./functions/server/bunext-req-handler";
|
||||
import generateRobotsTxt from "./utils/generate-robots-txt";
|
||||
import generateSitemap from "./utils/generate-sitemap";
|
||||
import { log } from "./utils/log";
|
||||
|
||||
@@ -8,5 +9,6 @@ const bunext = {
|
||||
bunextLog: log,
|
||||
bunextInit,
|
||||
bunextGenerateSitemap: generateSitemap,
|
||||
bunextGenerateRobotsTxt: generateRobotsTxt,
|
||||
};
|
||||
export default bunext;
|
||||
|
||||
@@ -114,6 +114,11 @@ export type BunextConfig = {
|
||||
* Options for the `bunext sitemap` command
|
||||
*/
|
||||
sitemap?: BunextSitemapConfig;
|
||||
/**
|
||||
* Options for the robots.txt written by `bunext sitemap`. Set to
|
||||
* `false` to skip generating it.
|
||||
*/
|
||||
robots?: BunextRobotsConfig | false;
|
||||
};
|
||||
|
||||
export type BunextConfigMiddlewareParams = {
|
||||
@@ -531,9 +536,64 @@ export type BunextGenerateSitemapParams = {
|
||||
};
|
||||
|
||||
export type BunextSitemapConfig = BunextGenerateSitemapParams & {
|
||||
/**
|
||||
* Maximum URLs per sitemap file. Above this, `bunext sitemap` writes
|
||||
* chunk files to `chunks_dir` and makes sitemap.xml a sitemap index
|
||||
* referencing them. Defaults to 10,000; capped at 50,000, the protocol
|
||||
* limit. Chunks are also split at 50MB.
|
||||
*/
|
||||
chunk_size?: number;
|
||||
/**
|
||||
* Directory for sitemap chunk files, relative to the project root.
|
||||
* Must be inside `public/` so the files are served. Defaults to
|
||||
* `public/sitemap-chunks`.
|
||||
*/
|
||||
chunks_dir?: string;
|
||||
/**
|
||||
* Output file, relative to the project root. Defaults to
|
||||
* `public/sitemap.xml`, which is served at `/sitemap.xml`.
|
||||
*/
|
||||
output?: string;
|
||||
};
|
||||
|
||||
export type BunextRobotsRule = {
|
||||
/**
|
||||
* One or more crawler names, e.g. `*` or `["GPTBot", "CCBot"]`
|
||||
*/
|
||||
user_agent: string | string[];
|
||||
allow?: string[];
|
||||
disallow?: string[];
|
||||
/**
|
||||
* Seconds between requests. Not supported by every crawler.
|
||||
*/
|
||||
crawl_delay?: number;
|
||||
};
|
||||
|
||||
export type BunextGenerateRobotsTxtParams = {
|
||||
/**
|
||||
* Base URL used to turn relative `sitemaps` paths into absolute URLs.
|
||||
* Defaults to `config.origin`, falling back to `http://localhost:<port>`.
|
||||
*/
|
||||
origin?: string;
|
||||
/**
|
||||
* Defaults to allowing every crawler everywhere.
|
||||
*/
|
||||
rules?: BunextRobotsRule[];
|
||||
/**
|
||||
* Sitemap URLs or paths, each written as a `Sitemap:` line.
|
||||
*/
|
||||
sitemaps?: string[];
|
||||
/**
|
||||
* Block every crawler from the whole site, ignoring `rules` and
|
||||
* `sitemaps`. Useful for staging deployments.
|
||||
*/
|
||||
disallow_all?: boolean;
|
||||
};
|
||||
|
||||
export type BunextRobotsConfig = BunextGenerateRobotsTxtParams & {
|
||||
/**
|
||||
* Output file, relative to the project root. Defaults to
|
||||
* `public/robots.txt`, which is served at `/robots.txt`.
|
||||
*/
|
||||
output?: string;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { BunextGenerateRobotsTxtParams } from "../types";
|
||||
import grabOrigin from "./grab-origin";
|
||||
|
||||
/**
|
||||
* First line of every generated robots.txt. `bunext sitemap` only
|
||||
* overwrites an existing robots.txt that starts with it, unless robots
|
||||
* is configured explicitly.
|
||||
*/
|
||||
export const RobotsTxtGeneratedMarker = "# Generated by bunext";
|
||||
|
||||
/**
|
||||
* # Generate a robots.txt string
|
||||
*/
|
||||
export default function generateRobotsTxt(
|
||||
params?: BunextGenerateRobotsTxtParams,
|
||||
): string {
|
||||
const lines = [RobotsTxtGeneratedMarker, ""];
|
||||
|
||||
if (params?.disallow_all) {
|
||||
lines.push("User-agent: *", "Disallow: /", "");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
const rules = params?.rules?.length
|
||||
? params.rules
|
||||
: [{ user_agent: "*", allow: ["/"] }];
|
||||
|
||||
for (const rule of rules) {
|
||||
const user_agents = [rule.user_agent].flat();
|
||||
|
||||
for (const user_agent of user_agents) {
|
||||
lines.push(`User-agent: ${sanitizeValue(user_agent)}`);
|
||||
}
|
||||
|
||||
for (const allow of rule.allow || []) {
|
||||
lines.push(`Allow: ${sanitizeValue(allow)}`);
|
||||
}
|
||||
|
||||
for (const disallow of rule.disallow || []) {
|
||||
lines.push(`Disallow: ${sanitizeValue(disallow)}`);
|
||||
}
|
||||
|
||||
// A group needs at least one rule; an empty Disallow allows all.
|
||||
if (!rule.allow?.length && !rule.disallow?.length) {
|
||||
lines.push("Disallow:");
|
||||
}
|
||||
|
||||
if (typeof rule.crawl_delay == "number") {
|
||||
lines.push(`Crawl-delay: ${rule.crawl_delay}`);
|
||||
}
|
||||
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (params?.sitemaps?.length) {
|
||||
const origin = (params.origin || grabOrigin()).replace(/\/+$/, "");
|
||||
|
||||
for (const sitemap of new Set(params.sitemaps)) {
|
||||
const url = sitemap.match(/^https?:\/\//)
|
||||
? sitemap
|
||||
: `${origin}/${sitemap.replace(/^\/+/, "")}`;
|
||||
|
||||
lines.push(`Sitemap: ${sanitizeValue(url)}`);
|
||||
}
|
||||
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Line breaks would let a value inject extra directives.
|
||||
*/
|
||||
function sanitizeValue(value: string) {
|
||||
return value.replace(/[\r\n]+/g, " ").trim();
|
||||
}
|
||||
+159
-23
@@ -12,56 +12,192 @@ import grabDirNames from "./grab-dir-names";
|
||||
import grabOrigin from "./grab-origin";
|
||||
import grabRelativePath from "./grab-relative-path";
|
||||
import isDevelopment from "./is-development";
|
||||
import { log } from "./log";
|
||||
|
||||
const ErrorPagePaths = ["/404", "/500"];
|
||||
|
||||
/**
|
||||
* Protocol limits for a single sitemap file
|
||||
* https://www.sitemaps.org/protocol.html#index
|
||||
*/
|
||||
export const SitemapMaxUrlsPerFile = 50_000;
|
||||
export const SitemapMaxBytesPerFile = 50 * 1024 * 1024;
|
||||
export const SitemapDefaultChunkSize = 10_000;
|
||||
|
||||
const SitemapXmlHeader = `<?xml version="1.0" encoding="UTF-8"?>`;
|
||||
const SitemapXmlns = `xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"`;
|
||||
|
||||
export type SitemapChunkFile = {
|
||||
file_name: string;
|
||||
xml: string;
|
||||
url_count: number;
|
||||
};
|
||||
|
||||
export type SitemapFiles = {
|
||||
/**
|
||||
* A `<urlset>` when everything fits in one file, otherwise a
|
||||
* `<sitemapindex>` referencing `chunks`.
|
||||
*/
|
||||
sitemap: string;
|
||||
chunks: SitemapChunkFile[];
|
||||
url_count: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* # Generate a sitemap.xml string for all pages
|
||||
* Dynamic routes are only included through a `sitemap` export in their
|
||||
* `.server.ts` file, or through `additional_paths`.
|
||||
* `.server.ts` file, or through `additional_paths`. Always returns a
|
||||
* single `<urlset>`; `bunext sitemap` splits large sitemaps into chunks.
|
||||
*/
|
||||
export default async function generateSitemap(
|
||||
params?: BunextGenerateSitemapParams,
|
||||
): Promise<string> {
|
||||
const entries = await grabSitemapEntries(params);
|
||||
const origin = (params?.origin || grabOrigin()).replace(/\/+$/, "");
|
||||
const origin = grabSitemapOrigin(params?.origin);
|
||||
|
||||
const urls = entries.map((entry) => {
|
||||
const loc = entry.path.match(/^https?:\/\//)
|
||||
? entry.path
|
||||
: `${origin}${entry.path}`;
|
||||
if (entries.length > SitemapMaxUrlsPerFile) {
|
||||
log.warn(
|
||||
`Sitemap has ${entries.length} URLs, above the ${SitemapMaxUrlsPerFile} ` +
|
||||
`per-file limit. Use \`bunext sitemap\` to split it into chunks.`,
|
||||
);
|
||||
}
|
||||
|
||||
const lines = [` <loc>${escapeXml(loc)}</loc>`];
|
||||
return renderUrlset(entries.map((entry) => renderUrl(entry, origin)));
|
||||
}
|
||||
|
||||
if (entry.lastmod) {
|
||||
const lastmod =
|
||||
entry.lastmod instanceof Date
|
||||
? entry.lastmod.toISOString()
|
||||
: entry.lastmod;
|
||||
lines.push(` <lastmod>${escapeXml(lastmod)}</lastmod>`);
|
||||
/**
|
||||
* # Generate sitemap.xml, split into chunk files when needed
|
||||
* Chunks hold at most `chunk_size` URLs and 50MB each. `chunks_url_path`
|
||||
* is the public URL path the chunk files are served from.
|
||||
*/
|
||||
export async function generateSitemapFiles({
|
||||
chunk_size = SitemapDefaultChunkSize,
|
||||
chunks_url_path,
|
||||
...params
|
||||
}: BunextGenerateSitemapParams & {
|
||||
chunk_size?: number;
|
||||
chunks_url_path: string;
|
||||
}): Promise<SitemapFiles> {
|
||||
const entries = await grabSitemapEntries(params);
|
||||
const origin = grabSitemapOrigin(params.origin);
|
||||
const max_urls = Math.min(
|
||||
Math.max(Math.floor(chunk_size) || 1, 1),
|
||||
SitemapMaxUrlsPerFile,
|
||||
);
|
||||
|
||||
const urls = entries.map((entry) => renderUrl(entry, origin));
|
||||
const empty_urlset_bytes = Buffer.byteLength(renderUrlset([]));
|
||||
|
||||
type Chunk = { urls: string[]; bytes: number; lastmod?: Date };
|
||||
const chunks: Chunk[] = [];
|
||||
let chunk: Chunk | undefined;
|
||||
|
||||
for (let i = 0; i < urls.length; i++) {
|
||||
const url_bytes = Buffer.byteLength(urls[i]) + 1;
|
||||
|
||||
if (
|
||||
!chunk ||
|
||||
chunk.urls.length >= max_urls ||
|
||||
chunk.bytes + url_bytes > SitemapMaxBytesPerFile
|
||||
) {
|
||||
chunk = { urls: [], bytes: empty_urlset_bytes };
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
if (entry.changefreq) {
|
||||
lines.push(` <changefreq>${entry.changefreq}</changefreq>`);
|
||||
chunk.urls.push(urls[i]);
|
||||
chunk.bytes += url_bytes;
|
||||
|
||||
const lastmod = parseLastmod(entries[i].lastmod);
|
||||
if (lastmod && (!chunk.lastmod || lastmod > chunk.lastmod)) {
|
||||
chunk.lastmod = lastmod;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof entry.priority == "number") {
|
||||
const priority = Math.min(Math.max(entry.priority, 0), 1);
|
||||
lines.push(` <priority>${priority.toFixed(1)}</priority>`);
|
||||
}
|
||||
if (chunks.length <= 1) {
|
||||
return {
|
||||
sitemap: renderUrlset(chunks[0]?.urls || []),
|
||||
chunks: [],
|
||||
url_count: urls.length,
|
||||
};
|
||||
}
|
||||
|
||||
return ` <url>\n${lines.join("\n")}\n </url>`;
|
||||
});
|
||||
const chunks_dir_path = chunks_url_path.replace(/^\/+|\/+$/g, "");
|
||||
const chunk_base_url = chunks_dir_path
|
||||
? `${origin}/${chunks_dir_path}`
|
||||
: origin;
|
||||
|
||||
const chunk_files: SitemapChunkFile[] = chunks.map((c, i) => ({
|
||||
file_name: `sitemap-${i + 1}.xml`,
|
||||
xml: renderUrlset(c.urls),
|
||||
url_count: c.urls.length,
|
||||
}));
|
||||
|
||||
const sitemap = [
|
||||
SitemapXmlHeader,
|
||||
`<sitemapindex ${SitemapXmlns}>`,
|
||||
...chunk_files.map((file, i) => {
|
||||
const loc = escapeXml(`${chunk_base_url}/${file.file_name}`);
|
||||
const lines = [` <loc>${loc}</loc>`];
|
||||
|
||||
const lastmod = chunks[i].lastmod?.toISOString();
|
||||
if (lastmod) lines.push(` <lastmod>${lastmod}</lastmod>`);
|
||||
|
||||
return ` <sitemap>\n${lines.join("\n")}\n </sitemap>`;
|
||||
}),
|
||||
`</sitemapindex>`,
|
||||
``,
|
||||
].join("\n");
|
||||
|
||||
return { sitemap, chunks: chunk_files, url_count: urls.length };
|
||||
}
|
||||
|
||||
function grabSitemapOrigin(origin?: string) {
|
||||
return (origin || grabOrigin()).replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function renderUrlset(urls: string[]) {
|
||||
return [
|
||||
`<?xml version="1.0" encoding="UTF-8"?>`,
|
||||
`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
|
||||
SitemapXmlHeader,
|
||||
`<urlset ${SitemapXmlns}>`,
|
||||
...urls,
|
||||
`</urlset>`,
|
||||
``,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function renderUrl(entry: BunextSitemapEntry, origin: string) {
|
||||
const loc = entry.path.match(/^https?:\/\//)
|
||||
? entry.path
|
||||
: `${origin}${entry.path}`;
|
||||
|
||||
const lines = [` <loc>${escapeXml(loc)}</loc>`];
|
||||
|
||||
if (entry.lastmod) {
|
||||
const lastmod =
|
||||
entry.lastmod instanceof Date
|
||||
? entry.lastmod.toISOString()
|
||||
: entry.lastmod;
|
||||
lines.push(` <lastmod>${escapeXml(lastmod)}</lastmod>`);
|
||||
}
|
||||
|
||||
if (entry.changefreq) {
|
||||
lines.push(` <changefreq>${entry.changefreq}</changefreq>`);
|
||||
}
|
||||
|
||||
if (typeof entry.priority == "number") {
|
||||
const priority = Math.min(Math.max(entry.priority, 0), 1);
|
||||
lines.push(` <priority>${priority.toFixed(1)}</priority>`);
|
||||
}
|
||||
|
||||
return ` <url>\n${lines.join("\n")}\n </url>`;
|
||||
}
|
||||
|
||||
function parseLastmod(lastmod?: Date | string) {
|
||||
if (!lastmod) return;
|
||||
const date = lastmod instanceof Date ? lastmod : new Date(lastmod);
|
||||
return isNaN(date.getTime()) ? undefined : date;
|
||||
}
|
||||
|
||||
export async function grabSitemapEntries(
|
||||
params?: BunextGenerateSitemapParams,
|
||||
): Promise<BunextSitemapEntry[]> {
|
||||
|
||||
Reference in New Issue
Block a user