Add robots.txt generation and sitemap chunking
- bunext sitemap now writes public/robots.txt (configurable via the robots config key; robots: false or --no-robots to skip). Hand-written robots.txt files are never overwritten without a robots config - Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with sitemap.xml as a sitemap index; stale chunks are cleaned up - Export bunextGenerateRobotsTxt - Run tests in publish.sh Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -41,6 +41,14 @@ function readOutput(file = "public/sitemap.xml") {
|
||||
return fs.readFileSync(path.join(projectDir, file), "utf-8");
|
||||
}
|
||||
|
||||
function readRobots(file = "public/robots.txt") {
|
||||
return fs.readFileSync(path.join(projectDir, file), "utf-8");
|
||||
}
|
||||
|
||||
function exists(file: string) {
|
||||
return fs.existsSync(path.join(projectDir, file));
|
||||
}
|
||||
|
||||
function locs(xml: string) {
|
||||
return [...xml.matchAll(/<loc>(.*?)<\/loc>/g)].map((m) => m[1]).sort();
|
||||
}
|
||||
@@ -221,4 +229,300 @@ describe("bunext sitemap", () => {
|
||||
expect(res.exit_code).toBe(1);
|
||||
expect(res.output).toContain("api down");
|
||||
});
|
||||
|
||||
describe("robots.txt", () => {
|
||||
it("writes an allow-all robots.txt pointing at the sitemap by default", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("robots.txt written to public/robots.txt");
|
||||
expect(readRobots()).toBe(
|
||||
[
|
||||
"# Generated by bunext",
|
||||
"",
|
||||
"User-agent: *",
|
||||
"Allow: /",
|
||||
"",
|
||||
"Sitemap: https://example.com/sitemap.xml",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("applies the robots config and writes to robots.output", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
robots: {
|
||||
output: "out/robots.txt",
|
||||
rules: [
|
||||
{ user_agent: "*", disallow: ["/admin"] },
|
||||
{ user_agent: "GPTBot", disallow: ["/"] },
|
||||
],
|
||||
sitemaps: ["/sitemaps/extra.xml"],
|
||||
},
|
||||
}`);
|
||||
|
||||
const res = await runSitemap();
|
||||
const robots = readRobots("out/robots.txt");
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
expect(robots).toContain("User-agent: *\nDisallow: /admin\n");
|
||||
expect(robots).toContain("User-agent: GPTBot\nDisallow: /\n");
|
||||
expect(robots).toContain("Sitemap: https://example.com/sitemap.xml");
|
||||
expect(robots).toContain(
|
||||
"Sitemap: https://example.com/sitemaps/extra.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("points at a custom sitemap output inside public/", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
sitemap: { output: "public/maps/site.xml" },
|
||||
}`);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toContain(
|
||||
"Sitemap: https://example.com/maps/site.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns and omits the Sitemap line when the sitemap is outside public/", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
|
||||
const res = await runSitemap("-o", "dist/sitemap.xml");
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("outside public/");
|
||||
expect(readRobots()).not.toContain("Sitemap:");
|
||||
});
|
||||
|
||||
it("uses --origin over robots.origin and sitemap.origin", async () => {
|
||||
writeConfig(`{
|
||||
sitemap: { origin: "https://sitemap.dev" },
|
||||
robots: { origin: "https://robots.dev" },
|
||||
}`);
|
||||
|
||||
await runSitemap("--origin", "https://cli.dev");
|
||||
|
||||
expect(readRobots()).toContain("Sitemap: https://cli.dev/sitemap.xml");
|
||||
});
|
||||
|
||||
it("uses robots.origin over sitemap.origin", async () => {
|
||||
writeConfig(`{
|
||||
sitemap: { origin: "https://sitemap.dev" },
|
||||
robots: { origin: "https://robots.dev" },
|
||||
}`);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toContain(
|
||||
"Sitemap: https://robots.dev/sitemap.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks all crawlers with disallow_all", async () => {
|
||||
writeConfig(`{ robots: { disallow_all: true } }`);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toBe(
|
||||
"# Generated by bunext\n\nUser-agent: *\nDisallow: /\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a hand-written robots.txt when robots is not configured", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n");
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("not generated by bunext");
|
||||
expect(readRobots()).toBe("User-agent: *\nDisallow: /x\n");
|
||||
});
|
||||
|
||||
it("overwrites a hand-written robots.txt when robots is configured", async () => {
|
||||
writeConfig(`{ robots: { rules: [{ user_agent: "*", allow: ["/"] }] } }`);
|
||||
writeProjectFile("public/robots.txt", "User-agent: *\nDisallow: /x\n");
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).toStartWith("# Generated by bunext");
|
||||
});
|
||||
|
||||
it("regenerates its own robots.txt without a robots config", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
writeProjectFile(
|
||||
"public/robots.txt",
|
||||
"# Generated by bunext\n\nUser-agent: *\nDisallow: /old\n",
|
||||
);
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(readRobots()).not.toContain("/old");
|
||||
expect(readRobots()).toContain("Allow: /");
|
||||
});
|
||||
|
||||
it("skips robots.txt with robots: false", async () => {
|
||||
writeConfig(`{ robots: false }`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(exists("public/sitemap.xml")).toBe(true);
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
expect(res.output).not.toContain("robots.txt");
|
||||
});
|
||||
|
||||
it("skips robots.txt with --no-robots", async () => {
|
||||
writeConfig(`{ robots: { disallow_all: true } }`);
|
||||
|
||||
await runSitemap("--no-robots");
|
||||
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
});
|
||||
|
||||
it("does not write robots.txt when sitemap generation fails", async () => {
|
||||
writeConfig(`{
|
||||
sitemap: {
|
||||
additional_paths: async () => { throw new Error("api down"); },
|
||||
},
|
||||
}`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(1);
|
||||
expect(exists("public/robots.txt")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunking", () => {
|
||||
const itemsServerFile = (count: number) =>
|
||||
`export const sitemap = () => Array.from({ length: ${count} }, (_, i) => "/items/" + i);`;
|
||||
|
||||
const chunkFiles = (dir = "public/sitemap-chunks") =>
|
||||
exists(dir)
|
||||
? fs
|
||||
.readdirSync(path.join(projectDir, dir))
|
||||
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
|
||||
: [];
|
||||
|
||||
beforeEach(() => {
|
||||
writeProjectFile("src/pages/items/[id].tsx", "export default () => null");
|
||||
});
|
||||
|
||||
it("writes chunks and a sitemap index above chunk_size", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
sitemap: { chunk_size: 1000 },
|
||||
}`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500));
|
||||
|
||||
const res = await runSitemap();
|
||||
const index = readOutput();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(res.output).toContain("2503 URLs in 3 chunks");
|
||||
expect(chunkFiles()).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
"sitemap-3.xml",
|
||||
]);
|
||||
expect(index).toContain("<sitemapindex");
|
||||
expect(locs(index)).toEqual([
|
||||
"https://example.com/sitemap-chunks/sitemap-1.xml",
|
||||
"https://example.com/sitemap-chunks/sitemap-2.xml",
|
||||
"https://example.com/sitemap-chunks/sitemap-3.xml",
|
||||
]);
|
||||
|
||||
const all_locs = chunkFiles().flatMap((f) =>
|
||||
locs(readOutput(`public/sitemap-chunks/${f}`)),
|
||||
);
|
||||
expect(all_locs).toHaveLength(2503);
|
||||
expect(new Set(all_locs).size).toBe(2503);
|
||||
expect(readRobots()).toContain(
|
||||
"Sitemap: https://example.com/sitemap.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("chunks 20,000+ URLs with the default chunk size", async () => {
|
||||
writeConfig(`{ origin: "https://example.com" }`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(25_000));
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(0);
|
||||
expect(chunkFiles()).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
"sitemap-3.xml",
|
||||
]);
|
||||
expect(
|
||||
locs(readOutput("public/sitemap-chunks/sitemap-3.xml")),
|
||||
).toHaveLength(5003);
|
||||
});
|
||||
|
||||
it("removes stale chunks but keeps other files in the chunks dir", async () => {
|
||||
writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(3500));
|
||||
writeProjectFile("public/sitemap-chunks/keep-me.txt", "mine");
|
||||
|
||||
await runSitemap();
|
||||
expect(chunkFiles()).toHaveLength(5);
|
||||
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500));
|
||||
await runSitemap();
|
||||
|
||||
expect(chunkFiles()).toEqual([
|
||||
"keep-me.txt",
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
]);
|
||||
});
|
||||
|
||||
it("goes back to a single sitemap and removes chunks when it shrinks", async () => {
|
||||
writeConfig(`{ origin: "https://example.com", sitemap: { chunk_size: 1000 } }`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(2500));
|
||||
await runSitemap();
|
||||
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(10));
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.output).toContain("Sitemap with 13 URLs");
|
||||
expect(readOutput()).toContain("<urlset");
|
||||
expect(chunkFiles()).toEqual([]);
|
||||
});
|
||||
|
||||
it("writes chunks to a custom chunks_dir inside public/", async () => {
|
||||
writeConfig(`{
|
||||
origin: "https://example.com",
|
||||
sitemap: { chunk_size: 1000, chunks_dir: "public/seo/maps" },
|
||||
}`);
|
||||
writeProjectFile("src/pages/items/[id].server.ts", itemsServerFile(1500));
|
||||
|
||||
await runSitemap();
|
||||
|
||||
expect(chunkFiles("public/seo/maps")).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
]);
|
||||
expect(locs(readOutput())).toContain(
|
||||
"https://example.com/seo/maps/sitemap-1.xml",
|
||||
);
|
||||
});
|
||||
|
||||
it("fails when chunks_dir is outside public/", async () => {
|
||||
writeConfig(`{ sitemap: { chunks_dir: "dist/chunks" } }`);
|
||||
|
||||
const res = await runSitemap();
|
||||
|
||||
expect(res.exit_code).toBe(1);
|
||||
expect(res.output).toContain("must be inside public/");
|
||||
expect(exists("public/sitemap.xml")).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, it, expect, beforeEach } from "bun:test";
|
||||
import generateRobotsTxt, {
|
||||
RobotsTxtGeneratedMarker,
|
||||
} from "../../utils/generate-robots-txt";
|
||||
|
||||
beforeEach(() => {
|
||||
(global as any).BUNEXT_CONFIG = { origin: "https://example.com" };
|
||||
});
|
||||
|
||||
describe("generateRobotsTxt", () => {
|
||||
it("allows everything by default, starting with the marker", () => {
|
||||
expect(generateRobotsTxt()).toBe(
|
||||
`${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nAllow: /\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("renders rule groups with multiple user agents and crawl delay", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
rules: [
|
||||
{
|
||||
user_agent: "*",
|
||||
allow: ["/"],
|
||||
disallow: ["/admin", "/api/"],
|
||||
crawl_delay: 5,
|
||||
},
|
||||
{ user_agent: ["GPTBot", "CCBot"], disallow: ["/"] },
|
||||
],
|
||||
});
|
||||
|
||||
expect(txt).toContain(
|
||||
"User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /api/\nCrawl-delay: 5\n",
|
||||
);
|
||||
expect(txt).toContain("User-agent: GPTBot\nUser-agent: CCBot\nDisallow: /\n");
|
||||
});
|
||||
|
||||
it("writes an empty Disallow for a rule with no paths", () => {
|
||||
const txt = generateRobotsTxt({ rules: [{ user_agent: "Bingbot" }] });
|
||||
expect(txt).toContain("User-agent: Bingbot\nDisallow:\n");
|
||||
});
|
||||
|
||||
it("resolves sitemap paths against the origin and dedupes them", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
sitemaps: [
|
||||
"/sitemap.xml",
|
||||
"sitemaps/blog.xml",
|
||||
"https://cdn.example.com/sitemap.xml",
|
||||
"/sitemap.xml",
|
||||
],
|
||||
});
|
||||
|
||||
expect(txt.match(/^Sitemap: .*$/gm)).toEqual([
|
||||
"Sitemap: https://example.com/sitemap.xml",
|
||||
"Sitemap: https://example.com/sitemaps/blog.xml",
|
||||
"Sitemap: https://cdn.example.com/sitemap.xml",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the origin param over config and strips its trailing slash", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
origin: "https://other.dev/",
|
||||
sitemaps: ["/sitemap.xml"],
|
||||
});
|
||||
expect(txt).toContain("Sitemap: https://other.dev/sitemap.xml");
|
||||
});
|
||||
|
||||
it("blocks everything with disallow_all, ignoring rules and sitemaps", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
disallow_all: true,
|
||||
rules: [{ user_agent: "*", allow: ["/"] }],
|
||||
sitemaps: ["/sitemap.xml"],
|
||||
});
|
||||
|
||||
expect(txt).toBe(
|
||||
`${RobotsTxtGeneratedMarker}\n\nUser-agent: *\nDisallow: /\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("strips line breaks so values cannot inject directives", () => {
|
||||
const txt = generateRobotsTxt({
|
||||
rules: [{ user_agent: "*", disallow: ["/a\nAllow: /secret"] }],
|
||||
});
|
||||
|
||||
expect(txt).not.toContain("\nAllow: /secret");
|
||||
expect(txt).toContain("Disallow: /a Allow: /secret");
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,9 @@ import path from "path";
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import generateSitemap, {
|
||||
generateSitemapFiles,
|
||||
grabSitemapEntries,
|
||||
SitemapMaxUrlsPerFile,
|
||||
} from "../../utils/generate-sitemap";
|
||||
|
||||
// grabAllPages skips anything under __tests__, so fixtures live in the OS tmp dir
|
||||
@@ -178,3 +180,101 @@ describe("generateSitemap", () => {
|
||||
expect(xml).toContain("<loc>https://other.dev/about</loc>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateSitemapFiles", () => {
|
||||
const manyPaths = (count: number) =>
|
||||
Array.from({ length: count }, (_, i) => `/items/${i}`);
|
||||
|
||||
const countUrls = (xml: string) => xml.match(/<url>/g)?.length || 0;
|
||||
|
||||
it("returns a single urlset with no chunks when within chunk_size", async () => {
|
||||
const { sitemap, chunks, url_count } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(chunks).toEqual([]);
|
||||
expect(sitemap).toContain("<urlset");
|
||||
expect(countUrls(sitemap)).toBe(url_count);
|
||||
});
|
||||
|
||||
it("splits into chunks referenced by a sitemap index", async () => {
|
||||
const { sitemap, chunks, url_count } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
chunk_size: 1000,
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
additional_paths: manyPaths(2500),
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(url_count).toBe(2500);
|
||||
expect(chunks.map((c) => c.file_name)).toEqual([
|
||||
"sitemap-1.xml",
|
||||
"sitemap-2.xml",
|
||||
"sitemap-3.xml",
|
||||
]);
|
||||
expect(chunks.map((c) => c.url_count)).toEqual([1000, 1000, 500]);
|
||||
expect(chunks.map((c) => countUrls(c.xml))).toEqual([1000, 1000, 500]);
|
||||
expect(chunks[0].xml).toContain("<loc>https://example.com/items/0</loc>");
|
||||
expect(chunks[2].xml).toContain(
|
||||
"<loc>https://example.com/items/2499</loc>",
|
||||
);
|
||||
|
||||
expect(sitemap).toContain("<sitemapindex");
|
||||
expect(sitemap).not.toContain("<url>");
|
||||
expect(sitemap.match(/<loc>.*?<\/loc>/g)).toEqual([
|
||||
"<loc>https://example.com/sitemap-chunks/sitemap-1.xml</loc>",
|
||||
"<loc>https://example.com/sitemap-chunks/sitemap-2.xml</loc>",
|
||||
"<loc>https://example.com/sitemap-chunks/sitemap-3.xml</loc>",
|
||||
]);
|
||||
});
|
||||
|
||||
it("sets each index entry's lastmod to the newest in its chunk", async () => {
|
||||
const { sitemap } = await generateSitemapFiles({
|
||||
chunks_url_path: "/maps/",
|
||||
origin: "https://other.dev/",
|
||||
chunk_size: 2,
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
use_file_lastmod: false,
|
||||
additional_paths: [
|
||||
{ path: "/items/a", lastmod: "2026-01-01" },
|
||||
{ path: "/items/b", lastmod: new Date("2026-03-01") },
|
||||
{ path: "/items/c" },
|
||||
{ path: "/items/d", lastmod: "not a date" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(sitemap).toContain(
|
||||
"<loc>https://other.dev/maps/sitemap-1.xml</loc>\n <lastmod>2026-03-01T00:00:00.000Z</lastmod>",
|
||||
);
|
||||
expect(sitemap).toContain(
|
||||
"<loc>https://other.dev/maps/sitemap-2.xml</loc>\n </sitemap>",
|
||||
);
|
||||
});
|
||||
|
||||
it("caps chunk_size at the protocol limit", async () => {
|
||||
const { chunks } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
chunk_size: 1_000_000,
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
additional_paths: manyPaths(SitemapMaxUrlsPerFile + 1),
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(chunks.map((c) => c.url_count)).toEqual([
|
||||
SitemapMaxUrlsPerFile,
|
||||
1,
|
||||
]);
|
||||
});
|
||||
|
||||
it("defaults to 10,000 URLs per chunk", async () => {
|
||||
const { chunks } = await generateSitemapFiles({
|
||||
chunks_url_path: "/sitemap-chunks",
|
||||
exclude: [/^(?!\/items\/)/],
|
||||
additional_paths: manyPaths(20_001),
|
||||
use_file_lastmod: false,
|
||||
});
|
||||
|
||||
expect(chunks.map((c) => c.url_count)).toEqual([10_000, 10_000, 1]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user