tbenandClaude Opus 5.5
be4bebaa22
- bunext sitemap now writes public/robots.txt (configurable via the robots config key; robots: false or --no-robots to skip). Hand-written robots.txt files are never overwritten without a robots config - Large sitemaps are split into public/sitemap-chunks/sitemap-N.xml (sitemap.chunk_size, default 10,000, max 50,000; 50MB per file) with sitemap.xml as a sitemap index; stale chunks are cleaned up - Export bunextGenerateRobotsTxt - Run tests in publish.sh Co-Authored-By: Claude Opus 5.5 <[email protected]>
58 lines
2.0 KiB
JavaScript
58 lines
2.0 KiB
JavaScript
import grabOrigin from "./grab-origin";
|
|
/**
|
|
* First line of every generated robots.txt. `bunext sitemap` only
|
|
* overwrites an existing robots.txt that starts with it, unless robots
|
|
* is configured explicitly.
|
|
*/
|
|
export const RobotsTxtGeneratedMarker = "# Generated by bunext";
|
|
/**
|
|
* # Generate a robots.txt string
|
|
*/
|
|
export default function generateRobotsTxt(params) {
|
|
const lines = [RobotsTxtGeneratedMarker, ""];
|
|
if (params?.disallow_all) {
|
|
lines.push("User-agent: *", "Disallow: /", "");
|
|
return lines.join("\n");
|
|
}
|
|
const rules = params?.rules?.length
|
|
? params.rules
|
|
: [{ user_agent: "*", allow: ["/"] }];
|
|
for (const rule of rules) {
|
|
const user_agents = [rule.user_agent].flat();
|
|
for (const user_agent of user_agents) {
|
|
lines.push(`User-agent: ${sanitizeValue(user_agent)}`);
|
|
}
|
|
for (const allow of rule.allow || []) {
|
|
lines.push(`Allow: ${sanitizeValue(allow)}`);
|
|
}
|
|
for (const disallow of rule.disallow || []) {
|
|
lines.push(`Disallow: ${sanitizeValue(disallow)}`);
|
|
}
|
|
// A group needs at least one rule; an empty Disallow allows all.
|
|
if (!rule.allow?.length && !rule.disallow?.length) {
|
|
lines.push("Disallow:");
|
|
}
|
|
if (typeof rule.crawl_delay == "number") {
|
|
lines.push(`Crawl-delay: ${rule.crawl_delay}`);
|
|
}
|
|
lines.push("");
|
|
}
|
|
if (params?.sitemaps?.length) {
|
|
const origin = (params.origin || grabOrigin()).replace(/\/+$/, "");
|
|
for (const sitemap of new Set(params.sitemaps)) {
|
|
const url = sitemap.match(/^https?:\/\//)
|
|
? sitemap
|
|
: `${origin}/${sitemap.replace(/^\/+/, "")}`;
|
|
lines.push(`Sitemap: ${sanitizeValue(url)}`);
|
|
}
|
|
lines.push("");
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
/**
|
|
* Line breaks would let a value inject extra directives.
|
|
*/
|
|
function sanitizeValue(value) {
|
|
return value.replace(/[\r\n]+/g, " ").trim();
|
|
}
|