import grabOrigin from "./grab-origin"; /** * First line of every generated robots.txt. `bunext sitemap` only * overwrites an existing robots.txt that starts with it, unless robots * is configured explicitly. */ export const RobotsTxtGeneratedMarker = "# Generated by bunext"; /** * # Generate a robots.txt string */ export default function generateRobotsTxt(params) { const lines = [RobotsTxtGeneratedMarker, ""]; if (params?.disallow_all) { lines.push("User-agent: *", "Disallow: /", ""); return lines.join("\n"); } const rules = params?.rules?.length ? params.rules : [{ user_agent: "*", allow: ["/"] }]; for (const rule of rules) { const user_agents = [rule.user_agent].flat(); for (const user_agent of user_agents) { lines.push(`User-agent: ${sanitizeValue(user_agent)}`); } for (const allow of rule.allow || []) { lines.push(`Allow: ${sanitizeValue(allow)}`); } for (const disallow of rule.disallow || []) { lines.push(`Disallow: ${sanitizeValue(disallow)}`); } // A group needs at least one rule; an empty Disallow allows all. if (!rule.allow?.length && !rule.disallow?.length) { lines.push("Disallow:"); } if (typeof rule.crawl_delay == "number") { lines.push(`Crawl-delay: ${rule.crawl_delay}`); } lines.push(""); } if (params?.sitemaps?.length) { const origin = (params.origin || grabOrigin()).replace(/\/+$/, ""); for (const sitemap of new Set(params.sitemaps)) { const url = sitemap.match(/^https?:\/\//) ? sitemap : `${origin}/${sitemap.replace(/^\/+/, "")}`; lines.push(`Sitemap: ${sanitizeValue(url)}`); } lines.push(""); } return lines.join("\n"); } /** * Line breaks would let a value inject extra directives. */ function sanitizeValue(value) { return value.replace(/[\r\n]+/g, " ").trim(); }