Add disable_seo config to keep preview deployments out of search engines

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/__tests__/functions/server/bunext-req-handler.test.ts

@@ -123,4 +123,29 @@ describe("bunext-req-handler", () => {
         const res = await bunextRequestHandler({ req, server });
         expect(await res.text()).toBe("web-pages");
     });
+
+    test("disable_seo blocks crawlers via header and robots.txt", async () => {
+        global.BUNEXT_CONSTANTS.config.disable_seo = true;
+
+        const page = await bunextRequestHandler({
+            req: new Request("http://localhost/about"),
+            server,
+        });
+        expect(page.headers.get("X-Robots-Tag")).toBe("noindex, nofollow");
+
+        const robots = await bunextRequestHandler({
+            req: new Request("http://localhost/robots.txt"),
+            server,
+        });
+        expect(await robots.text()).toContain("User-agent: *\nDisallow: /");
+        expect(robots.headers.get("X-Robots-Tag")).toBe("noindex, nofollow");
+    });
+
+    test("no X-Robots-Tag without disable_seo", async () => {
+        const res = await bunextRequestHandler({
+            req: new Request("http://localhost/about"),
+            server,
+        });
+        expect(res.headers.get("X-Robots-Tag")).toBeNull();
+    });
 });

src/commands/sitemap/index.ts

@@ -41,6 +41,13 @@ export default function () {
             global.BUNEXT_CONFIG = (await grabConfig()) || {};
             global.BUNEXT_CONSTANTS = grabConstants();
 
+            if (global.BUNEXT_CONFIG.disable_seo) {
+                log.warn(
+                    `\`disable_seo\` is set: the server blocks all crawlers at ` +
+                        `/robots.txt, so the generated robots.txt is not served.`,
+                );
+            }
+
             const { output, chunk_size, chunks_dir, ...sitemap_config } =
                 global.BUNEXT_CONFIG.sitemap || {};
 

src/functions/server/bunext-req-handler.ts

@@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data";
 import { log } from "../../utils/log";
 import fullRebuild from "./full-rebuild";
 import grabRouter from "../../utils/grab-router";
+import { NoSEORobotsValue } from "../../utils/disable-seo";
+import generateRobotsTxt from "../../utils/generate-robots-txt";
 
 const HMR_RETRY_COOLDOWN_MS = 5000;
 let lastHmrRetryTime = 0;
@@ -65,8 +67,14 @@ export default async function bunextRequestHandler({
             return new Response("Modules Rebuilt");
         }
 
+        const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo;
+
         if (url.pathname === "/__hmr" && is_dev) {
             return handleHmr({ req });
+        } else if (disable_seo && url.pathname == "/robots.txt") {
+            response = new Response(generateRobotsTxt({ disallow_all: true }), {
+                headers: { "Content-Type": "text/plain; charset=utf-8" },
+            });
         } else if (url.pathname.startsWith("/.bunext")) {
             response = await handleBunextPublicAssets({ req });
         } else if (url.pathname.startsWith("/api/")) {
@@ -91,6 +99,10 @@ export default async function bunextRequestHandler({
             throw new Error(`No Response generated`);
         }
 
+        if (disable_seo) {
+            response.headers.set("X-Robots-Tag", NoSEORobotsValue);
+        }
+
         if (is_dev) {
             response.headers.set(
                 "Cache-Control",

src/functions/server/web-pages/generate-web-html.tsx

@@ -8,6 +8,7 @@ import { log } from "../../../utils/log";
 import { AppData } from "../../../data/app-data";
 import _ from "lodash";
 import grabDirNames from "../../../utils/grab-dir-names";
+import { NoSEORobotsValue } from "../../../utils/disable-seo";
 
 const { ROOT_DIR } = grabDirNames();
 
@@ -69,7 +70,9 @@ export default async function genWebHTML({
 
     const dev = isDevelopment();
 
-    const final_meta = _.merge(root_meta, page_meta);
+    const final_meta = global.BUNEXT_CONFIG.disable_seo
+        ? { ..._.merge(root_meta, page_meta), robots: NoSEORobotsValue }
+        : _.merge(root_meta, page_meta);
 
     const public_envs = Object.fromEntries(
         Object.entries(process.env).filter(([k]) =>

src/types/index.ts

@@ -119,6 +119,13 @@ export type BunextConfig = {
      * `false` to skip generating it.
      */
     robots?: BunextRobotsConfig | false;
+    /**
+     * Keep the whole site out of search engines, e.g. for preview
+     * deployments. Every response gets `X-Robots-Tag: noindex, nofollow`,
+     * pages get a matching robots meta tag (overriding `meta.robots`), and
+     * `/robots.txt` disallows all crawlers regardless of `public/robots.txt`.
+     */
+    disable_seo?: boolean;
 };
 
 export type BunextConfigMiddlewareParams = {

src/utils/disable-seo.ts

@@ -0,0 +1,5 @@
+/**
+ * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo`
+ * is set.
+ */
+export const NoSEORobotsValue = "noindex, nofollow";