Add disable_seo config to keep preview deployments out of search engines

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dist/functions/server/bunext-req-handler.js

@@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data";
 import { log } from "../../utils/log";
 import fullRebuild from "./full-rebuild";
 import grabRouter from "../../utils/grab-router";
+import { NoSEORobotsValue } from "../../utils/disable-seo";
+import generateRobotsTxt from "../../utils/generate-robots-txt";
 const HMR_RETRY_COOLDOWN_MS = 5000;
 let lastHmrRetryTime = 0;
 export default async function bunextRequestHandler({ req: initial_req, server, }) {
@@ -43,9 +45,15 @@ export default async function bunextRequestHandler({ req: initial_req, server, }
             await fullRebuild({ msg: `HMR Retry Rebuild ...` });
             return new Response("Modules Rebuilt");
         }
+        const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo;
         if (url.pathname === "/__hmr" && is_dev) {
             return handleHmr({ req });
         }
+        else if (disable_seo && url.pathname == "/robots.txt") {
+            response = new Response(generateRobotsTxt({ disallow_all: true }), {
+                headers: { "Content-Type": "text/plain; charset=utf-8" },
+            });
+        }
         else if (url.pathname.startsWith("/.bunext")) {
             response = await handleBunextPublicAssets({ req });
         }
@@ -69,6 +77,9 @@ export default async function bunextRequestHandler({ req: initial_req, server, }
         if (!response) {
             throw new Error(`No Response generated`);
         }
+        if (disable_seo) {
+            response.headers.set("X-Robots-Tag", NoSEORobotsValue);
+        }
         if (is_dev) {
             response.headers.set("Cache-Control", "no-cache, no-store, must-revalidate");
         }