Add disable_seo config to keep preview deployments out of search engines

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/functions/server/bunext-req-handler.ts

@@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data";
 import { log } from "../../utils/log";
 import fullRebuild from "./full-rebuild";
 import grabRouter from "../../utils/grab-router";
+import { NoSEORobotsValue } from "../../utils/disable-seo";
+import generateRobotsTxt from "../../utils/generate-robots-txt";
 
 const HMR_RETRY_COOLDOWN_MS = 5000;
 let lastHmrRetryTime = 0;
@@ -65,8 +67,14 @@ export default async function bunextRequestHandler({
             return new Response("Modules Rebuilt");
         }
 
+        const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo;
+
         if (url.pathname === "/__hmr" && is_dev) {
             return handleHmr({ req });
+        } else if (disable_seo && url.pathname == "/robots.txt") {
+            response = new Response(generateRobotsTxt({ disallow_all: true }), {
+                headers: { "Content-Type": "text/plain; charset=utf-8" },
+            });
         } else if (url.pathname.startsWith("/.bunext")) {
             response = await handleBunextPublicAssets({ req });
         } else if (url.pathname.startsWith("/api/")) {
@@ -91,6 +99,10 @@ export default async function bunextRequestHandler({
             throw new Error(`No Response generated`);
         }
 
+        if (disable_seo) {
+            response.headers.set("X-Robots-Tag", NoSEORobotsValue);
+        }
+
         if (is_dev) {
             response.headers.set(
                 "Cache-Control",