Add disable_seo config to keep preview deployments out of search engines

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dist/commands/sitemap/index.js

@@ -18,6 +18,10 @@ export default function () {
         const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
         global.BUNEXT_CONFIG = (await grabConfig()) || {};
         global.BUNEXT_CONSTANTS = grabConstants();
+        if (global.BUNEXT_CONFIG.disable_seo) {
+            log.warn(`\`disable_seo\` is set: the server blocks all crawlers at ` +
+                `/robots.txt, so the generated robots.txt is not served.`);
+        }
         const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
         if (!existsSync(grabPageLastmodFile(sitemap_config.lastmod_file))) {
             log.warn(`No page lastmod record found. Run \`bunext build\` first ` +

dist/functions/server/bunext-req-handler.js

@@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data";
 import { log } from "../../utils/log";
 import fullRebuild from "./full-rebuild";
 import grabRouter from "../../utils/grab-router";
+import { NoSEORobotsValue } from "../../utils/disable-seo";
+import generateRobotsTxt from "../../utils/generate-robots-txt";
 const HMR_RETRY_COOLDOWN_MS = 5000;
 let lastHmrRetryTime = 0;
 export default async function bunextRequestHandler({ req: initial_req, server, }) {
@@ -43,9 +45,15 @@ export default async function bunextRequestHandler({ req: initial_req, server, }
             await fullRebuild({ msg: `HMR Retry Rebuild ...` });
             return new Response("Modules Rebuilt");
         }
+        const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo;
         if (url.pathname === "/__hmr" && is_dev) {
             return handleHmr({ req });
         }
+        else if (disable_seo && url.pathname == "/robots.txt") {
+            response = new Response(generateRobotsTxt({ disallow_all: true }), {
+                headers: { "Content-Type": "text/plain; charset=utf-8" },
+            });
+        }
         else if (url.pathname.startsWith("/.bunext")) {
             response = await handleBunextPublicAssets({ req });
         }
@@ -69,6 +77,9 @@ export default async function bunextRequestHandler({ req: initial_req, server, }
         if (!response) {
             throw new Error(`No Response generated`);
         }
+        if (disable_seo) {
+            response.headers.set("X-Robots-Tag", NoSEORobotsValue);
+        }
         if (is_dev) {
             response.headers.set("Cache-Control", "no-cache, no-store, must-revalidate");
         }

dist/functions/server/web-pages/generate-web-html.js

@@ -8,6 +8,7 @@ import { log } from "../../../utils/log";
 import { AppData } from "../../../data/app-data";
 import _ from "lodash";
 import grabDirNames from "../../../utils/grab-dir-names";
+import { NoSEORobotsValue } from "../../../utils/disable-seo";
 const { ROOT_DIR } = grabDirNames();
 export default async function genWebHTML({ component: Main, pageProps, bundledMap, module, routeParams, debug, root_module, }) {
     const { ClientRootElementIDName, ClientWindowPagePropsName } = grabContants();
@@ -46,7 +47,9 @@ export default async function genWebHTML({ component: Main, pageProps, bundledMa
     const Head = module?.Head;
     const RootHead = root_module?.Head;
     const dev = isDevelopment();
-    const final_meta = _.merge(root_meta, page_meta);
+    const final_meta = global.BUNEXT_CONFIG.disable_seo
+        ? { ..._.merge(root_meta, page_meta), robots: NoSEORobotsValue }
+        : _.merge(root_meta, page_meta);
     const public_envs = Object.fromEntries(Object.entries(process.env).filter(([k]) => k.startsWith("BUNEXT_PUBLIC_")));
     const client_process = {
         env: {

dist/types/index.d.ts

@@ -102,6 +102,13 @@ export type BunextConfig = {
      * `false` to skip generating it.
      */
     robots?: BunextRobotsConfig | false;
+    /**
+     * Keep the whole site out of search engines, e.g. for preview
+     * deployments. Every response gets `X-Robots-Tag: noindex, nofollow`,
+     * pages get a matching robots meta tag (overriding `meta.robots`), and
+     * `/robots.txt` disallows all crawlers regardless of `public/robots.txt`.
+     */
+    disable_seo?: boolean;
 };
 export type BunextConfigMiddlewareParams = {
     req: Request;

dist/utils/disable-seo.d.ts

@@ -0,0 +1,5 @@
+/**
+ * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo`
+ * is set.
+ */
+export declare const NoSEORobotsValue = "noindex, nofollow";

dist/utils/disable-seo.js

@@ -0,0 +1,5 @@
+/**
+ * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo`
+ * is set.
+ */
+export const NoSEORobotsValue = "noindex, nofollow";