Add disable_seo config to keep preview deployments out of search engines

Co-Authored-By: Claude Opus 5.5 <[email protected]>

README.md

@@ -827,6 +827,25 @@ const config: BunextConfig = {
 
 To serve it from a route instead, `bunext.bunextGenerateRobotsTxt()` takes the same options except `output` and returns the text.
 
+### Disabling SEO
+
+For preview or test deployments that must never be indexed, set `disable_seo`:
+
+```ts
+// bunext.config.ts
+const config: BunextConfig = {
+    disable_seo: process.env.PREVIEW === "true",
+};
+```
+
+With it on, the server:
+
+- adds `X-Robots-Tag: noindex, nofollow` to every response (pages, API routes and files),
+- renders `<meta name="robots" content="noindex, nofollow">` on every page, replacing any `meta.robots`,
+- answers `/robots.txt` with `Disallow: /` for all crawlers, ignoring `public/robots.txt`.
+
+Nothing is written to disk, so the same build can be deployed to production with the option off. Responses returned directly by [middleware](#middleware) are left untouched.
+
 ---
 
 ## Static Files
@@ -971,6 +990,7 @@ export default config;
 | `serverOptions`      | `ServeOptions`                                                                    | —                | Extra options passed to `Bun.serve()` (excluding `fetch`) — see [Server Options](#server-options) |
 | `sitemap`            | `BunextSitemapConfig`                                                             | —                | Options for `bunext sitemap` — see [Sitemap](#sitemap)                                            |
 | `robots`             | `BunextRobotsConfig \| false`                                                     | —                | robots.txt options for `bunext sitemap`, or `false` to skip it — see [robots.txt](#robotstxt)     |
+| `disable_seo`        | `boolean`                                                                         | `false`          | Keep the whole site out of search engines (e.g. preview deployments) — see [Disabling SEO](#disabling-seo) |
 
 ### Middleware
 

dist/commands/sitemap/index.js

@@ -18,6 +18,10 @@ export default function () {
         const { ROOT_DIR, PUBLIC_DIR } = grabDirNames();
         global.BUNEXT_CONFIG = (await grabConfig()) || {};
         global.BUNEXT_CONSTANTS = grabConstants();
+        if (global.BUNEXT_CONFIG.disable_seo) {
+            log.warn(`\`disable_seo\` is set: the server blocks all crawlers at ` +
+                `/robots.txt, so the generated robots.txt is not served.`);
+        }
         const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {};
         if (!existsSync(grabPageLastmodFile(sitemap_config.lastmod_file))) {
             log.warn(`No page lastmod record found. Run \`bunext build\` first ` +

dist/functions/server/bunext-req-handler.js

@@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data";
 import { log } from "../../utils/log";
 import fullRebuild from "./full-rebuild";
 import grabRouter from "../../utils/grab-router";
+import { NoSEORobotsValue } from "../../utils/disable-seo";
+import generateRobotsTxt from "../../utils/generate-robots-txt";
 const HMR_RETRY_COOLDOWN_MS = 5000;
 let lastHmrRetryTime = 0;
 export default async function bunextRequestHandler({ req: initial_req, server, }) {
@@ -43,9 +45,15 @@ export default async function bunextRequestHandler({ req: initial_req, server, }
             await fullRebuild({ msg: `HMR Retry Rebuild ...` });
             return new Response("Modules Rebuilt");
         }
+        const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo;
         if (url.pathname === "/__hmr" && is_dev) {
             return handleHmr({ req });
         }
+        else if (disable_seo && url.pathname == "/robots.txt") {
+            response = new Response(generateRobotsTxt({ disallow_all: true }), {
+                headers: { "Content-Type": "text/plain; charset=utf-8" },
+            });
+        }
         else if (url.pathname.startsWith("/.bunext")) {
             response = await handleBunextPublicAssets({ req });
         }
@@ -69,6 +77,9 @@ export default async function bunextRequestHandler({ req: initial_req, server, }
         if (!response) {
             throw new Error(`No Response generated`);
         }
+        if (disable_seo) {
+            response.headers.set("X-Robots-Tag", NoSEORobotsValue);
+        }
         if (is_dev) {
             response.headers.set("Cache-Control", "no-cache, no-store, must-revalidate");
         }

dist/functions/server/web-pages/generate-web-html.js

@@ -8,6 +8,7 @@ import { log } from "../../../utils/log";
 import { AppData } from "../../../data/app-data";
 import _ from "lodash";
 import grabDirNames from "../../../utils/grab-dir-names";
+import { NoSEORobotsValue } from "../../../utils/disable-seo";
 const { ROOT_DIR } = grabDirNames();
 export default async function genWebHTML({ component: Main, pageProps, bundledMap, module, routeParams, debug, root_module, }) {
     const { ClientRootElementIDName, ClientWindowPagePropsName } = grabContants();
@@ -46,7 +47,9 @@ export default async function genWebHTML({ component: Main, pageProps, bundledMa
     const Head = module?.Head;
     const RootHead = root_module?.Head;
     const dev = isDevelopment();
-    const final_meta = _.merge(root_meta, page_meta);
+    const final_meta = global.BUNEXT_CONFIG.disable_seo
+        ? { ..._.merge(root_meta, page_meta), robots: NoSEORobotsValue }
+        : _.merge(root_meta, page_meta);
     const public_envs = Object.fromEntries(Object.entries(process.env).filter(([k]) => k.startsWith("BUNEXT_PUBLIC_")));
     const client_process = {
         env: {

dist/types/index.d.ts

@@ -102,6 +102,13 @@ export type BunextConfig = {
      * `false` to skip generating it.
      */
     robots?: BunextRobotsConfig | false;
+    /**
+     * Keep the whole site out of search engines, e.g. for preview
+     * deployments. Every response gets `X-Robots-Tag: noindex, nofollow`,
+     * pages get a matching robots meta tag (overriding `meta.robots`), and
+     * `/robots.txt` disallows all crawlers regardless of `public/robots.txt`.
+     */
+    disable_seo?: boolean;
 };
 export type BunextConfigMiddlewareParams = {
     req: Request;

dist/utils/disable-seo.d.ts

@@ -0,0 +1,5 @@
+/**
+ * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo`
+ * is set.
+ */
+export declare const NoSEORobotsValue = "noindex, nofollow";

dist/utils/disable-seo.js

@@ -0,0 +1,5 @@
+/**
+ * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo`
+ * is set.
+ */
+export const NoSEORobotsValue = "noindex, nofollow";

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@moduletrace/bunext",
-    "version": "1.2.0",
+    "version": "1.2.1",
     "main": "dist/index.js",
     "module": "index.ts",
     "dependencies": {

src/__tests__/functions/server/bunext-req-handler.test.ts

@@ -123,4 +123,29 @@ describe("bunext-req-handler", () => {
         const res = await bunextRequestHandler({ req, server });
         expect(await res.text()).toBe("web-pages");
     });
+
+    test("disable_seo blocks crawlers via header and robots.txt", async () => {
+        global.BUNEXT_CONSTANTS.config.disable_seo = true;
+
+        const page = await bunextRequestHandler({
+            req: new Request("http://localhost/about"),
+            server,
+        });
+        expect(page.headers.get("X-Robots-Tag")).toBe("noindex, nofollow");
+
+        const robots = await bunextRequestHandler({
+            req: new Request("http://localhost/robots.txt"),
+            server,
+        });
+        expect(await robots.text()).toContain("User-agent: *\nDisallow: /");
+        expect(robots.headers.get("X-Robots-Tag")).toBe("noindex, nofollow");
+    });
+
+    test("no X-Robots-Tag without disable_seo", async () => {
+        const res = await bunextRequestHandler({
+            req: new Request("http://localhost/about"),
+            server,
+        });
+        expect(res.headers.get("X-Robots-Tag")).toBeNull();
+    });
 });

src/commands/sitemap/index.ts

@@ -41,6 +41,13 @@ export default function () {
             global.BUNEXT_CONFIG = (await grabConfig()) || {};
             global.BUNEXT_CONSTANTS = grabConstants();
 
+            if (global.BUNEXT_CONFIG.disable_seo) {
+                log.warn(
+                    `\`disable_seo\` is set: the server blocks all crawlers at ` +
+                        `/robots.txt, so the generated robots.txt is not served.`,
+                );
+            }
+
             const { output, chunk_size, chunks_dir, ...sitemap_config } =
                 global.BUNEXT_CONFIG.sitemap || {};
 

src/functions/server/bunext-req-handler.ts

@@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data";
 import { log } from "../../utils/log";
 import fullRebuild from "./full-rebuild";
 import grabRouter from "../../utils/grab-router";
+import { NoSEORobotsValue } from "../../utils/disable-seo";
+import generateRobotsTxt from "../../utils/generate-robots-txt";
 
 const HMR_RETRY_COOLDOWN_MS = 5000;
 let lastHmrRetryTime = 0;
@@ -65,8 +67,14 @@ export default async function bunextRequestHandler({
             return new Response("Modules Rebuilt");
         }
 
+        const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo;
+
         if (url.pathname === "/__hmr" && is_dev) {
             return handleHmr({ req });
+        } else if (disable_seo && url.pathname == "/robots.txt") {
+            response = new Response(generateRobotsTxt({ disallow_all: true }), {
+                headers: { "Content-Type": "text/plain; charset=utf-8" },
+            });
         } else if (url.pathname.startsWith("/.bunext")) {
             response = await handleBunextPublicAssets({ req });
         } else if (url.pathname.startsWith("/api/")) {
@@ -91,6 +99,10 @@ export default async function bunextRequestHandler({
             throw new Error(`No Response generated`);
         }
 
+        if (disable_seo) {
+            response.headers.set("X-Robots-Tag", NoSEORobotsValue);
+        }
+
         if (is_dev) {
             response.headers.set(
                 "Cache-Control",

src/functions/server/web-pages/generate-web-html.tsx

@@ -8,6 +8,7 @@ import { log } from "../../../utils/log";
 import { AppData } from "../../../data/app-data";
 import _ from "lodash";
 import grabDirNames from "../../../utils/grab-dir-names";
+import { NoSEORobotsValue } from "../../../utils/disable-seo";
 
 const { ROOT_DIR } = grabDirNames();
 
@@ -69,7 +70,9 @@ export default async function genWebHTML({
 
     const dev = isDevelopment();
 
-    const final_meta = _.merge(root_meta, page_meta);
+    const final_meta = global.BUNEXT_CONFIG.disable_seo
+        ? { ..._.merge(root_meta, page_meta), robots: NoSEORobotsValue }
+        : _.merge(root_meta, page_meta);
 
     const public_envs = Object.fromEntries(
         Object.entries(process.env).filter(([k]) =>

src/types/index.ts

@@ -119,6 +119,13 @@ export type BunextConfig = {
      * `false` to skip generating it.
      */
     robots?: BunextRobotsConfig | false;
+    /**
+     * Keep the whole site out of search engines, e.g. for preview
+     * deployments. Every response gets `X-Robots-Tag: noindex, nofollow`,
+     * pages get a matching robots meta tag (overriding `meta.robots`), and
+     * `/robots.txt` disallows all crawlers regardless of `public/robots.txt`.
+     */
+    disable_seo?: boolean;
 };
 
 export type BunextConfigMiddlewareParams = {

src/utils/disable-seo.ts

@@ -0,0 +1,5 @@
+/**
+ * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo`
+ * is set.
+ */
+export const NoSEORobotsValue = "noindex, nofollow";