From 23a309005a61d6147bcc394fc8de3cbf63a01db2 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Thu, 15 Oct 2026 05:29:08 +0100 Subject: [PATCH] Add disable_seo config to keep preview deployments out of search engines Co-Authored-By: Claude Opus 5.5 --- README.md | 20 +++++++++++++++ dist/commands/sitemap/index.js | 4 +++ dist/functions/server/bunext-req-handler.js | 11 ++++++++ .../server/web-pages/generate-web-html.js | 5 +++- dist/types/index.d.ts | 7 ++++++ dist/utils/disable-seo.d.ts | 5 ++++ dist/utils/disable-seo.js | 5 ++++ package.json | 2 +- .../server/bunext-req-handler.test.ts | 25 +++++++++++++++++++ src/commands/sitemap/index.ts | 7 ++++++ src/functions/server/bunext-req-handler.ts | 12 +++++++++ .../server/web-pages/generate-web-html.tsx | 5 +++- src/types/index.ts | 7 ++++++ src/utils/disable-seo.ts | 5 ++++ 14 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 dist/utils/disable-seo.d.ts create mode 100644 dist/utils/disable-seo.js create mode 100644 src/utils/disable-seo.ts diff --git a/README.md b/README.md index d65f7f3..8c32851 100644 --- a/README.md +++ b/README.md @@ -827,6 +827,25 @@ const config: BunextConfig = { To serve it from a route instead, `bunext.bunextGenerateRobotsTxt()` takes the same options except `output` and returns the text. +### Disabling SEO + +For preview or test deployments that must never be indexed, set `disable_seo`: + +```ts +// bunext.config.ts +const config: BunextConfig = { + disable_seo: process.env.PREVIEW === "true", +}; +``` + +With it on, the server: + +- adds `X-Robots-Tag: noindex, nofollow` to every response (pages, API routes and files), +- renders `` on every page, replacing any `meta.robots`, +- answers `/robots.txt` with `Disallow: /` for all crawlers, ignoring `public/robots.txt`. + +Nothing is written to disk, so the same build can be deployed to production with the option off. Responses returned directly by [middleware](#middleware) are left untouched. + --- ## Static Files @@ -971,6 +990,7 @@ export default config; | `serverOptions` | `ServeOptions` | — | Extra options passed to `Bun.serve()` (excluding `fetch`) — see [Server Options](#server-options) | | `sitemap` | `BunextSitemapConfig` | — | Options for `bunext sitemap` — see [Sitemap](#sitemap) | | `robots` | `BunextRobotsConfig \| false` | — | robots.txt options for `bunext sitemap`, or `false` to skip it — see [robots.txt](#robotstxt) | +| `disable_seo` | `boolean` | `false` | Keep the whole site out of search engines (e.g. preview deployments) — see [Disabling SEO](#disabling-seo) | ### Middleware diff --git a/dist/commands/sitemap/index.js b/dist/commands/sitemap/index.js index 560ed00..a4e9277 100644 --- a/dist/commands/sitemap/index.js +++ b/dist/commands/sitemap/index.js @@ -18,6 +18,10 @@ export default function () { const { ROOT_DIR, PUBLIC_DIR } = grabDirNames(); global.BUNEXT_CONFIG = (await grabConfig()) || {}; global.BUNEXT_CONSTANTS = grabConstants(); + if (global.BUNEXT_CONFIG.disable_seo) { + log.warn(`\`disable_seo\` is set: the server blocks all crawlers at ` + + `/robots.txt, so the generated robots.txt is not served.`); + } const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {}; if (!existsSync(grabPageLastmodFile(sitemap_config.lastmod_file))) { log.warn(`No page lastmod record found. Run \`bunext build\` first ` + diff --git a/dist/functions/server/bunext-req-handler.js b/dist/functions/server/bunext-req-handler.js index d6d0bde..dc112f1 100644 --- a/dist/functions/server/bunext-req-handler.js +++ b/dist/functions/server/bunext-req-handler.js @@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data"; import { log } from "../../utils/log"; import fullRebuild from "./full-rebuild"; import grabRouter from "../../utils/grab-router"; +import { NoSEORobotsValue } from "../../utils/disable-seo"; +import generateRobotsTxt from "../../utils/generate-robots-txt"; const HMR_RETRY_COOLDOWN_MS = 5000; let lastHmrRetryTime = 0; export default async function bunextRequestHandler({ req: initial_req, server, }) { @@ -43,9 +45,15 @@ export default async function bunextRequestHandler({ req: initial_req, server, } await fullRebuild({ msg: `HMR Retry Rebuild ...` }); return new Response("Modules Rebuilt"); } + const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo; if (url.pathname === "/__hmr" && is_dev) { return handleHmr({ req }); } + else if (disable_seo && url.pathname == "/robots.txt") { + response = new Response(generateRobotsTxt({ disallow_all: true }), { + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }); + } else if (url.pathname.startsWith("/.bunext")) { response = await handleBunextPublicAssets({ req }); } @@ -69,6 +77,9 @@ export default async function bunextRequestHandler({ req: initial_req, server, } if (!response) { throw new Error(`No Response generated`); } + if (disable_seo) { + response.headers.set("X-Robots-Tag", NoSEORobotsValue); + } if (is_dev) { response.headers.set("Cache-Control", "no-cache, no-store, must-revalidate"); } diff --git a/dist/functions/server/web-pages/generate-web-html.js b/dist/functions/server/web-pages/generate-web-html.js index 638da89..d854506 100644 --- a/dist/functions/server/web-pages/generate-web-html.js +++ b/dist/functions/server/web-pages/generate-web-html.js @@ -8,6 +8,7 @@ import { log } from "../../../utils/log"; import { AppData } from "../../../data/app-data"; import _ from "lodash"; import grabDirNames from "../../../utils/grab-dir-names"; +import { NoSEORobotsValue } from "../../../utils/disable-seo"; const { ROOT_DIR } = grabDirNames(); export default async function genWebHTML({ component: Main, pageProps, bundledMap, module, routeParams, debug, root_module, }) { const { ClientRootElementIDName, ClientWindowPagePropsName } = grabContants(); @@ -46,7 +47,9 @@ export default async function genWebHTML({ component: Main, pageProps, bundledMa const Head = module?.Head; const RootHead = root_module?.Head; const dev = isDevelopment(); - const final_meta = _.merge(root_meta, page_meta); + const final_meta = global.BUNEXT_CONFIG.disable_seo + ? { ..._.merge(root_meta, page_meta), robots: NoSEORobotsValue } + : _.merge(root_meta, page_meta); const public_envs = Object.fromEntries(Object.entries(process.env).filter(([k]) => k.startsWith("BUNEXT_PUBLIC_"))); const client_process = { env: { diff --git a/dist/types/index.d.ts b/dist/types/index.d.ts index 9db2978..bd6e334 100644 --- a/dist/types/index.d.ts +++ b/dist/types/index.d.ts @@ -102,6 +102,13 @@ export type BunextConfig = { * `false` to skip generating it. */ robots?: BunextRobotsConfig | false; + /** + * Keep the whole site out of search engines, e.g. for preview + * deployments. Every response gets `X-Robots-Tag: noindex, nofollow`, + * pages get a matching robots meta tag (overriding `meta.robots`), and + * `/robots.txt` disallows all crawlers regardless of `public/robots.txt`. + */ + disable_seo?: boolean; }; export type BunextConfigMiddlewareParams = { req: Request; diff --git a/dist/utils/disable-seo.d.ts b/dist/utils/disable-seo.d.ts new file mode 100644 index 0000000..219917d --- /dev/null +++ b/dist/utils/disable-seo.d.ts @@ -0,0 +1,5 @@ +/** + * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo` + * is set. + */ +export declare const NoSEORobotsValue = "noindex, nofollow"; diff --git a/dist/utils/disable-seo.js b/dist/utils/disable-seo.js new file mode 100644 index 0000000..9747440 --- /dev/null +++ b/dist/utils/disable-seo.js @@ -0,0 +1,5 @@ +/** + * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo` + * is set. + */ +export const NoSEORobotsValue = "noindex, nofollow"; diff --git a/package.json b/package.json index d573dc3..702e1a2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@moduletrace/bunext", - "version": "1.2.0", + "version": "1.2.1", "main": "dist/index.js", "module": "index.ts", "dependencies": { diff --git a/src/__tests__/functions/server/bunext-req-handler.test.ts b/src/__tests__/functions/server/bunext-req-handler.test.ts index f3592c8..37f4864 100644 --- a/src/__tests__/functions/server/bunext-req-handler.test.ts +++ b/src/__tests__/functions/server/bunext-req-handler.test.ts @@ -123,4 +123,29 @@ describe("bunext-req-handler", () => { const res = await bunextRequestHandler({ req, server }); expect(await res.text()).toBe("web-pages"); }); + + test("disable_seo blocks crawlers via header and robots.txt", async () => { + global.BUNEXT_CONSTANTS.config.disable_seo = true; + + const page = await bunextRequestHandler({ + req: new Request("http://localhost/about"), + server, + }); + expect(page.headers.get("X-Robots-Tag")).toBe("noindex, nofollow"); + + const robots = await bunextRequestHandler({ + req: new Request("http://localhost/robots.txt"), + server, + }); + expect(await robots.text()).toContain("User-agent: *\nDisallow: /"); + expect(robots.headers.get("X-Robots-Tag")).toBe("noindex, nofollow"); + }); + + test("no X-Robots-Tag without disable_seo", async () => { + const res = await bunextRequestHandler({ + req: new Request("http://localhost/about"), + server, + }); + expect(res.headers.get("X-Robots-Tag")).toBeNull(); + }); }); diff --git a/src/commands/sitemap/index.ts b/src/commands/sitemap/index.ts index 9f4808c..aac0da5 100644 --- a/src/commands/sitemap/index.ts +++ b/src/commands/sitemap/index.ts @@ -41,6 +41,13 @@ export default function () { global.BUNEXT_CONFIG = (await grabConfig()) || {}; global.BUNEXT_CONSTANTS = grabConstants(); + if (global.BUNEXT_CONFIG.disable_seo) { + log.warn( + `\`disable_seo\` is set: the server blocks all crawlers at ` + + `/robots.txt, so the generated robots.txt is not served.`, + ); + } + const { output, chunk_size, chunks_dir, ...sitemap_config } = global.BUNEXT_CONFIG.sitemap || {}; diff --git a/src/functions/server/bunext-req-handler.ts b/src/functions/server/bunext-req-handler.ts index 342565e..33aa81b 100644 --- a/src/functions/server/bunext-req-handler.ts +++ b/src/functions/server/bunext-req-handler.ts @@ -10,6 +10,8 @@ import { AppData } from "../../data/app-data"; import { log } from "../../utils/log"; import fullRebuild from "./full-rebuild"; import grabRouter from "../../utils/grab-router"; +import { NoSEORobotsValue } from "../../utils/disable-seo"; +import generateRobotsTxt from "../../utils/generate-robots-txt"; const HMR_RETRY_COOLDOWN_MS = 5000; let lastHmrRetryTime = 0; @@ -65,8 +67,14 @@ export default async function bunextRequestHandler({ return new Response("Modules Rebuilt"); } + const disable_seo = global.BUNEXT_CONSTANTS.config?.disable_seo; + if (url.pathname === "/__hmr" && is_dev) { return handleHmr({ req }); + } else if (disable_seo && url.pathname == "/robots.txt") { + response = new Response(generateRobotsTxt({ disallow_all: true }), { + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }); } else if (url.pathname.startsWith("/.bunext")) { response = await handleBunextPublicAssets({ req }); } else if (url.pathname.startsWith("/api/")) { @@ -91,6 +99,10 @@ export default async function bunextRequestHandler({ throw new Error(`No Response generated`); } + if (disable_seo) { + response.headers.set("X-Robots-Tag", NoSEORobotsValue); + } + if (is_dev) { response.headers.set( "Cache-Control", diff --git a/src/functions/server/web-pages/generate-web-html.tsx b/src/functions/server/web-pages/generate-web-html.tsx index ffb39b2..e0013ad 100644 --- a/src/functions/server/web-pages/generate-web-html.tsx +++ b/src/functions/server/web-pages/generate-web-html.tsx @@ -8,6 +8,7 @@ import { log } from "../../../utils/log"; import { AppData } from "../../../data/app-data"; import _ from "lodash"; import grabDirNames from "../../../utils/grab-dir-names"; +import { NoSEORobotsValue } from "../../../utils/disable-seo"; const { ROOT_DIR } = grabDirNames(); @@ -69,7 +70,9 @@ export default async function genWebHTML({ const dev = isDevelopment(); - const final_meta = _.merge(root_meta, page_meta); + const final_meta = global.BUNEXT_CONFIG.disable_seo + ? { ..._.merge(root_meta, page_meta), robots: NoSEORobotsValue } + : _.merge(root_meta, page_meta); const public_envs = Object.fromEntries( Object.entries(process.env).filter(([k]) => diff --git a/src/types/index.ts b/src/types/index.ts index 5b64242..47c8266 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -119,6 +119,13 @@ export type BunextConfig = { * `false` to skip generating it. */ robots?: BunextRobotsConfig | false; + /** + * Keep the whole site out of search engines, e.g. for preview + * deployments. Every response gets `X-Robots-Tag: noindex, nofollow`, + * pages get a matching robots meta tag (overriding `meta.robots`), and + * `/robots.txt` disallows all crawlers regardless of `public/robots.txt`. + */ + disable_seo?: boolean; }; export type BunextConfigMiddlewareParams = { diff --git a/src/utils/disable-seo.ts b/src/utils/disable-seo.ts new file mode 100644 index 0000000..9747440 --- /dev/null +++ b/src/utils/disable-seo.ts @@ -0,0 +1,5 @@ +/** + * `robots` meta / `X-Robots-Tag` value used when `config.disable_seo` + * is set. + */ +export const NoSEORobotsValue = "noindex, nofollow";