Fix server component stale caching. Add server components to SSR bundler

dist/functions/bundler/all-pages-esbuild-context-bundler.js

@@ -61,6 +61,7 @@ export default async function allPagesESBuildContextBundler(params) {
                 "react-dom/client",
                 "react/jsx-runtime",
                 "react/jsx-dev-runtime",
+                ...(global.CONFIG.page_compiler_excludes || []),
             ],
             logLevel: did_process_exit_because_of_bundler_error
                 ? "silent"

dist/functions/bundler/pages-ssr-bundler.js

@@ -7,16 +7,27 @@ import grabPageReactComponentString from "../server/web-pages/grab-page-react-co
 import grabRootFilePath from "../server/web-pages/grab-root-file-path";
 import ssrVirtualFilesPlugin from "./plugins/ssr-virtual-files-plugin";
 import ssrCTXArtifactTracker from "./plugins/ssr-ctx-artifact-tracker";
-const { BUNX_CWD_MODULE_CACHE_DIR } = grabDirNames();
+import { writeFileSync } from "fs";
+import path from "path";
+import { log } from "../../utils/log";
+const { BUNX_CWD_MODULE_CACHE_DIR, BUNX_TMP_DIR } = grabDirNames();
 export default async function pagesSSRBundler(params) {
-    const pages = grabAllPages();
+    const pages = grabAllPages({
+        include_server: true,
+    });
     const dev = isDevelopment();
+    const config = global.CONFIG;
+    try {
+        writeFileSync(path.join(BUNX_TMP_DIR, "ssr-pages.json"), JSON.stringify(pages, null, 4));
+    }
+    catch (error) { }
     const entryToPage = new Map();
     const { root_file_path } = grabRootFilePath();
     for (const page of pages) {
-        if (page.local_path.match(/\/pages\/api\//)) {
+        if (page.local_path.match(/\/pages\/api\//) ||
+            page.local_path.match(/\.server\.tsx?$/)) {
             const ts = await Bun.file(page.local_path).text();
-            if (ts.match(/(export default)|(export \w+ handler)/)) {
+            if (ts.match(/(export default)|(export \w+ handler)|(export \w+ server)/)) {
                 entryToPage.set(page.local_path, { ...page, tsx: ts });
             }
             continue;
@@ -32,6 +43,11 @@ export default async function pagesSSRBundler(params) {
         entryToPage.set(page.local_path, { ...page, tsx });
     }
     const entryPoints = [...entryToPage.keys()].map((e) => `ssr-virtual:${e}`);
+    try {
+        writeFileSync(path.join(BUNX_TMP_DIR, "ssr-entry-to-page.json"), JSON.stringify(Object(entryToPage), null, 4));
+        writeFileSync(path.join(BUNX_TMP_DIR, "ssr-entrypoints.json"), JSON.stringify(entryPoints, null, 4));
+    }
+    catch (error) { }
     await esbuild.build({
         entryPoints,
         outdir: BUNX_CWD_MODULE_CACHE_DIR,
@@ -62,6 +78,9 @@ export default async function pagesSSRBundler(params) {
             "react/jsx-runtime",
             "react/jsx-dev-runtime",
             "bun:*",
+            "sqlite-vec",
+            "better-sqlite3",
+            ...(config.ssr_compiler_excludes || []),
         ],
         splitting: true,
         // logLevel: "silent",

dist/functions/bundler/plugins/ssr-ctx-artifact-tracker.js

@@ -1,8 +1,12 @@
 import {} from "esbuild";
 import grabArtifactsFromBundledResults from "../grab-artifacts-from-bundled-result";
+import { writeFileSync } from "fs";
+import path from "path";
+import grabDirNames from "../../../utils/grab-dir-names";
 let build_start = 0;
 let build_starts = 0;
 const MAX_BUILD_STARTS = 2;
+const { BUNX_TMP_DIR } = grabDirNames();
 export default function ssrCTXArtifactTracker({ entryToPage, post_build_fn, }) {
     const artifactTracker = {
         name: "ssr-artifact-tracker",
@@ -41,6 +45,10 @@ export default function ssrCTXArtifactTracker({ entryToPage, post_build_fn, }) {
                     // );
                     // log.success(`SSR [Built] in ${elapsed}ms`);
                 }
+                try {
+                    writeFileSync(path.join(BUNX_TMP_DIR, "ctx-map.json"), JSON.stringify(global.SSR_BUNDLER_CTX_MAP, null, 4));
+                }
+                catch (error) { }
                 global.SSR_BUNDLER_CTX_DISPOSED = false;
             });
         },

dist/functions/server/web-pages/grab-page-combined-server-res.js

@@ -3,14 +3,21 @@ import { log } from "../../../utils/log";
 import grabRootFilePath from "./grab-root-file-path";
 import grabPageServerRes from "./grab-page-server-res";
 import grabPageServerPath from "./grab-page-server-path";
+import path from "path";
+import grabDirNames from "../../../utils/grab-dir-names";
+const { ROOT_DIR } = grabDirNames();
 export default async function grabPageCombinedServerRes({ file_path, debug, url, query, routeParams, }) {
     const now = Date.now();
     const { root_file_path } = grabRootFilePath();
     const { server_file_path: root_server_file_path } = root_file_path
         ? grabPageServerPath({ file_path: root_file_path })
         : {};
-    const root_server_module = root_server_file_path
+    const root_server_ctx_map = global.SSR_BUNDLER_CTX_MAP[root_server_file_path || ""];
-        ? await import(`${root_server_file_path}?t=${now}`)
+    const final_root_server_path = root_server_ctx_map?.local_path
+        ? path.join(ROOT_DIR, root_server_ctx_map.path)
+        : root_server_file_path;
+    const root_server_module = final_root_server_path
+        ? await import(`${final_root_server_path}?t=${now}`)
         : undefined;
     const root_server_fn = root_server_module?.default || root_server_module?.server;
     const rootServerRes = await grabPageServerRes({
@@ -23,8 +30,12 @@ export default async function grabPageCombinedServerRes({ file_path, debug, url,
         log.info(`rootServerRes:`, rootServerRes);
     }
     const { server_file_path } = grabPageServerPath({ file_path });
-    const server_module = server_file_path
+    const page_server_ctx = global.SSR_BUNDLER_CTX_MAP[server_file_path || ""];
-        ? await import(`${server_file_path}?t=${now}`)
+    const final_page_server_path = page_server_ctx?.local_path
+        ? path.join(ROOT_DIR, page_server_ctx.path)
+        : root_server_file_path;
+    const server_module = final_page_server_path
+        ? await import(`${final_page_server_path}?t=${now}`)
         : undefined;
     const server_fn = server_module?.default || server_module?.server;
     const serverRes = await grabPageServerRes({

dist/types/index.d.ts

@@ -54,6 +54,18 @@ export type BunextConfig = {
      */
     pages_exclude_patterns?: RegExp[];
     max_logs?: number;
+    /**
+     * Patterns to exclude from the SSR bundler. This is
+     * required for modules that aren't compatible with esbuild
+     * bundler. Eg. `sqlite-vec`
+     */
+    ssr_compiler_excludes?: string[];
+    /**
+     * Patterns to exclude from the Main page bundler. This is
+     * required for modules that aren't compatible with esbuild
+     * bundler for the browser. Eg. `react/jsx-dev-runtime`
+     */
+    page_compiler_excludes?: string[];
 };
 export type BunextConfigMiddlewareParams = {
     req: Request;

dist/utils/grab-all-pages.d.ts

@@ -2,6 +2,7 @@ import type { PageFiles } from "../types";
 type Params = {
     exclude_api?: boolean;
     api_only?: boolean;
+    include_server?: boolean;
 };
 export default function grabAllPages(params?: Params): PageFiles[];
 export {};

dist/utils/grab-all-pages.js

@@ -5,7 +5,10 @@ import AppNames from "./grab-app-names";
 import checkExcludedPatterns from "./check-excluded-patterns";
 export default function grabAllPages(params) {
     const { PAGES_DIR } = grabDirNames();
-    const pages = grabPageDirRecursively({ page_dir: PAGES_DIR });
+    const pages = grabPageDirRecursively({
+        page_dir: PAGES_DIR,
+        include_server: params?.include_server,
+    });
     if (params?.exclude_api) {
         return pages.filter((p) => !Boolean(p.url_path.startsWith("/api/")));
     }
@@ -14,7 +17,7 @@ export default function grabAllPages(params) {
     }
     return pages;
 }
-function grabPageDirRecursively({ page_dir }) {
+function grabPageDirRecursively({ page_dir, include_server, }) {
     const pages = readdirSync(page_dir);
     const pages_files = [];
     const root_pages_file = grabPageFileObject({ file_path: `` });
@@ -28,13 +31,17 @@ function grabPageDirRecursively({ page_dir }) {
         if (!existsSync(full_page_path) || !page_name) {
             continue;
         }
-        if (page.match(new RegExp(`${AppNames["RootPagesComponentName"]}`))) {
+        if (page.match(new RegExp(`${AppNames["RootPagesComponentName"]}`)) &&
+            !page_name.match(/\.server\.tsx?/)) {
             continue;
         }
-        if (checkExcludedPatterns({ path: full_page_path })) {
+        const is_page_excluded = checkExcludedPatterns({
+            path: full_page_path,
+        });
+        if (is_page_excluded) {
             continue;
         }
-        if (page_name.match(/\.server\.tsx?/)) {
+        if (page_name.match(/\.server\.tsx?/) && !include_server) {
             continue;
         }
         const page_stat = statSync(full_page_path);
@@ -43,6 +50,7 @@ function grabPageDirRecursively({ page_dir }) {
                 continue;
             const new_page_files = grabPageDirRecursively({
                 page_dir: full_page_path,
+                include_server,
             });
             pages_files.push(...new_page_files);
         }

dist/utils/write-logs.d.ts

@@ -0,0 +1 @@
+export default function writeLogs(log: any): void;

dist/utils/write-logs.js

@@ -0,0 +1,22 @@
+import { writeFileSync } from "fs";
+import grabDirNames from "./grab-dir-names";
+import path from "path";
+const { BUNX_LOGS_DIR } = grabDirNames();
+export default function writeLogs(log) {
+    try {
+        const now = Date.now();
+        let new_log_name = `${now}`;
+        let log_content = log.toString();
+        try {
+            const json = JSON.stringify(log, null, 4);
+            new_log_name += `.json`;
+            log_content = json;
+        }
+        catch (error) {
+            new_log_name += `.log`;
+        }
+        const log_path = path.join(BUNX_LOGS_DIR, new_log_name);
+        writeFileSync(log_path, log_content);
+    }
+    catch (error) { }
+}

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@moduletrace/bunext",
-    "version": "1.0.84",
+    "version": "1.0.85",
     "main": "dist/index.js",
     "module": "index.ts",
     "dependencies": {

src/functions/bundler/all-pages-esbuild-context-bundler.ts

@@ -93,6 +93,7 @@ export default async function allPagesESBuildContextBundler(params?: Params) {
                 "react-dom/client",
                 "react/jsx-runtime",
                 "react/jsx-dev-runtime",
+                ...(global.CONFIG.page_compiler_excludes || []),
             ],
             logLevel: did_process_exit_because_of_bundler_error
                 ? "silent"

src/functions/bundler/pages-ssr-bundler.ts

@@ -8,24 +8,44 @@ import grabPageReactComponentString from "../server/web-pages/grab-page-react-co
 import grabRootFilePath from "../server/web-pages/grab-root-file-path";
 import ssrVirtualFilesPlugin from "./plugins/ssr-virtual-files-plugin";
 import ssrCTXArtifactTracker from "./plugins/ssr-ctx-artifact-tracker";
+import { writeFileSync } from "fs";
+import path from "path";
+import { log } from "../../utils/log";
 
-const { BUNX_CWD_MODULE_CACHE_DIR } = grabDirNames();
+const { BUNX_CWD_MODULE_CACHE_DIR, BUNX_TMP_DIR } = grabDirNames();
 
 type Params = {
     post_build_fn?: (params: { artifacts: any[] }) => Promise<void> | void;
 };
 
 export default async function pagesSSRBundler(params?: Params) {
-    const pages = grabAllPages();
+    const pages = grabAllPages({
+        include_server: true,
+    });
     const dev = isDevelopment();
+    const config = global.CONFIG;
+
+    try {
+        writeFileSync(
+            path.join(BUNX_TMP_DIR, "ssr-pages.json"),
+            JSON.stringify(pages, null, 4),
+        );
+    } catch (error) {}
 
     const entryToPage = new Map<string, PageFiles & { tsx: string }>();
     const { root_file_path } = grabRootFilePath();
 
     for (const page of pages) {
-        if (page.local_path.match(/\/pages\/api\//)) {
+        if (
+            page.local_path.match(/\/pages\/api\//) ||
+            page.local_path.match(/\.server\.tsx?$/)
+        ) {
             const ts = await Bun.file(page.local_path).text();
-            if (ts.match(/(export default)|(export \w+ handler)/)) {
+            if (
+                ts.match(
+                    /(export default)|(export \w+ handler)|(export \w+ server)/,
+                )
+            ) {
                 entryToPage.set(page.local_path, { ...page, tsx: ts });
             }
             continue;
@@ -44,6 +64,17 @@ export default async function pagesSSRBundler(params?: Params) {
 
     const entryPoints = [...entryToPage.keys()].map((e) => `ssr-virtual:${e}`);
 
+    try {
+        writeFileSync(
+            path.join(BUNX_TMP_DIR, "ssr-entry-to-page.json"),
+            JSON.stringify(Object(entryToPage), null, 4),
+        );
+        writeFileSync(
+            path.join(BUNX_TMP_DIR, "ssr-entrypoints.json"),
+            JSON.stringify(entryPoints, null, 4),
+        );
+    } catch (error) {}
+
     await esbuild.build({
         entryPoints,
         outdir: BUNX_CWD_MODULE_CACHE_DIR,
@@ -76,6 +107,9 @@ export default async function pagesSSRBundler(params?: Params) {
             "react/jsx-runtime",
             "react/jsx-dev-runtime",
             "bun:*",
+            "sqlite-vec",
+            "better-sqlite3",
+            ...(config.ssr_compiler_excludes || []),
         ],
         splitting: true,
         // logLevel: "silent",

src/functions/bundler/plugins/ssr-ctx-artifact-tracker.ts

@@ -1,11 +1,16 @@
 import { type Plugin } from "esbuild";
 import type { PageFiles } from "../../../types";
 import grabArtifactsFromBundledResults from "../grab-artifacts-from-bundled-result";
+import { writeFileSync } from "fs";
+import path from "path";
+import grabDirNames from "../../../utils/grab-dir-names";
 
 let build_start = 0;
 let build_starts = 0;
 const MAX_BUILD_STARTS = 2;
 
+const { BUNX_TMP_DIR } = grabDirNames();
+
 type Params = {
     entryToPage: Map<
         string,
@@ -65,6 +70,13 @@ export default function ssrCTXArtifactTracker({
                     // log.success(`SSR [Built] in ${elapsed}ms`);
                 }
 
+                try {
+                    writeFileSync(
+                        path.join(BUNX_TMP_DIR, "ctx-map.json"),
+                        JSON.stringify(global.SSR_BUNDLER_CTX_MAP, null, 4),
+                    );
+                } catch (error) {}
+
                 global.SSR_BUNDLER_CTX_DISPOSED = false;
             });
         },

src/functions/server/web-pages/grab-page-combined-server-res.ts

@@ -8,6 +8,10 @@ import { log } from "../../../utils/log";
 import grabRootFilePath from "./grab-root-file-path";
 import grabPageServerRes from "./grab-page-server-res";
 import grabPageServerPath from "./grab-page-server-path";
+import path from "path";
+import grabDirNames from "../../../utils/grab-dir-names";
+
+const { ROOT_DIR } = grabDirNames();
 
 type Params = {
     file_path: string;
@@ -30,8 +34,14 @@ export default async function grabPageCombinedServerRes({
     const { server_file_path: root_server_file_path } = root_file_path
         ? grabPageServerPath({ file_path: root_file_path })
         : {};
-    const root_server_module: BunextPageServerModule = root_server_file_path
+    const root_server_ctx_map =
-        ? await import(`${root_server_file_path}?t=${now}`)
+        global.SSR_BUNDLER_CTX_MAP[root_server_file_path || ""];
+    const final_root_server_path = root_server_ctx_map?.local_path
+        ? path.join(ROOT_DIR, root_server_ctx_map.path)
+        : root_server_file_path;
+
+    const root_server_module: BunextPageServerModule = final_root_server_path
+        ? await import(`${final_root_server_path}?t=${now}`)
         : undefined;
 
     const root_server_fn =
@@ -50,8 +60,13 @@ export default async function grabPageCombinedServerRes({
     }
 
     const { server_file_path } = grabPageServerPath({ file_path });
-    const server_module: BunextPageServerModule = server_file_path
+    const page_server_ctx = global.SSR_BUNDLER_CTX_MAP[server_file_path || ""];
-        ? await import(`${server_file_path}?t=${now}`)
+    const final_page_server_path = page_server_ctx?.local_path
+        ? path.join(ROOT_DIR, page_server_ctx.path)
+        : root_server_file_path;
+
+    const server_module: BunextPageServerModule = final_page_server_path
+        ? await import(`${final_page_server_path}?t=${now}`)
         : undefined;
 
     const server_fn = server_module?.default || server_module?.server;

src/types/index.ts

@@ -71,6 +71,18 @@ export type BunextConfig = {
      */
     pages_exclude_patterns?: RegExp[];
     max_logs?: number;
+    /**
+     * Patterns to exclude from the SSR bundler. This is
+     * required for modules that aren't compatible with esbuild
+     * bundler. Eg. `sqlite-vec`
+     */
+    ssr_compiler_excludes?: string[];
+    /**
+     * Patterns to exclude from the Main page bundler. This is
+     * required for modules that aren't compatible with esbuild
+     * bundler for the browser. Eg. `react/jsx-dev-runtime`
+     */
+    page_compiler_excludes?: string[];
 };
 
 export type BunextConfigMiddlewareParams = {

src/utils/grab-all-pages.ts

@@ -8,12 +8,16 @@ import checkExcludedPatterns from "./check-excluded-patterns";
 type Params = {
     exclude_api?: boolean;
     api_only?: boolean;
+    include_server?: boolean;
 };
 
 export default function grabAllPages(params?: Params) {
     const { PAGES_DIR } = grabDirNames();
 
-    const pages = grabPageDirRecursively({ page_dir: PAGES_DIR });
+    const pages = grabPageDirRecursively({
+        page_dir: PAGES_DIR,
+        include_server: params?.include_server,
+    });
 
     if (params?.exclude_api) {
         return pages.filter((p) => !Boolean(p.url_path.startsWith("/api/")));
@@ -26,7 +30,13 @@ export default function grabAllPages(params?: Params) {
     return pages;
 }
 
-function grabPageDirRecursively({ page_dir }: { page_dir: string }) {
+function grabPageDirRecursively({
+    page_dir,
+    include_server,
+}: {
+    page_dir: string;
+    include_server?: boolean;
+}) {
     const pages = readdirSync(page_dir);
     const pages_files: PageFiles[] = [];
 
@@ -45,15 +55,22 @@ function grabPageDirRecursively({ page_dir }: { page_dir: string }) {
             continue;
         }
 
-        if (page.match(new RegExp(`${AppNames["RootPagesComponentName"]}`))) {
+        if (
+            page.match(new RegExp(`${AppNames["RootPagesComponentName"]}`)) &&
+            !page_name.match(/\.server\.tsx?/)
+        ) {
             continue;
         }
 
-        if (checkExcludedPatterns({ path: full_page_path })) {
+        const is_page_excluded = checkExcludedPatterns({
+            path: full_page_path,
+        });
+
+        if (is_page_excluded) {
             continue;
         }
 
-        if (page_name.match(/\.server\.tsx?/)) {
+        if (page_name.match(/\.server\.tsx?/) && !include_server) {
             continue;
         }
 
@@ -63,6 +80,7 @@ function grabPageDirRecursively({ page_dir }: { page_dir: string }) {
             if (checkExcludedPatterns({ path: full_page_path })) continue;
             const new_page_files = grabPageDirRecursively({
                 page_dir: full_page_path,
+                include_server,
             });
 
             pages_files.push(...new_page_files);

src/utils/write-logs.ts

@@ -0,0 +1,26 @@
+import { writeFileSync } from "fs";
+import grabDirNames from "./grab-dir-names";
+import path from "path";
+
+const { BUNX_LOGS_DIR } = grabDirNames();
+
+export default function writeLogs(log: any) {
+    try {
+        const now = Date.now();
+        let new_log_name = `${now}`;
+
+        let log_content = log.toString();
+
+        try {
+            const json = JSON.stringify(log, null, 4);
+            new_log_name += `.json`;
+            log_content = json;
+        } catch (error) {
+            new_log_name += `.log`;
+        }
+
+        const log_path = path.join(BUNX_LOGS_DIR, new_log_name);
+
+        writeFileSync(log_path, log_content);
+    } catch (error) {}
+}