From b3057119dfdc290d232b76f25d4eba104600cab3 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Tue, 20 Oct 2026 19:33:15 +0100 Subject: [PATCH] Download the whole app build behind a loading page In production, a page request without the running build's cookie gets a small loading page instead of the app. It downloads every file in .bunext/public (listed at /build-info.json) into that build's cache, checks they are all there, hands over to a new service worker waiting behind an older one, sets the cookie and reloads. The service worker then serves the app's code from the cache, so later visits load it from the device. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 +- public/loading.html | 250 ++++++++++++++++ public/sw.js | 91 +++--- src/data/app-data.ts | 26 ++ src/data/pwa-data.ts | 15 + .../pwa/build-precache-manifest.test.ts | 55 ++++ .../server/pwa/build-precache-manifest.ts | 55 ++++ .../server/pwa/grab-precache-manifest.ts | 27 ++ src/functions/server/pwa/handle-build-info.ts | 27 ++ .../server/pwa/handle-precache-gate.ts | 63 ++++ src/scripts/generate-brand-assets.ts | 279 ++++++++++++++++++ src/server.ts | 23 ++ src/types/index.ts | 9 + src/utils/is-hashed-asset-path.ts | 5 +- src/utils/needs-precache-page.test.ts | 50 ++++ src/utils/needs-precache-page.ts | 54 ++++ 16 files changed, 991 insertions(+), 45 deletions(-) create mode 100644 public/loading.html create mode 100644 src/functions/server/pwa/build-precache-manifest.test.ts create mode 100644 src/functions/server/pwa/build-precache-manifest.ts create mode 100644 src/functions/server/pwa/grab-precache-manifest.ts create mode 100644 src/functions/server/pwa/handle-build-info.ts create mode 100644 src/functions/server/pwa/handle-precache-gate.ts create mode 100644 src/utils/needs-precache-page.test.ts create mode 100644 src/utils/needs-precache-page.ts diff --git a/AGENTS.md b/AGENTS.md index 85e2804..0e8eaca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ This is **Cobble** (`cobble.moduletrace.org`): an open-source crew of small AI w `src/server.ts` - turns on SQLite foreign keys (`enableForeignKeys`, so the schema's cascades fire; off by default per connection), runs `bunextInit()`, starts `Bun.serve`, then starts cron: - `/media/*` short-circuits to the media server handler -- bunext's responses go through `compressResponse` (`src/functions/server/compress-response.ts`): page scripts and styles bunext names after their content (`isHashedAssetPath`) are cached `immutable` and compressed once (brotli or gzip, kept in memory within `CompressCacheMaxFiles` and `CompressCacheMaxBytes`, with parallel requests sharing one pass); pages and API JSON are gzipped as they stream; media, streams and small bodies go out as they are. Every variant carries `Vary: Accept-Encoding`. `public/sw.js` serves those same assets cache-first. Keep page props small: every page is a full server render, so anything only a modal needs loads when the modal opens. The chat header's models modal fetches the install-wide catalog once (`api/admin/harness/catalog`) and the conversation's small setting separately (`api/admin/bots/harness-overview`), both kept in `localStorage` (the catalog for `AppData.HarnessCatalogCacheMs`, the setting for `AppData.HarnessOverviewCacheMs`) and cleared on logout or after a model save (`src/utils/harness-cache.ts`). `bun run check:bundle` (run in CI after a production build) fails if the conversation page's initial JS passes its budget or the full Lucide set returns. Never render a lucide icon by name (twui's `LucideIcon`, a string `beforeIcon`): that ships every icon; import the icon +- bunext's responses go through `compressResponse` (`src/functions/server/compress-response.ts`): page scripts and styles bunext names after their content (`isHashedAssetPath`) are cached `immutable` and compressed once (brotli or gzip, kept in memory within `CompressCacheMaxFiles` and `CompressCacheMaxBytes`, with parallel requests sharing one pass); pages and API JSON are gzipped as they stream; media, streams and small bodies go out as they are. Every variant carries `Vary: Accept-Encoding`. `public/sw.js` serves every `.bunext/public` file from the build's own cache (see PWA). Keep page props small: every page is a full server render, so anything only a modal needs loads when the modal opens. The chat header's models modal fetches the install-wide catalog once (`api/admin/harness/catalog`) and the conversation's small setting separately (`api/admin/bots/harness-overview`), both kept in `localStorage` (the catalog for `AppData.HarnessCatalogCacheMs`, the setting for `AppData.HarnessOverviewCacheMs`) and cleared on logout or after a model save (`src/utils/harness-cache.ts`). `bun run check:bundle` (run in CI after a production build) fails if the conversation page's initial JS passes its budget or the full Lucide set returns. Never render a lucide icon by name (twui's `LucideIcon`, a string `beforeIcon`): that ships every icon; import the icon - `/ws` upgrades only after `userAuth` succeeds (session frozen into the socket at upgrade time); the `websocket` handler object lives in `src/websocket/` - everything else goes to `bunext.bunextRequestHandler` - `ensureDockerNetwork()` makes sure the computers' docker network (`AppData.DockerNetworkName`) exists before the server starts; without docker the process exits (computers are containers, so nothing works without it). Creating a computer's container checks it again first @@ -133,9 +133,10 @@ Envs are set in Settings, under Environment (super admin only), not in `.env`. T ### PWA and notifications -- **Generated assets:** `public/manifest.webmanifest`, `public/icons/*.png` and `public/offline.html` are generated from `public/logos/logo.svg` and the theme tokens by `bun run gen:favicon` (`src/scripts/generate-brand-assets.ts` reads `src/styles/main.css`, so the manifest colors and the offline page never drift from the theme). The outputs are committed like the favicons. `src/data/pwa-data.ts` holds the shared paths, icon list, start path (`/admin/bots`) and shortcuts. The apple touch icon is opaque on paper (a transparent one renders black on iOS). +- **Generated assets:** `public/manifest.webmanifest`, `public/icons/*.png`, `public/offline.html` and `public/loading.html` are generated from `public/logos/logo.svg` and the theme tokens by `bun run gen:favicon` (`src/scripts/generate-brand-assets.ts` reads `src/styles/main.css`, so the manifest colors and the offline page never drift from the theme). The outputs are committed like the favicons. `src/data/pwa-data.ts` holds the shared paths, icon list, start path (`/admin/bots`) and shortcuts. The apple touch icon is opaque on paper (a transparent one renders black on iOS). - **Head and install:** `src/pages/__root.tsx` `Head` links the manifest, sets the apple/mobile meta tags and renders `PwaInitScript`, which captures `beforeinstallprompt` before hydration (twui's hook reads it later), points `theme-color` at `--c-paper` (following the theme without repeating the color), and rewrites bunext's fixed viewport meta to add `viewport-fit=cover` and `interactive-widget=resizes-content`. `use-install-prompt` exposes installable/standalone/iOS state; the rail user menu shows Install app while installable, hides once installed, and opens `install-app-modal` with the Safari Share steps on iOS. -- **Service worker:** `public/sw.js`, served by `src/server.ts` through `handlePwaFile` with `no-store` and `Service-Worker-Allowed: /` (bunext's one-hour public cache would delay every deploy, including through Cloudflare). Navigations are network-only with `public/offline.html` as the fallback; authenticated HTML is never cached. `/.bunext/`, `/public/`, icons and logos are stale-while-revalidate; `/api/`, `/ws`, `/media/`, `/computers/` are never intercepted. `ServiceWorkerRegistrar` in `__root.tsx` registers it in production only (development HMR is never cached) and offers a reload when a new version waits; it never reloads on its own, so a running reply isn't cut off. +- **Service worker:** `public/sw.js`, served by `src/server.ts` through `handlePwaFile` with `no-store` and `Service-Worker-Allowed: /` (bunext's one-hour public cache would delay every deploy, including through Cloudflare). Navigations are network-only with `public/offline.html` as the fallback; authenticated HTML is never cached. `/public/`, icons and logos are stale-while-revalidate; `/api/`, `/ws`, `/media/`, `/computers/` and `/build-info.json` are never intercepted. `ServiceWorkerRegistrar` in `__root.tsx` registers it in production only (development HMR is never cached) and offers a reload when a new version waits; it never reloads on its own, so a running reply isn't cut off. +- **Precache (production):** the whole of `.bunext/public` is downloaded once per build before the app is used. On start the server reads the build id from `.bunext/build-info.json` and lists every file in `.bunext/public` (`grabPrecacheManifest`, logged, read once). `handlePrecacheGate` (in `src/server.ts`, before bunext) answers a page navigation whose `AppData.BuildCookieName` cookie isn't that build with the generated `public/loading.html` at the same address (`no-store`, `{{build}}` filled in; `needsPrecachePage`: `Sec-Fetch-Mode: navigate` or `Sec-Fetch-Dest: document`, since a navigation the worker passes on arrives as `empty`, never `/api/`, `/.bunext/`, `/media`, `/computers/`, `/ws` or a path with an extension). The page reads the build's files from `GET /build-info.json` (`handle-build-info.ts`, `no-store`) and downloads them into `cobble-assets-` itself (`AppData.PrecacheConcurrency` at a time), so it works whichever worker version is in charge. It then checks every file is in the cache, downloading any missing ones again up to `AppData.PrecacheVerifyPasses` more times, before it writes the marker (`PwaPrecacheMarkerPath`). It deletes the old build's cache, tells a new worker waiting behind an older one to take over (`SKIP_WAITING`, after `reg.update()`, for up to `AppData.PrecacheWorkerWaitMs`), sets the cookie and reloads into the app. Without Cache Storage, when files are still missing after the last pass, or once nothing has landed for `AppData.PrecacheStallMs`, it sets the cookie anyway, so the app runs from the network. The worker only serves `/.bunext/public/*` cache-first from the one `cobble-assets-*` cache. `sw.js` never changes per build, and React knows nothing of any of this. The loading page is generated with the offline page (`bun run gen:favicon`). - **Phone shell:** below `md` the rail is a 280px slide-over with a scrim, Escape to close, and a close on every route change (`RailDrawerContext`); it is a 64px icon rail at `md` and 260px at `lg`. Rail internals use the `@container` `@rail:` variant (`--container-rail` in `src/styles/main.css`), not viewport `lg:`, so the drawer shows the full rail. `RailDrawerButton` sits in both pane headers. The shell is `h-dvh` with safe-area insets, and the headers/thread/composer use mobile padding. - **Connection recovery:** twui's `useWebSocket` stops retrying after three attempts, so `LiveSocket` is remounted on `online`, `pageshow` and a hidden tab becoming visible (`use-connection-epoch`). `useOnlineStatus` drives the offline strip and disables sending: nothing is queued offline. - **Web Push:** `push_subscriptions` (one row per endpoint and user; the endpoint is unique and a capability URL, never sent to the client) and `user_presence` (one row per user, refreshed by the socket keepalive in `src/websocket/index.ts`). VAPID keys are app settings in the `envs` table (`VAPID_PUBLIC_KEY` not secret, `VAPID_PRIVATE_KEY` secret), generated by `ensureVapidKeys` on the main server's first start; the subject is `SiteData.SiteURL`. `pushBotNotice` is the one trigger: bot messages (never the user's) and a needs-you face change without a message. It is skipped while `user_presence.last_seen_at` is within `AppData.PushPresenceWindowMs`, so an app open anywhere wins. `src/dict/push-dict.ts` owns payload limits and tags (one conversation replaces its own notification); sending never blocks or fails the job, 404/410 removes the device, and other failures are recorded on the subscription. API in `api/admin/push/` (subscribe, unsubscribe, test, remove-device, overview); Settings > Notifications (`notifications/`) has the per-device switch, a test send, and every device this user has. Permission is only asked from the switch (a denial is hard to undo, and iOS requires the gesture); iOS also needs the installed app. Development has no service worker, so the page says notifications need the installed production app. diff --git a/public/loading.html b/public/loading.html new file mode 100644 index 0000000..a6dec0e --- /dev/null +++ b/public/loading.html @@ -0,0 +1,250 @@ + + + + + + + +cobble + + + + +
+ + + ember-v2 + + + + + + + +

Getting cobble ready

+

Saving the app on this device, so it opens fast from now on.

+
+

+
+ + + diff --git a/public/sw.js b/public/sw.js index 6f4b11d..2bc6036 100644 --- a/public/sw.js +++ b/public/sw.js @@ -4,14 +4,28 @@ * The app is server-rendered and private, so authenticated HTML is never * cached: navigating offline lands on the generated offline page instead * of a stale conversation or a cached login redirect. Only the shell - * (icons, manifest, offline page) and versioned app assets are cached. + * (icons, manifest, offline page) and the app's own code are cached. + * + * The app's code (`.bunext/public`) is downloaded whole, once per build, + * by the loading page (`public/loading.html`, served in place of the app + * until the browser has the build's cookie) into that build's cache, + * `cobble-assets-`, the only one kept. The worker serves the + * build's files from it. This file stays the same across deploys. * * Bump `CACHE_VERSION` when this file changes in a way that must drop the - * old caches. + * old shell cache. */ -const CACHE_VERSION = "cobble-v2"; + +const CACHE_VERSION = "cobble-v3"; const CACHE_NAME = `cobble-shell-${CACHE_VERSION}`; +/** + * `PwaBuildInfoPath` and `PwaPrecacheCachePrefix` in + * `src/data/pwa-data.ts`. + */ +const BUILD_INFO_URL = "/build-info.json"; +const ASSET_CACHE_PREFIX = "cobble-assets-"; + const OFFLINE_URL = "/offline.html"; const SHELL = [ @@ -27,9 +41,15 @@ const SHELL = [ ]; /** - * Versioned app code and public assets: safe to refresh in the background. + * The build's code: served from its own cache, so a build's files never + * mix with another's. */ -const ASSET_PREFIXES = ["/.bunext/", "/public/", "/icons/", "/logos/", "/brand/"]; +const BUILD_PREFIX = "/.bunext/public/"; + +/** + * Public assets: safe to refresh in the background. + */ +const ASSET_PREFIXES = ["/public/", "/icons/", "/logos/", "/brand/"]; const ASSET_PATHS = ["/favicon.ico", "/favicon.png", "/apple-touch-icon.png"]; /** @@ -37,17 +57,7 @@ const ASSET_PATHS = ["/favicon.ico", "/favicon.png", "/apple-touch-icon.png"]; * auth side effects, and the worker itself. */ const BYPASS_PREFIXES = ["/api/", "/ws", "/media", "/computers/"]; -const BYPASS_PATHS = ["/sw.js", "/manifest.webmanifest", "/auth/logout"]; - -/** - * Page scripts and styles bunext names after their content: one at such - * a path never changes, so it is served from the cache without asking the - * network again (`src/utils/is-hashed-asset-path.ts` has the same - * pattern). Each deploy adds new names, so the oldest are dropped past - * `MAX_HASHED_ASSETS`. - */ -const HASHED_ASSET = /^\/\.bunext\/public\/pages\/(?:.*\/)?(?:chunk-)?[A-Z0-9]{8}\.(?:js|css)$/; -const MAX_HASHED_ASSETS = 400; +const BYPASS_PATHS = ["/sw.js", "/manifest.webmanifest", "/auth/logout", BUILD_INFO_URL]; const DEFAULT_ICON = "/icons/icon-192.png"; const DEFAULT_BADGE = "/icons/icon-192.png"; @@ -74,6 +84,17 @@ self.addEventListener("install", (event) => { ); }); +/** + * The cache of the build the worker serves: the one `cobble-assets-*` + * cache, since the loading page deletes the others. Looked up every time: + * the loading page switches builds while the worker runs. + */ +async function grabAssetCacheName() { + const keys = await caches.keys(); + + return keys.find((key) => key.startsWith(ASSET_CACHE_PREFIX)) || null; +} + self.addEventListener("activate", (event) => { event.waitUntil( caches @@ -81,7 +102,9 @@ self.addEventListener("activate", (event) => { .then((keys) => Promise.all( keys - .filter((key) => key !== CACHE_NAME) + .filter( + (key) => key !== CACHE_NAME && !key.startsWith(ASSET_CACHE_PREFIX), + ) .map((key) => caches.delete(key)), ), ) @@ -113,13 +136,13 @@ self.addEventListener("fetch", (event) => { return; } - if (!isAssetRequest(url)) return; - - if (HASHED_ASSET.test(url.pathname)) { + if (url.pathname.startsWith(BUILD_PREFIX)) { event.respondWith(cacheFirst(request)); return; } + if (!isAssetRequest(url)) return; + event.respondWith( caches.open(CACHE_NAME).then(async (cache) => { const cached = await cache.match(request); @@ -139,7 +162,11 @@ self.addEventListener("fetch", (event) => { }); async function cacheFirst(request) { - const cache = await caches.open(CACHE_NAME); + const name = await grabAssetCacheName(); + + if (!name) return fetch(request); + + const cache = await caches.open(name); const cached = await cache.match(request); if (cached) return cached; @@ -148,31 +175,15 @@ async function cacheFirst(request) { if (response.ok && response.type === "basic") { await cache.put(request, response.clone()).catch(() => undefined); - trimHashedAssets(cache).catch(() => undefined); } return response; } /** - * Cache keys come back in insertion order, so the first hashed ones are - * the oldest. - */ -async function trimHashedAssets(cache) { - const hashed = (await cache.keys()).filter((request) => - HASHED_ASSET.test(new URL(request.url).pathname), - ); - - await Promise.all( - hashed - .slice(0, Math.max(0, hashed.length - MAX_HASHED_ASSETS)) - .map((request) => cache.delete(request)), - ); -} - -/** - * The page asks for a new version from its update notice. Nothing skips - * the wait until the user says so. + * The page asks for a new version from its update notice: nothing skips + * the wait until the user says so. The loading page asks too, since it + * isn't the app and nothing of the app's is running. */ self.addEventListener("message", (event) => { if (event.data?.type === "SKIP_WAITING") { diff --git a/src/data/app-data.ts b/src/data/app-data.ts index 557008e..ce50e2a 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -6,6 +6,25 @@ export const AppData = { * Shared with twui's `ColorSchemeSelector`, so it must stay `theme`. */ ThemeStorageKey: "theme", + /** + * How many of a build's files the loading page downloads at once. + */ + PrecacheConcurrency: 6, + /** + * How many more times the loading page downloads files it finds + * missing from the cache after a pass, before it gives up on them. + */ + PrecacheVerifyPasses: 2, + /** + * How long the loading page waits for a new service worker to take + * over before it reloads into the app anyway. + */ + PrecacheWorkerWaitMs: 10_000, + /** + * The loading page gives up and lets the app run from the network + * when no new file lands for this long. + */ + PrecacheStallMs: 30_000, /** * Prefix of the composer's unsent drafts in `localStorage`, one per * bot and workspace. @@ -38,6 +57,13 @@ export const AppData = { TlsCaProfileFilename: `${SiteData["SiteSlug"]}-ca.mobileconfig`, AuthKeyCookieName: `${SiteData["SiteSlug"]}_auth_key`, AuthCSRFCookieName: `${SiteData["SiteSlug"]}_auth_csrf`, + /** + * The build this browser has been through the loading page for + * (`handlePrecacheGate`). Set by the page itself, so not `HttpOnly`; + * it holds nothing private. + */ + BuildCookieName: `${SiteData["SiteSlug"]}_build`, + BuildCookieMaxAgeSeconds: 365 * 24 * 60 * 60, AuthExpiryDays: 60, APICrudBasePath: "/api/admin/crud", ForcePrivateHeaderKey: `x-${SiteData["SiteSlug"]}-force-private`, diff --git a/src/data/pwa-data.ts b/src/data/pwa-data.ts index 2d89229..4fddec1 100644 --- a/src/data/pwa-data.ts +++ b/src/data/pwa-data.ts @@ -9,6 +9,21 @@ export const PwaScope = "/"; export const PwaManifestPath = "/manifest.webmanifest"; export const PwaServiceWorkerPath = "/sw.js"; export const PwaOfflinePath = "/offline.html"; +/** + * The page a browser gets instead of the app while it downloads a build + * it hasn't yet (`handlePrecacheGate`), generated like the offline page. + */ +export const PwaLoadingPath = "/loading.html"; + +/** + * The running build's id and files (`handle-build-info.ts`). Each build's + * `.bunext/public` files are downloaded into their own cache, + * ``, which holds the marker once every file is in. + * `public/sw.js` repeats all three. + */ +export const PwaBuildInfoPath = "/build-info.json"; +export const PwaPrecacheCachePrefix = "cobble-assets-"; +export const PwaPrecacheMarkerPath = "/__cobble-precached"; export type PwaIcon = { src: string; diff --git a/src/functions/server/pwa/build-precache-manifest.test.ts b/src/functions/server/pwa/build-precache-manifest.test.ts new file mode 100644 index 0000000..ed6c58e --- /dev/null +++ b/src/functions/server/pwa/build-precache-manifest.test.ts @@ -0,0 +1,55 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import path from "path"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import buildPrecacheManifest from "./build-precache-manifest"; + +const root = mkdtempSync(path.join(tmpdir(), "precache-")); + +afterAll(() => rmSync(root, { recursive: true, force: true })); + +function writeBuild({ dir, files }: { dir: string; files: string[] }) { + writeFileSync( + path.join(dir, "build-info.json"), + JSON.stringify({ build_id: "build-1" }), + ); + + for (const file of files) { + const full_path = path.join(dir, "public", file); + mkdirSync(path.dirname(full_path), { recursive: true }); + writeFileSync(full_path, "x"); + } +} + +describe("buildPrecacheManifest", () => { + test("lists every public file under its served path", () => { + const dir = path.join(root, "full"); + mkdirSync(dir); + writeBuild({ + dir, + files: [ + "vendor/react.js", + "pages/chunk-AEHOZS24.js", + "pages/admin/bots/[bot_id]/VXCMP4GM.js", + "pages/auth/login/X4TQCLSA.css", + ], + }); + + expect(buildPrecacheManifest({ bunext_dir: dir })).toEqual({ + build_id: "build-1", + files: [ + "/.bunext/public/pages/admin/bots/[bot_id]/VXCMP4GM.js", + "/.bunext/public/pages/auth/login/X4TQCLSA.css", + "/.bunext/public/pages/chunk-AEHOZS24.js", + "/.bunext/public/vendor/react.js", + ], + }); + }); + + test("is null without a build", () => { + const dir = path.join(root, "empty"); + mkdirSync(dir); + + expect(buildPrecacheManifest({ bunext_dir: dir })).toBeNull(); + }); +}); diff --git a/src/functions/server/pwa/build-precache-manifest.ts b/src/functions/server/pwa/build-precache-manifest.ts new file mode 100644 index 0000000..a144936 --- /dev/null +++ b/src/functions/server/pwa/build-precache-manifest.ts @@ -0,0 +1,55 @@ +import path from "path"; +import { existsSync, readdirSync, readFileSync, statSync } from "fs"; +import type { PrecacheManifest } from "@/src/types"; + +type Params = { + /** + * bunext's build directory (`.bunext`). + */ + bunext_dir: string; +}; + +/** + * Lists every file of one bunext build's `public/` directory as the path + * the browser asks for (`/.bunext/public/...`), with the build's id, so + * the service worker can download the whole build before the app runs. + * Null without a build. + */ +export default function buildPrecacheManifest({ + bunext_dir, +}: Params): PrecacheManifest | null { + const public_dir = path.join(bunext_dir, "public"); + const build_info_path = path.join(bunext_dir, "build-info.json"); + + if (!existsSync(public_dir) || !existsSync(build_info_path)) return null; + + const build_id = JSON.parse(readFileSync(build_info_path, "utf-8")) + ?.build_id as string | undefined; + + if (!build_id) return null; + + const files: string[] = []; + + const walk = (dir: string) => { + for (const name of readdirSync(dir)) { + const full_path = path.join(dir, name); + + if (statSync(full_path).isDirectory()) { + walk(full_path); + continue; + } + + const relative = path + .relative(bunext_dir, full_path) + .split(path.sep) + .join("/"); + + files.push(`/.bunext/${relative}`); + } + }; + + walk(public_dir); + files.sort(); + + return { build_id, files }; +} diff --git a/src/functions/server/pwa/grab-precache-manifest.ts b/src/functions/server/pwa/grab-precache-manifest.ts new file mode 100644 index 0000000..cd132d5 --- /dev/null +++ b/src/functions/server/pwa/grab-precache-manifest.ts @@ -0,0 +1,27 @@ +import path from "path"; +import grabDirNames from "@/src/utils/grab-dir-names"; +import type { PrecacheManifest } from "@/src/types"; +import buildPrecacheManifest from "./build-precache-manifest"; + +let manifest: PrecacheManifest | null | undefined; + +/** + * The running build's precache manifest, production only (development + * rebuilds on every change and is never cached). Read once per process: + * a build never changes under a running server. + */ +export default function grabPrecacheManifest(): PrecacheManifest | null { + if (process.env.NODE_ENV != "production") return null; + if (manifest !== undefined) return manifest; + + try { + manifest = buildPrecacheManifest({ + bunext_dir: path.join(grabDirNames().ROOT_DIR, ".bunext"), + }); + } catch (error) { + console.log("Couldn't read the build for precaching:", error); + manifest = null; + } + + return manifest; +} diff --git a/src/functions/server/pwa/handle-build-info.ts b/src/functions/server/pwa/handle-build-info.ts new file mode 100644 index 0000000..a36a6dc --- /dev/null +++ b/src/functions/server/pwa/handle-build-info.ts @@ -0,0 +1,27 @@ +import { PwaBuildInfoPath } from "@/src/data/pwa-data"; +import grabPrecacheManifest from "./grab-precache-manifest"; + +/** + * Serves the running build (`.bunext/build-info.json`'s id) with every + * file in `.bunext/public`, for the loading page to download before the + * app runs. Never cached: a deploy must show at once. Not found in + * development, which is never precached. + */ +export default function handleBuildInfo({ + pathname, +}: { + pathname: string; +}): Response | null { + if (pathname != PwaBuildInfoPath) return null; + + const manifest = grabPrecacheManifest(); + + if (!manifest) return new Response("Not Found", { status: 404 }); + + return Response.json(manifest, { + headers: { + "Cache-Control": "no-store, max-age=0, must-revalidate", + "X-Content-Type-Options": "nosniff", + }, + }); +} diff --git a/src/functions/server/pwa/handle-precache-gate.ts b/src/functions/server/pwa/handle-precache-gate.ts new file mode 100644 index 0000000..e470262 --- /dev/null +++ b/src/functions/server/pwa/handle-precache-gate.ts @@ -0,0 +1,63 @@ +import path from "path"; +import { AppData } from "@/src/data/app-data"; +import { PwaLoadingPath } from "@/src/data/pwa-data"; +import { getCookie } from "@/src/utils/cookies"; +import needsPrecachePage from "@/src/utils/needs-precache-page"; +import grabPrecacheManifest from "./grab-precache-manifest"; + +const LOADING_PAGE = path.join( + path.resolve(import.meta.dir, "../../../../public"), + PwaLoadingPath, +); + +/** + * The placeholder `public/loading.html` carries for the running build. + */ +const BuildPlaceholder = "{{build}}"; + +/** + * Production only: a browser opening a page without the running build in + * its cookie gets the loading page in its place, at the same address. The + * page downloads the build's `.bunext/public` files into the build's + * cache, checks they are all there, sets the cookie and reloads, and the + * service worker then serves the app's code from the device. Null lets + * the request through. + */ +export default async function handlePrecacheGate({ + req, + url, +}: { + req: Request; + url: URL; +}): Promise { + const manifest = grabPrecacheManifest(); + + const needs_page = needsPrecachePage({ + method: req.method, + pathname: url.pathname, + fetch_mode: req.headers.get("Sec-Fetch-Mode"), + fetch_dest: req.headers.get("Sec-Fetch-Dest"), + accept: req.headers.get("Accept"), + cookie_build_id: getCookie(req, AppData["BuildCookieName"]), + build_id: manifest?.build_id, + }); + + if (!needs_page || !manifest) return null; + + const file = Bun.file(LOADING_PAGE); + + if (!(await file.exists())) return null; + + const build = JSON.stringify({ + build_id: manifest.build_id, + total: manifest.files.length, + }).replace(/ + + + + + + +${name} + + + + +
+${logo} +

Getting ${name} ready

+

Saving the app on this device, so it opens fast from now on.

+
+

+
+ + + +`; +} + /** * The logo inline, so the offline page has no request to make while * offline. Its own width and height give way to ours. @@ -347,6 +620,11 @@ async function main() { buildOfflinePage({ colors, logo: inlineLogo(await Bun.file(LOGO_SVG).text()) }), ); + writeFileSync( + path.join(PUBLIC_DIR, "loading.html"), + buildLoadingPage({ colors, logo: inlineLogo(await Bun.file(LOGO_SVG).text()) }), + ); + console.log("Generated brand and PWA assets from public/logos/logo.svg:"); console.log(" public/favicon.ico (16/24/32/48 multi-size)"); console.log(" public/favicon.png (32x32)"); @@ -358,6 +636,7 @@ async function main() { console.log(" public/icons/maskable-512.png (maskable, safe zone)"); console.log(" public/manifest.webmanifest (name, colors and icons)"); console.log(" public/offline.html (self-contained offline page)"); + console.log(" public/loading.html (precache loading page)"); } main() diff --git a/src/server.ts b/src/server.ts index e03229e..d5be5c5 100644 --- a/src/server.ts +++ b/src/server.ts @@ -11,6 +11,9 @@ import liveRelay from "./live-relay"; import enableForeignKeys from "./functions/backend/db/enable-foreign-keys"; import handleComputerProxy from "./functions/server/computers/handle-computer-proxy"; import handlePwaFile from "./functions/server/pwa/handle-pwa-file"; +import handleBuildInfo from "./functions/server/pwa/handle-build-info"; +import handlePrecacheGate from "./functions/server/pwa/handle-precache-gate"; +import grabPrecacheManifest from "./functions/server/pwa/grab-precache-manifest"; import compressResponse from "./functions/server/compress-response"; import serveLocalCa from "./functions/server/serve-local-ca"; import startHttpRedirect from "./functions/server/start-http-redirect"; @@ -30,6 +33,18 @@ const port = process.env.PORT || 7227; await enableForeignKeys(); await bunext.bunextInit(); +/** + * The running build's id and files, read once: browsers whose build + * cookie doesn't match get the loading page first (production only). + */ +const precache = grabPrecacheManifest(); + +if (precache) { + bunext.bunextLog.info( + `Build ${precache.build_id}: ${precache.files.length} files to precache`, + ); +} + /** * Web Push needs a key pair; the first start of the main install creates * it (in the Environment settings) so notifications work without setup. @@ -111,6 +126,14 @@ const server = Bun.serve({ if (pwa_file) return pwa_file; + const build_info = handleBuildInfo({ pathname: url.pathname }); + + if (build_info) return build_info; + + const loading_page = await handlePrecacheGate({ req, url }); + + if (loading_page) return loading_page; + const local_ca = await serveLocalCa({ pathname: url.pathname }); if (local_ca) return local_ca; diff --git a/src/types/index.ts b/src/types/index.ts index 711dfd2..12ef1c3 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -697,6 +697,15 @@ export type BotTextModel = { provider_name: string; }; +/** + * One build's `.bunext/public` files, downloaded by the service worker + * before the app runs (`grabPrecacheManifest`). + */ +export type PrecacheManifest = { + build_id: string; + files: string[]; +}; + export type PagePropsType = { user?: User | null; user_types?: BUN_SQLITE_COBBLE_USER_TYPES[] | null; diff --git a/src/utils/is-hashed-asset-path.ts b/src/utils/is-hashed-asset-path.ts index 7d95c25..2c69816 100644 --- a/src/utils/is-hashed-asset-path.ts +++ b/src/utils/is-hashed-asset-path.ts @@ -3,9 +3,10 @@ * (`chunk-AEHOZS24.js`, `admin/bots/H6VDJMBN.js`, `X4TQCLSA.css`): one * at such a path never changes, so it can be cached for good. The * vendor files (`/.bunext/public/vendor/react.js`) keep fixed names and - * are not. `public/sw.js` repeats this pattern. + * are not. The loading page (`public/loading.html`) is generated with + * this pattern. */ -const HashedAssetPattern = /^\/\.bunext\/public\/pages\/(?:.*\/)?(?:chunk-)?[A-Z0-9]{8}\.(?:js|css)$/; +export const HashedAssetPattern = /^\/\.bunext\/public\/pages\/(?:.*\/)?(?:chunk-)?[A-Z0-9]{8}\.(?:js|css)$/; type Params = { pathname: string; diff --git a/src/utils/needs-precache-page.test.ts b/src/utils/needs-precache-page.test.ts new file mode 100644 index 0000000..a63359a --- /dev/null +++ b/src/utils/needs-precache-page.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from "bun:test"; +import needsPrecachePage from "./needs-precache-page"; + +const page = { + method: "GET", + pathname: "/admin/bots/1", + fetch_dest: "document", + build_id: "build-2", +}; + +describe("needsPrecachePage", () => { + test("stands in for a page on a build the browser hasn't downloaded", () => { + expect(needsPrecachePage(page)).toBe(true); + expect(needsPrecachePage({ ...page, cookie_build_id: "build-1" })).toBe(true); + expect( + needsPrecachePage({ ...page, fetch_dest: "empty", fetch_mode: "navigate" }), + ).toBe(true); + expect( + needsPrecachePage({ ...page, fetch_dest: null, accept: "text/html,*/*" }), + ).toBe(true); + }); + + test("lets the current build through", () => { + expect(needsPrecachePage({ ...page, cookie_build_id: "build-2" })).toBe(false); + }); + + test("never runs in development", () => { + expect(needsPrecachePage({ ...page, build_id: null })).toBe(false); + }); + + test("leaves everything that isn't a page navigation", () => { + expect(needsPrecachePage({ ...page, method: "POST" })).toBe(false); + expect(needsPrecachePage({ ...page, fetch_dest: "script" })).toBe(false); + expect( + needsPrecachePage({ + ...page, + fetch_dest: "empty", + fetch_mode: "cors", + accept: "text/html", + }), + ).toBe(false); + expect(needsPrecachePage({ ...page, fetch_dest: null, accept: "*/*" })).toBe(false); + expect(needsPrecachePage({ ...page, pathname: "/api/admin/bots/send-message" })).toBe(false); + expect(needsPrecachePage({ ...page, pathname: "/.bunext/public/vendor/react.js" })).toBe(false); + expect(needsPrecachePage({ ...page, pathname: "/computers/2/vnc/vnc.html" })).toBe(false); + expect(needsPrecachePage({ ...page, pathname: "/media/1/private/a" })).toBe(false); + expect(needsPrecachePage({ ...page, pathname: "/build-info.json" })).toBe(false); + expect(needsPrecachePage({ ...page, pathname: "/sw.js" })).toBe(false); + }); +}); diff --git a/src/utils/needs-precache-page.ts b/src/utils/needs-precache-page.ts new file mode 100644 index 0000000..d5da573 --- /dev/null +++ b/src/utils/needs-precache-page.ts @@ -0,0 +1,54 @@ +/** + * Paths the loading page never stands in for: the app's code, live data, + * media, proxied computers and the files the loading page itself needs. + */ +const OpenPrefixes = ["/.bunext/", "/api/", "/ws", "/media", "/computers/"]; + +type Params = { + method: string; + pathname: string; + /** + * `Sec-Fetch-Mode` and `Sec-Fetch-Dest`, when the browser sends them. + * A navigation the service worker passes on keeps `navigate` but + * arrives with the destination `empty`. + */ + fetch_mode?: string | null; + fetch_dest?: string | null; + accept?: string | null; + /** + * The build in the browser's cookie, if any. + */ + cookie_build_id?: string | null; + /** + * The running build; null in development. + */ + build_id?: string | null; +}; + +/** + * Whether a request is a browser opening a page of the app on a build it + * hasn't downloaded yet, so it gets the loading page instead. Only page + * navigations: anything with a file extension (`/favicon.ico`, + * `/sw.js`, `/build-info.json`) goes through as usual. + */ +export default function needsPrecachePage({ + method, + pathname, + fetch_mode, + fetch_dest, + accept, + cookie_build_id, + build_id, +}: Params): boolean { + if (!build_id) return false; + if (cookie_build_id == build_id) return false; + if (method != "GET") return false; + if (OpenPrefixes.some((prefix) => pathname.startsWith(prefix))) return false; + if (pathname.split("/").pop()?.includes(".")) return false; + + if (fetch_mode || fetch_dest) { + return fetch_mode == "navigate" || fetch_dest == "document"; + } + + return Boolean(accept?.includes("text/html")); +}