Take the Gitea registry out of CI and mirror it separately

The Gitea registry is on a VPN, which a GitHub runner can never reach, so
the mirror step in the release workflow could only ever fail: its last four
releases reported a step that did nothing. The workflow now builds and
pushes to GHCR alone, which is the registry the installer defaults to, and
nothing in CI mentions Gitea.

The image reaches the Gitea registry through `scripts/mirror-gitea.sh`
(`bun run mirror:gitea`) instead, run in parallel with a release and from the
VPN side: it copies from the Gitea host over ssh, where skopeo can see both
registries, so nothing has to be installed there. `skopeo copy --all` keeps
the manifest list and its signatures, and every destination tag is read back
and compared with GHCR's digest, so a partial copy fails loudly instead of
being assumed to have worked. `latest` and `major.minor` are only moved while
that release still owns them in GHCR, so mirroring an older release never
rolls them back.

Neither direction can fail the other now, which is the point: a registry that
is down, slow or full no longer takes a release with it.

.github/workflows/release.yml

@@ -1,5 +1,12 @@
 name: Release image
 
+# Builds and pushes the multi-arch image to GHCR, the registry the installer
+# defaults to.
+#
+# The Gitea registry is on a VPN, which a GitHub runner can never reach, so CI
+# does not touch it at all: `scripts/mirror-gitea.sh` copies a release into it
+# from the VPN side, in parallel and after the fact. Keeping the two apart is
+# what makes a registry outage unable to fail a release.
 on:
   release:
     types: [published]
@@ -30,13 +37,6 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Log in to Gitea
-        uses: docker/login-action@v3
-        with:
-          registry: git.tben.me
-          username: ${{ secrets.GITEA_USERNAME }}
-          password: ${{ secrets.GITEA_TOKEN }}
-
       # The version the app reports on Settings > General when the build
       # isn't a published release.
       - name: Read the source version
@@ -45,6 +45,11 @@ jobs:
           set -euo pipefail
           echo "version=$(sed -n -E 's/^[[:space:]]*"version": "([^"]+)",?$/\1/p' package.json | head -n1)" >> "$GITHUB_OUTPUT"
 
+      # The same four tags for every kind of build: a release gets its own
+      # version plus the moving `latest` and `major.minor`, so that
+      # scripts/mirror-gitea.sh can copy whichever of them still belong to
+      # it, and a dispatch build gets the commit tag only, since it is not a
+      # release and must not move `latest`.
       - name: Work out image tags
         id: tags
         env:
@@ -57,32 +62,24 @@ jobs:
           if [ "$EVENT" = "release" ] && [ -n "$RELEASE_TAG" ]; then
             version="$RELEASE_TAG"
             minor="$(printf '%s' "$version" | sed -E 's/^v?([0-9]+)\.([0-9]+).*/\1.\2/')"
-            ghcr="$(printf '%s\n' \
+            tags="$(printf '%s\n' \
               "ghcr.io/moduletrace/cobble:latest" \
               "ghcr.io/moduletrace/cobble:${version}" \
               "ghcr.io/moduletrace/cobble:${minor}" \
               "ghcr.io/moduletrace/cobble:${sha}")"
-            gitea="$(printf '%s\n' \
-              "git.tben.me/moduletrace/cobble:latest" \
-              "git.tben.me/moduletrace/cobble:${version}" \
-              "git.tben.me/moduletrace/cobble:${minor}" \
-              "git.tben.me/moduletrace/cobble:${sha}")"
           else
             version="$SOURCE_VERSION"
-            ghcr="ghcr.io/moduletrace/cobble:${sha}"
+            tags="ghcr.io/moduletrace/cobble:${sha}"
-            gitea="git.tben.me/moduletrace/cobble:${sha}"
           fi
           {
             echo "version=$version"
-            echo "ghcr<<EOF"
+            echo "tags<<EOF"
-            echo "$ghcr"
+            echo "$tags"
-            echo "EOF"
-            echo "gitea<<EOF"
-            echo "$gitea"
             echo "EOF"
           } >> "$GITHUB_OUTPUT"
 
-      # The registry the installer defaults to, so this one must succeed.
+      # The only push CI makes. The installer defaults to this registry, so
+      # this step failing has to fail the release.
       - name: Build and push to GHCR
         uses: docker/build-push-action@v6
         with:
@@ -92,62 +89,8 @@ jobs:
           push: true
           build-args: |
             COBBLE_VERSION=${{ steps.tags.outputs.version }}
-          tags: ${{ steps.tags.outputs.ghcr }}
+          tags: ${{ steps.tags.outputs.tags }}
           provenance: false
           sbom: false
           cache-from: type=gha
           cache-to: type=gha,mode=max
-
-      # The mirror's registry, from the image just built: no second build,
-      # and Cloudflare rate limiting git.tben.me (429/1015 on blob uploads)
-      # can never fail a release the installer depends on. It must still be
-      # loud: `continue-on-error` alone leaves the check green when the
-      # mirror does nothing at all, which is how the 413 below went unseen
-      # for four releases.
-      - name: Mirror to Gitea
-        id: mirror
-        continue-on-error: true
-        env:
-          GITEA_TAGS: ${{ steps.tags.outputs.gitea }}
-          GHCR_IMAGES: ${{ steps.tags.outputs.ghcr }}
-        run: |
-          set -uo pipefail
-          first_ghcr="$(printf '%s\n' "$GHCR_IMAGES" | head -n1)"
-          args=()
-          while IFS= read -r tag; do
-            [ -n "$tag" ] || continue
-            args+=(--tag "${tag}")
-          done <<< "$GITEA_TAGS"
-
-          output="$(mktemp)"
-          status=0
-          docker buildx imagetools create "${args[@]}" "${first_ghcr}" 2>&1 | tee "$output" || status=$?
-
-          if [ "$status" -eq 0 ]; then
-            {
-              echo "### Gitea mirror"
-              echo
-              echo "Mirrored \`${first_ghcr}\` to:"
-              echo
-              printf -- '- \`%s\`\n' $GITEA_TAGS
-            } >> "$GITHUB_STEP_SUMMARY"
-            exit 0
-          fi
-
-          # The usual cause: git.tben.me sits behind Cloudflare, whose
-          # request-body cap is 100 MB, and this image has a 243 MiB
-          # `bun install` layer that no route through it can carry.
-          reason="$(grep -oE '[0-9]{3} [A-Za-z ]{0,40}' "$output" | tail -n1)"
-          {
-            echo "### Gitea mirror failed"
-            echo
-            echo "\`${first_ghcr}\` did **not** reach the Gitea registry. The release itself is fine: the installer defaults to GHCR."
-            echo
-            echo '```'
-            tail -n 12 "$output"
-            echo '```'
-            echo
-            echo "See the [registry push limits](https://developers.cloudflare.com/workers/platform/limits/): git.tben.me is behind Cloudflare, which refuses any request body over 100 MB, and this image's \`bun install\` layer alone is about 243 MiB."
-          } >> "$GITHUB_STEP_SUMMARY"
-          echo "::warning title=Gitea mirror failed::${reason:-docker buildx imagetools create failed}; the image did not reach git.tben.me. See the job summary."
-          exit "$status"

AGENTS.md

@@ -134,7 +134,7 @@ The app serves plain HTTP only; TLS is the reverse proxy's or tunnel's job, and
 
 The container runs with `--network cobble`, `-p BIND:PORT`, `.env` at `/app/.env` read only, the host data directory at `/data`, the docker socket at `/var/run/docker.sock`, `MAIN_SERVER=1` and `PORT`, `--init`, `--cpus 1` and `--memory 2048m` (a light orchestration process: the heavy work happens in each computer's own container; overridable with `--cpus`/`--memory`), a `unless-stopped` restart policy and bounded json-file logs. `/data` must stay a bind mount: `resolveHostPath` turns paths under the app's own mounts into host paths for computer containers, and a named volume would break them. `DATA_DIR=/data` and `DB_DIR=/data/db` come from the Dockerfile; `entrypoint.sh` keeps bun-sqlite's schema record on the same volume.
 
-Publishing is `scripts/publish-github.sh` and `scripts/publish-gitea.sh`: the shared flow (argument parsing, the release gate, tag preparation, the push, the release notes) lives in `scripts/lib/publish-common.sh` and `scripts/lib/publish-release.sh`, and only the release object differs (`gh release create` for GitHub, which is what fires the image build, and `curl` against the Gitea API with a `GITEA_TOKEN` or the token `tea login` stored). Both are marked-block scripts tested by `scripts/publish-tools.test.ts`, which lifts the blocks out with the `install.test.ts` approach. Images are built by GitHub Actions on a published release (`.github/workflows/release.yml`), for `linux/amd64` and `linux/arm64`, and pushed to both `ghcr.io/moduletrace/cobble` and `git.tben.me/moduletrace/cobble` with `latest`, version, `major.minor` and commit-sha tags. The version they are built with is passed as the `COBBLE_VERSION` build arg (the published tag, else `package.json`'s version read from `<version>`), which the image turns into `ENV COBBLE_VERSION` and `runningVersion` (`src/utils/running-version.ts`) reports on Settings > General; bump `package.json`'s `version` with a release, since a local build and a workflow-dispatch build fall back to it. `.github/workflows/ci.yml` runs `bun run typecheck` and `bun run test` on every push and pull request, and checks that `install.sh` parses. The Dockerfile pins Bun 1.4.3 to match CI. The repository is mirrored on Gitea, and the installer defaults to the GHCR image with `--image git.tben.me/moduletrace/cobble` as the Gitea alternative.
+Publishing is `scripts/publish-github.sh` and `scripts/publish-gitea.sh`: the shared flow (argument parsing, the release gate, tag preparation, the push, the release notes) lives in `scripts/lib/publish-common.sh` and `scripts/lib/publish-release.sh`, and only the release object differs (`gh release create` for GitHub, which is what fires the image build, and `curl` against the Gitea API with a `GITEA_TOKEN` or the token `tea login` stored). Both are marked-block scripts tested by `scripts/publish-tools.test.ts`, which lifts the blocks out with the `install.test.ts` approach. Images are built by GitHub Actions on a published release (`.github/workflows/release.yml`), for `linux/amd64` and `linux/arm64`, and pushed to `ghcr.io/moduletrace/cobble` with `latest`, version, `major.minor` and commit-sha tags. CI never touches the Gitea registry: it is on a VPN, which a runner cannot reach, so `scripts/mirror-gitea.sh` (`bun run mirror:gitea`) copies a release into `git.tben.me/moduletrace/cobble` afterwards, from the Gitea host over ssh, running skopeo there in a container and reading every destination tag's digest back against GHCR's; it only moves `latest` and `major.minor` while that release still owns them in GHCR, so mirroring an older one never rolls them back. The two run in parallel and neither can fail the other, which is the point: the installer defaults to GHCR, so a registry outage cannot fail a release. The version they are built with is passed as the `COBBLE_VERSION` build arg (the published tag, else `package.json`'s version read from `<version>`), which the image turns into `ENV COBBLE_VERSION` and `runningVersion` (`src/utils/running-version.ts`) reports on Settings > General; bump `package.json`'s `version` with a release, since a local build and a workflow-dispatch build fall back to it. `.github/workflows/ci.yml` runs `bun run typecheck` and `bun run test` on every push and pull request, and checks that `install.sh` parses. The Dockerfile pins Bun 1.4.3 to match CI. The repository is mirrored on Gitea, and the installer defaults to the GHCR image with `--image git.tben.me/moduletrace/cobble` as the Gitea alternative.
 
 ### Envs
 

README.md

@@ -245,15 +245,16 @@ queue and Web Push.
 
 ## Images
 
-Every release publishes a multi-arch image to both registries:
+Every release publishes a multi-arch image to
+`ghcr.io/moduletrace/cobble`, which is what the installer pulls. Tags are
+`latest`, the release version, `major.minor` and the commit sha.
 
-- `ghcr.io/moduletrace/cobble`
+The same image is copied into `git.tben.me/moduletrace/cobble` by
-- `git.tben.me/moduletrace/cobble`
+`bun run mirror:gitea`, which runs after the build over the VPN (the Gitea
-
+registry is not on the public internet, so GitHub Actions cannot reach it).
-Tags are `latest`, the release version, `major.minor` and the commit sha.
+Install from Gitea with `--image git.tben.me/moduletrace/cobble`, or build
-Install from Gitea with
+from source with `docker build -t cobble:local .` and
-`--image git.tben.me/moduletrace/cobble`, or build from source with
+`--image cobble --version local --no-pull`.
-`docker build -t cobble:local .` and `--image cobble --version local --no-pull`.
 
 A release is published from a clean `main` with the two scripts, which tag the
 `package.json` version, push and create the release objects:
@@ -264,7 +265,9 @@ scripts/publish-gitea.sh    # or: bun run publish:gitea
 ```
 
 Both take `--version VERSION`, `--dry-run` and `--yes`. Only the GitHub release
-builds the image; see [RELEASING.md](RELEASING.md) for the full checklist.
+builds the image, and the Gitea registry is filled in afterwards, once that
+build has finished and from the VPN, by `bun run mirror:gitea`; see
+[RELEASING.md](RELEASING.md) for the full checklist.
 
 ## Development
 

RELEASING.md

@@ -62,16 +62,24 @@ them in that order, since only the GitHub release builds images.
        gh release create v0.1.0 --title v0.1.0 --notes-file notes.md --latest
 
 7. [ ] The release workflow builds `linux/amd64` and `linux/arm64` and
-       pushes to both registries:
+       pushes to `ghcr.io/moduletrace/cobble`, the registry the installer
-       - `ghcr.io/moduletrace/cobble`
+       defaults to. It does not touch Gitea: that registry is on the VPN,
-       - `git.tben.me/moduletrace/cobble`
+       and a GitHub runner cannot reach it.
 8. [ ] Check the manifests:
 
        docker buildx imagetools inspect ghcr.io/moduletrace/cobble:latest
-       docker buildx imagetools inspect git.tben.me/moduletrace/cobble:latest
 
 9. [ ] Confirm the GHCR package is public and pull it anonymously.
-10. [ ] Publish the release to Gitea as well:
+10. [ ] Once the build has finished, copy the image into the Gitea registry
+        from the VPN:
+
+        GITEA_TOKEN=... bun run mirror:gitea   # or: scripts/mirror-gitea.sh
+
+        It copies the version and commit tags, plus `latest` and
+        `major.minor` while this release still owns them, and reads every
+        destination digest back. A failure here does not affect the release:
+        the installer keeps working from GHCR.
+11. [ ] Publish the release to Gitea as well:
 
         scripts/publish-gitea.sh                  # or: bun run publish:gitea
 
@@ -79,7 +87,7 @@ them in that order, since only the GitHub release builds images.
         git push gitea main
         git push gitea refs/tags/v0.1.0
 
-11. [ ] Update every install with the one-liner:
+12. [ ] Update every install with the one-liner:
 
         curl -fsSL https://url.moduletrace.com/cobble | sudo bash
 

package.json

@@ -19,7 +19,8 @@
         "test": "bun test",
         "check:bundle": "bun src/scripts/check-bundle-size.ts",
         "publish:github": "bash scripts/publish-github.sh",
-        "publish:gitea": "bash scripts/publish-gitea.sh"
+        "publish:gitea": "bash scripts/publish-gitea.sh",
+        "mirror:gitea": "bash scripts/mirror-gitea.sh"
     },
     "devDependencies": {
         "@next/mdx": "^16.3.0",

scripts/mirror-gitea.sh

@@ -0,0 +1,318 @@
+#!/usr/bin/env bash
+# Copies a published Cobble image from GHCR into the Gitea registry.
+#
+#   scripts/mirror-gitea.sh
+#
+# The Gitea registry is on the VPN and GitHub Actions runners are on the
+# public internet, so CI can never reach it. The release workflow therefore
+# builds and pushes to GHCR only, and this script mirrors the result
+# separately: a registry that is unreachable, slow or full can never fail a
+# release the installer depends on.
+#
+# It copies from the Gitea host itself, because that host reaches GHCR over
+# the public internet and its own registry over the docker network it shares
+# with it. Nothing has to be installed on that host: skopeo runs there in a
+# container.
+#
+# `skopeo copy --all` keeps the manifest list and its signatures, so every
+# architecture comes across and the digest is unchanged. Each destination tag
+# is then read back and compared, so a partial copy is reported as a failure
+# instead of being assumed to have worked.
+#
+# Run it after scripts/publish-github.sh: that release is what builds the
+# image, so there is nothing to copy until it has finished.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/publish-common.sh
+. "$SCRIPT_DIR/lib/publish-common.sh"
+# shellcheck source=lib/publish-release.sh
+. "$SCRIPT_DIR/lib/publish-release.sh"
+
+SOURCE_REPO="${MIRROR_SOURCE_REPO:-moduletrace/cobble}"
+SOURCE_REGISTRY_HOST="${MIRROR_SOURCE_REGISTRY_HOST:-ghcr.io}"
+HOST="${MIRROR_HOST:-10.7.0.4}"
+HOST_USER="${MIRROR_HOST_USER:-root}"
+REGISTRY_HOST="${MIRROR_REGISTRY_HOST:-gitea:3000}"
+REGISTRY_REPO="${MIRROR_REGISTRY_REPO:-moduletrace/cobble}"
+# The name a person types, which is not the one the copy uses: the host
+# reaches its registry by docker network name, while an install reaches it
+# through the proxy in front of it.
+REGISTRY_PUBLIC_HOST="${MIRROR_REGISTRY_PUBLIC_HOST:-git.tben.me}"
+NETWORK="${MIRROR_NETWORK:-cloudflare}"
+SKOPEO_IMAGE="${MIRROR_SKOPEO_IMAGE:-quay.io/skopeo/stable:latest}"
+MIRROR_USER="${MIRROR_USER:-cobble}"
+CONNECT_TIMEOUT="${MIRROR_CONNECT_TIMEOUT:-10}"
+
+usage() {
+    cat <<EOF
+Copies a published Cobble image from GHCR into the Gitea registry.
+
+Usage: scripts/mirror-gitea.sh [options]
+
+Options:
+  --version VERSION   Version to mirror, with or without the leading v.
+                      Defaults to package.json's version.
+  --yes, -y           Do not prompt.
+  --dry-run           Print what would run, copy nothing.
+  -h, --help          Show this help.
+
+The version is copied as vX.Y.Z and sha-<commit>. It is copied as latest and
+major.minor too, but only while it is still the newest release in GHCR, so
+mirroring an older release never rolls those two tags backwards. Every
+destination is read back and its digest compared with GHCR's.
+
+Run it after scripts/publish-github.sh, and after the image build for that
+release has finished.
+
+Environment:
+  GITEA_TOKEN              Token for the registry (required).
+  MIRROR_HOST              The Gitea host to copy from over the VPN.
+                           Default: 10.7.0.4
+  MIRROR_HOST_USER         Its ssh user. Default: root
+  MIRROR_REGISTRY_HOST     The registry as that host sees it, by its docker
+                           network name. Default: gitea:3000
+  MIRROR_REGISTRY_PUBLIC_HOST
+                           The registry's public name, for the pull command
+                           it prints. Default: git.tben.me
+  MIRROR_SOURCE_REGISTRY_HOST
+                           Where the release workflow pushed.
+                           Default: ghcr.io
+  MIRROR_NETWORK           The docker network the registry is on, so the
+                           copy container can resolve it.
+                           Default: cloudflare
+  MIRROR_USER              The registry user for basic auth. Default: cobble
+  MIRROR_SKOPEO_IMAGE      The skopeo image the host runs.
+                           Default: quay.io/skopeo/stable:latest
+  DRY_RUN=1, ASSUME_YES=1  Same as --dry-run and --yes.
+EOF
+}
+
+parse_publish_args "$@"
+if [ "$HELP" = "1" ]; then
+    usage
+    exit 0
+fi
+reject_unknown_args "$0"
+
+# MARK:mirror-tags
+# The moving tag a release owns: major.minor, read from the version the same
+# way the release workflow reads it, so the two never disagree.
+mirror_minor_tag() {
+    printf '%s' "${1#v}" | sed -E 's/^([0-9]+)\.([0-9]+).*/\1.\2/'
+}
+
+# The commit tag for a release: the first 12 characters of the commit the
+# tag points at, which is `sha-${GITHUB_SHA::12}` in the workflow.
+mirror_sha_tag() {
+    local sha
+    sha="$(git rev-parse -q --verify "refs/tags/$1^{commit}" 2>/dev/null || true)"
+    if [ -z "$sha" ]; then
+        return 1
+    fi
+    printf 'sha-%s' "${sha:0:12}"
+}
+
+# The tags this release owns outright, which no newer release claims.
+mirror_fixed_tags() {
+    local sha_tag
+    printf '%s\n' "$1"
+    if sha_tag="$(mirror_sha_tag "$1")"; then
+        printf '%s\n' "$sha_tag"
+    fi
+}
+
+# The tags a newer release takes over.
+mirror_moving_tags() {
+    printf 'latest\n%s\n' "$(mirror_minor_tag "$1")"
+}
+# END:mirror-tags
+
+# MARK:mirror-guard
+# Whether this release may still write the moving tags. It may while GHCR's
+# `latest` is its own image: once a newer release has moved `latest`, copying
+# an older version into the Gitea registry must not drag it back.
+mirror_owns_moving_tags() {
+    if [ -z "$2" ]; then
+        # Nothing to compare against, so this release is taken as the newest.
+        return 0
+    fi
+    [ "$1" = "$2" ]
+}
+# END:mirror-guard
+
+# MARK:mirror-digest
+MANIFEST_ACCEPT="application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"
+
+# Reads a tag's digest from GHCR over the public internet. The token is
+# anonymous: the image is public, and a registry answers a bare manifest
+# request with 401 until it is given one.
+github_registry_digest() {
+    local tag="$1"
+    local token
+    token="$(
+        curl -sSf "https://$SOURCE_REGISTRY_HOST/token?scope=repository:$SOURCE_REPO:pull" 2>/dev/null |
+            sed -n 's/.*"token":"\([^"]*\)".*/\1/p'
+    )" || return 1
+    if [ -z "$token" ]; then
+        return 1
+    fi
+    curl -sSf -o /dev/null -D- \
+        -H "Authorization: Bearer $token" \
+        -H "Accept: $MANIFEST_ACCEPT" \
+        "https://$SOURCE_REGISTRY_HOST/v2/$SOURCE_REPO/manifests/$tag" 2>/dev/null |
+        tr -d '\r' | sed -n 's/^docker-content-digest: //Ip' | head -n1
+}
+# END:mirror-digest
+
+# MARK:mirror-remote
+# Runs a bash script on the Gitea host, which is the one place that reaches
+# both registries. The script arrives on stdin, so nothing is installed there
+# and no quoting has to survive a round trip through ssh.
+mirror_on_host() {
+    ssh -o BatchMode=yes -o ConnectTimeout="$CONNECT_TIMEOUT" "$HOST_USER@$HOST" \
+        "MIRROR_NETWORK='$NETWORK' MIRROR_SKOPEO_IMAGE='$SKOPEO_IMAGE' bash -s"
+}
+
+# Reads a destination tag's digest back from the registry. Empty means the
+# tag is not there, which a caller reports differently from a mismatch.
+mirror_digest_on_host() {
+    mirror_on_host <<REMOTE
+set -uo pipefail
+docker run --rm --network "\$MIRROR_NETWORK" "\$MIRROR_SKOPEO_IMAGE" \\
+    inspect --tls-verify=false --creds '$MIRROR_USER:$GITEA_TOKEN' \\
+    --format '{{.Digest}}' 'docker://$REGISTRY_HOST/$REGISTRY_REPO:$1' 2>/dev/null | tail -n1
+REMOTE
+}
+
+# Copies one tag, leaving skopeo's output in MIRROR_COPY_LOG so a failure can
+# be shown without streaming its progress into the report.
+mirror_copy_on_host() {
+    MIRROR_COPY_LOG="$(mktemp -t cobble-mirror.XXXXXX)"
+    mirror_on_host <<REMOTE >"$MIRROR_COPY_LOG" 2>&1
+set -uo pipefail
+docker run --rm --network "\$MIRROR_NETWORK" "\$MIRROR_SKOPEO_IMAGE" \\
+    copy --all --retry-times 5 \\
+    --src-tls-verify=true --dest-tls-verify=false \\
+    --dest-creds '$MIRROR_USER:$GITEA_TOKEN' \\
+    'docker://$SOURCE_REGISTRY_HOST/$SOURCE_REPO:$1' \\
+    'docker://$REGISTRY_HOST/$REGISTRY_REPO:$2'
+REMOTE
+}
+# END:mirror-remote
+
+# MARK:mirror-main
+require_repo_root
+
+step "Resolving the version"
+if [ -z "$VERSION_RAW" ]; then
+    VERSION_RAW="$(source_version)"
+    if [ -z "$VERSION_RAW" ]; then
+        fail "No version in package.json and none passed with --version."
+    fi
+    detail "Read from package.json: $VERSION_RAW"
+fi
+validate_version "$VERSION_RAW"
+TAG="$(normalize_version "$VERSION_RAW")"
+ok "Mirroring $TAG."
+
+require_cmd git
+require_cmd ssh "Install an ssh client."
+require_cmd curl "Install curl."
+
+if [ -z "${GITEA_TOKEN:-}" ]; then
+    fail "No Gitea token. Set GITEA_TOKEN to a token whose user can push packages to the $REGISTRY_REPO owner."
+fi
+
+step "Checking the released image in $SOURCE_REGISTRY_HOST"
+SOURCE_DIGEST="$(github_registry_digest "$TAG")" ||
+    fail "No $TAG in $SOURCE_REGISTRY_HOST/$SOURCE_REPO. Publish the GitHub release first: scripts/publish-github.sh builds the image."
+if [ -z "$SOURCE_DIGEST" ]; then
+    fail "No $TAG in $SOURCE_REGISTRY_HOST/$SOURCE_REPO. Publish the GitHub release first: scripts/publish-github.sh builds the image."
+fi
+ok "$TAG is $SOURCE_DIGEST."
+
+LATEST_DIGEST="$(github_registry_digest latest || true)"
+
+TAGS_TO_COPY="$(mirror_fixed_tags "$TAG")"
+MOVING_SKIPPED=""
+if mirror_owns_moving_tags "$SOURCE_DIGEST" "$LATEST_DIGEST"; then
+    TAGS_TO_COPY="$(
+        printf '%s\n%s\n' "$TAGS_TO_COPY" "$(mirror_moving_tags "$TAG")" |
+            awk 'NF && !seen[$0]++'
+    )"
+else
+    MOVING_SKIPPED="$(mirror_moving_tags "$TAG" | tr '\n' ' ')"
+fi
+
+step "Mirroring $TAG into the Gitea registry"
+printf '    %shost%s     %s\n' "$C_DIM" "$C_RESET" "$HOST_USER@$HOST"
+printf '    %sregistry%s %s/%s\n' "$C_DIM" "$C_RESET" "$REGISTRY_HOST" "$REGISTRY_REPO"
+printf '    %ssource%s   %s/%s\n' "$C_DIM" "$C_RESET" "$SOURCE_REGISTRY_HOST" "$SOURCE_REPO"
+printf '    %stags%s     %s\n' "$C_DIM" "$C_RESET" "$(printf '%s' "$TAGS_TO_COPY" | tr '\n' ' ')"
+if [ -n "$MOVING_SKIPPED" ]; then
+    warn "$SOURCE_REGISTRY_HOST/$SOURCE_REPO:latest belongs to a newer release, so ${MOVING_SKIPPED% } is left where it is."
+fi
+
+step "Checking the Gitea host"
+if is_dry_run; then
+    detail "[dry run] ssh -o BatchMode=yes $HOST_USER@$HOST 'bash -s'"
+    remote_hostname="$HOST"
+else
+    remote_hostname="$(
+        mirror_on_host <<'REMOTE'
+hostname
+REMOTE
+    )" || fail "Could not reach $HOST_USER@$HOST over ssh. The Gitea host is on the VPN: connect to it first."
+    ok "Reached $remote_hostname."
+fi
+
+confirm "Mirror $TAG into $REGISTRY_HOST/$REGISTRY_REPO?"
+
+failed=0
+for dest_tag in $TAGS_TO_COPY; do
+    step "Copying $TAG to $dest_tag"
+
+    if is_dry_run; then
+        detail "[dry run] skopeo copy --all $SOURCE_REGISTRY_HOST/$SOURCE_REPO:$TAG -> $REGISTRY_HOST/$REGISTRY_REPO:$dest_tag"
+        continue
+    fi
+
+    if ! mirror_copy_on_host "$TAG" "$dest_tag"; then
+        sed -n '$!{/^$/d};p' "${MIRROR_COPY_LOG:-/dev/null}" 2>/dev/null | tail -n 8 >&2
+        warn "The copy to $dest_tag failed."
+        failed=1
+        continue
+    fi
+
+    # The digest is what makes the two registries the same image: a copy that
+    # stored a different manifest list would read back differently here.
+    copied_digest="$(mirror_digest_on_host "$dest_tag")"
+    if [ "$copied_digest" != "$SOURCE_DIGEST" ]; then
+        warn "$dest_tag reads back as ${copied_digest:-nothing}, not $SOURCE_DIGEST."
+        failed=1
+        continue
+    fi
+    ok "$dest_tag is $SOURCE_DIGEST."
+done
+
+if [ -n "${MIRROR_COPY_LOG:-}" ] && [ -f "$MIRROR_COPY_LOG" ]; then
+    rm -f "$MIRROR_COPY_LOG"
+fi
+
+step "Mirrored $TAG"
+printf '    %sregistry%s %s/%s\n' "$C_DIM" "$C_RESET" "$REGISTRY_HOST" "$REGISTRY_REPO"
+printf '    %stags%s     %s\n' "$C_DIM" "$C_RESET" "$(printf '%s' "$TAGS_TO_COPY" | tr '\n' ' ')"
+printf '    %sdigest%s   %s\n' "$C_DIM" "$C_RESET" "$SOURCE_DIGEST"
+if [ "$DRY_RUN" = "1" ]; then
+    printf '\n%sDry run: copied nothing.%s\n' "$C_YELLOW" "$C_RESET"
+fi
+
+if [ "$failed" = "1" ]; then
+    fail "The Gitea registry does not match $SOURCE_REGISTRY_HOST/$SOURCE_REPO. The release itself is unaffected: the installer defaults to GHCR."
+fi
+
+printf '\nAn install that can reach the Gitea registry can pull it with:\n'
+printf '    curl -fsSL https://url.moduletrace.com/cobble | sudo bash -s -- --image %s/%s:%s\n' \
+    "$REGISTRY_PUBLIC_HOST" "$REGISTRY_REPO" "$TAG"
+# END:mirror-main

scripts/publish-gitea.sh

@@ -8,9 +8,10 @@
 # creates the Gitea release over the API.
 #
 # Gitea has no release workflow, so nothing is built here: the image is built
-# once by .github/workflows/release.yml and mirrored to
+# once by .github/workflows/release.yml and copied into the Gitea registry by
-# git.tben.me/moduletrace/cobble. This script only publishes the source and
+# scripts/mirror-gitea.sh, run from the VPN side, since a GitHub runner can
-# the release object.
+# never reach it. This script only publishes the source and the release
+# object.
 #
 # The token comes from GITEA_TOKEN, or from the Gitea section of ~/.config/tea
 # if the tea CLI is installed. It needs the `write:repository` scope on the
@@ -221,6 +222,5 @@ else
 fi
 
 print_release_summary "$REMOTE" "$BRANCH" "https://${REMOTE_HOST}/${REPO_SLUG}/releases/tag/$TAG"
-printf '\nThe image is built by the GitHub release; mirror it into the Gitea registry with:\n'
+printf '\nThe image is built by the GitHub release, which cannot reach this registry.\nOnce that build has finished, copy it in over the VPN with:\n'
-printf '    docker buildx imagetools create --tag git.tben.me/moduletrace/cobble:latest \\\n'
+printf '    bun run mirror:gitea\n'
-printf '        ghcr.io/moduletrace/cobble:latest\n'

scripts/publish-github.sh

@@ -7,11 +7,12 @@
 # installer syntax), tags HEAD with the package.json version, pushes main and
 # the tag to the `origin` remote, then creates the GitHub release. Publishing
 # that release is what fires .github/workflows/release.yml, which builds the
-# multi-arch image and pushes it to GHCR (and mirrors it to Gitea).
+# multi-arch image and pushes it to GHCR, the registry the installer defaults
+# to. It never pushes to Gitea: that registry is on the VPN, out of a
+# runner's reach, so scripts/mirror-gitea.sh copies the image separately.
 #
 # Run scripts/publish-gitea.sh afterwards for the source mirror's own release
-# object. There is no existing publish script in the repo: this and its Gitea
+# object, and scripts/mirror-gitea.sh once the image build has finished.
-# twin are it.
 set -Eeuo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

scripts/publish-tools.test.ts

@@ -4,6 +4,7 @@ import path from "path";
 const ROOT = path.join(import.meta.dir, "..");
 const COMMON = path.join(ROOT, "scripts/lib/publish-common.sh");
 const RELEASE = path.join(ROOT, "scripts/lib/publish-release.sh");
+const MIRROR = path.join(ROOT, "scripts/mirror-gitea.sh");
 const SCRIPTS = [
     path.join(ROOT, "scripts/publish-github.sh"),
     path.join(ROOT, "scripts/publish-gitea.sh"),
@@ -99,13 +100,21 @@ async function pathWithoutJq(): Promise<string> {
 }
 
 describe("publish scripts syntax", () => {
-    for (const script of [...SCRIPTS, COMMON, RELEASE]) {
+    for (const script of [...SCRIPTS, MIRROR, COMMON, RELEASE]) {
         test(`${path.relative(ROOT, script)} parses`, async () => {
             const proc = Bun.spawn({ cmd: ["bash", "-n", script] });
             expect(await proc.exited).toBe(0);
         });
     }
 
+    test("the mirror script sources the shared libraries", async () => {
+        const source = await Bun.file(MIRROR).text();
+
+        expect(source).toContain('SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"');
+        expect(source).toContain('. "$SCRIPT_DIR/lib/publish-common.sh"');
+        expect(source).toContain('. "$SCRIPT_DIR/lib/publish-release.sh"');
+    });
+
     test("both publish scripts source the shared library", async () => {
         for (const script of SCRIPTS) {
             const source = await Bun.file(script).text();
@@ -141,6 +150,156 @@ describe("publish scripts remote wiring", () => {
     });
 });
 
+describe("mirror script wiring", () => {
+    test("CI never pushes to the Gitea registry", async () => {
+        const workflow = await Bun.file(
+            path.join(ROOT, ".github/workflows/release.yml"),
+        ).text();
+
+        expect(workflow).not.toContain("git.tben.me");
+        expect(workflow).not.toContain("GITEA_TOKEN");
+        expect(workflow).not.toContain("imagetools create");
+        expect(workflow).toContain("Build and push to GHCR");
+    });
+
+    test("the mirror script copies every tag the workflow publishes", async () => {
+        const [workflow, mirror] = await Promise.all([
+            Bun.file(path.join(ROOT, ".github/workflows/release.yml")).text(),
+            Bun.file(MIRROR).text(),
+        ]);
+
+        // The workflow's own four tags, so a mirror tag can never be invented
+        // here and go missing there.
+        for (const tag of ["latest", "${version}", "${minor}", "${sha}"]) {
+            expect(workflow).toContain(tag);
+        }
+        expect(mirror).toContain("mirror_minor_tag");
+        expect(mirror).toContain("mirror_sha_tag");
+        expect(mirror).toContain("mirror_fixed_tags");
+        expect(mirror).toContain("mirror_moving_tags");
+    });
+
+    test("the mirror script asks before copying and verifies each digest", async () => {
+        const source = await Bun.file(MIRROR).text();
+
+        expect(source).toContain("confirm \"Mirror $TAG into");
+        expect(source).toContain("mirror_digest_on_host");
+        expect(source).toContain("skopeo");
+        expect(source).toContain("--all");
+    });
+
+    test("the pull command it prints names the public host, not the docker one", async () => {
+        const source = await Bun.file(MIRROR).text();
+
+        // The copy reaches the registry by docker network name, which is not
+        // resolvable for an install; printing that would hand out a broken
+        // command.
+        expect(source).toContain("MIRROR_REGISTRY_PUBLIC_HOST");
+        expect(source).toContain('"$REGISTRY_PUBLIC_HOST" "$REGISTRY_REPO" "$TAG"');
+        expect(source).not.toContain('"${REGISTRY_HOST%%:*}" "$REGISTRY_REPO" "$TAG"');
+    });
+
+    test("without a token the mirror script refuses instead of guessing", async () => {
+        const proc = Bun.spawn({
+            cmd: ["bash", MIRROR, "--yes"],
+            cwd: ROOT,
+            env: { ...process.env, GITEA_TOKEN: "" },
+            stdout: "pipe",
+            stderr: "pipe",
+        });
+        const [stderr, code] = await Promise.all([
+            new Response(proc.stderr).text(),
+            proc.exited,
+        ]);
+
+        expect(code).toBe(1);
+        expect(stderr).toContain("No Gitea token");
+    });
+});
+
+describe("mirror lib tag handling", () => {
+    test("major.minor is read exactly as the release workflow reads it", async () => {
+        const block = await blockCaller({ file: MIRROR, name: "mirror-tags" });
+
+        for (const [version, expected] of [
+            ["v0.1.5", "0.1"],
+            ["v1.2.3", "1.2"],
+            ["v10.20.30", "10.20"],
+            ["v0.1.5-rc.1", "0.1"],
+        ] as const) {
+            const out = await runBash({
+                script: [block, `mirror_minor_tag '${version}'`].join("\n"),
+            });
+            expect(out).toBe(expected);
+        }
+    });
+
+    test("fixed tags are the version and its commit, moving ones the rest", async () => {
+        const block = await blockCaller({ file: MIRROR, name: "mirror-tags" });
+
+        const fixed = await runBash({
+            script: [block, "mirror_fixed_tags v0.1.5"].join("\n"),
+        });
+        expect(fixed.split("\n")[0]).toBe("v0.1.5");
+
+        const moving = await runBash({
+            script: [block, "mirror_moving_tags v0.1.5"].join("\n"),
+        });
+        expect(moving.split("\n")).toEqual(["latest", "0.1"]);
+    });
+
+    test("a tag with no commit on disk still mirrors its version", async () => {
+        const block = await blockCaller({ file: MIRROR, name: "mirror-tags" });
+
+        const out = await runBash({
+            script: [block, "mirror_fixed_tags v99.99.99"].join("\n"),
+        });
+
+        expect(out).toBe("v99.99.99");
+    });
+});
+
+describe("mirror lib moving tag guard", () => {
+    test("a release may move latest while it owns it", async () => {
+        const block = await blockCaller({ file: MIRROR, name: "mirror-guard" });
+
+        const out = await runBash({
+            script: [
+                block,
+                'mirror_owns_moving_tags sha256:aa sha256:aa && printf owns',
+            ].join("\n"),
+        });
+
+        expect(out).toBe("owns");
+    });
+
+    test("a newer release's latest is never rolled back", async () => {
+        const block = await blockCaller({ file: MIRROR, name: "mirror-guard" });
+
+        const out = await runBash({
+            script: [
+                block,
+                'if mirror_owns_moving_tags sha256:aa sha256:bb; then printf owns; else printf refuses; fi',
+            ].join("\n"),
+        });
+
+        expect(out).toBe("refuses");
+    });
+
+    test("with no latest to compare, the release is taken as newest", async () => {
+        const block = await blockCaller({ file: MIRROR, name: "mirror-guard" });
+
+        const out = await runBash({
+            script: [
+                block,
+                'mirror_owns_moving_tags sha256:aa "" && printf owns',
+            ].join("\n"),
+        });
+
+        expect(out).toBe("owns");
+    });
+});
+
 describe("publish lib version handling", () => {
     test("normalizes a version with and without the v", async () => {
         const block = await blockCaller({ name: "version" });