ci(docker): build on self-hosted runner with a warm BuildKit cache

.github/workflows/docker.yaml

@@ -9,17 +9,24 @@ on:
         description: image tag
         required: true
       platforms:
+        # amd64-only by default: keeps the persistent cache to a single target dir, which
+        # matters on this disk-constrained host. Pass "linux/amd64, linux/arm64" (or just
+        # linux/arm64) explicitly when an arm image is needed.
         description: image platforms
         required: true
-        default: linux/amd64, linux/arm64
+        default: linux/amd64
 
 env:
   CI: true
 
 jobs:
   publish:
     name: docker-publish
-    runs-on: ubuntu-latest
+    # Self-hosted so the persistent BuildKit builder (dev-server-cache) and its warm cargo
+    # cache mounts survive across runs — that is what makes the Rust build incremental.
+    # Safe here because this workflow is workflow_dispatch-only (maintainer-triggered): no
+    # untrusted PR code runs on the host. Do NOT move the PR-triggered tests workflow here.
+    runs-on: [self-hosted, dev-server]
     steps:
       - name: Docker login
         uses: docker/login-action@v3
@@ -30,8 +37,32 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        # Registers binfmt so arm64 can be built via emulation when the platforms input asks
+        # for it. No-op / cheap when only linux/amd64 is requested.
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          # Fixed name + keep-state + no cleanup => the builder and its cache (including the
+          # Dockerfile cargo cache mounts) persist across jobs on this host.
+          name: dev-server-cache
+          keep-state: true
+          cleanup: false
+          # Bound the on-disk cache so it can't grow without limit: BuildKit GC keeps the cache
+          # under ~15GB (LRU eviction) and drops entries older than 7 days. 15GB comfortably
+          # holds one arch's warm cargo cache (~8-14GB) and is sized for this host's tight free
+          # space (~32GB, 90% full). The github-runner Ansible role enforces the same cap from
+          # the host side as a belt-and-suspenders.
+          buildkitd-config-inline: |
+            [worker.oci]
+              gc = true
+              [[worker.oci.gcpolicy]]
+                all = true
+                keepBytes = "15GB"
+                keepDuration = "168h"
 
       - name: Build & Publish
         uses: docker/build-push-action@v6

Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 FROM rust:1.89-bookworm AS rust
 
 
@@ -11,36 +12,69 @@ ADD Cargo.toml .
 ADD Cargo.lock .
 ADD crates crates
 
+# BuildKit cache mounts turn every build into an incremental one: only changed crates
+# recompile instead of the whole dependency graph. They require a persistent builder to be
+# useful — see .github/workflows/docker.yaml (the `dev-server-cache` buildx builder) and the
+# github-runner Ansible role that keeps the builder's cache volume alive across jobs.
+#
+#   - /usr/local/cargo/{registry,git}: downloaded crate sources; shared across targets/arches.
+#   - /app/target: compiled artifacts; scoped per-arch via id=...-$TARGETARCH so amd64 and
+#     arm64 don't clobber each other's object files when both are built at once.
+#   - sharing=locked: two runners share one builder, so serialize access to keep cargo from
+#     corrupting the shared dirs when two builds overlap.
+#
+# The target/ dir is a cache mount, so its contents are NOT part of the image layer. Each
+# builder stage copies the finished binaries out to /out, and the runtime stage picks them up
+# from there.
+
 
 FROM builder AS hotblocks-builder
-RUN cargo build -p sqd-hotblocks -p reclaim-measure --release
+ARG TARGETARCH
+RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \
+    --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \
+    cargo build -p sqd-hotblocks -p reclaim-measure --release \
+    && mkdir -p /out \
+    && cp target/release/sqd-hotblocks target/release/reclaim-measure /out/
 
 
 FROM rust AS hotblocks
 WORKDIR /app
-COPY --from=hotblocks-builder /app/target/release/sqd-hotblocks .
+COPY --from=hotblocks-builder /out/sqd-hotblocks .
 # Read-only probe: opens the live db as a RocksDB secondary instance and reports how much
 # --startup-disk-reclaim would free. Run it in the pod before turning that flag on.
-COPY --from=hotblocks-builder /app/target/release/reclaim-measure .
+COPY --from=hotblocks-builder /out/reclaim-measure .
 ENTRYPOINT ["/app/sqd-hotblocks"]
 
 
 FROM builder AS hotblocks-retain-builder
-RUN cargo build -p sqd-hotblocks-retain --release
+ARG TARGETARCH
+RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \
+    --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \
+    cargo build -p sqd-hotblocks-retain --release \
+    && mkdir -p /out \
+    && cp target/release/sqd-hotblocks-retain /out/
 
 
 FROM rust AS hotblocks-retain
 WORKDIR /app
-COPY --from=hotblocks-retain-builder /app/target/release/sqd-hotblocks-retain .
+COPY --from=hotblocks-retain-builder /out/sqd-hotblocks-retain .
 ENTRYPOINT ["/app/sqd-hotblocks-retain"]
 
 
 FROM builder AS archive-builder
-RUN cargo build -p sqd-archive --release
+ARG TARGETARCH
+RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
+    --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \
+    --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \
+    cargo build -p sqd-archive --release \
+    && mkdir -p /out \
+    && cp target/release/sqd-archive /out/
 
 
 FROM debian:bookworm-slim AS sqd-archive
 RUN apt-get update && apt-get install ca-certificates -y
 WORKDIR /app
-COPY --from=archive-builder /app/target/release/sqd-archive .
+COPY --from=archive-builder /out/sqd-archive .
 ENTRYPOINT ["/app/sqd-archive"]