diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 53e38dde..3f6e7e73 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -9,9 +9,12 @@ on: description: image tag required: true platforms: + # amd64-only by default: keeps the persistent cache to a single target dir, which + # matters on this disk-constrained host. Pass "linux/amd64, linux/arm64" (or just + # linux/arm64) explicitly when an arm image is needed. description: image platforms required: true - default: linux/amd64, linux/arm64 + default: linux/amd64 env: CI: true @@ -19,7 +22,11 @@ env: jobs: publish: name: docker-publish - runs-on: ubuntu-latest + # Self-hosted so the persistent BuildKit builder (dev-server-cache) and its warm cargo + # cache mounts survive across runs — that is what makes the Rust build incremental. + # Safe here because this workflow is workflow_dispatch-only (maintainer-triggered): no + # untrusted PR code runs on the host. Do NOT move the PR-triggered tests workflow here. + runs-on: [self-hosted, dev-server] steps: - name: Docker login uses: docker/login-action@v3 @@ -30,8 +37,32 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + # Registers binfmt so arm64 can be built via emulation when the platforms input asks + # for it. No-op / cheap when only linux/amd64 is requested. + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + # Fixed name + keep-state + no cleanup => the builder and its cache (including the + # Dockerfile cargo cache mounts) persist across jobs on this host. + name: dev-server-cache + keep-state: true + cleanup: false + # Bound the on-disk cache so it can't grow without limit: BuildKit GC keeps the cache + # under ~15GB (LRU eviction) and drops entries older than 7 days. 15GB comfortably + # holds one arch's warm cargo cache (~8-14GB) and is sized for this host's tight free + # space (~32GB, 90% full). The github-runner Ansible role enforces the same cap from + # the host side as a belt-and-suspenders. + buildkitd-config-inline: | + [worker.oci] + gc = true + [[worker.oci.gcpolicy]] + all = true + keepBytes = "15GB" + keepDuration = "168h" - name: Build & Publish uses: docker/build-push-action@v6 diff --git a/Dockerfile b/Dockerfile index f22f7007..d9c757e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM rust:1.89-bookworm AS rust @@ -11,36 +12,69 @@ ADD Cargo.toml . ADD Cargo.lock . ADD crates crates +# BuildKit cache mounts turn every build into an incremental one: only changed crates +# recompile instead of the whole dependency graph. They require a persistent builder to be +# useful — see .github/workflows/docker.yaml (the `dev-server-cache` buildx builder) and the +# github-runner Ansible role that keeps the builder's cache volume alive across jobs. +# +# - /usr/local/cargo/{registry,git}: downloaded crate sources; shared across targets/arches. +# - /app/target: compiled artifacts; scoped per-arch via id=...-$TARGETARCH so amd64 and +# arm64 don't clobber each other's object files when both are built at once. +# - sharing=locked: two runners share one builder, so serialize access to keep cargo from +# corrupting the shared dirs when two builds overlap. +# +# The target/ dir is a cache mount, so its contents are NOT part of the image layer. Each +# builder stage copies the finished binaries out to /out, and the runtime stage picks them up +# from there. + FROM builder AS hotblocks-builder -RUN cargo build -p sqd-hotblocks -p reclaim-measure --release +ARG TARGETARCH +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \ + cargo build -p sqd-hotblocks -p reclaim-measure --release \ + && mkdir -p /out \ + && cp target/release/sqd-hotblocks target/release/reclaim-measure /out/ FROM rust AS hotblocks WORKDIR /app -COPY --from=hotblocks-builder /app/target/release/sqd-hotblocks . +COPY --from=hotblocks-builder /out/sqd-hotblocks . # Read-only probe: opens the live db as a RocksDB secondary instance and reports how much # --startup-disk-reclaim would free. Run it in the pod before turning that flag on. -COPY --from=hotblocks-builder /app/target/release/reclaim-measure . +COPY --from=hotblocks-builder /out/reclaim-measure . ENTRYPOINT ["/app/sqd-hotblocks"] FROM builder AS hotblocks-retain-builder -RUN cargo build -p sqd-hotblocks-retain --release +ARG TARGETARCH +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \ + cargo build -p sqd-hotblocks-retain --release \ + && mkdir -p /out \ + && cp target/release/sqd-hotblocks-retain /out/ FROM rust AS hotblocks-retain WORKDIR /app -COPY --from=hotblocks-retain-builder /app/target/release/sqd-hotblocks-retain . +COPY --from=hotblocks-retain-builder /out/sqd-hotblocks-retain . ENTRYPOINT ["/app/sqd-hotblocks-retain"] FROM builder AS archive-builder -RUN cargo build -p sqd-archive --release +ARG TARGETARCH +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \ + cargo build -p sqd-archive --release \ + && mkdir -p /out \ + && cp target/release/sqd-archive /out/ FROM debian:bookworm-slim AS sqd-archive RUN apt-get update && apt-get install ca-certificates -y WORKDIR /app -COPY --from=archive-builder /app/target/release/sqd-archive . +COPY --from=archive-builder /out/sqd-archive . ENTRYPOINT ["/app/sqd-archive"]