diff --git a/.boringcache.toml b/.boringcache.toml new file mode 100644 index 000000000..160655d82 --- /dev/null +++ b/.boringcache.toml @@ -0,0 +1,13 @@ +workspace = "discourse/discourse-docker" + +[proxy] +metadata-hints = ["project=discourse-docker"] + +[adapters.docker] +tag = "discourse-images" +cache-mode = "max" +metadata-hints = ["tool=oci", "lane=image"] + +[adapters.ccache] +tag = "discourse-native-image" +metadata-hints = ["tool=ccache", "lane=image"] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f14b3d8b5..d55151c83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,13 +22,19 @@ jobs: runs-on: ubuntu-latest outputs: timestamp: ${{ steps.timestamp.outputs.timestamp }} + datestamp: ${{ steps.timestamp.outputs.datestamp }} steps: - id: timestamp run: | timestamp=$(date +%Y%m%d-%H%M) + datestamp=$(echo $timestamp | sed -e 's/-.*$//') echo "timestamp=$timestamp" + echo "datestamp=$datestamp" echo "timestamp=$timestamp" >> $GITHUB_OUTPUT + echo "datestamp=$datestamp" >> $GITHUB_OUTPUT + # run all targets individually for cache, otherwise this may break caching + # possibley related to https://github.com/docker/buildx/issues/414 base: # `cdck-linux-8-core` for amd64 builds # `ubuntu-24.04-8core-arm` for arm64 builds @@ -43,53 +49,126 @@ jobs: working-directory: image env: TIMESTAMP: ${{ needs.timestamp.outputs.timestamp }} + DATESTAMP: ${{ needs.timestamp.outputs.datestamp }} ARCH: ${{ matrix.arch }} + CACHE_IMAGE: ghcr.io/${{ github.repository }} + BORINGCACHE_ENABLED: ${{ vars.BORINGCACHE_ENABLED == 'true' && (secrets.BORINGCACHE_RESTORE_TOKEN != '' || (github.event_name != 'pull_request' && secrets.BORINGCACHE_SAVE_TOKEN != '')) && 'true' || 'false' }} + # The setup action owns the managed builder; the later CLI-owned Bake + # commands need the same credentials for tool and mount caches. + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: ${{ github.event_name != 'pull_request' && secrets.BORINGCACHE_SAVE_TOKEN || '' }} steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx + if: env.BORINGCACHE_ENABLED != 'true' + uses: docker/setup-buildx-action@v3 + + - name: Login to cache registry + if: env.BORINGCACHE_ENABLED != 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up BoringCache + if: env.BORINGCACHE_ENABLED == 'true' + uses: boringcache/one@e24257b122813ad11d53b9ed024b474ca4946ad2 # v1.19.1 + with: + cli-version: v1.19.1 + trust-policy: auto + setup: none + mode: docker + working-directory: image + docker-command: setup + # Leave the CLI's default 22243 port free for the later Bake wrappers. + proxy-port: 22242 + diagnostics: summary + verify: none + fail-on-cache-error: false + metadata-hints: workflow=build,architecture=${{ matrix.arch }} + - name: Login to Docker Hub if: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build runtime deps + - name: Build deps run: | - docker buildx bake base-runtime-deps --no-cache --load + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-runtime-deps --load + else + docker buildx bake base-runtime-deps -f docker-bake.hcl -f docker-bake.cache.hcl --load + fi - name: Build slim image run: | - docker buildx bake base-slim --load + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-slim-main + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-slim-stable + else + docker buildx bake base-slim-main -f docker-bake.hcl -f docker-bake.cache.hcl + docker buildx bake base-slim-stable -f docker-bake.hcl -f docker-bake.cache.hcl + fi - name: Build web only image run: | - docker buildx bake base-web-only --load + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-web-only-main --load + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-web-only-stable --load + else + docker buildx bake base-web-only-main -f docker-bake.hcl -f docker-bake.cache.hcl --load + docker buildx bake base-web-only-stable -f docker-bake.hcl -f docker-bake.cache.hcl --load + fi - name: Build release image run: | - docker buildx bake base-release --load + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-release-main --load + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-release-stable --load + else + docker buildx bake base-release-main -f docker-bake.hcl -f docker-bake.cache.hcl --load + docker buildx bake base-release-stable -f docker-bake.hcl -f docker-bake.cache.hcl --load + fi - name: Build test images run: | - docker buildx bake test-release-${{ matrix.arch }} --load + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake test-release --load + else + docker buildx bake test-release -f docker-bake.hcl -f docker-bake.cache.hcl --load + fi - name: run specs for `main` branch if: vars.SKIP_TESTS != '1' run: | - docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 -e DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS=1 ${TEST_IMAGE}:release-${{ matrix.arch }} + docker run --rm -e CI=1 -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 -e DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS=1 ${TEST_IMAGE}:release - name: Build and push test image if: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} run: | - docker buildx bake test --set="*.tags=${TEST_IMAGE}" --set="*.output=type=registry,push-by-digest=true" --metadata-file=/tmp/test.json + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake test-slim -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/test-slim.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake test-slim-browsers -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/test-slim-browsers.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake test-release -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/test-release.json + else + docker buildx bake test-slim -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/test-slim.json + docker buildx bake test-slim-browsers -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/test-slim-browsers.json + docker buildx bake test-release -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/test-release.json + fi - name: Build and push dev image if: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} run: | - docker buildx bake dev --set="*.tags=${DEV_IMAGE}" --set="*.output=type=registry,push-by-digest=true" --allow=fs.read=../templates --metadata-file=/tmp/dev.json + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake --allow=fs.read=../templates dev -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/dev.json + else + docker buildx bake --allow=fs.read=../templates dev -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/dev.json + fi - name: Print summary run: | @@ -97,57 +176,79 @@ jobs: - name: Print `docker history` summary for main branch image run: | - docker history ${BASE_IMAGE}:release-main-${{ matrix.arch }} + docker history ${BASE_IMAGE}:release-main - name: push to dockerhub if: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} run: | - docker buildx bake base --set="*.tags=${BASE_IMAGE}" --set="*.output=type=registry,push-by-digest=true" --metadata-file=/tmp/base.json + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-runtime-deps -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-runtime-deps.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-slim-main -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-slim-main.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-slim-stable -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-slim-stable.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-web-only-main -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-web-only-main.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-web-only-stable -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-web-only-stable.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-release-main -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-release-main.json + boringcache docker --mount-cache --tool-cache ccache -- docker buildx bake base-release-stable -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/base-release-stable.json + else + docker buildx bake base-runtime-deps -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-runtime-deps.json + docker buildx bake base-slim-main -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-slim-main.json + docker buildx bake base-slim-stable -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-slim-stable.json + docker buildx bake base-web-only-main -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-web-only-main.json + docker buildx bake base-web-only-stable -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-web-only-stable.json + docker buildx bake base-release-main -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-release-main.json + docker buildx bake base-release-stable -f docker-bake.hcl -f docker-bake.push.hcl -f docker-bake.cache.hcl -f docker-bake.cache-write.hcl --metadata-file=/tmp/base-release-stable.json + fi - name: save image digests if: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} id: metadata run: | - test_slim_${{ matrix.arch }}=$(jq -r '."test-slim-${{ matrix.arch }}"["containerimage.digest"]' /tmp/test.json) + test_slim_${{ matrix.arch }}=$(jq -r '."test-slim"["containerimage.digest"]' /tmp/test-slim.json) echo "test_slim_${{ matrix.arch }}=$test_slim_${{ matrix.arch }}" echo "test_slim_${{ matrix.arch }}=$test_slim_${{ matrix.arch }}" >> $GITHUB_OUTPUT - test_slim_browsers_${{ matrix.arch }}=$(jq -r '."test-slim-browsers-${{ matrix.arch }}"["containerimage.digest"]' /tmp/test.json) + test_slim_browsers_${{ matrix.arch }}=$(jq -r '."test-slim-browsers"["containerimage.digest"]' /tmp/test-slim-browsers.json) echo "test_slim_browsers_${{ matrix.arch }}=$test_slim_browsers_${{ matrix.arch }}" echo "test_slim_browsers_${{ matrix.arch }}=$test_slim_browsers_${{ matrix.arch }}" >> $GITHUB_OUTPUT - test_release_${{ matrix.arch }}=$(jq -r '."test-release-${{ matrix.arch }}"["containerimage.digest"]' /tmp/test.json) + test_release_${{ matrix.arch }}=$(jq -r '."test-release"["containerimage.digest"]' /tmp/test-release.json) echo "test_release_${{ matrix.arch }}=$test_release_${{ matrix.arch }}" echo "test_release_${{ matrix.arch }}=$test_release_${{ matrix.arch }}" >> $GITHUB_OUTPUT - dev_${{ matrix.arch }}=$(jq -r '."dev-${{ matrix.arch }}"["containerimage.digest"]' /tmp/dev.json) + dev_${{ matrix.arch }}=$(jq -r '."dev"["containerimage.digest"]' /tmp/dev.json) echo "dev_${{ matrix.arch }}=$dev_${{ matrix.arch }}" echo "dev_${{ matrix.arch }}=$dev_${{ matrix.arch }}" >> $GITHUB_OUTPUT - base_slim_main_${{ matrix.arch }}=$(jq -r '."base-slim-main-${{ matrix.arch }}"["containerimage.digest"]' /tmp/base.json) + base_runtime_deps_${{ matrix.arch }}=$(jq -r '."base-runtime-deps"["containerimage.digest"]' /tmp/base-runtime-deps.json) + echo "base_runtime_deps_${{ matrix.arch }}=$base_runtime_deps_${{ matrix.arch }}" + echo "base_runtime_deps_${{ matrix.arch }}=$base_runtime_deps_${{ matrix.arch }}" >> $GITHUB_OUTPUT + + base_slim_main_${{ matrix.arch }}=$(jq -r '."base-slim-main"["containerimage.digest"]' /tmp/base-slim-main.json) echo "base_slim_main_${{ matrix.arch }}=$base_slim_main_${{ matrix.arch }}" echo "base_slim_main_${{ matrix.arch }}=$base_slim_main_${{ matrix.arch }}" >> $GITHUB_OUTPUT - base_slim_stable_${{ matrix.arch }}=$(jq -r '."base-slim-stable-${{ matrix.arch }}"["containerimage.digest"]' /tmp/base.json) + base_slim_stable_${{ matrix.arch }}=$(jq -r '."base-slim-stable"["containerimage.digest"]' /tmp/base-slim-stable.json) echo "base_slim_stable_${{ matrix.arch }}=$base_slim_stable_${{ matrix.arch }}" echo "base_slim_stable_${{ matrix.arch }}=$base_slim_stable_${{ matrix.arch }}" >> $GITHUB_OUTPUT - base_web_only_main_${{ matrix.arch }}=$(jq -r '."base-web-only-main-${{ matrix.arch }}"["containerimage.digest"]' /tmp/base.json) + base_web_only_main_${{ matrix.arch }}=$(jq -r '."base-web-only-main"["containerimage.digest"]' /tmp/base-web-only-main.json) echo "base_web_only_main_${{ matrix.arch }}=$base_web_only_main_${{ matrix.arch }}" echo "base_web_only_main_${{ matrix.arch }}=$base_web_only_main_${{ matrix.arch }}" >> $GITHUB_OUTPUT - base_web_only_stable_${{ matrix.arch }}=$(jq -r '."base-web-only-stable-${{ matrix.arch }}"["containerimage.digest"]' /tmp/base.json) + base_web_only_stable_${{ matrix.arch }}=$(jq -r '."base-web-only-stable"["containerimage.digest"]' /tmp/base-web-only-stable.json) echo "base_web_only_stable_${{ matrix.arch }}=$base_web_only_stable_${{ matrix.arch }}" echo "base_web_only_stable_${{ matrix.arch }}=$base_web_only_stable_${{ matrix.arch }}" >> $GITHUB_OUTPUT - base_release_main_${{ matrix.arch }}=$(jq -r '."base-release-main-${{ matrix.arch }}"["containerimage.digest"]' /tmp/base.json) + base_release_main_${{ matrix.arch }}=$(jq -r '."base-release-main"["containerimage.digest"]' /tmp/base-release-main.json) echo "base_release_main_${{ matrix.arch }}=$base_release_main_${{ matrix.arch }}" echo "base_release_main_${{ matrix.arch }}=$base_release_main_${{ matrix.arch }}" >> $GITHUB_OUTPUT - base_release_stable_${{ matrix.arch }}=$(jq -r '."base-release-stable-${{ matrix.arch }}"["containerimage.digest"]' /tmp/base.json) + base_release_stable_${{ matrix.arch }}=$(jq -r '."base-release-stable"["containerimage.digest"]' /tmp/base-release-stable.json) echo "base_release_stable_${{ matrix.arch }}=$base_release_stable_${{ matrix.arch }}" echo "base_release_stable_${{ matrix.arch }}=$base_release_stable_${{ matrix.arch }}" >> $GITHUB_OUTPUT outputs: + base_runtime_deps_amd64: ${{ steps.metadata.outputs.base_runtime_deps_amd64 }} + base_runtime_deps_arm64: ${{ steps.metadata.outputs.base_runtime_deps_arm64 }} base_slim_main_amd64: ${{ steps.metadata.outputs.base_slim_main_amd64 }} base_slim_main_arm64: ${{ steps.metadata.outputs.base_slim_main_arm64 }} base_slim_stable_amd64: ${{ steps.metadata.outputs.base_slim_stable_amd64 }} @@ -180,13 +281,36 @@ jobs: working-directory: image env: ARCH: ${{ matrix.arch }} + BORINGCACHE_ENABLED: ${{ vars.BORINGCACHE_ENABLED == 'true' && secrets.BORINGCACHE_RESTORE_TOKEN != '' && secrets.BORINGCACHE_SAVE_TOKEN != '' && 'true' || 'false' }} + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: ${{ github.event_name != 'pull_request' && secrets.BORINGCACHE_SAVE_TOKEN || '' }} timeout-minutes: 15 steps: - - uses: docker/setup-buildx-action@v3 - uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Set up Docker Buildx + if: env.BORINGCACHE_ENABLED != 'true' + uses: docker/setup-buildx-action@v3 + + - name: Set up BoringCache + if: env.BORINGCACHE_ENABLED == 'true' + uses: boringcache/one@e24257b122813ad11d53b9ed024b474ca4946ad2 # v1.19.1 + with: + cli-version: v1.19.1 + trust-policy: auto + setup: none + mode: docker + working-directory: image + docker-command: setup + # Leave the CLI's default 22243 port free for the later Bake wrapper. + proxy-port: 22242 + diagnostics: summary + verify: none + fail-on-cache-error: false + metadata-hints: workflow=setup-wizard,architecture=${{ matrix.arch }} + - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -195,11 +319,16 @@ jobs: - name: build and push to dockerhub run: | - docker buildx bake setup-wizard --set="*.tags=${SETUP_WIZARD_IMAGE}" --set="*.output=type=registry,push-by-digest=true" --metadata-file=/tmp/setup-wizard.json + if [[ "$BORINGCACHE_ENABLED" == "true" ]]; then + boringcache docker -- docker buildx bake setup-wizard -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/setup-wizard.json + else + docker buildx bake setup-wizard -f docker-bake.hcl -f docker-bake.push.hcl --metadata-file=/tmp/setup-wizard.json + fi + - name: save image digests id: metadata run: | - ${{ matrix.arch }}=$(jq -r '."setup-wizard-${{ matrix.arch }}"["containerimage.digest"]' /tmp/setup-wizard.json) + ${{ matrix.arch }}=$(jq -r '."setup-wizard"["containerimage.digest"]' /tmp/setup-wizard.json) echo "${{ matrix.arch }}=$${{ matrix.arch }}" echo "${{ matrix.arch }}=$${{ matrix.arch }}" >> $GITHUB_OUTPUT outputs: @@ -232,6 +361,16 @@ jobs: ${{ needs.base.outputs.base_slim_main_amd64 }} \ ${{ needs.base.outputs.base_slim_main_arm64 }} + # runtime-deps timestamped + docker buildx imagetools create -t ${BASE_IMAGE}:2.0.${{ env.TIMESTAMP }}-runtime-deps \ + ${{ needs.base.outputs.base_runtime_deps_amd64 }} \ + ${{ needs.base.outputs.base_runtime_deps_arm64 }} + + # runtime-deps release + docker buildx imagetools create -t ${BASE_IMAGE}:runtime-deps \ + ${{ needs.base.outputs.base_runtime_deps_amd64 }} \ + ${{ needs.base.outputs.base_runtime_deps_arm64 }} + # Web-Only `main` timestamped docker buildx imagetools create -t ${BASE_IMAGE}:2.0.${{ env.TIMESTAMP }}-web-only \ ${{ needs.base.outputs.base_web_only_main_amd64 }} \ diff --git a/image/base/Dockerfile b/image/base/Dockerfile index d58217595..85e9fdbfc 100644 --- a/image/base/Dockerfile +++ b/image/base/Dockerfile @@ -4,8 +4,10 @@ ARG DEBIAN_RELEASE=trixie ARG RUBY_VERSION=3.4.10 ARG FROM_DOCKER_IMAGE_TAG=${RUBY_VERSION}-${DEBIAN_RELEASE}-slim +ARG DATESTAMP=0 FROM discourse/ruby:${FROM_DOCKER_IMAGE_TAG} AS builder +ARG DATESTAMP ARG DEBIAN_RELEASE ENV DEBIAN_RELEASE=${DEBIAN_RELEASE} RUN echo "deb http://deb.debian.org/debian ${DEBIAN_RELEASE}-backports main" > "/etc/apt/sources.list.d/${DEBIAN_RELEASE}-backports.list" @@ -13,46 +15,78 @@ RUN apt update &&\ DEBIAN_FRONTEND=noninteractive apt-get -y install wget \ autoconf build-essential \ git \ + ccache \ cmake \ gnupg \ libpcre2-dev \ libfreetype-dev \ libbrotli-dev +# BoringCache v1.19.1 targets ccache 4.13.6's @-attribute syntax. Keep +# Debian's compiler launchers, but use the same released ccache binary on both +# native runner architectures. +ARG CCACHE_VERSION=4.13.6 +RUN set -eux; \ + architecture="$(dpkg --print-architecture)"; \ + case "$architecture" in \ + amd64) release_arch=x86_64; checksum=567b1b648411819590f918f045218c92da14418bdec3b30db94a3b4f5d77cf13 ;; \ + arm64) release_arch=aarch64; checksum=fae67fb810e1f0d390409af6603355483572229e19183e68574cd0f851a6fb98 ;; \ + *) echo "Unsupported ccache architecture: $architecture" >&2; exit 1 ;; \ + esac; \ + archive="ccache-${CCACHE_VERSION}-linux-${release_arch}-glibc.tar.gz"; \ + wget --quiet --tries=8 "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/${archive}" -O "/tmp/${archive}"; \ + echo "${checksum} /tmp/${archive}" | sha256sum --check; \ + tar xzf "/tmp/${archive}" -C /tmp; \ + install -m 0755 "/tmp/ccache-${CCACHE_VERSION}-linux-${release_arch}-glibc/ccache" /usr/bin/ccache; \ + ccache --version | grep -F "ccache version ${CCACHE_VERSION}"; \ + rm -rf "/tmp/${archive}" "/tmp/ccache-${CCACHE_VERSION}-linux-${release_arch}-glibc" + +ENV PATH="/usr/lib/ccache:${PATH}" \ + CCACHE_COMPILERCHECK=content + FROM builder AS libheif-builder -ADD install-libheif /tmp/install-libheif -RUN /tmp/install-libheif +COPY install-libheif /tmp/install-libheif +RUN /tmp/install-libheif && ccache --show-stats FROM libheif-builder AS imagemagick_builder -ADD install-imagemagick /tmp/install-imagemagick -RUN /tmp/install-imagemagick +COPY install-imagemagick /tmp/install-imagemagick +RUN /tmp/install-imagemagick && ccache --show-stats FROM libheif-builder AS jpegli-builder -ADD install-jpegli /tmp/install-jpegli -RUN /tmp/install-jpegli +COPY install-jpegli /tmp/install-jpegli +RUN /tmp/install-jpegli && ccache --show-stats FROM libheif-builder AS vips-builder COPY --from=jpegli-builder /usr/local/lib/jpegli /usr/local/lib/jpegli -ADD install-vips /tmp/install-vips -RUN /tmp/install-vips +COPY install-vips /tmp/install-vips +RUN /tmp/install-vips && ccache --show-stats + +FROM builder AS nginx-builder +# From https://nginx.org/en/pgp_keys.html +COPY nginx_public_keys.key /tmp/nginx_public_keys.key +COPY install-nginx /tmp/install-nginx +RUN gpg --import /tmp/nginx_public_keys.key &&\ + rm /tmp/nginx_public_keys.key &&\ + /tmp/install-nginx &&\ + ccache --show-stats FROM builder AS thpoff-builder # This tool allows us to disable huge page support for our current process # since the flag is preserved through forks and execs it can be used on any # process -ADD thpoff.c /src/thpoff.c -RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c +COPY thpoff.c /src/thpoff.c +RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c && ccache --show-stats FROM builder AS jemalloc-builder -ADD install-jemalloc /tmp/install-jemalloc -RUN /tmp/install-jemalloc +COPY install-jemalloc /tmp/install-jemalloc +RUN /tmp/install-jemalloc && ccache --show-stats FROM builder AS oxipng-builder -ADD install-oxipng /tmp/install-oxipng -RUN /tmp/install-oxipng +COPY install-oxipng /tmp/install-oxipng +RUN /tmp/install-oxipng && ccache --show-stats FROM discourse/ruby:${FROM_DOCKER_IMAGE_TAG} AS discourse-runtime-base - +ARG DATESTAMP ARG DEBIAN_RELEASE ARG PG_MAJOR=18 ARG PG_MAJOR_OLD=15 @@ -198,9 +232,14 @@ COPY etc/ /etc COPY sbin/ /sbin FROM discourse-runtime-base AS discourse-build-base +COPY --from=builder /usr/bin/ccache /usr/bin/ccache +COPY --from=builder /usr/lib/ccache /usr/lib/ccache +ENV PATH="/usr/lib/ccache:${PATH}" \ + CCACHE_COMPILERCHECK=content + # From https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key # fingerprint: 6F71 F525 2828 41EE DAF8 51B4 2F59 B5F9 9B1B E0B4 -ADD nodesource-repo.gpg.key /usr/share/keyrings/nodesource.asc +COPY nodesource-repo.gpg.key /usr/share/keyrings/nodesource.asc RUN --mount=type=tmpfs,target=/var/log \ --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/var/lib/apt \ @@ -236,16 +275,38 @@ RUN install -dm 0755 -o discourse -g discourse /var/www/discourse &&\ FROM discourse-slim AS discourse-web-only ENV RAILS_ENV=production -RUN cd /var/www/discourse &&\ +RUN --mount=type=cache,id=discourse-bundle-${DISCOURSE_BRANCH},target=/home/discourse/.cache/bundle,sharing=locked \ + chown discourse:discourse /home/discourse/.cache/bundle &&\ + cd /var/www/discourse &&\ + echo "Installed bundle cache before install:" &&\ + sudo -u discourse du -sh /home/discourse/.cache/bundle &&\ sudo -u discourse bundle config --local deployment true &&\ - sudo -u discourse bundle config --local path ./vendor/bundle &&\ + sudo -u discourse bundle config --local path /home/discourse/.cache/bundle &&\ sudo -u discourse bundle config --local without test development &&\ sudo -u discourse bundle install --jobs $(nproc --ignore=1) &&\ + ccache --show-stats &&\ + echo "Installed bundle cache after install:" &&\ + sudo -u discourse du -sh /home/discourse/.cache/bundle &&\ + rm -rf /var/www/discourse/vendor/bundle &&\ + install -dm 0755 -o discourse -g discourse /var/www/discourse/vendor/bundle &&\ + cp -a /home/discourse/.cache/bundle/. /var/www/discourse/vendor/bundle/ &&\ + sudo -u discourse bundle config --local path ./vendor/bundle &&\ + sudo -u discourse bundle install --jobs $(nproc --ignore=1) &&\ + sudo -u discourse bundle check &&\ find /var/www/discourse/vendor/bundle -name cache -not -path '*/gems/*' -type d -exec rm -rf {} + &&\ find /var/www/discourse/vendor/bundle -name tmp -type d -exec rm -rf {} + -RUN cd /var/www/discourse &&\ - sudo -u discourse /bin/bash -c 'pnpm install --frozen-lockfile' +RUN install -dm 0755 -o discourse -g discourse /var/www/discourse/.pnpm-store +RUN --mount=type=cache,id=discourse-pnpm-${DISCOURSE_BRANCH},target=/var/www/discourse/.pnpm-store,sharing=locked \ + chown discourse:discourse /var/www/discourse/.pnpm-store &&\ + cd /var/www/discourse &&\ + pnpm_store="$(sudo -u discourse pnpm store path --store-dir=/var/www/discourse/.pnpm-store)" &&\ + case "$pnpm_store" in /var/www/discourse/.pnpm-store/*) ;; *) echo "Unexpected pnpm store: $pnpm_store" >&2; exit 1 ;; esac &&\ + echo "pnpm store before install: $pnpm_store" &&\ + du -sh /var/www/discourse/.pnpm-store &&\ + sudo -u discourse /bin/bash -c 'pnpm install --store-dir=/var/www/discourse/.pnpm-store --frozen-lockfile' &&\ + echo "pnpm store after install: $pnpm_store" &&\ + du -sh /var/www/discourse/.pnpm-store FROM discourse-web-only AS discourse-release RUN --mount=type=tmpfs,target=/var/log \ diff --git a/image/discourse_dev/Dockerfile b/image/discourse_dev/Dockerfile index 6f4989008..509a62cd6 100644 --- a/image/discourse_dev/Dockerfile +++ b/image/discourse_dev/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y install redis RUN rm -rf /var/www/* # Give discourse user no-passwd sudo permissions (for bundle install) -ADD sudoers.discourse /etc/sudoers.d/discourse +COPY sudoers.discourse /etc/sudoers.d/discourse RUN sudo -u discourse bundle config set --global path /home/discourse/.bundle/gems @@ -49,7 +49,7 @@ RUN sed -e 's/\(db_name: discourse\)/\1_development/' /pups/postgres.template.ym RUN LANG=en_US.UTF-8 /pups/bin/pups /pups/postgres.yml # add dev databases -ADD postgres_dev.template.yml /pups/postgres_dev.yml +COPY postgres_dev.template.yml /pups/postgres_dev.yml RUN /pups/bin/pups /pups/postgres_dev.yml # move default postgres_data out of the way @@ -57,7 +57,7 @@ RUN mv /shared/postgres_data /shared/postgres_data_orig # re-instantiate data on boot if needed (this will allow it to persist across # invocations when used with a mounted volume) -ADD ensure-database /etc/runit/1.d/ensure-database +COPY ensure-database /etc/runit/1.d/ensure-database # Install Mailpit, retaining the old binary name for discourse/bin/docker compatibility COPY --from=mailpit --chmod=0755 /mailpit /usr/local/bin/mailpit diff --git a/image/discourse_test/Dockerfile b/image/discourse_test/Dockerfile index ca821cdc7..049f615eb 100644 --- a/image/discourse_test/Dockerfile +++ b/image/discourse_test/Dockerfile @@ -29,12 +29,12 @@ FROM base AS with_browsers ENV TESTEM_DEFAULT_BROWSER=Chrome # From https://dl.google.com/linux/linux_signing_key.pub # fingerprint: EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796 -ADD google-linux-signing-key.pub /usr/share/keyrings/google-chrome.asc -ADD install-chrome /tmp/install-chrome +COPY google-linux-signing-key.pub /usr/share/keyrings/google-chrome.asc +COPY install-chrome /tmp/install-chrome # From https://archive.mozilla.org/pub/firefox/releases//KEY # fingerprint: 14F2 6682 D091 6CDD 81E3 7B6D 61B7 B526 D98F 0353 -ADD mozilla-release-key.asc /tmp/mozilla-release-key.asc -ADD install-firefox /tmp/install-firefox +COPY mozilla-release-key.asc /tmp/mozilla-release-key.asc +COPY install-firefox /tmp/install-firefox RUN /tmp/install-chrome &&\ apt update &&\ apt install -y libxss1 firefox-esr &&\ @@ -43,9 +43,35 @@ RUN /tmp/install-chrome &&\ FROM with_browsers AS release -RUN cd /var/www/discourse &&\ +RUN --mount=type=cache,id=discourse-bundle-test-main,target=/home/discourse/.cache/bundle,sharing=locked \ + chown discourse:discourse /home/discourse/.cache/bundle &&\ + cd /var/www/discourse &&\ + echo "Test installed bundle cache before install:" &&\ + sudo -u discourse du -sh /home/discourse/.cache/bundle &&\ + sudo -u discourse bundle config --local deployment true &&\ + sudo -u discourse bundle config --local path /home/discourse/.cache/bundle &&\ sudo -u discourse bundle install --jobs $(nproc --ignore=1) &&\ - sudo -E -u discourse -H /bin/bash -c 'CI=1 pnpm install' + ccache --show-stats &&\ + echo "Test installed bundle cache after install:" &&\ + sudo -u discourse du -sh /home/discourse/.cache/bundle &&\ + rm -rf /var/www/discourse/vendor/bundle &&\ + install -dm 0755 -o discourse -g discourse /var/www/discourse/vendor/bundle &&\ + cp -a /home/discourse/.cache/bundle/. /var/www/discourse/vendor/bundle/ &&\ + sudo -u discourse bundle config --local path ./vendor/bundle &&\ + sudo -u discourse bundle install --jobs $(nproc --ignore=1) &&\ + sudo -u discourse bundle check + +RUN install -dm 0755 -o discourse -g discourse /var/www/discourse/.pnpm-store +RUN --mount=type=cache,id=discourse-pnpm-main,target=/var/www/discourse/.pnpm-store,sharing=locked \ + chown discourse:discourse /var/www/discourse/.pnpm-store &&\ + cd /var/www/discourse &&\ + pnpm_store="$(sudo -E -u discourse -H pnpm store path --store-dir=/var/www/discourse/.pnpm-store)" &&\ + case "$pnpm_store" in /var/www/discourse/.pnpm-store/*) ;; *) echo "Unexpected pnpm store: $pnpm_store" >&2; exit 1 ;; esac &&\ + echo "Test pnpm store before install: $pnpm_store" &&\ + du -sh /var/www/discourse/.pnpm-store &&\ + sudo -E -u discourse -H /bin/bash -c 'CI=1 pnpm install --store-dir=/var/www/discourse/.pnpm-store' &&\ + echo "Test pnpm store after install: $pnpm_store" &&\ + du -sh /var/www/discourse/.pnpm-store RUN cd /var/www/discourse && sudo -E -u discourse -H bundle exec rake plugin:install_all_official &&\ LOAD_PLUGINS=1 sudo -E -u discourse -H bundle exec rake plugin:install_all_gems &&\ diff --git a/image/docker-bake.cache-write.hcl b/image/docker-bake.cache-write.hcl new file mode 100644 index 000000000..4916c6306 --- /dev/null +++ b/image/docker-bake.cache-write.hcl @@ -0,0 +1,67 @@ +target "base-runtime-deps" { + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-runtime-deps-${ARCH}" + mode = "max" + }] +} +target "base-build-deps" { + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-build-deps-${ARCH}" + mode = "max" + }] +} +target "base-slim" { + name = "base-slim-${branch}" + matrix = { + branch = ["main", "stable"] + } + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-slim-${branch}-${ARCH}" + mode = "max" + }] +} +target "base-web-only" { + name = "base-web-only-${branch}" + matrix = { + branch = ["main", "stable"] + } + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-web-only-${branch}-${ARCH}" + mode = "max" + }] +} +target "base-release" { + name = "base-release-${branch}" + matrix = { + branch = ["main", "stable"] + } + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-release-${branch}-${ARCH}" + mode = "max" + }] +} + +target "test" { + name = "test-${tag}" + matrix = { + tag = ["slim", "slim-browsers", "release"] + } + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-test-${tag}-${ARCH}" + mode = "max" + }] +} + +target "dev" { + cache-to = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-dev-${ARCH}" + mode = "max" + }] +} diff --git a/image/docker-bake.cache.hcl b/image/docker-bake.cache.hcl new file mode 100644 index 000000000..e039e227f --- /dev/null +++ b/image/docker-bake.cache.hcl @@ -0,0 +1,88 @@ +variable "CACHE_IMAGE" { + default = "" + validation { + condition = CACHE_IMAGE != "" + error_message = "The variable 'CACHE_IMAGE' must not be empty." + } +} + +variable "ARCH" { + default = "" + validation { + condition = ARCH != "" + error_message = "The variable 'ARCH' must not be empty." + } +} + +target "base-runtime-deps" { + cache-from = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-runtime-deps-${ARCH}" + }] +} +target "base-build-deps" { + cache-from = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-build-deps-${ARCH}" + }] +} +target "base-slim" { + name = "base-slim-${branch}" + matrix = { + branch = ["main", "stable"] + } + cache-from = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-slim-${branch}-${ARCH}" + }] +} +target "base-web-only" { + name = "base-web-only-${branch}" + matrix = { + branch = ["main", "stable"] + } + cache-from = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-web-only-${branch}-${ARCH}" + }] +} +target "base-release" { + name = "base-release-${branch}" + matrix = { + branch = ["main", "stable"] + } + cache-from = [{ + type = "registry" + ref = "${CACHE_IMAGE}:cache-release-${branch}-${ARCH}" + }] +} + +target "test" { + name = "test-${tag}" + matrix = { + tag = ["slim", "slim-browsers", "release"] + } + cache-from = [ + { + type = "registry" + ref = "${CACHE_IMAGE}:cache-test-${tag}-${ARCH}" + }, + { + type = "registry" + ref = "${CACHE_IMAGE}:cache-slim-main-${ARCH}" + } + ] +} + +target "dev" { + cache-from = [ + { + type = "registry" + ref = "${CACHE_IMAGE}:cache-dev-${ARCH}" + }, + { + type = "registry" + ref = "${CACHE_IMAGE}:cache-slim-main-${ARCH}" + } + ] +} diff --git a/image/docker-bake.hcl b/image/docker-bake.hcl index bfb76b526..8c5f363ca 100644 --- a/image/docker-bake.hcl +++ b/image/docker-bake.hcl @@ -1,11 +1,3 @@ -variable "ARCH" { - default = "amd64,arm64" -} - -variable "ARCH_ARRAY" { - default = split(",", ARCH) -} - variable "BASE_IMAGE" { default = "local_discourse/base" } @@ -22,96 +14,77 @@ variable "DEV_IMAGE" { default = "local_discourse/discourse_dev" } -group "base" { - targets = ["base-slim", "base-web-only", "base-release"] +variable "DATESTAMP" { + default = "0" } target "base-runtime-deps" { - name = "base-runtime-deps-${arch}" - matrix = { - arch = ARCH_ARRAY - } context = "./base" - tags = ["${BASE_IMAGE}:runtime-deps-${arch}"] + tags = ["${BASE_IMAGE}:runtime-deps"] target = "discourse-runtime-base" - platforms = ["linux/${arch}"] + args = { + "DATESTAMP" = DATESTAMP + } } target "base-build-deps" { - name = "base-build-deps-${arch}" - matrix = { - arch = ARCH_ARRAY - } context = "./base" - tags = ["${BASE_IMAGE}:build-deps-${arch}"] + tags = ["${BASE_IMAGE}:build-deps"] target = "discourse-build-base" - platforms = ["linux/${arch}"] } target "base-slim" { - name = "base-slim-${branch}-${arch}" + name = "base-slim-${branch}" matrix = { - arch = ARCH_ARRAY branch = ["main", "stable"] } context = "./base" - tags = ["${BASE_IMAGE}:slim-${branch}-${arch}"] + tags = ["${BASE_IMAGE}:slim-${branch}"] target = "discourse-slim" - platforms = ["linux/${arch}"] args = { "DISCOURSE_BRANCH" = "${branch}" } } target "base-web-only" { - name = "base-web-only-${branch}-${arch}" + name = "base-web-only-${branch}" matrix = { - arch = ARCH_ARRAY branch = ["main", "stable"] } context = "./base" - tags = ["${BASE_IMAGE}:web-only-${branch}-${arch}"] + tags = ["${BASE_IMAGE}:web-only-${branch}"] target = "discourse-web-only" - platforms = ["linux/${arch}"] args = { "DISCOURSE_BRANCH" = "${branch}" } } target "base-release" { - name = "base-release-${branch}-${arch}" + name = "base-release-${branch}" matrix = { - arch = ARCH_ARRAY branch = ["main", "stable"] } context = "./base" - tags = ["${BASE_IMAGE}:release-${branch}-${arch}"] + tags = ["${BASE_IMAGE}:release-${branch}"] target = "discourse-release" - platforms = ["linux/${arch}"] args = { "DISCOURSE_BRANCH" = "${branch}" } } -# depends on raw arch image, canary build for test images when building base images target "test" { - name = "test-${build_target.tag}-${arch}" + name = "test-${build_target.tag}" matrix = { - arch = ARCH_ARRAY - branch = ["main"] build_target = [ { - from_tag = "slim" target_name = "base" tag = "slim" }, { - from_tag = "slim" target_name = "with_browsers" tag = "slim-browsers" }, { - from_tag = "release" target_name = "release" tag = "release" } @@ -119,40 +92,30 @@ target "test" { } target = build_target.target_name context = "./discourse_test" - platforms = ["linux/${arch}"] - tags = ["${TEST_IMAGE}:${build_target.tag}-${arch}"] + tags = ["${TEST_IMAGE}:${build_target.tag}"] args = { "from_tag" = "from" + "SOURCE_DATE_EPOCH" = 0 } contexts = { - from = "target:base-${build_target.from_tag}-${branch}-${arch}" + from = "target:base-slim-main" } } target "dev" { - name = "dev-${arch}" - matrix = { - arch = ARCH_ARRAY - branch = ["main"] - } context = "./discourse_dev" - tags = ["${DEV_IMAGE}:release-${arch}"] - platforms = ["linux/${arch}"] + tags = ["${DEV_IMAGE}:release"] args = { "from_tag" = "from" + "SOURCE_DATE_EPOCH" = 0 } contexts = { - from = "target:base-slim-${branch}-${arch}" + from = "target:base-slim-main" templates = "../templates" } } target "setup-wizard" { - name = "setup-wizard-${arch}" - matrix = { - arch = ARCH_ARRAY - } context = "./setup_wizard" - tags = ["${SETUP_WIZARD_IMAGE}:release-${arch}"] - platforms = ["linux/${arch}"] + tags = ["${SETUP_WIZARD_IMAGE}:release"] } diff --git a/image/docker-bake.push.hcl b/image/docker-bake.push.hcl new file mode 100644 index 000000000..170796901 --- /dev/null +++ b/image/docker-bake.push.hcl @@ -0,0 +1,56 @@ +# push targets, push by digests +target "_common-push" { + output = [{ + type = "registry", + push-by-digest = true + }] +} + +target "base-runtime-deps" { + inherits = ["_common-push"] + tags = [BASE_IMAGE] +} +target "base-build-deps" { + inherits = ["_common-push"] + tags = [BASE_IMAGE] +} +target "base-slim" { + name = "base-slim-${branch}" + matrix = { + branch = ["main", "stable"] + } + inherits = ["_common-push"] + tags = [BASE_IMAGE] +} +target "base-web-only" { + name = "base-web-only-${branch}" + matrix = { + branch = ["main", "stable"] + } + inherits = ["_common-push"] + tags = [BASE_IMAGE] +} +target "base-release" { + name = "base-release-${branch}" + matrix = { + branch = ["main", "stable"] + } + inherits = ["_common-push"] + tags = [BASE_IMAGE] +} +target "dev" { + inherits = ["dev", "_common-push"] + tags = [DEV_IMAGE] +} +target "test" { + name = "test-${tag}" + matrix = { + tag = ["slim", "slim-browsers", "release"] + } + inherits = ["_common-push"] + tags = [TEST_IMAGE] +} +target "setup-wizard" { + inherits = ["_common-push"] + tags = [SETUP_WIZARD_IMAGE] +}