diff --git a/.boringcache.toml b/.boringcache.toml index 57fa8d1..6362ae4 100644 --- a/.boringcache.toml +++ b/.boringcache.toml @@ -1,26 +1,31 @@ workspace = "boringcache/benchmark-zed" -# Cargo's registry and git sources are common dependency state. The Cargo -# adapter composes them with the correctness-preserving target snapshot and -# its native sccache backend in one lifecycle. +# The root plan owns the persistent rolling chain. The release matrix uses +# separate committed plans under plans/ so each Cargo layer can be judged +# independently without rewriting this file in a workflow. [entries.cargo-registry] -tag = "zed-cargo-registry-local" +tag = "zed-cargo-rolling-main-registry" [entries.cargo-git] -tag = "zed-cargo-git-local" +tag = "zed-cargo-rolling-main-git" [entries.zed-target] kind = "cargo-target" -tag = "zed-target-local" +tag = "zed-cargo-rolling-main-target" path = "upstream/target" [profiles.cargo-product] entries = ["cargo-registry", "cargo-git", "zed-target"] +[adapters.sccache] +tag = "zed-cargo-rolling-main-sccache" +no-git = true +metadata-hints = ["benchmark=zed", "lane=rolling", "layer=compiler"] + [adapters.cargo] -tag = "zed-sccache-local" profiles = ["cargo-product"] +compiler-cache = "sccache" no-git = true metadata-hints = ["benchmark=zed", "lane=cargo-product"] command = ["cargo", "build", "--release", "--locked", "--message-format=json-render-diagnostics"] diff --git a/.github/workflows/zed-cargo-product.yml b/.github/workflows/zed-cargo-product.yml index f7926a2..7a81ca3 100644 --- a/.github/workflows/zed-cargo-product.yml +++ b/.github/workflows/zed-cargo-product.yml @@ -1,11 +1,21 @@ -name: "Zed Cargo product run" -run-name: ${{ format('Zed Cargo fresh | CLI {0}', inputs.cli_version || 'Action default') }} +name: "Zed Cargo layer matrix" +run-name: ${{ format('Zed Cargo layers | CLI {0}', inputs.cli_version || 'Action default') }} on: workflow_dispatch: inputs: cli_version: - description: "Exact immutable BoringCache CLI release tag." + description: Optional exact immutable BoringCache CLI release tag. + required: false + type: string + default: "" + base_sha: + description: Seed commit. Defaults to benchmark-source.env. + required: false + type: string + default: "" + head_sha: + description: Adjacent commit built by every warm lane. Defaults to benchmark-source.env. required: false type: string default: "" @@ -14,305 +24,374 @@ on: required: false type: string default: "" - schedule: - - cron: "0 4 * * 5" - pull_request: - paths: - - ".github/workflows/zed-cargo-product.yml" - - ".gitmodules" - - "scripts/**" - - "upstream" permissions: actions: read contents: read +concurrency: + group: benchmark-zed-cargo-layer-matrix + cancel-in-progress: false + env: - BORINGCACHE_CLI_VERSION: ${{ inputs.cli_version }} - CACHE_SCOPE: zed-cargo-r${{ github.run_id }}-a${{ github.run_attempt }} + CARGO_INCREMENTAL: "0" + ZED_BUNDLE: "true" + CC: clang jobs: - publish-base: - name: BoringCache Zed Cargo cold - if: inputs.runner_label == '' || github.ref_name == 'main' - runs-on: ${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' && inputs.runner_label || 'ubuntu-24.04' }} - timeout-minutes: 240 + source: + name: Resolve adjacent Zed source + runs-on: ubuntu-24.04 outputs: base_sha: ${{ steps.source.outputs.base_sha }} head_sha: ${{ steps.source.outputs.head_sha }} steps: - - name: Start the workflow timer - id: workflow_timer - run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - - name: Checkout benchmark repository - uses: actions/checkout@v6 + - uses: actions/checkout@v6 - - name: Checkout the pinned Zed commit and its parent + - name: Select the reviewed source pair id: source + env: + INPUT_BASE_SHA: ${{ inputs.base_sha }} + INPUT_HEAD_SHA: ${{ inputs.head_sha }} run: | set -euo pipefail - git submodule sync --recursive - git -c protocol.version=2 submodule update --init --depth 2 --recommend-shallow --jobs 4 upstream - head_sha="$(git -C upstream rev-parse HEAD)" - if ! base_sha="$(git -C upstream rev-parse HEAD^ 2>/dev/null)"; then - git -C upstream fetch --deepen=1 origin "$head_sha" - base_sha="$(git -C upstream rev-parse HEAD^)" - fi - git -C upstream checkout --detach "$base_sha" - echo "base_sha=${base_sha}" >> "$GITHUB_OUTPUT" - echo "head_sha=${head_sha}" >> "$GITHUB_OUTPUT" - - - name: Prepare the base source tree - run: ./scripts/prepare-source.sh base - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libxkbcommon-dev libxkbcommon-x11-dev libwayland-dev libvulkan-dev \ - libasound2-dev libfontconfig1-dev libfreetype6-dev \ - libglib2.0-dev libssl-dev pkg-config cmake \ - libx11-dev libx11-xcb-dev libxcb1-dev \ - libxcursor-dev libxinerama-dev libxi-dev libxrandr-dev + source benchmark-source.env + echo "base_sha=${INPUT_BASE_SHA:-$ZED_BASE_SHA}" >> "$GITHUB_OUTPUT" + echo "head_sha=${INPUT_HEAD_SHA:-$ZED_HEAD_SHA}" >> "$GITHUB_OUTPUT" - - name: Install Zed's pinned Rust toolchain - working-directory: upstream - run: rustup show active-toolchain - - - name: Free runner disk space - run: ./scripts/free-runner-disk.sh + cold: + name: "Cold: target + sccache seed" + needs: source + if: inputs.runner_label == '' || github.ref_name == 'main' + runs-on: ${{ inputs.runner_label || 'ubuntu-24.04' }} + timeout-minutes: 240 + steps: + - uses: actions/checkout@v6 - - name: Scope the Cargo product plan - run: ./scripts/scope-boringcache-run.sh "$CACHE_SCOPE" + - name: Prepare the seed source and release toolchain + run: ./scripts/prepare-zed-lane.sh "${{ needs.source.outputs.base_sha }}" - - name: Require a cold target + - name: Require an empty target run: test ! -e upstream/target - - name: Start the build timer - id: build_timer + - name: Start lane timing + id: timer run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - - name: Run the released Cargo product - id: cache + + - name: Select the cold primary CLI plan + run: ./scripts/activate-cargo-plan.sh cold primary + + - name: Build the primary seed through the combined Cargo plan + id: primary uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 with: - cli-version: ${{ env.BORINGCACHE_CLI_VERSION }} + cli-version: ${{ inputs.cli_version }} setup: none mode: cargo trust-policy: publish working-directory: upstream fail-on-cache-error: true - metadata-hints: phase=base + diagnostics: summary + metadata-hints: phase=matrix-cold-primary env: BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} BORINGCACHE_SAVE_TOKEN: ${{ secrets.BORINGCACHE_SAVE_TOKEN }} + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib' - - name: Capture the build timing - id: build_timing - env: - BUILD_STARTED_AT: ${{ steps.build_timer.outputs.started_at }} - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" - - - name: Verify the Zed build - run: | - set -euo pipefail - test -x upstream/target/release/zed - test -z "$(git -C upstream status --porcelain=v1 --untracked-files=all)" + - name: Select the cold remote-server CLI plan + run: ./scripts/activate-cargo-plan.sh cold remote-server - - name: Write the benchmark phase evidence + - name: Build the remote server seed through the combined Cargo plan + id: remote + uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 + with: + cli-version: ${{ inputs.cli_version }} + setup: none + mode: cargo + trust-policy: publish + working-directory: upstream + fail-on-cache-error: true + diagnostics: summary + metadata-hints: phase=matrix-cold-remote-server env: - WORKFLOW_STARTED_AT: ${{ steps.workflow_timer.outputs.started_at }} - BUILD_SECONDS: ${{ steps.build_timing.outputs.build_seconds }} - CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} - CACHE_TAG: ${{ steps.cache.outputs.cache-tag }} - CACHE_WORKSPACE: ${{ steps.cache.outputs.workspace }} - EVIDENCE_PATH: ${{ steps.cache.outputs.evidence-path }} - SOURCE_SHA: ${{ steps.source.outputs.base_sha }} + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: ${{ secrets.BORINGCACHE_SAVE_TOKEN }} + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib -C target-feature=+crt-static' + CC_x86_64_unknown_linux_musl: musl-gcc + + - name: Report cold-layer evidence run: | - set -euo pipefail - source_repository="$(sed -n 's/^ZED_SOURCE_REPOSITORY=//p' benchmark-source.env)" - python3 ./scripts/benchmark-report.py phase \ - --benchmark zed-cargo \ - --strategy boringcache \ - --lane fresh \ - --phase cold \ - --mode cargo \ - --build-seconds "$BUILD_SECONDS" \ - --workflow-seconds "$(( $(date +%s) - WORKFLOW_STARTED_AT ))" \ - --cache-hit "$CACHE_HIT" \ - --cache-tag "$CACHE_TAG" \ - --workspace "$CACHE_WORKSPACE" \ - --source-repository "$source_repository" \ - --source-sha "$SOURCE_SHA" \ - --evidence "$EVIDENCE_PATH" - - - name: Retain the benchmark phase evidence - uses: actions/upload-artifact@v6 - with: - name: phase-zed-cargo-boringcache-fresh-cold - path: benchmark-results/ - if-no-files-found: error - - - name: Retain base product evidence + echo "lane_seconds=$(( $(date +%s) - ${{ steps.timer.outputs.started_at }} ))" >> "$GITHUB_STEP_SUMMARY" + ./scripts/summarize-cargo-evidence.py \ + cold-primary "${{ steps.primary.outputs.evidence-path }}" \ + cold-remote-server "${{ steps.remote.outputs.evidence-path }}" + + - name: Retain cold product evidence uses: actions/upload-artifact@v6 with: - name: zed-cargo-product-base - path: ${{ steps.cache.outputs.evidence-path }} + name: zed-cargo-layers-cold + path: | + ${{ steps.primary.outputs.evidence-path }} + ${{ steps.remote.outputs.evidence-path }} if-no-files-found: error - consume-head: - name: BoringCache Zed Cargo warm - needs: publish-base - runs-on: ${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' && inputs.runner_label || 'ubuntu-24.04' }} + target-only: + name: "Target only: compiler cache disabled" + needs: [source, cold] + runs-on: ${{ inputs.runner_label || 'ubuntu-24.04' }} timeout-minutes: 240 steps: - - name: Start the workflow timer - id: workflow_timer + - uses: actions/checkout@v6 + + - name: Prepare the adjacent source and release toolchain + run: ./scripts/prepare-zed-lane.sh "${{ needs.source.outputs.head_sha }}" "${{ needs.source.outputs.base_sha }}" + + - name: Require an empty target + run: test ! -e upstream/target + + - name: Start lane timing + id: timer run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - - name: Checkout benchmark repository - uses: actions/checkout@v6 - - name: Checkout the pinned Zed head - run: | - set -euo pipefail - git submodule sync --recursive - git -c protocol.version=2 submodule update --init --depth 2 --recommend-shallow --jobs 4 upstream - test "$(git -C upstream rev-parse HEAD)" = "${{ needs.publish-base.outputs.head_sha }}" + - name: Select the target-only primary CLI plan + run: ./scripts/activate-cargo-plan.sh target-only primary + + - name: Build primary with the target-only Cargo plan + id: primary + uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 + with: + cli-version: ${{ inputs.cli_version }} + setup: none + mode: cargo + trust-policy: restore + working-directory: upstream + fail-on-cache-error: true + diagnostics: summary + metadata-hints: phase=matrix-target-only-primary + env: + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: "" + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib' - - name: Prepare the head source tree - run: ./scripts/prepare-source.sh warm1 + - name: Select the target-only remote-server CLI plan + run: ./scripts/activate-cargo-plan.sh target-only remote-server - - name: Install system dependencies + - name: Build remote server with the target-only Cargo plan + id: remote + uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 + with: + cli-version: ${{ inputs.cli_version }} + setup: none + mode: cargo + trust-policy: restore + working-directory: upstream + fail-on-cache-error: true + diagnostics: summary + metadata-hints: phase=matrix-target-only-remote-server + env: + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: "" + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib -C target-feature=+crt-static' + CC_x86_64_unknown_linux_musl: musl-gcc + + - name: Report target-only evidence run: | - sudo apt-get update - sudo apt-get install -y \ - libxkbcommon-dev libxkbcommon-x11-dev libwayland-dev libvulkan-dev \ - libasound2-dev libfontconfig1-dev libfreetype6-dev \ - libglib2.0-dev libssl-dev pkg-config cmake \ - libx11-dev libx11-xcb-dev libxcb1-dev \ - libxcursor-dev libxinerama-dev libxi-dev libxrandr-dev + test "${{ steps.primary.outputs.cache-hit }}" = "true" + echo "lane_seconds=$(( $(date +%s) - ${{ steps.timer.outputs.started_at }} ))" >> "$GITHUB_STEP_SUMMARY" + ./scripts/summarize-cargo-evidence.py \ + target-only-primary "${{ steps.primary.outputs.evidence-path }}" \ + target-only-remote-server "${{ steps.remote.outputs.evidence-path }}" - - name: Install Zed's pinned Rust toolchain - working-directory: upstream - run: rustup show active-toolchain + - name: Retain target-only product evidence + uses: actions/upload-artifact@v6 + with: + name: zed-cargo-layers-target-only + path: | + ${{ steps.primary.outputs.evidence-path }} + ${{ steps.remote.outputs.evidence-path }} + if-no-files-found: error - - name: Free runner disk space - run: ./scripts/free-runner-disk.sh + sccache-only: + name: "sccache only: target excluded" + needs: [source, cold] + runs-on: ${{ inputs.runner_label || 'ubuntu-24.04' }} + timeout-minutes: 240 + steps: + - uses: actions/checkout@v6 - - name: Scope the same Cargo product plan - run: ./scripts/scope-boringcache-run.sh "$CACHE_SCOPE" + - name: Prepare the adjacent source and release toolchain + run: ./scripts/prepare-zed-lane.sh "${{ needs.source.outputs.head_sha }}" "${{ needs.source.outputs.base_sha }}" - - name: Require an empty destination + - name: Require an empty target run: test ! -e upstream/target - - name: Start the build timer - id: build_timer + - name: Start lane timing + id: timer run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" - - - name: Run the released Cargo product - id: cache + + - name: Select the sccache-only primary CLI plan + run: ./scripts/activate-cargo-plan.sh sccache-only primary + + - name: Build primary with the sccache-only Cargo plan + id: primary uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 with: - cli-version: ${{ env.BORINGCACHE_CLI_VERSION }} + cli-version: ${{ inputs.cli_version }} setup: none mode: cargo trust-policy: restore working-directory: upstream fail-on-cache-error: true - metadata-hints: phase=head + diagnostics: summary + metadata-hints: phase=matrix-sccache-only-primary env: BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} BORINGCACHE_SAVE_TOKEN: "" + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib' - - name: Capture the build timing - id: build_timing - env: - BUILD_STARTED_AT: ${{ steps.build_timer.outputs.started_at }} - run: echo "build_seconds=$(( $(date +%s) - BUILD_STARTED_AT ))" >> "$GITHUB_OUTPUT" - - - name: Verify the Zed build - run: | - set -euo pipefail - test -x upstream/target/release/zed - test -z "$(git -C upstream status --porcelain=v1 --untracked-files=all)" + - name: Select the sccache-only remote-server CLI plan + run: ./scripts/activate-cargo-plan.sh sccache-only remote-server - - name: Write the benchmark phase evidence + - name: Build remote server with the sccache-only Cargo plan + id: remote + uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 + with: + cli-version: ${{ inputs.cli_version }} + setup: none + mode: cargo + trust-policy: restore + working-directory: upstream + fail-on-cache-error: true + diagnostics: summary + metadata-hints: phase=matrix-sccache-only-remote-server env: - WORKFLOW_STARTED_AT: ${{ steps.workflow_timer.outputs.started_at }} - BUILD_SECONDS: ${{ steps.build_timing.outputs.build_seconds }} - CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} - CACHE_TAG: ${{ steps.cache.outputs.cache-tag }} - CACHE_WORKSPACE: ${{ steps.cache.outputs.workspace }} - EVIDENCE_PATH: ${{ steps.cache.outputs.evidence-path }} - SOURCE_SHA: ${{ needs.publish-base.outputs.head_sha }} + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: "" + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib -C target-feature=+crt-static' + CC_x86_64_unknown_linux_musl: musl-gcc + + - name: Report sccache-only evidence run: | - set -euo pipefail - source_repository="$(sed -n 's/^ZED_SOURCE_REPOSITORY=//p' benchmark-source.env)" - python3 ./scripts/benchmark-report.py phase \ - --benchmark zed-cargo \ - --strategy boringcache \ - --lane fresh \ - --phase warm \ - --mode cargo \ - --build-seconds "$BUILD_SECONDS" \ - --workflow-seconds "$(( $(date +%s) - WORKFLOW_STARTED_AT ))" \ - --cache-hit "$CACHE_HIT" \ - --cache-tag "$CACHE_TAG" \ - --workspace "$CACHE_WORKSPACE" \ - --source-repository "$source_repository" \ - --source-sha "$SOURCE_SHA" \ - --evidence "$EVIDENCE_PATH" - - - name: Retain the benchmark phase evidence + echo "lane_seconds=$(( $(date +%s) - ${{ steps.timer.outputs.started_at }} ))" >> "$GITHUB_STEP_SUMMARY" + ./scripts/summarize-cargo-evidence.py \ + sccache-only-primary "${{ steps.primary.outputs.evidence-path }}" \ + sccache-only-remote-server "${{ steps.remote.outputs.evidence-path }}" + + - name: Retain sccache-only product evidence uses: actions/upload-artifact@v6 with: - name: phase-zed-cargo-boringcache-fresh-warm - path: benchmark-results/ + name: zed-cargo-layers-sccache-only + path: | + ${{ steps.primary.outputs.evidence-path }} + ${{ steps.remote.outputs.evidence-path }} if-no-files-found: error - - - name: Publish product run summary + + combined: + name: "Combined: target + sccache" + needs: [source, cold] + runs-on: ${{ inputs.runner_label || 'ubuntu-24.04' }} + timeout-minutes: 240 + steps: + - uses: actions/checkout@v6 + + - name: Prepare the adjacent source and release toolchain + run: ./scripts/prepare-zed-lane.sh "${{ needs.source.outputs.head_sha }}" "${{ needs.source.outputs.base_sha }}" + + - name: Require an empty target + run: test ! -e upstream/target + + - name: Start lane timing + id: timer + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + + - name: Select the combined primary CLI plan + run: ./scripts/activate-cargo-plan.sh combined primary + + - name: Build primary with the combined Cargo plan + id: primary + uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 + with: + cli-version: ${{ inputs.cli_version }} + setup: none + mode: cargo + trust-policy: restore + working-directory: upstream + fail-on-cache-error: true + diagnostics: summary + metadata-hints: phase=matrix-combined-primary + env: + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: "" + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib' + + - name: Select the combined remote-server CLI plan + run: ./scripts/activate-cargo-plan.sh combined remote-server + + - name: Build remote server with the combined Cargo plan + id: remote + uses: boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56 # v1 / 1.18.1 + with: + cli-version: ${{ inputs.cli_version }} + setup: none + mode: cargo + trust-policy: restore + working-directory: upstream + fail-on-cache-error: true + diagnostics: summary + metadata-hints: phase=matrix-combined-remote-server + env: + BORINGCACHE_RESTORE_TOKEN: ${{ secrets.BORINGCACHE_RESTORE_TOKEN }} + BORINGCACHE_SAVE_TOKEN: "" + RUSTFLAGS: '-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib -C target-feature=+crt-static' + CC_x86_64_unknown_linux_musl: musl-gcc + + - name: Report combined evidence run: | - { - echo "## Zed Cargo product run" - echo - echo "- CLI: \`${BORINGCACHE_CLI_VERSION}\`" - echo "- Source: \`${{ needs.publish-base.outputs.base_sha }}\` → \`${{ needs.publish-base.outputs.head_sha }}\`" - echo "- Signed target available before the adjacent build: \`${{ steps.cache.outputs.cache-hit }}\`" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Retain head product evidence + test "${{ steps.primary.outputs.cache-hit }}" = "true" + echo "lane_seconds=$(( $(date +%s) - ${{ steps.timer.outputs.started_at }} ))" >> "$GITHUB_STEP_SUMMARY" + ./scripts/summarize-cargo-evidence.py \ + combined-primary "${{ steps.primary.outputs.evidence-path }}" \ + combined-remote-server "${{ steps.remote.outputs.evidence-path }}" + + - name: Retain combined product evidence uses: actions/upload-artifact@v6 with: - name: zed-cargo-product-head - path: ${{ steps.cache.outputs.evidence-path }} + name: zed-cargo-layers-combined + path: | + ${{ steps.primary.outputs.evidence-path }} + ${{ steps.remote.outputs.evidence-path }} if-no-files-found: error report: - name: Zed Cargo fresh report - needs: consume-head + name: Compare Cargo layers + needs: [cold, target-only, sccache-only, combined] runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - uses: actions/checkout@v6 - - name: Download the benchmark phase evidence + - name: Download every lane's product evidence uses: actions/download-artifact@v6 with: - pattern: phase-zed-cargo-* - path: phase-evidence + pattern: zed-cargo-layers-* + path: layer-evidence - - name: Publish the Zed Cargo fresh benchmark report + - name: Publish the independent layer report run: | - python3 ./scripts/benchmark-report.py summarize \ - --title "Zed Cargo fresh benchmark" \ - --input-dir phase-evidence \ - --output-dir benchmark-results + set -euo pipefail + mkdir -p benchmark-results + args=() + while IFS= read -r evidence; do + label="$(basename "$(dirname "$evidence")")/$(basename "$evidence")" + args+=("$label" "$evidence") + done < <(find layer-evidence -type f -name '*.json' -print | sort) + test "${#args[@]}" -gt 0 + ./scripts/summarize-cargo-evidence.py "${args[@]}" | + tee benchmark-results/zed-cargo-layer-matrix.md - - name: Retain the Zed Cargo fresh result + - name: Retain the layer report uses: actions/upload-artifact@v6 with: - name: benchmark-zed-cargo-boringcache-fresh - path: benchmark-results/zed-cargo-boringcache-fresh.json + name: benchmark-zed-cargo-layer-matrix + path: benchmark-results/zed-cargo-layer-matrix.md if-no-files-found: error diff --git a/.github/workflows/zed-cargo-rolling-chain.yml b/.github/workflows/zed-cargo-rolling-chain.yml index 1bd2d6e..4581ae0 100644 --- a/.github/workflows/zed-cargo-rolling-chain.yml +++ b/.github/workflows/zed-cargo-rolling-chain.yml @@ -12,10 +12,6 @@ on: required: false type: string default: "" - cache_scope: - description: Existing rolling scope whose target was published from base_sha. - required: true - type: string base_sha: description: Source commit currently represented by the published target. required: true @@ -45,7 +41,6 @@ jobs: outputs: base_sha: ${{ steps.source.outputs.ZED_BASE_SHA }} head_sha: ${{ steps.source.outputs.ZED_HEAD_SHA }} - cache_scope: ${{ steps.scope.outputs.cache_scope }} source_repository: ${{ steps.source.outputs.ZED_SOURCE_REPOSITORY }} steps: - uses: actions/checkout@v6 @@ -53,16 +48,6 @@ jobs: id: source run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" - - name: Name the rolling cohort - id: scope - run: | - set -euo pipefail - ref_slug="$(printf '%s' "$GITHUB_REF_NAME" | tr -c 'A-Za-z0-9._-' '-')" - ref_slug="${ref_slug#-}" - ref_slug="${ref_slug%-}" - ref_slug="${ref_slug:-main}" - echo "cache_scope=zed-cargo-rolling-${ref_slug}" >> "$GITHUB_OUTPUT" - advance: name: BoringCache Zed Cargo commit needs: source @@ -72,7 +57,6 @@ jobs: env: RESOLVED_BASE_SHA: ${{ github.event_name == 'push' && needs.source.outputs.base_sha || inputs.base_sha }} RESOLVED_HEAD_SHA: ${{ github.event_name == 'push' && needs.source.outputs.head_sha || inputs.head_sha }} - RESOLVED_CACHE_SCOPE: ${{ github.event_name == 'push' && needs.source.outputs.cache_scope || inputs.cache_scope }} steps: - name: Start the workflow timer id: workflow_timer @@ -110,9 +94,6 @@ jobs: - name: Free runner disk space run: ./scripts/free-runner-disk.sh - - name: Scope the persistent Cargo product chain - run: ./scripts/scope-boringcache-run.sh "$RESOLVED_CACHE_SCOPE" - - name: Require an empty destination run: test ! -e upstream/target diff --git a/README.md b/README.md index b9aab14..0fe4b8e 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,15 @@ This repository contains the BoringCache benchmark for Zed. -Benchmark workflows are in [`.github/workflows/`](.github/workflows/), with configuration in [`.boringcache.toml`](.boringcache.toml). +The release matrix compares one cold seed with target-only, sccache-only, and +combined Cargo plans on fresh runners. Each layer choice and both of Zed's Linux +release commands live in committed `.boringcache.toml` files under `plans/`. +The Action activates the selected plan inside the clean Zed checkout so the CLI +measures source freshness against Zed itself, while the workflow remains only a +lane/phase selector. +The GitHub workflow selects those plans and leaves cache identity, target +selection, and compiler-cache selection to the BoringCache CLI. + +The root [`.boringcache.toml`](.boringcache.toml) owns the persistent rolling +chain. [`.github/workflows/zed-cargo-product.yml`](.github/workflows/zed-cargo-product.yml) +owns the independent release matrix. diff --git a/plans/cold/primary/.boringcache.toml b/plans/cold/primary/.boringcache.toml new file mode 100644 index 0000000..9a7f5d4 --- /dev/null +++ b/plans/cold/primary/.boringcache.toml @@ -0,0 +1,37 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git", "zed-target"] + +[adapters.sccache] +tag = "zed-cargo-layers-1181-d4010e9-r3-sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "layer=compiler"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=cold", "phase=primary"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-gnu", + "--package", + "zed", + "--package", + "cli", +] diff --git a/plans/cold/remote-server/.boringcache.toml b/plans/cold/remote-server/.boringcache.toml new file mode 100644 index 0000000..cd6d39d --- /dev/null +++ b/plans/cold/remote-server/.boringcache.toml @@ -0,0 +1,35 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git", "zed-target"] + +[adapters.sccache] +tag = "zed-cargo-layers-1181-d4010e9-r3-sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "layer=compiler"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=cold", "phase=remote-server"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-musl", + "--package", + "remote_server", +] diff --git a/plans/combined/primary/.boringcache.toml b/plans/combined/primary/.boringcache.toml new file mode 100644 index 0000000..6d12cd9 --- /dev/null +++ b/plans/combined/primary/.boringcache.toml @@ -0,0 +1,37 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git", "zed-target"] + +[adapters.sccache] +tag = "zed-cargo-layers-1181-d4010e9-r3-sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "layer=compiler"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=combined", "phase=primary"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-gnu", + "--package", + "zed", + "--package", + "cli", +] diff --git a/plans/combined/remote-server/.boringcache.toml b/plans/combined/remote-server/.boringcache.toml new file mode 100644 index 0000000..1a3512b --- /dev/null +++ b/plans/combined/remote-server/.boringcache.toml @@ -0,0 +1,35 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git", "zed-target"] + +[adapters.sccache] +tag = "zed-cargo-layers-1181-d4010e9-r3-sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "layer=compiler"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=combined", "phase=remote-server"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-musl", + "--package", + "remote_server", +] diff --git a/plans/sccache-only/primary/.boringcache.toml b/plans/sccache-only/primary/.boringcache.toml new file mode 100644 index 0000000..30a653e --- /dev/null +++ b/plans/sccache-only/primary/.boringcache.toml @@ -0,0 +1,37 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git"] + +[adapters.sccache] +tag = "zed-cargo-layers-1181-d4010e9-r3-sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "layer=compiler"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=sccache-only", "phase=primary"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-gnu", + "--package", + "zed", + "--package", + "cli", +] diff --git a/plans/sccache-only/remote-server/.boringcache.toml b/plans/sccache-only/remote-server/.boringcache.toml new file mode 100644 index 0000000..d656145 --- /dev/null +++ b/plans/sccache-only/remote-server/.boringcache.toml @@ -0,0 +1,35 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git"] + +[adapters.sccache] +tag = "zed-cargo-layers-1181-d4010e9-r3-sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "layer=compiler"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "sccache" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=sccache-only", "phase=remote-server"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-musl", + "--package", + "remote_server", +] diff --git a/plans/target-only/primary/.boringcache.toml b/plans/target-only/primary/.boringcache.toml new file mode 100644 index 0000000..abf52cd --- /dev/null +++ b/plans/target-only/primary/.boringcache.toml @@ -0,0 +1,32 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git", "zed-target"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "none" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=target-only", "phase=primary"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-gnu", + "--package", + "zed", + "--package", + "cli", +] diff --git a/plans/target-only/remote-server/.boringcache.toml b/plans/target-only/remote-server/.boringcache.toml new file mode 100644 index 0000000..85a60aa --- /dev/null +++ b/plans/target-only/remote-server/.boringcache.toml @@ -0,0 +1,30 @@ +workspace = "boringcache/benchmark-zed" + +[entries.cargo-registry] +tag = "zed-cargo-layers-1181-d4010e9-r3-registry" + +[entries.cargo-git] +tag = "zed-cargo-layers-1181-d4010e9-r3-git" + +[entries.zed-target] +kind = "cargo-target" +tag = "zed-cargo-layers-1181-d4010e9-r3-target" +path = "target" + +[profiles.cargo-product] +entries = ["cargo-registry", "cargo-git", "zed-target"] + +[adapters.cargo] +profiles = ["cargo-product"] +compiler-cache = "none" +no-git = true +metadata-hints = ["benchmark=zed-cargo-layers", "lane=target-only", "phase=remote-server"] +command = [ + "cargo", + "build", + "--release", + "--target", + "x86_64-unknown-linux-musl", + "--package", + "remote_server", +] diff --git a/scripts/activate-cargo-plan.sh b/scripts/activate-cargo-plan.sh new file mode 100755 index 0000000..0894219 --- /dev/null +++ b/scripts/activate-cargo-plan.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Expose one committed CLI plan at the upstream checkout where Cargo freshness +# must be measured. The symlink is Git-ignored runtime plumbing; the selected +# plan remains immutable repository data under plans/. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +lane="${1:?expected a Cargo layer lane}" +phase="${2:?expected a Cargo release phase}" +plan="${repo_root}/plans/${lane}/${phase}/.boringcache.toml" +upstream="${repo_root}/upstream" + +test -f "${plan}" +test -d "${upstream}/.git" || test -f "${upstream}/.git" + +exclude="$(git -C "${upstream}" rev-parse --absolute-git-dir)/info/exclude" +if ! grep -Fqx '/.boringcache.toml' "${exclude}" 2>/dev/null; then + echo '/.boringcache.toml' >> "${exclude}" +fi + +ln -sfn "../plans/${lane}/${phase}/.boringcache.toml" \ + "${upstream}/.boringcache.toml" +test -f "${upstream}/.boringcache.toml" +test -z "$(git -C "${upstream}" status --porcelain=v1 --untracked-files=all)" +echo "Activated the ${lane}/${phase} Cargo plan in the upstream checkout." diff --git a/scripts/prepare-zed-lane.sh b/scripts/prepare-zed-lane.sh new file mode 100755 index 0000000..ee39632 --- /dev/null +++ b/scripts/prepare-zed-lane.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Shared setup for every cache-matrix lane: check out a pinned Zed commit, install +# Zed's build dependencies and pinned toolchain, re-verify the release recipe, and +# mirror the bundle's RELEASE_VERSION. Every lane must be identical up to the cache +# strategy under test, so this lives in one place rather than being restated per job. +# +# prepare-zed-lane.sh [expected-parent-sha] +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +sha="${1:?expected a Zed commit sha}" +expected_parent="${2:-}" + +git submodule sync --recursive +git -c protocol.version=2 submodule update --init --depth 2 --recommend-shallow --jobs 4 upstream +git -C upstream fetch --no-tags --depth 2 origin "${sha}" +git -C upstream checkout --detach "${sha}" +if [ -n "${expected_parent}" ]; then + test "$(git -C upstream rev-parse HEAD^)" = "${expected_parent}" +fi +git -C upstream clean -fdx +test -z "$(git -C upstream status --porcelain=v1 --untracked-files=all)" + +sudo apt-get update +sudo apt-get install -y \ + libxkbcommon-dev libxkbcommon-x11-dev libwayland-dev libvulkan-dev \ + libasound2-dev libfontconfig1-dev libfreetype6-dev \ + libglib2.0-dev libssl-dev pkg-config cmake \ + libx11-dev libx11-xcb-dev libxcb1-dev \ + libxcursor-dev libxinerama-dev libxi-dev libxrandr-dev \ + musl-tools clang + +pinned="$(sed -n 's/^channel = "\(.*\)"$/\1/p' upstream/rust-toolchain.toml)" +test -n "${pinned}" +(cd upstream && rustup show active-toolchain) +rustup target add --toolchain "${pinned}" x86_64-unknown-linux-musl + +./scripts/verify-zed-release-recipe.py upstream + +version="$(cd upstream && script/get-crate-version zed)" +test -n "${version}" +echo "RELEASE_VERSION=${version}" >> "${GITHUB_ENV:-/dev/null}" + +./scripts/free-runner-disk.sh diff --git a/scripts/summarize-cargo-evidence.py b/scripts/summarize-cargo-evidence.py new file mode 100755 index 0000000..38c9e29 --- /dev/null +++ b/scripts/summarize-cargo-evidence.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Surface the Cargo product's rebuild set and sccache result into the run summary. + +The actions/cache control lane reports how many Cargo units it compiled from +`run-zed-release-phases.sh`. The BoringCache lane reports the equivalent as +`compile_requests` inside the adapter's evidence JSON, alongside the sccache hit +rate. Printing both in the same shape is what makes the two lanes comparable on +rebuild set rather than only on wall time. +""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + + +def summarize(label: str, evidence_path: Path) -> str: + evidence = json.loads(evidence_path.read_text()) + phase = evidence.get("phases", {}).get("restore", {}) + mode = phase.get("mode_evidence", {}) + native = mode.get("native_tool") or {} + compiler_cache = mode.get("cargo_cache", {}).get("compiler_cache") + + requests = native.get("compile_requests") + executed = native.get("compile_requests_executed") + hits = native.get("cache_hits") + misses = native.get("cache_misses") + rate = native.get("hit_rate") + write_errors = native.get("cache_write_errors") + elapsed = mode.get("elapsed_seconds") + target_hit = mode.get("target_cache_hit") + + lines = [f"- `{label}`:"] + if elapsed is not None: + lines.append(f" - elapsed: {elapsed:.0f}s") + if target_hit is not None: + lines.append(f" - target snapshot restored: `{target_hit}`") + if compiler_cache == "none": + lines.append(" - compiler cache: disabled") + if requests is not None: + lines.append( + f" - Cargo units compiled: {executed if executed is not None else requests}" + f" ({requests} compile requests)" + ) + if hits is not None: + lines.append( + f" - sccache: {hits} hits / {misses} misses" + f" ({(rate or 0):.1f}% hit rate)" + ) + if write_errors: + lines.append(f" - sccache write errors: {write_errors}") + return "\n".join(lines) + + +def main() -> int: + if len(sys.argv) < 2 or len(sys.argv) % 2 != 1: + print( + "Usage: summarize-cargo-evidence.py LABEL EVIDENCE_PATH [LABEL EVIDENCE_PATH ...]", + file=sys.stderr, + ) + return 2 + + blocks = ["## Cargo rebuild set", ""] + for index in range(1, len(sys.argv), 2): + label = sys.argv[index] + path = Path(sys.argv[index + 1]) + if not path.is_file(): + blocks.append(f"- `{label}`: evidence missing at {path}") + continue + try: + blocks.append(summarize(label, path)) + except (json.JSONDecodeError, OSError) as error: + blocks.append(f"- `{label}`: unreadable evidence ({error})") + + report = "\n".join(blocks) + print(report) + + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a", encoding="utf-8") as handle: + handle.write(report + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-zed-release-recipe.py b/scripts/verify-zed-release-recipe.py new file mode 100755 index 0000000..4cad310 --- /dev/null +++ b/scripts/verify-zed-release-recipe.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Verify that every committed Cargo layer plan matches Zed's Linux release.""" + +from __future__ import annotations + +import re +import shlex +import sys +import tomllib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +LANES = { + "cold": {"target": True, "compiler": "sccache"}, + "target-only": {"target": True, "compiler": "none"}, + "sccache-only": {"target": False, "compiler": "sccache"}, + "combined": {"target": True, "compiler": "sccache"}, +} +PHASES = { + "primary": ("x86_64-unknown-linux-gnu", ("zed", "cli")), + "remote-server": ("x86_64-unknown-linux-musl", ("remote_server",)), +} + + +class RecipeMismatch(RuntimeError): + pass + + +def require(condition: bool, message: str) -> None: + if not condition: + raise RecipeMismatch(message) + + +def read_settings(path: Path) -> dict[str, str]: + settings: dict[str, str] = {} + for number, raw_line in enumerate(path.read_text().splitlines(), 1): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + require("=" in line, f"Invalid setting at {path}:{number}") + key, raw_value = line.split("=", 1) + values = shlex.split(raw_value) + require(len(values) == 1, f"Expected one value for {key} at {path}:{number}") + settings[key] = values[0] + return settings + + +def expected_command(phase: str) -> list[str]: + target, packages = PHASES[phase] + command = [ + "cargo", + "build", + "--release", + "--target", + target, + ] + for package in packages: + command.extend(("--package", package)) + return command + + +def verify_upstream(upstream: Path, contract: dict[str, str]) -> None: + toolchain_path = upstream / "rust-toolchain.toml" + bundle_path = upstream / contract["ZED_UPSTREAM_RELEASE_SCRIPT"] + require(toolchain_path.is_file(), f"Missing {toolchain_path}") + require(bundle_path.is_file(), f"Missing {bundle_path}") + + channel_match = re.search( + r'^channel = "([^"]+)"$', toolchain_path.read_text(), re.MULTILINE + ) + require(channel_match is not None, "Missing channel in upstream/rust-toolchain.toml") + + bundle = bundle_path.read_text() + for fragment in ( + "target_triple=${host_line#*: }", + "musl_triple=${target_triple%-gnu}-musl", + 'remote_server_triple=${REMOTE_SERVER_TARGET:-"${musl_triple}"}', + "export ZED_BUNDLE=true", + "export CC=${CC:-$(which clang)}", + ): + require(fragment in bundle, f"Zed's bundle-linux changed: {fragment}") + + base_flags = contract["ZED_BASE_RUSTFLAGS"].replace("$ORIGIN", r"\$ORIGIN") + require( + f'export RUSTFLAGS="${{RUSTFLAGS:-}} {base_flags}"' in bundle, + "Zed's primary release RUSTFLAGS changed", + ) + require( + f'export RUSTFLAGS="${{RUSTFLAGS:-}} {contract["ZED_MUSL_RUSTFLAGS"]}"' + in bundle, + "Zed's remote-server RUSTFLAGS changed", + ) + require( + f'export "$musl_cc_var"={contract["ZED_MUSL_CC"]}' in bundle, + "Zed's remote-server compiler changed", + ) + + expected_builds = [ + 'cargo build --release --target "${target_triple}" ' + f'--package {contract["ZED_PRIMARY_PACKAGE"]} ' + f'--package {contract["ZED_PRIMARY_COMPANION_PACKAGE"]}', + 'cargo build --release --target "${remote_server_triple}" ' + f'--package {contract["ZED_REMOTE_SERVER_PACKAGE"]}', + ] + actual_builds = [ + line.strip() + for line in bundle.splitlines() + if line.strip().startswith("cargo build ") + ] + require( + actual_builds == expected_builds, + "Zed's Linux release commands changed; update the committed plans", + ) + + +def verify_plans() -> None: + target_tags: set[str] = set() + compiler_tags: set[str] = set() + dependency_tag_pairs: set[tuple[str, str]] = set() + + for lane, expected in LANES.items(): + for phase in PHASES: + path = ROOT / "plans" / lane / phase / ".boringcache.toml" + require(path.is_file(), f"Missing {path}") + plan = tomllib.loads(path.read_text()) + entries = plan["profiles"]["cargo-product"]["entries"] + cargo = plan["adapters"]["cargo"] + + require( + cargo["compiler-cache"] == expected["compiler"], + f"{path} has the wrong compiler-cache selector", + ) + require( + ("zed-target" in entries) is expected["target"], + f"{path} has the wrong target-layer selector", + ) + require( + cargo["command"] == expected_command(phase), + f"{path} does not match Zed's {phase} release command", + ) + require(cargo["no-git"] is True, f"{path} must use stable reviewed tags") + + config_entries = plan["entries"] + target_tags.add(config_entries["zed-target"]["tag"]) + dependency_tag_pairs.add( + ( + config_entries["cargo-registry"]["tag"], + config_entries["cargo-git"]["tag"], + ) + ) + if expected["compiler"] == "sccache": + require( + "sccache" in plan["adapters"], + f"{path} must give sccache an independent identity", + ) + compiler_tags.add(plan["adapters"]["sccache"]["tag"]) + else: + require( + "sccache" not in plan["adapters"], + f"{path} must not configure a disabled compiler layer", + ) + + require(len(target_tags) == 1, "Every target-bearing lane must share one seed tag") + require(len(compiler_tags) == 1, "Every sccache lane must share one seed tag") + require( + len(dependency_tag_pairs) == 1, + "Every lane must hold Cargo dependency transport constant", + ) + + +def verify_workflows() -> None: + matrix_path = ROOT / ".github/workflows/zed-cargo-product.yml" + rolling_path = ROOT / ".github/workflows/zed-cargo-rolling-chain.yml" + matrix = matrix_path.read_text() + rolling = rolling_path.read_text() + all_workflows = matrix + rolling + + require("scope-boringcache-run" not in all_workflows, "Workflows must not rewrite tags") + require("mode: sccache" not in all_workflows, "Cargo owns the composed lifecycle") + require( + "boringcache/one@ab52a39d3d7358c22b359a6ffbf86cf74be9bf56" in matrix, + "The matrix must use released One 1.18.1", + ) + for lane in LANES: + for phase in PHASES: + require( + f"activate-cargo-plan.sh {lane} {phase}" in matrix, + f"The matrix does not select plans/{lane}/{phase}", + ) + require( + matrix.count("working-directory: upstream") == 8, + "Every matrix phase must run Cargo from the upstream Git checkout", + ) + require( + matrix.count("mode: cargo") == 8, + "The four lanes must each run both Cargo release phases", + ) + + root_plan = tomllib.loads((ROOT / ".boringcache.toml").read_text()) + require( + root_plan["adapters"]["cargo"]["compiler-cache"] == "sccache", + "The rolling Cargo plan must select its compiler layer explicitly", + ) + require( + "sccache" in root_plan["adapters"], + "The rolling plan must give sccache an independent identity", + ) + + +def verify(upstream: Path) -> str: + contract = read_settings(ROOT / "scripts/zed-release-recipe.env") + source = read_settings(ROOT / "benchmark-source.env") + verify_upstream(upstream, contract) + verify_plans() + verify_workflows() + return source["ZED_HEAD_SHA"] + + +def main() -> int: + upstream = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / "upstream" + try: + source_sha = verify(upstream.resolve()) + except (KeyError, OSError, RecipeMismatch, tomllib.TOMLDecodeError) as error: + print(f"Zed release recipe mismatch: {error}", file=sys.stderr) + return 1 + + print(f"Verified Zed Cargo layer plans at {source_sha}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/zed-release-recipe.env b/scripts/zed-release-recipe.env new file mode 100644 index 0000000..ccad08e --- /dev/null +++ b/scripts/zed-release-recipe.env @@ -0,0 +1,9 @@ +ZED_UPSTREAM_RELEASE_SCRIPT=script/bundle-linux +ZED_HOST_TARGET=x86_64-unknown-linux-gnu +ZED_REMOTE_SERVER_TARGET=x86_64-unknown-linux-musl +ZED_PRIMARY_PACKAGE=zed +ZED_PRIMARY_COMPANION_PACKAGE=cli +ZED_REMOTE_SERVER_PACKAGE=remote_server +ZED_BASE_RUSTFLAGS='-C link-args=-Wl,--disable-new-dtags,-rpath,$ORIGIN/../lib' +ZED_MUSL_RUSTFLAGS='-C target-feature=+crt-static' +ZED_MUSL_CC=musl-gcc diff --git a/test/test_cargo_product.py b/test/test_cargo_product.py index 4c18f86..2bf4327 100644 --- a/test/test_cargo_product.py +++ b/test/test_cargo_product.py @@ -1,12 +1,21 @@ +import json import os import subprocess +import sys import tempfile +import tomllib import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[1] WORKFLOWS = ROOT / ".github" / "workflows" +LANES = { + "cold": ("sccache", True), + "target-only": ("none", True), + "sccache-only": ("sccache", False), + "combined": ("sccache", True), +} class SourceSyncTest(unittest.TestCase): @@ -47,76 +56,153 @@ def test_advances_exactly_one_upstream_commit(self): self.assertEqual(settings["ZED_HEAD_SHA"], following) -class CargoProductWorkflowTest(unittest.TestCase): - def test_workflows_use_zeds_checked_in_rust_toolchain(self): - workflow_text = "\n".join( - (WORKFLOWS / name).read_text() - for name in ["zed-cargo-product.yml", "zed-cargo-rolling-chain.yml"] - ) - source = (ROOT / "benchmark-source.env").read_text() +class CargoLayerPlanTest(unittest.TestCase): + def test_each_layer_choice_is_a_committed_cli_plan(self): + target_tags = set() + compiler_tags = set() - self.assertEqual(workflow_text.count("rustup show active-toolchain"), 3) - self.assertNotIn("rustup toolchain install", workflow_text) - self.assertNotIn("ZED_RUST_VERSION", source + workflow_text) + for lane, (compiler, includes_target) in LANES.items(): + for phase in ("primary", "remote-server"): + path = ROOT / "plans" / lane / phase / ".boringcache.toml" + plan = tomllib.loads(path.read_text()) + profile = plan["profiles"]["cargo-product"]["entries"] + cargo = plan["adapters"]["cargo"] - def test_cargo_is_the_only_live_boringcache_rust_lifecycle(self): - workflow_text = "\n".join( - path.read_text() for path in sorted(WORKFLOWS.glob("*.yml")) - ) - config = (ROOT / ".boringcache.toml").read_text() + self.assertEqual(cargo["compiler-cache"], compiler) + self.assertEqual("zed-target" in profile, includes_target) + self.assertEqual(cargo["profiles"], ["cargo-product"]) + self.assertNotIn("--manifest-path", cargo["command"]) + target_tags.add(plan["entries"]["zed-target"]["tag"]) - self.assertIn("[adapters.cargo]", config) - self.assertNotIn("[adapters.sccache]", config) - self.assertNotIn("mode: sccache", workflow_text) - self.assertNotIn("uses: boringcache/one@", workflow_text) - - def test_weekly_fresh_workflow_owns_publish_and_consume(self): - workflow = (WORKFLOWS / "zed-cargo-product.yml").read_text() - - self.assertNotIn("push:", workflow) - self.assertNotIn("pull_request:", workflow) - self.assertNotIn("workflow_dispatch:", workflow) - self.assertIn("workflow_call:", workflow) - self.assertIn("schedule:", workflow) - self.assertIn('cron: "0 4 * * 5"', workflow) - self.assertEqual(workflow.count("boringcache cargo \\"), 2) - self.assertIn("--write", workflow) - self.assertIn("--read-only", workflow) - self.assertIn("cargo-freshness-v2.json", workflow) - self.assertIn("--native-tool-evidence-json", workflow) - - def test_source_updates_run_the_persistent_rolling_chain(self): - dispatcher = (WORKFLOWS / "zed-rust-cache-proof.yml").read_text() + if compiler == "sccache": + self.assertIn("sccache", plan["adapters"]) + compiler_tags.add(plan["adapters"]["sccache"]["tag"]) + else: + self.assertNotIn("sccache", plan["adapters"]) + + self.assertEqual(len(target_tags), 1) + self.assertEqual(len(compiler_tags), 1) + + def test_action_selects_plans_but_does_not_plan_layers(self): + matrix = (WORKFLOWS / "zed-cargo-product.yml").read_text() rolling = (WORKFLOWS / "zed-cargo-rolling-chain.yml").read_text() - sync = (WORKFLOWS / "sync.yml").read_text() - source = (ROOT / "benchmark-source.env").read_text() - - self.assertIn('- "benchmark-source.env"', dispatcher) - self.assertIn("uses: ./.github/workflows/zed-cargo-rolling-chain.yml", dispatcher) - self.assertNotIn("zed-cargo-product.yml", dispatcher) - self.assertIn("ZED_ROLLING_CACHE_SCOPE=", source) - self.assertIn('cron: "*/30 * * * *"', sync) - self.assertIn("advance-source-pair.sh benchmark-source.env ZED", sync) - self.assertIn("Require the previous rolling benchmark to be green", sync) - self.assertIn("steps.previous.outputs.ready == 'true'", sync) - self.assertIn("git add benchmark-source.env upstream", sync) - self.assertIn("group: benchmark-zed-cargo-rolling-chain", rolling) - - settings = dict(line.split("=", 1) for line in source.splitlines()) - gitlink = subprocess.run( - ["git", "ls-files", "-s", "upstream"], + workflow_text = matrix + rolling + + self.assertNotIn("scope-boringcache-run", workflow_text) + self.assertNotIn("mode: sccache", workflow_text) + self.assertNotIn("sed -i", workflow_text) + self.assertNotIn("fail-on-cache-miss", matrix) + self.assertNotIn("cache_scope:", rolling) + self.assertEqual(matrix.count("mode: cargo"), 8) + self.assertIn("inputs.cli_version", matrix) + self.assertIn("Action default", matrix) + + for lane in LANES: + for phase in ("primary", "remote-server"): + self.assertIn( + f"activate-cargo-plan.sh {lane} {phase}", + matrix, + ) + self.assertEqual(matrix.count("working-directory: upstream"), 8) + + def test_rolling_plan_owns_its_stable_compiler_identity(self): + plan = tomllib.loads((ROOT / ".boringcache.toml").read_text()) + + self.assertEqual(plan["adapters"]["cargo"]["compiler-cache"], "sccache") + self.assertIn("sccache", plan["adapters"]) + self.assertEqual( + plan["adapters"]["sccache"]["tag"], + "zed-cargo-rolling-main-sccache", + ) + + def test_source_pair_matches_the_pinned_submodule(self): + source = dict( + line.split("=", 1) + for line in (ROOT / "benchmark-source.env").read_text().splitlines() + ) + head = subprocess.run( + ["git", "-C", "upstream", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True, - ).stdout.split()[1] - self.assertEqual(settings["ZED_HEAD_SHA"], gitlink) + ).stdout.strip() + + self.assertEqual(source["ZED_HEAD_SHA"], head) - def test_canary_defaults_to_the_cargo_product(self): - workflow = (WORKFLOWS / "canary-dispatch.yml").read_text() + def test_release_recipe_and_layer_contract(self): + subprocess.run( + [sys.executable, str(ROOT / "scripts/verify-zed-release-recipe.py"), "upstream"], + cwd=ROOT, + check=True, + ) + + def test_native_hit_rate_is_already_a_percentage(self): + evidence = { + "phases": { + "restore": { + "mode_evidence": { + "elapsed_seconds": 12.4, + "target_cache_hit": True, + "native_tool": { + "compile_requests": 101, + "compile_requests_executed": 100, + "cache_hits": 96, + "cache_misses": 4, + "hit_rate": 96.0, + }, + } + } + } + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "evidence.json" + path.write_text(json.dumps(evidence)) + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/summarize-cargo-evidence.py"), + "combined", + str(path), + ], + check=True, + text=True, + capture_output=True, + ) + + self.assertIn("96.0% hit rate", result.stdout) + self.assertNotIn("9600", result.stdout) + + def test_report_handles_a_disabled_compiler_cache(self): + evidence = { + "phases": { + "restore": { + "mode_evidence": { + "elapsed_seconds": 12.4, + "target_cache_hit": True, + "native_tool": None, + "cargo_cache": {"compiler_cache": "none"}, + } + } + } + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "evidence.json" + path.write_text(json.dumps(evidence)) + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/summarize-cargo-evidence.py"), + "target-only", + str(path), + ], + check=True, + text=True, + capture_output=True, + ) - self.assertIn("uses: ./.github/workflows/zed-cargo-product.yml", workflow) - self.assertNotIn("zed-sccache", workflow) + self.assertIn("target snapshot restored: `True`", result.stdout) + self.assertIn("compiler cache: disabled", result.stdout) if __name__ == "__main__":