diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index 8e8863c30..aa066bbd5 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -1,9 +1,14 @@ name: Bench ABBA tiebreaker # Drift-free paired (A/B/B/A) prover benchmark for resolving small (~1%) deltas the -# cheap PR benchmark can't confirm. It builds both binaries and runs ~20 interleaved -# pairs, so it OCCUPIES THE SINGLE BENCH SERVER FOR ~30-40 MIN. For that reason it -# NEVER auto-triggers -- it runs only on an explicit `/bench-abba` comment on a PR. +# cheap PR benchmark can't confirm. At the default workload it OCCUPIES THE SINGLE +# BENCH SERVER FOR SEVERAL HOURS, so it NEVER auto-triggers -- it runs only on an +# explicit `/bench-abba` comment on a PR. +# +# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (default: 20 pairs, ethrex 100tx +# --continuations; "cont10" is the quick coarse option, "mono[TX]" the legacy +# monolithic prove). Cont proofs need rkyv pointer_width_64 on both sides, so PR +# branches older than that fix must rebase before a cont bench. on: issue_comment: types: [created] @@ -26,45 +31,112 @@ jobs: startsWith(github.event.comment.body, '/bench-abba') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: [self-hosted, bench] - # Generous ceiling so a hang/OOM can't strand the single bench runner; the - # workload itself is ~30-40 min at the default 20 pairs (clamped to <=40). - timeout-minutes: 120 + # Hang guardrail, not expected duration: a cont100 CPU prove is ~7 min, so the + # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds. + timeout-minutes: 720 steps: - - name: Acknowledge (react + occupancy notice) - uses: actions/github-script@v7 - with: - script: | - await github.rest.reactions.createForIssueComment({ - owner: context.repo.owner, repo: context.repo.repo, - comment_id: context.payload.comment.id, content: 'eyes' - }); - await github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.issue.number, - body: '⏳ **ABBA tiebreaker started** on the bench server (~30–40 min). The bench server is occupied until it finishes.' - }); - - - name: Resolve PR head + pair count + - name: Resolve PR head + bench config id: cfg env: GH_TOKEN: ${{ github.token }} PR_NUM: ${{ github.event.issue.number }} COMMENT_BODY: ${{ github.event.comment.body }} run: | + # The runner is persistent self-hosted: drop the previous run's logs so + # the always() result comment never tails a stale /tmp/abba_out.txt. + rm -f /tmp/abba_out.txt /tmp/abba_result.txt # Resolve the head SHA (not the branch name): pinning the commit works for # fork PRs too (the branch lives in the fork, not origin/) and avoids a # force-push race mid-run. HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" - # Optional pair count, e.g. "/bench-abba 32"; default 20. Clamp to [2,40] - # so a "/bench-abba 10000" can't monopolize the single bench server. - N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-abba[[:space:]]*\([0-9]\+\).*|\1|p') - N=${N:-20} - if [ "$N" -lt 2 ] 2>/dev/null || [ "$N" -gt 40 ] 2>/dev/null; then - echo "::warning::pair count $N out of range [2,40]; using 20" - N=20 + # Everything after "/bench-abba" on its line, tokens in any order: a number = + # pair count; cont[TX]/mono[TX] = workload. + ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1) + PAIRS=20; CONTINUATIONS=1; TX_COUNT=100 + set -f # tokens must not glob-expand against the runner's CWD + for tok in $ARGS; do + case "$tok" in + cont) CONTINUATIONS=1; TX_COUNT=100 ;; + mono) CONTINUATIONS=0; TX_COUNT=5 ;; + cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;; + mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;; + [0-9]*) PAIRS="$tok" ;; + *) echo "::warning::ignoring unrecognized token '$tok'" ;; + esac + done + # Digits-only + clamps: PAIRS capped so one comment can't monopolize the + # single bench server; mono capped at 5tx (monolithic peak heap grows with + # the trace; 20tx needs ~78 GB). + if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi + case "$TX_COUNT" in + ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; + esac + if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then + echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT" + TX_COUNT=$TX_DEFAULT + fi + case "$PAIRS" in + ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 20"; PAIRS=20 ;; + esac + if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 40 ]; then + echo "::warning::pair count $PAIRS out of range [2,40]; using 20" + PAIRS=20 + fi + # Even is ideal so the AB/BA orders balance; round an odd request up by one. + if [ "$((PAIRS % 2))" -ne 0 ]; then + PAIRS=$((PAIRS + 1)) + echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" + fi + if [ "$CONTINUATIONS" = "1" ]; then + WORKLOAD="ethrex ${TX_COUNT}tx continuations" + else + WORKLOAD="ethrex ${TX_COUNT}tx monolithic" fi - echo "pairs=$N" >> "$GITHUB_OUTPUT" + # Outputs land before the fail-fast below so the always() result + # comment is fully labeled even when this step exits early. + { + echo "pairs=$PAIRS" + echo "continuations=$CONTINUATIONS" + echo "tx_count=$TX_COUNT" + echo "workload=$WORKLOAD" + } >> "$GITHUB_OUTPUT" + # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx + # continuation proof exceeds rkyv's old 2 GiB cap and only dies after + # blocking the single bench server for hours. Skip the check when the + # fetch fails: gh api prints HTTP error bodies to stdout, so gate on + # its exit status AND on the payload looking like the manifest (it + # declares rkyv) — never treat an error blob as a missing feature. + # Purely textual; deletable once every open branch postdates the fix. + if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then + if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \ + -H "Accept: application/vnd.github.raw" 2>/dev/null) \ + && printf '%s' "$SIDE" | grep -q '^rkyv' \ + && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then + MSG="PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." + echo "$MSG" > /tmp/abba_out.txt # surfaces in the result comment + echo "::error::$MSG" + exit 1 + fi + fi + echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" + + - name: Acknowledge (react + occupancy notice) + uses: actions/github-script@v7 + env: + PAIRS: ${{ steps.cfg.outputs.pairs }} + WORKLOAD: ${{ steps.cfg.outputs.workload }} + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: context.payload.comment.id, content: 'eyes' + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: context.issue.number, + body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} (a cont100 pair is ~15 min, so the default 20 pairs runs ~4.5-5 hr; pass a smaller pair count or \`cont10\` for a quicker, coarser run). The bench server is occupied until it finishes.` + }); - name: Checkout (full history for ref resolution) uses: actions/checkout@v4 @@ -84,6 +156,8 @@ jobs: env: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} + CONTINUATIONS: ${{ steps.cfg.outputs.continuations }} + TX_COUNT: ${{ steps.cfg.outputs.tx_count }} run: | export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" set -o pipefail @@ -100,12 +174,14 @@ jobs: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} OUTCOME: ${{ steps.run.outcome }} + WORKLOAD: ${{ steps.cfg.outputs.workload }} with: script: | const fs = require('fs'); const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } }; const head = (process.env.HEAD_SHA || '').slice(0, 10), pairs = process.env.PAIRS; - let body = `## ABBA tiebreaker — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`; + const workload = process.env.WORKLOAD || 'ethrex'; + let body = `## ABBA tiebreaker — \`${head}\` vs \`main\` (${pairs} pairs, ${workload})\n\n`; if (process.env.OUTCOME === 'success') { const res = read('/tmp/abba_result.txt') || read('/tmp/abba_out.txt'); body += '```\n' + res + '\n```\n'; diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 6928255d9..040b51c89 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -6,9 +6,12 @@ name: Benchmark GPU (PR) # It builds the cli at the PR head and at main, runs N interleaved pairs on the GPU, # posts the paired-t + Wilcoxon verdict back to the PR, then ALWAYS destroys the box. # -# Triggered by a "/bench-gpu [N]" comment on a PR (N = pair count, default 14) or via -# workflow_dispatch. Orchestration runs on a GitHub-hosted runner; all GPU work happens -# on the rented Vast box (provisioned by the template onstart). +# Triggered by a "/bench-gpu [N] [cont[TX]|mono[TX]]" comment on a PR (N = pair count, +# default 14) or via workflow_dispatch. Workload default: ethrex 100tx --continuations; +# "mono[TX]" = legacy monolithic prove. Cont proofs need rkyv pointer_width_64 on both +# sides, so PR branches older than that fix must rebase before a cont bench. +# Orchestration runs on a GitHub-hosted runner; all GPU work happens on the rented +# Vast box (provisioned by the template onstart). # # Requires repo secrets: # VAST_API_KEY — https://cloud.vast.ai/manage-keys/ @@ -20,6 +23,9 @@ on: pairs: description: "Number of A/B/B/A pairs" default: "14" + mode: + description: "Workload: cont[TX] (--continuations, TX defaults to 100) or mono[TX] (monolithic, TX defaults to 5)" + default: "cont100" issue_comment: types: [created] @@ -57,12 +63,12 @@ jobs: github.event.issue.pull_request && startsWith(github.event.comment.body, '/bench-gpu') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)) - # ABBA on the GPU: provisioning + dual cuda build (~30 min) + 2*pairs proves - # (~95s each). At the max 32 pairs (64 proves) a slow-provision box runs ~3 hr, - # so allow headroom over that; teardown still always destroys the box. - timeout-minutes: 210 + # Provisioning + dual cuda build (~30 min) + 2*pairs proves (~3.5 min each at + # the default cont100). Sized for the 32-pair worst case (~4.5 hr) with headroom; + # teardown still always destroys the box. + timeout-minutes: 330 steps: - - name: Resolve PR ref + pair count + - name: Resolve PR ref + bench config id: config env: GH_TOKEN: ${{ github.token }} @@ -70,24 +76,54 @@ jobs: COMMENT_BODY: ${{ github.event.comment.body }} PR_NUM: ${{ github.event.issue.number }} DISPATCH_PAIRS: ${{ github.event.inputs.pairs }} + DISPATCH_MODE: ${{ github.event.inputs.mode }} DISPATCH_REF: ${{ github.ref_name }} run: | if [ "$EVENT_NAME" = "issue_comment" ]; then # Pin the head SHA (works for fork PRs; avoids a force-push race mid-run). HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) OUT_PR_NUM="$PR_NUM"; OUT_HEAD_SHA="$HEAD_SHA"; OUT_BRANCH="" - # "/bench-gpu 20" -> 20 pairs; otherwise default. - N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-gpu[[:space:]]*\([0-9]\+\).*|\1|p') - PAIRS=${N:-14} + # Everything after "/bench-gpu" on its line, tokens in any order: + # a number = pair count; cont[TX]/mono[TX] = workload (see loop below). + ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-gpu||p' | head -n1) + PAIRS=14 else # workflow_dispatch: compare this branch vs main. OUT_PR_NUM=""; OUT_HEAD_SHA=""; OUT_BRANCH="$DISPATCH_REF" + ARGS="$DISPATCH_MODE" PAIRS=${DISPATCH_PAIRS:-14} fi + # Defaults: continuations with the 100-transfer fixture (production mode). + CONTINUATIONS=1; TX_COUNT=100 + set -f # tokens must not glob-expand against the runner's CWD + for tok in $ARGS; do + case "$tok" in + cont) CONTINUATIONS=1; TX_COUNT=100 ;; + mono) CONTINUATIONS=0; TX_COUNT=5 ;; + cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;; + mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;; + [0-9]*) PAIRS="$tok" ;; + *) echo "::warning::ignoring unrecognized token '$tok'" ;; + esac + done + # TX_COUNT and PAIRS are interpolated into the remote bash -lc below: enforce + # digits-only. Mono is capped at 5tx (monolithic peak heap grows with the + # trace; 20tx needs ~78 GB, over the 48 GB box floor). + if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi + case "$TX_COUNT" in + ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; + esac + if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then + echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT" + TX_COUNT=$TX_DEFAULT + fi + case "$PAIRS" in + ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 14"; PAIRS=14 ;; + esac # Clamp to [2,32]; out-of-range -> default. 14 ~ resolves a 2% delta. The ceiling # keeps the worst-case run (64 proves + provisioning + dual build) under the job # timeout above. - if [ "$PAIRS" -lt 2 ] 2>/dev/null || [ "$PAIRS" -gt 32 ] 2>/dev/null; then + if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 32 ]; then echo "::warning::pair count out of range [2,32], defaulting to 14" PAIRS=14 fi @@ -96,19 +132,47 @@ jobs: PAIRS=$((PAIRS + 1)) echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" fi + if [ "$CONTINUATIONS" = "1" ]; then + WORKLOAD="ethrex ${TX_COUNT}tx continuations" + else + WORKLOAD="ethrex ${TX_COUNT}tx monolithic" + fi + # Outputs land before the fail-fast below so the always() result + # comment is fully labeled even when this step exits early. { echo "pr_num=$OUT_PR_NUM" echo "head_sha=$OUT_HEAD_SHA" echo "branch=$OUT_BRANCH" echo "pairs=$PAIRS" + echo "continuations=$CONTINUATIONS" + echo "tx_count=$TX_COUNT" + echo "workload=$WORKLOAD" } >> "$GITHUB_OUTPUT" - echo "Using $PAIRS A/B/B/A pairs" + # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx + # continuation proof exceeds rkyv's old 2 GiB cap and only dies after the + # full dual build (~1 hr of GPU rental). Skip the check when the fetch + # fails: gh api prints HTTP error bodies to stdout, so gate on its exit + # status AND on the payload looking like the manifest (it declares + # rkyv) — never treat an error blob as a missing feature. Purely + # textual; deletable once every open branch postdates the fix. + if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then + REF="${OUT_HEAD_SHA:-$OUT_BRANCH}" + if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$REF" \ + -H "Accept: application/vnd.github.raw" 2>/dev/null) \ + && printf '%s' "$SIDE" | grep -q '^rkyv' \ + && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then + echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." + exit 1 + fi + fi + echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" - name: Acknowledge (react + occupancy notice) if: github.event_name == 'issue_comment' uses: actions/github-script@v7 env: PAIRS: ${{ steps.config.outputs.pairs }} + WORKLOAD: ${{ steps.config.outputs.workload }} with: script: | await github.rest.reactions.createForIssueComment({ @@ -118,7 +182,7 @@ jobs: // Post the "started" notice under the SAME marker the result step uses, so the // result updates this comment in place (and re-runs reuse it rather than stacking). const marker = 'GPU Benchmark (ABBA)'; - const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) on the CUDA prover path. This takes ~1 hr; the result will replace this comment.`; + const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. This takes ~2.5 hr at the default workload; the result will replace this comment.`; const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, per_page: 100, @@ -170,10 +234,10 @@ jobs: # because vast can't numerically compare the driver_version string server-side. MIN_DRIVER: "580" run: | - # cpu_ram filter is in GB. Floor 48 GB: the bench workload moved to the - # 5-transfer ethrex fixture (executor/tests/ethrex_5_transfers.bin), far smaller - # than the old 20-transfer prove (~78 GB heap) that set the previous 96 GB floor. - # 48 GB widens the dedicated pool (~15 vs ~11 offers). + # cpu_ram filter is in GB. Floor 48 GB: continuation proves are flat-memory + # (~10 GB) and the legacy 5tx monolithic prove also fits — far below the old + # 20-transfer prove (~78 GB heap) that set the previous 96 GB floor. 48 GB + # widens the dedicated pool (~15 vs ~11 offers). # gpu_frac=1 requires a WHOLE-MACHINE offer (you rent every GPU on the host), so # Vast places no other tenant on the box: CPU cores, RAM/memory bandwidth, PCIe, # and NVMe are fully dedicated. Without it the "most expensive" sort below lands on @@ -203,7 +267,7 @@ jobs: sleep "$OFFER_INTERVAL" done if [ -z "$OFFER_ID" ]; then - echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=96GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)" + echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=48GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)" exit 1 fi echo "id=$OFFER_ID" >> "$GITHUB_OUTPUT" @@ -331,6 +395,8 @@ jobs: HEAD_SHA: ${{ steps.config.outputs.head_sha }} BRANCH: ${{ steps.config.outputs.branch }} PAIRS: ${{ steps.config.outputs.pairs }} + CONTINUATIONS: ${{ steps.config.outputs.continuations }} + TX_COUNT: ${{ steps.config.outputs.tx_count }} run: | SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST" @@ -355,8 +421,8 @@ jobs: # explicit and robust to the template default changing.) The harness still builds the # cli at REF_A (the PR) and origin/main in isolated worktrees, runs PAIRS interleaved # A/B/B/A proves, and prints the paired-t CI + Wilcoxon verdict. BENCH_FEATURES routes - # the build through the CUDA prover path. NOTE: requires this PR's bench_abba.sh change - # (the BENCH_FEATURES env) to be on main — i.e. it only takes effect after merge. + # the build through the CUDA prover path; CONTINUATIONS/TX_COUNT pick the workload. + # The harness runs from main, so workflow/script changes take effect post-merge. # REBUILD=1: each Vast box is fresh, GPU-specific hardware — always rebuild both # binaries (cubin is compiled for the detected arch); never trust a cached binary. # CUDARC_PIN: compat shim for pre-pin baseline shas. cudarc's CUDA version is now pinned @@ -371,6 +437,7 @@ jobs: git fetch --force origin main; $FETCH; \ git checkout -f origin/main; \ REBUILD=1 CUDARC_PIN=cuda-12080 SYSROOT_DIR=/opt/lambda-vm-sysroot BENCH_FEATURES='$BENCH_FEATURES' \ + CONTINUATIONS=$CONTINUATIONS TX_COUNT=$TX_COUNT \ scripts/bench_abba.sh $REF_A origin/main $PAIRS" # pipefail so a failed remote bench (e.g. a prove that dies) propagates through the @@ -386,9 +453,10 @@ jobs: if: always() && (steps.bench.outcome == 'success' || steps.bench.outcome == 'failure') env: OUTCOME: ${{ steps.bench.outcome }} + WORKLOAD: ${{ steps.config.outputs.workload }} run: | { - echo "## GPU ABBA — ethrex 20 transfers (vs main)" + echo "## GPU ABBA — ${WORKLOAD:-ethrex} (vs main)" if [ "$OUTCOME" = "success" ] && [ -s "$RUNNER_TEMP/abba_result.txt" ]; then echo '```' cat "$RUNNER_TEMP/abba_result.txt" @@ -410,6 +478,7 @@ jobs: OUTCOME: ${{ steps.bench.outcome }} GPU_NAME: ${{ env.GPU_NAME }} OFFER_PRICE: ${{ steps.offer.outputs.price }} + WORKLOAD: ${{ steps.config.outputs.workload }} with: script: | const fs = require('fs'); @@ -419,9 +488,10 @@ jobs: const pairs = process.env.PAIRS; const gpu = (process.env.GPU_NAME || '').replace('_', ' '); const price = process.env.OFFER_PRICE; + const workload = process.env.WORKLOAD || 'ethrex'; let body = `## GPU Benchmark (ABBA) — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`; - body += `${gpu} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · drift-free A/B/B/A\n\n`; + body += `${gpu} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · ${workload} · drift-free A/B/B/A\n\n`; if (process.env.OUTCOME === 'success') { const res = read(`${tmp}/abba_result.txt`) || read(`${tmp}/abba_out.txt`); body += '```\n' + res + '\n```\n'; diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml index f612949a8..cc4d00a70 100644 --- a/bench_vs/lambda/recursion/Cargo.toml +++ b/bench_vs/lambda/recursion/Cargo.toml @@ -65,7 +65,8 @@ lambda-vm-prover = { path = "../../../prover", default-features = false, feature "profile-markers", ] } lambda-vm-syscalls = { path = "../../../syscalls" } -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } [profile.release] debug = 2 diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml index e4fcdb7fd..71e89beef 100644 --- a/bin/cli/Cargo.toml +++ b/bin/cli/Cargo.toml @@ -9,7 +9,8 @@ executor = { path = "../../executor" } prover = { path = "../../prover", package = "lambda-vm-prover" } stark = { path = "../../crypto/stark" } clap = { version = "4.3.10", features = ["derive"] } -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } tempfile = "3" tikv-jemallocator = "0.6" tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true } diff --git a/crypto/crypto/Cargo.toml b/crypto/crypto/Cargo.toml index 6b78f81e7..9a36b2614 100644 --- a/crypto/crypto/Cargo.toml +++ b/crypto/crypto/Cargo.toml @@ -22,10 +22,12 @@ rand_chacha = { version = "0.3.1", default-features = false } memmap2 = { version = "0.9", optional = true } tempfile = { version = "3", optional = true } libc = { version = "0.2", optional = true } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. rkyv = { version = "0.8.10", default-features = false, features = [ "alloc", "bytecheck", "aligned", + "pointer_width_64", ], optional = true } [target.'cfg(target_arch = "riscv64")'.dependencies] diff --git a/crypto/math/Cargo.toml b/crypto/math/Cargo.toml index df43ea975..a161056ca 100644 --- a/crypto/math/Cargo.toml +++ b/crypto/math/Cargo.toml @@ -25,10 +25,12 @@ num-traits = { version = "0.2.19", default-features = false } # rkyv zero-copy (de)serialization. Optional; used by the recursion verifier to # read a proof straight from its byte buffer with no deserialization pass. +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. rkyv = { version = "0.8.10", default-features = false, features = [ "alloc", "bytecheck", "aligned", + "pointer_width_64", ], optional = true } [dev-dependencies] diff --git a/crypto/stark/Cargo.toml b/crypto/stark/Cargo.toml index 78d95be67..09a5c1d9d 100644 --- a/crypto/stark/Cargo.toml +++ b/crypto/stark/Cargo.toml @@ -20,7 +20,8 @@ log = "0.4.17" digest = "0.10.7" serde = { version = "1.0", features = ["derive"] } itertools = "0.11.0" -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } # Parallelization crates rayon = { version = "1.8.0", optional = true } diff --git a/crypto/stark/src/proof/stark.rs b/crypto/stark/src/proof/stark.rs index ba4aca2dc..9ce3ed32f 100644 --- a/crypto/stark/src/proof/stark.rs +++ b/crypto/stark/src/proof/stark.rs @@ -16,6 +16,17 @@ use crate::{ // `tests/bus_tests/completeness_tests.rs`. Do not add a production serde // dependency on these types. +// With no pointer-width feature enabled rkyv silently falls back to 32-bit +// rel-ptrs, capping an archive at ~2 GiB — which large continuation proofs +// exceed, and which CI round-trips (all under 2 GiB) can't catch. Pinned here, +// where the archived proof types live, so standalone builds of this crate fail +// if a Cargo.toml loses `pointer_width_64`; lambda-vm-prover repeats the +// assert to cover the host + riscv64 guest graphs. +const _: () = assert!( + size_of::() == 8, + "proof wire format requires rkyv's pointer_width_64 feature on every proof-format crate", +); + #[derive( Debug, Clone, diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 15344d138..748ddd9a8 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -24,7 +24,9 @@ rayon = { version = "1.8.0", optional = true } sysinfo = { version = "0.31", default-features = false, features = ["system"] } log = "0.4" digest = "0.10.7" -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: 32-bit rel-ptrs cap an archive at ~2 GiB, which large +# continuation proofs exceed. Keep in sync across all proof-format crates. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } [dev-dependencies] env_logger = "*" diff --git a/prover/src/lib.rs b/prover/src/lib.rs index ff9601bb4..a8e89f989 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -209,8 +209,9 @@ pub struct GuestInput { /// 4-byte magic identifying a lambda-vm recursion input blob ("LVMR"). pub const RECURSION_INPUT_MAGIC: [u8; 4] = *b"LVMR"; -/// Wire-format version of the recursion input blob. -pub const RECURSION_INPUT_VERSION: u32 = 1; +/// Wire-format version of the recursion input blob. v2: rkyv pointer_width_64 +/// (64-bit rel-ptrs) — v1 archives use 32-bit offsets and are incompatible. +pub const RECURSION_INPUT_VERSION: u32 = 2; /// Required alignment (bytes) of the archive's first byte in guest memory. pub const RECURSION_INPUT_ALIGN: usize = 16; @@ -234,6 +235,18 @@ const _: () = { ); }; +// With no pointer-width feature enabled rkyv silently falls back to 32-bit +// rel-ptrs, capping an archive at ~2 GiB — which large continuation proofs +// exceed, and which nothing else catches: every CI round-trip fits 32-bit +// offsets, and RECURSION_INPUT_VERSION can't flag it since host and guest +// compile the constant from this same file. This compiles into both the host +// and the riscv64 guest graph (the recursion guests path-depend on this +// crate), so a Cargo.toml losing the feature fails the build here. +const _: () = assert!( + size_of::() == 8, + "proof wire format v2 requires rkyv's pointer_width_64 feature on every proof-format crate", +); + /// Encode a [`GuestInput`] into the on-wire blob: a 12-byte /// `magic + version + reserved` prefix followed by the rkyv archive. The prefix /// both aligns the archive in guest memory (so in-place reads don't trap) and @@ -251,8 +264,9 @@ pub fn encode_recursion_input(input: &GuestInput) -> Result, Error> { } /// Validate the wire prefix and return the archive bytes (zero-copy slice). -/// Returns `None` if the magic or version doesn't match — the caller should -/// halt cleanly rather than proceed into an `access_unchecked`. +/// Returns `None` if the blob is too short or the magic or version doesn't +/// match — callers halt with a legible wrong-format error instead of +/// surfacing whatever bytecheck makes of old-format bytes. pub fn recursion_archive_bytes(blob: &[u8]) -> Option<&[u8]> { if blob.len() < RECURSION_INPUT_PREFIX_LEN { return None; diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index 1f4800011..90482a3a4 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -594,6 +594,54 @@ fn run_recursion_pipeline( ); } +/// The wire-prefix rejection path: a stale version (v1 blobs predate rkyv +/// pointer_width_64), a corrupted magic, and a blob shorter than the prefix +/// must all yield `None` — the clean "bad magic or version" error the +/// breaking-change story leans on, rather than a bytecheck error over +/// old-format bytes. Pure function, no proving needed. +#[test] +fn test_recursion_prefix_rejects_wrong_magic_version_and_short_blobs() { + let archive = [0xAAu8; 16]; + let mut blob = Vec::with_capacity(crate::RECURSION_INPUT_PREFIX_LEN + archive.len()); + blob.extend_from_slice(&crate::RECURSION_INPUT_MAGIC); + blob.extend_from_slice(&crate::RECURSION_INPUT_VERSION.to_le_bytes()); + blob.extend_from_slice(&[0u8; 4]); // reserved + blob.extend_from_slice(&archive); + + // Baseline: a well-formed prefix passes and returns exactly the archive. + assert_eq!( + crate::recursion_archive_bytes(&blob), + Some(&archive[..]), + "well-formed prefix must expose the archive bytes" + ); + + // (a) Stale wire version: a v1 blob (32-bit rel-ptrs) must be rejected. + let mut stale = blob.clone(); + stale[4..8].copy_from_slice(&1u32.to_le_bytes()); + assert_eq!( + crate::recursion_archive_bytes(&stale), + None, + "v1 blob must be rejected by the version check" + ); + + // (b) Corrupted magic. + let mut bad_magic = blob.clone(); + bad_magic[0] ^= 0xFF; + assert_eq!( + crate::recursion_archive_bytes(&bad_magic), + None, + "flipped magic byte must be rejected" + ); + + // (c) Shorter than the 12-byte prefix (including empty). + assert_eq!( + crate::recursion_archive_bytes(&blob[..crate::RECURSION_INPUT_PREFIX_LEN - 1]), + None, + "blob shorter than the prefix must be rejected" + ); + assert_eq!(crate::recursion_archive_bytes(&[]), None); +} + /// Decode the blob on the host and mirror the guest's verify+attest, then run /// the consumer check — a cheap guard on the encode/decode/attest contract /// without running the VM.