diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml
index 8e8863c30..aa066bbd5 100644
--- a/.github/workflows/bench-abba.yml
+++ b/.github/workflows/bench-abba.yml
@@ -1,9 +1,14 @@
name: Bench ABBA tiebreaker
# Drift-free paired (A/B/B/A) prover benchmark for resolving small (~1%) deltas the
-# cheap PR benchmark can't confirm. It builds both binaries and runs ~20 interleaved
-# pairs, so it OCCUPIES THE SINGLE BENCH SERVER FOR ~30-40 MIN. For that reason it
-# NEVER auto-triggers -- it runs only on an explicit `/bench-abba` comment on a PR.
+# cheap PR benchmark can't confirm. At the default workload it OCCUPIES THE SINGLE
+# BENCH SERVER FOR SEVERAL HOURS, so it NEVER auto-triggers -- it runs only on an
+# explicit `/bench-abba` comment on a PR.
+#
+# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (default: 20 pairs, ethrex 100tx
+# --continuations; "cont10" is the quick coarse option, "mono[TX]" the legacy
+# monolithic prove). Cont proofs need rkyv pointer_width_64 on both sides, so PR
+# branches older than that fix must rebase before a cont bench.
on:
issue_comment:
types: [created]
@@ -26,45 +31,112 @@ jobs:
startsWith(github.event.comment.body, '/bench-abba') &&
contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)
runs-on: [self-hosted, bench]
- # Generous ceiling so a hang/OOM can't strand the single bench runner; the
- # workload itself is ~30-40 min at the default 20 pairs (clamped to <=40).
- timeout-minutes: 120
+ # Hang guardrail, not expected duration: a cont100 CPU prove is ~7 min, so the
+ # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds.
+ timeout-minutes: 720
steps:
- - name: Acknowledge (react + occupancy notice)
- uses: actions/github-script@v7
- with:
- script: |
- await github.rest.reactions.createForIssueComment({
- owner: context.repo.owner, repo: context.repo.repo,
- comment_id: context.payload.comment.id, content: 'eyes'
- });
- await github.rest.issues.createComment({
- owner: context.repo.owner, repo: context.repo.repo,
- issue_number: context.issue.number,
- body: '⏳ **ABBA tiebreaker started** on the bench server (~30–40 min). The bench server is occupied until it finishes.'
- });
-
- - name: Resolve PR head + pair count
+ - name: Resolve PR head + bench config
id: cfg
env:
GH_TOKEN: ${{ github.token }}
PR_NUM: ${{ github.event.issue.number }}
COMMENT_BODY: ${{ github.event.comment.body }}
run: |
+ # The runner is persistent self-hosted: drop the previous run's logs so
+ # the always() result comment never tails a stale /tmp/abba_out.txt.
+ rm -f /tmp/abba_out.txt /tmp/abba_result.txt
# Resolve the head SHA (not the branch name): pinning the commit works for
# fork PRs too (the branch lives in the fork, not origin/) and avoids a
# force-push race mid-run.
HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
- # Optional pair count, e.g. "/bench-abba 32"; default 20. Clamp to [2,40]
- # so a "/bench-abba 10000" can't monopolize the single bench server.
- N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-abba[[:space:]]*\([0-9]\+\).*|\1|p')
- N=${N:-20}
- if [ "$N" -lt 2 ] 2>/dev/null || [ "$N" -gt 40 ] 2>/dev/null; then
- echo "::warning::pair count $N out of range [2,40]; using 20"
- N=20
+ # Everything after "/bench-abba" on its line, tokens in any order: a number =
+ # pair count; cont[TX]/mono[TX] = workload.
+ ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1)
+ PAIRS=20; CONTINUATIONS=1; TX_COUNT=100
+ set -f # tokens must not glob-expand against the runner's CWD
+ for tok in $ARGS; do
+ case "$tok" in
+ cont) CONTINUATIONS=1; TX_COUNT=100 ;;
+ mono) CONTINUATIONS=0; TX_COUNT=5 ;;
+ cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;;
+ mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;;
+ [0-9]*) PAIRS="$tok" ;;
+ *) echo "::warning::ignoring unrecognized token '$tok'" ;;
+ esac
+ done
+ # Digits-only + clamps: PAIRS capped so one comment can't monopolize the
+ # single bench server; mono capped at 5tx (monolithic peak heap grows with
+ # the trace; 20tx needs ~78 GB).
+ if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi
+ case "$TX_COUNT" in
+ ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;;
+ esac
+ if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then
+ echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT"
+ TX_COUNT=$TX_DEFAULT
+ fi
+ case "$PAIRS" in
+ ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 20"; PAIRS=20 ;;
+ esac
+ if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 40 ]; then
+ echo "::warning::pair count $PAIRS out of range [2,40]; using 20"
+ PAIRS=20
+ fi
+ # Even is ideal so the AB/BA orders balance; round an odd request up by one.
+ if [ "$((PAIRS % 2))" -ne 0 ]; then
+ PAIRS=$((PAIRS + 1))
+ echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance"
+ fi
+ if [ "$CONTINUATIONS" = "1" ]; then
+ WORKLOAD="ethrex ${TX_COUNT}tx continuations"
+ else
+ WORKLOAD="ethrex ${TX_COUNT}tx monolithic"
fi
- echo "pairs=$N" >> "$GITHUB_OUTPUT"
+ # Outputs land before the fail-fast below so the always() result
+ # comment is fully labeled even when this step exits early.
+ {
+ echo "pairs=$PAIRS"
+ echo "continuations=$CONTINUATIONS"
+ echo "tx_count=$TX_COUNT"
+ echo "workload=$WORKLOAD"
+ } >> "$GITHUB_OUTPUT"
+ # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx
+ # continuation proof exceeds rkyv's old 2 GiB cap and only dies after
+ # blocking the single bench server for hours. Skip the check when the
+ # fetch fails: gh api prints HTTP error bodies to stdout, so gate on
+ # its exit status AND on the payload looking like the manifest (it
+ # declares rkyv) — never treat an error blob as a missing feature.
+ # Purely textual; deletable once every open branch postdates the fix.
+ if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then
+ if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \
+ -H "Accept: application/vnd.github.raw" 2>/dev/null) \
+ && printf '%s' "$SIDE" | grep -q '^rkyv' \
+ && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then
+ MSG="PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono."
+ echo "$MSG" > /tmp/abba_out.txt # surfaces in the result comment
+ echo "::error::$MSG"
+ exit 1
+ fi
+ fi
+ echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD"
+
+ - name: Acknowledge (react + occupancy notice)
+ uses: actions/github-script@v7
+ env:
+ PAIRS: ${{ steps.cfg.outputs.pairs }}
+ WORKLOAD: ${{ steps.cfg.outputs.workload }}
+ with:
+ script: |
+ await github.rest.reactions.createForIssueComment({
+ owner: context.repo.owner, repo: context.repo.repo,
+ comment_id: context.payload.comment.id, content: 'eyes'
+ });
+ await github.rest.issues.createComment({
+ owner: context.repo.owner, repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} (a cont100 pair is ~15 min, so the default 20 pairs runs ~4.5-5 hr; pass a smaller pair count or \`cont10\` for a quicker, coarser run). The bench server is occupied until it finishes.`
+ });
- name: Checkout (full history for ref resolution)
uses: actions/checkout@v4
@@ -84,6 +156,8 @@ jobs:
env:
HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
PAIRS: ${{ steps.cfg.outputs.pairs }}
+ CONTINUATIONS: ${{ steps.cfg.outputs.continuations }}
+ TX_COUNT: ${{ steps.cfg.outputs.tx_count }}
run: |
export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
set -o pipefail
@@ -100,12 +174,14 @@ jobs:
HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
PAIRS: ${{ steps.cfg.outputs.pairs }}
OUTCOME: ${{ steps.run.outcome }}
+ WORKLOAD: ${{ steps.cfg.outputs.workload }}
with:
script: |
const fs = require('fs');
const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } };
const head = (process.env.HEAD_SHA || '').slice(0, 10), pairs = process.env.PAIRS;
- let body = `## ABBA tiebreaker — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`;
+ const workload = process.env.WORKLOAD || 'ethrex';
+ let body = `## ABBA tiebreaker — \`${head}\` vs \`main\` (${pairs} pairs, ${workload})\n\n`;
if (process.env.OUTCOME === 'success') {
const res = read('/tmp/abba_result.txt') || read('/tmp/abba_out.txt');
body += '```\n' + res + '\n```\n';
diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml
index 6928255d9..040b51c89 100644
--- a/.github/workflows/benchmark-gpu.yml
+++ b/.github/workflows/benchmark-gpu.yml
@@ -6,9 +6,12 @@ name: Benchmark GPU (PR)
# It builds the cli at the PR head and at main, runs N interleaved pairs on the GPU,
# posts the paired-t + Wilcoxon verdict back to the PR, then ALWAYS destroys the box.
#
-# Triggered by a "/bench-gpu [N]" comment on a PR (N = pair count, default 14) or via
-# workflow_dispatch. Orchestration runs on a GitHub-hosted runner; all GPU work happens
-# on the rented Vast box (provisioned by the template onstart).
+# Triggered by a "/bench-gpu [N] [cont[TX]|mono[TX]]" comment on a PR (N = pair count,
+# default 14) or via workflow_dispatch. Workload default: ethrex 100tx --continuations;
+# "mono[TX]" = legacy monolithic prove. Cont proofs need rkyv pointer_width_64 on both
+# sides, so PR branches older than that fix must rebase before a cont bench.
+# Orchestration runs on a GitHub-hosted runner; all GPU work happens on the rented
+# Vast box (provisioned by the template onstart).
#
# Requires repo secrets:
# VAST_API_KEY — https://cloud.vast.ai/manage-keys/
@@ -20,6 +23,9 @@ on:
pairs:
description: "Number of A/B/B/A pairs"
default: "14"
+ mode:
+ description: "Workload: cont[TX] (--continuations, TX defaults to 100) or mono[TX] (monolithic, TX defaults to 5)"
+ default: "cont100"
issue_comment:
types: [created]
@@ -57,12 +63,12 @@ jobs:
github.event.issue.pull_request &&
startsWith(github.event.comment.body, '/bench-gpu') &&
contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association))
- # ABBA on the GPU: provisioning + dual cuda build (~30 min) + 2*pairs proves
- # (~95s each). At the max 32 pairs (64 proves) a slow-provision box runs ~3 hr,
- # so allow headroom over that; teardown still always destroys the box.
- timeout-minutes: 210
+ # Provisioning + dual cuda build (~30 min) + 2*pairs proves (~3.5 min each at
+ # the default cont100). Sized for the 32-pair worst case (~4.5 hr) with headroom;
+ # teardown still always destroys the box.
+ timeout-minutes: 330
steps:
- - name: Resolve PR ref + pair count
+ - name: Resolve PR ref + bench config
id: config
env:
GH_TOKEN: ${{ github.token }}
@@ -70,24 +76,54 @@ jobs:
COMMENT_BODY: ${{ github.event.comment.body }}
PR_NUM: ${{ github.event.issue.number }}
DISPATCH_PAIRS: ${{ github.event.inputs.pairs }}
+ DISPATCH_MODE: ${{ github.event.inputs.mode }}
DISPATCH_REF: ${{ github.ref_name }}
run: |
if [ "$EVENT_NAME" = "issue_comment" ]; then
# Pin the head SHA (works for fork PRs; avoids a force-push race mid-run).
HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
OUT_PR_NUM="$PR_NUM"; OUT_HEAD_SHA="$HEAD_SHA"; OUT_BRANCH=""
- # "/bench-gpu 20" -> 20 pairs; otherwise default.
- N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-gpu[[:space:]]*\([0-9]\+\).*|\1|p')
- PAIRS=${N:-14}
+ # Everything after "/bench-gpu" on its line, tokens in any order:
+ # a number = pair count; cont[TX]/mono[TX] = workload (see loop below).
+ ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-gpu||p' | head -n1)
+ PAIRS=14
else
# workflow_dispatch: compare this branch vs main.
OUT_PR_NUM=""; OUT_HEAD_SHA=""; OUT_BRANCH="$DISPATCH_REF"
+ ARGS="$DISPATCH_MODE"
PAIRS=${DISPATCH_PAIRS:-14}
fi
+ # Defaults: continuations with the 100-transfer fixture (production mode).
+ CONTINUATIONS=1; TX_COUNT=100
+ set -f # tokens must not glob-expand against the runner's CWD
+ for tok in $ARGS; do
+ case "$tok" in
+ cont) CONTINUATIONS=1; TX_COUNT=100 ;;
+ mono) CONTINUATIONS=0; TX_COUNT=5 ;;
+ cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;;
+ mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;;
+ [0-9]*) PAIRS="$tok" ;;
+ *) echo "::warning::ignoring unrecognized token '$tok'" ;;
+ esac
+ done
+ # TX_COUNT and PAIRS are interpolated into the remote bash -lc below: enforce
+ # digits-only. Mono is capped at 5tx (monolithic peak heap grows with the
+ # trace; 20tx needs ~78 GB, over the 48 GB box floor).
+ if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi
+ case "$TX_COUNT" in
+ ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;;
+ esac
+ if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then
+ echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT"
+ TX_COUNT=$TX_DEFAULT
+ fi
+ case "$PAIRS" in
+ ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 14"; PAIRS=14 ;;
+ esac
# Clamp to [2,32]; out-of-range -> default. 14 ~ resolves a 2% delta. The ceiling
# keeps the worst-case run (64 proves + provisioning + dual build) under the job
# timeout above.
- if [ "$PAIRS" -lt 2 ] 2>/dev/null || [ "$PAIRS" -gt 32 ] 2>/dev/null; then
+ if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 32 ]; then
echo "::warning::pair count out of range [2,32], defaulting to 14"
PAIRS=14
fi
@@ -96,19 +132,47 @@ jobs:
PAIRS=$((PAIRS + 1))
echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance"
fi
+ if [ "$CONTINUATIONS" = "1" ]; then
+ WORKLOAD="ethrex ${TX_COUNT}tx continuations"
+ else
+ WORKLOAD="ethrex ${TX_COUNT}tx monolithic"
+ fi
+ # Outputs land before the fail-fast below so the always() result
+ # comment is fully labeled even when this step exits early.
{
echo "pr_num=$OUT_PR_NUM"
echo "head_sha=$OUT_HEAD_SHA"
echo "branch=$OUT_BRANCH"
echo "pairs=$PAIRS"
+ echo "continuations=$CONTINUATIONS"
+ echo "tx_count=$TX_COUNT"
+ echo "workload=$WORKLOAD"
} >> "$GITHUB_OUTPUT"
- echo "Using $PAIRS A/B/B/A pairs"
+ # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx
+ # continuation proof exceeds rkyv's old 2 GiB cap and only dies after the
+ # full dual build (~1 hr of GPU rental). Skip the check when the fetch
+ # fails: gh api prints HTTP error bodies to stdout, so gate on its exit
+ # status AND on the payload looking like the manifest (it declares
+ # rkyv) — never treat an error blob as a missing feature. Purely
+ # textual; deletable once every open branch postdates the fix.
+ if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then
+ REF="${OUT_HEAD_SHA:-$OUT_BRANCH}"
+ if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$REF" \
+ -H "Accept: application/vnd.github.raw" 2>/dev/null) \
+ && printf '%s' "$SIDE" | grep -q '^rkyv' \
+ && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then
+ echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono."
+ exit 1
+ fi
+ fi
+ echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD"
- name: Acknowledge (react + occupancy notice)
if: github.event_name == 'issue_comment'
uses: actions/github-script@v7
env:
PAIRS: ${{ steps.config.outputs.pairs }}
+ WORKLOAD: ${{ steps.config.outputs.workload }}
with:
script: |
await github.rest.reactions.createForIssueComment({
@@ -118,7 +182,7 @@ jobs:
// Post the "started" notice under the SAME marker the result step uses, so the
// result updates this comment in place (and re-runs reuse it rather than stacking).
const marker = 'GPU Benchmark (ABBA)';
- const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) on the CUDA prover path. This takes ~1 hr; the result will replace this comment.`;
+ const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. This takes ~2.5 hr at the default workload; the result will replace this comment.`;
const comments = await github.paginate(github.rest.issues.listComments, {
owner: context.repo.owner, repo: context.repo.repo,
issue_number: context.issue.number, per_page: 100,
@@ -170,10 +234,10 @@ jobs:
# because vast can't numerically compare the driver_version string server-side.
MIN_DRIVER: "580"
run: |
- # cpu_ram filter is in GB. Floor 48 GB: the bench workload moved to the
- # 5-transfer ethrex fixture (executor/tests/ethrex_5_transfers.bin), far smaller
- # than the old 20-transfer prove (~78 GB heap) that set the previous 96 GB floor.
- # 48 GB widens the dedicated pool (~15 vs ~11 offers).
+ # cpu_ram filter is in GB. Floor 48 GB: continuation proves are flat-memory
+ # (~10 GB) and the legacy 5tx monolithic prove also fits — far below the old
+ # 20-transfer prove (~78 GB heap) that set the previous 96 GB floor. 48 GB
+ # widens the dedicated pool (~15 vs ~11 offers).
# gpu_frac=1 requires a WHOLE-MACHINE offer (you rent every GPU on the host), so
# Vast places no other tenant on the box: CPU cores, RAM/memory bandwidth, PCIe,
# and NVMe are fully dedicated. Without it the "most expensive" sort below lands on
@@ -203,7 +267,7 @@ jobs:
sleep "$OFFER_INTERVAL"
done
if [ -z "$OFFER_ID" ]; then
- echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=96GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)"
+ echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=48GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)"
exit 1
fi
echo "id=$OFFER_ID" >> "$GITHUB_OUTPUT"
@@ -331,6 +395,8 @@ jobs:
HEAD_SHA: ${{ steps.config.outputs.head_sha }}
BRANCH: ${{ steps.config.outputs.branch }}
PAIRS: ${{ steps.config.outputs.pairs }}
+ CONTINUATIONS: ${{ steps.config.outputs.continuations }}
+ TX_COUNT: ${{ steps.config.outputs.tx_count }}
run: |
SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST"
@@ -355,8 +421,8 @@ jobs:
# explicit and robust to the template default changing.) The harness still builds the
# cli at REF_A (the PR) and origin/main in isolated worktrees, runs PAIRS interleaved
# A/B/B/A proves, and prints the paired-t CI + Wilcoxon verdict. BENCH_FEATURES routes
- # the build through the CUDA prover path. NOTE: requires this PR's bench_abba.sh change
- # (the BENCH_FEATURES env) to be on main — i.e. it only takes effect after merge.
+ # the build through the CUDA prover path; CONTINUATIONS/TX_COUNT pick the workload.
+ # The harness runs from main, so workflow/script changes take effect post-merge.
# REBUILD=1: each Vast box is fresh, GPU-specific hardware — always rebuild both
# binaries (cubin is compiled for the detected arch); never trust a cached binary.
# CUDARC_PIN: compat shim for pre-pin baseline shas. cudarc's CUDA version is now pinned
@@ -371,6 +437,7 @@ jobs:
git fetch --force origin main; $FETCH; \
git checkout -f origin/main; \
REBUILD=1 CUDARC_PIN=cuda-12080 SYSROOT_DIR=/opt/lambda-vm-sysroot BENCH_FEATURES='$BENCH_FEATURES' \
+ CONTINUATIONS=$CONTINUATIONS TX_COUNT=$TX_COUNT \
scripts/bench_abba.sh $REF_A origin/main $PAIRS"
# pipefail so a failed remote bench (e.g. a prove that dies) propagates through the
@@ -386,9 +453,10 @@ jobs:
if: always() && (steps.bench.outcome == 'success' || steps.bench.outcome == 'failure')
env:
OUTCOME: ${{ steps.bench.outcome }}
+ WORKLOAD: ${{ steps.config.outputs.workload }}
run: |
{
- echo "## GPU ABBA — ethrex 20 transfers (vs main)"
+ echo "## GPU ABBA — ${WORKLOAD:-ethrex} (vs main)"
if [ "$OUTCOME" = "success" ] && [ -s "$RUNNER_TEMP/abba_result.txt" ]; then
echo '```'
cat "$RUNNER_TEMP/abba_result.txt"
@@ -410,6 +478,7 @@ jobs:
OUTCOME: ${{ steps.bench.outcome }}
GPU_NAME: ${{ env.GPU_NAME }}
OFFER_PRICE: ${{ steps.offer.outputs.price }}
+ WORKLOAD: ${{ steps.config.outputs.workload }}
with:
script: |
const fs = require('fs');
@@ -419,9 +488,10 @@ jobs:
const pairs = process.env.PAIRS;
const gpu = (process.env.GPU_NAME || '').replace('_', ' ');
const price = process.env.OFFER_PRICE;
+ const workload = process.env.WORKLOAD || 'ethrex';
let body = `## GPU Benchmark (ABBA) — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`;
- body += `${gpu} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · drift-free A/B/B/A\n\n`;
+ body += `${gpu} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · ${workload} · drift-free A/B/B/A\n\n`;
if (process.env.OUTCOME === 'success') {
const res = read(`${tmp}/abba_result.txt`) || read(`${tmp}/abba_out.txt`);
body += '```\n' + res + '\n```\n';
diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml
index f612949a8..cc4d00a70 100644
--- a/bench_vs/lambda/recursion/Cargo.toml
+++ b/bench_vs/lambda/recursion/Cargo.toml
@@ -65,7 +65,8 @@ lambda-vm-prover = { path = "../../../prover", default-features = false, feature
"profile-markers",
] }
lambda-vm-syscalls = { path = "../../../syscalls" }
-rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] }
+# pointer_width_64: proof-format pointer width — see prover/Cargo.toml.
+rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] }
[profile.release]
debug = 2
diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml
index e4fcdb7fd..71e89beef 100644
--- a/bin/cli/Cargo.toml
+++ b/bin/cli/Cargo.toml
@@ -9,7 +9,8 @@ executor = { path = "../../executor" }
prover = { path = "../../prover", package = "lambda-vm-prover" }
stark = { path = "../../crypto/stark" }
clap = { version = "4.3.10", features = ["derive"] }
-rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] }
+# pointer_width_64: proof-format pointer width — see prover/Cargo.toml.
+rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] }
tempfile = "3"
tikv-jemallocator = "0.6"
tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true }
diff --git a/crypto/crypto/Cargo.toml b/crypto/crypto/Cargo.toml
index 6b78f81e7..9a36b2614 100644
--- a/crypto/crypto/Cargo.toml
+++ b/crypto/crypto/Cargo.toml
@@ -22,10 +22,12 @@ rand_chacha = { version = "0.3.1", default-features = false }
memmap2 = { version = "0.9", optional = true }
tempfile = { version = "3", optional = true }
libc = { version = "0.2", optional = true }
+# pointer_width_64: proof-format pointer width — see prover/Cargo.toml.
rkyv = { version = "0.8.10", default-features = false, features = [
"alloc",
"bytecheck",
"aligned",
+ "pointer_width_64",
], optional = true }
[target.'cfg(target_arch = "riscv64")'.dependencies]
diff --git a/crypto/math/Cargo.toml b/crypto/math/Cargo.toml
index df43ea975..a161056ca 100644
--- a/crypto/math/Cargo.toml
+++ b/crypto/math/Cargo.toml
@@ -25,10 +25,12 @@ num-traits = { version = "0.2.19", default-features = false }
# rkyv zero-copy (de)serialization. Optional; used by the recursion verifier to
# read a proof straight from its byte buffer with no deserialization pass.
+# pointer_width_64: proof-format pointer width — see prover/Cargo.toml.
rkyv = { version = "0.8.10", default-features = false, features = [
"alloc",
"bytecheck",
"aligned",
+ "pointer_width_64",
], optional = true }
[dev-dependencies]
diff --git a/crypto/stark/Cargo.toml b/crypto/stark/Cargo.toml
index 78d95be67..09a5c1d9d 100644
--- a/crypto/stark/Cargo.toml
+++ b/crypto/stark/Cargo.toml
@@ -20,7 +20,8 @@ log = "0.4.17"
digest = "0.10.7"
serde = { version = "1.0", features = ["derive"] }
itertools = "0.11.0"
-rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] }
+# pointer_width_64: proof-format pointer width — see prover/Cargo.toml.
+rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] }
# Parallelization crates
rayon = { version = "1.8.0", optional = true }
diff --git a/crypto/stark/src/proof/stark.rs b/crypto/stark/src/proof/stark.rs
index ba4aca2dc..9ce3ed32f 100644
--- a/crypto/stark/src/proof/stark.rs
+++ b/crypto/stark/src/proof/stark.rs
@@ -16,6 +16,17 @@ use crate::{
// `tests/bus_tests/completeness_tests.rs`. Do not add a production serde
// dependency on these types.
+// With no pointer-width feature enabled rkyv silently falls back to 32-bit
+// rel-ptrs, capping an archive at ~2 GiB — which large continuation proofs
+// exceed, and which CI round-trips (all under 2 GiB) can't catch. Pinned here,
+// where the archived proof types live, so standalone builds of this crate fail
+// if a Cargo.toml loses `pointer_width_64`; lambda-vm-prover repeats the
+// assert to cover the host + riscv64 guest graphs.
+const _: () = assert!(
+ size_of::() == 8,
+ "proof wire format requires rkyv's pointer_width_64 feature on every proof-format crate",
+);
+
#[derive(
Debug,
Clone,
diff --git a/prover/Cargo.toml b/prover/Cargo.toml
index 15344d138..748ddd9a8 100644
--- a/prover/Cargo.toml
+++ b/prover/Cargo.toml
@@ -24,7 +24,9 @@ rayon = { version = "1.8.0", optional = true }
sysinfo = { version = "0.31", default-features = false, features = ["system"] }
log = "0.4"
digest = "0.10.7"
-rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] }
+# pointer_width_64: 32-bit rel-ptrs cap an archive at ~2 GiB, which large
+# continuation proofs exceed. Keep in sync across all proof-format crates.
+rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] }
[dev-dependencies]
env_logger = "*"
diff --git a/prover/src/lib.rs b/prover/src/lib.rs
index ff9601bb4..a8e89f989 100644
--- a/prover/src/lib.rs
+++ b/prover/src/lib.rs
@@ -209,8 +209,9 @@ pub struct GuestInput {
/// 4-byte magic identifying a lambda-vm recursion input blob ("LVMR").
pub const RECURSION_INPUT_MAGIC: [u8; 4] = *b"LVMR";
-/// Wire-format version of the recursion input blob.
-pub const RECURSION_INPUT_VERSION: u32 = 1;
+/// Wire-format version of the recursion input blob. v2: rkyv pointer_width_64
+/// (64-bit rel-ptrs) — v1 archives use 32-bit offsets and are incompatible.
+pub const RECURSION_INPUT_VERSION: u32 = 2;
/// Required alignment (bytes) of the archive's first byte in guest memory.
pub const RECURSION_INPUT_ALIGN: usize = 16;
@@ -234,6 +235,18 @@ const _: () = {
);
};
+// With no pointer-width feature enabled rkyv silently falls back to 32-bit
+// rel-ptrs, capping an archive at ~2 GiB — which large continuation proofs
+// exceed, and which nothing else catches: every CI round-trip fits 32-bit
+// offsets, and RECURSION_INPUT_VERSION can't flag it since host and guest
+// compile the constant from this same file. This compiles into both the host
+// and the riscv64 guest graph (the recursion guests path-depend on this
+// crate), so a Cargo.toml losing the feature fails the build here.
+const _: () = assert!(
+ size_of::() == 8,
+ "proof wire format v2 requires rkyv's pointer_width_64 feature on every proof-format crate",
+);
+
/// Encode a [`GuestInput`] into the on-wire blob: a 12-byte
/// `magic + version + reserved` prefix followed by the rkyv archive. The prefix
/// both aligns the archive in guest memory (so in-place reads don't trap) and
@@ -251,8 +264,9 @@ pub fn encode_recursion_input(input: &GuestInput) -> Result, Error> {
}
/// Validate the wire prefix and return the archive bytes (zero-copy slice).
-/// Returns `None` if the magic or version doesn't match — the caller should
-/// halt cleanly rather than proceed into an `access_unchecked`.
+/// Returns `None` if the blob is too short or the magic or version doesn't
+/// match — callers halt with a legible wrong-format error instead of
+/// surfacing whatever bytecheck makes of old-format bytes.
pub fn recursion_archive_bytes(blob: &[u8]) -> Option<&[u8]> {
if blob.len() < RECURSION_INPUT_PREFIX_LEN {
return None;
diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs
index 1f4800011..90482a3a4 100644
--- a/prover/src/tests/recursion_smoke_test.rs
+++ b/prover/src/tests/recursion_smoke_test.rs
@@ -594,6 +594,54 @@ fn run_recursion_pipeline(
);
}
+/// The wire-prefix rejection path: a stale version (v1 blobs predate rkyv
+/// pointer_width_64), a corrupted magic, and a blob shorter than the prefix
+/// must all yield `None` — the clean "bad magic or version" error the
+/// breaking-change story leans on, rather than a bytecheck error over
+/// old-format bytes. Pure function, no proving needed.
+#[test]
+fn test_recursion_prefix_rejects_wrong_magic_version_and_short_blobs() {
+ let archive = [0xAAu8; 16];
+ let mut blob = Vec::with_capacity(crate::RECURSION_INPUT_PREFIX_LEN + archive.len());
+ blob.extend_from_slice(&crate::RECURSION_INPUT_MAGIC);
+ blob.extend_from_slice(&crate::RECURSION_INPUT_VERSION.to_le_bytes());
+ blob.extend_from_slice(&[0u8; 4]); // reserved
+ blob.extend_from_slice(&archive);
+
+ // Baseline: a well-formed prefix passes and returns exactly the archive.
+ assert_eq!(
+ crate::recursion_archive_bytes(&blob),
+ Some(&archive[..]),
+ "well-formed prefix must expose the archive bytes"
+ );
+
+ // (a) Stale wire version: a v1 blob (32-bit rel-ptrs) must be rejected.
+ let mut stale = blob.clone();
+ stale[4..8].copy_from_slice(&1u32.to_le_bytes());
+ assert_eq!(
+ crate::recursion_archive_bytes(&stale),
+ None,
+ "v1 blob must be rejected by the version check"
+ );
+
+ // (b) Corrupted magic.
+ let mut bad_magic = blob.clone();
+ bad_magic[0] ^= 0xFF;
+ assert_eq!(
+ crate::recursion_archive_bytes(&bad_magic),
+ None,
+ "flipped magic byte must be rejected"
+ );
+
+ // (c) Shorter than the 12-byte prefix (including empty).
+ assert_eq!(
+ crate::recursion_archive_bytes(&blob[..crate::RECURSION_INPUT_PREFIX_LEN - 1]),
+ None,
+ "blob shorter than the prefix must be rejected"
+ );
+ assert_eq!(crate::recursion_archive_bytes(&[]), None);
+}
+
/// Decode the blob on the host and mirror the guest's verify+attest, then run
/// the consumer check — a cheap guard on the encode/decode/attest contract
/// without running the VM.