Repository navigation
Backend: bounded application cache with expiry and metrics #1274
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance baselines | |
| # Two gates keep their baselines this way, with one set of rules | |
| # (vm/selfhost/perf_baseline.py): the ParparVM performance gate in | |
| # vm/selfhost/perf-baseline/, and the Flutter benchmark's regression gate in | |
| # scripts/flutter-bench/baseline/ (scripts/flutter-bench/flutter_baseline.py). | |
| # | |
| # The ParparVM performance gate's baselines live in vm/selfhost/perf-baseline/: | |
| # base/ holds the consolidated rows, and each pull request that changes a baseline | |
| # writes only its own pr/<number>.json, so baseline changes stay in the pull | |
| # request's diff without two branches ever editing the same file | |
| # (vm/selfhost/perf_baseline.py explains the layout). | |
| # | |
| # check every pull request: the overlays resolve against base/ without | |
| # contradicting each other, and the pull request touched only its own | |
| # overlay (base/ is the fold's alone). No paths filter: several of this | |
| # repository's pull requests exceed GitHub's paths-filter diff limit, and a | |
| # filtered check would silently not run on exactly those. | |
| # fold nightly on master: every overlay on master belongs to a merged pull | |
| # request, so it is folded into base/ and deleted, in one commit that | |
| # provably changes no verdict. Then the website is rebuilt, because the | |
| # Port Status page's ParparVM vs JDK 25 table is rendered from these rows. | |
| # A push made with GITHUB_TOKEN triggers no workflow, hence the dispatch. | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| push: | |
| branches: | |
| - master | |
| schedule: | |
| - cron: '20 3 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| check: | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| # The pull request's base commit has to be present for the diff. | |
| fetch-depth: 0 | |
| - name: The gate's own unit tests | |
| working-directory: vm/selfhost | |
| run: python3 -m unittest test_perf_gate | |
| - name: The Flutter benchmark gate's unit tests | |
| working-directory: scripts/flutter-bench | |
| run: python3 -m unittest test_flutter_baseline | |
| - name: Check the baselines | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| # A pull_request run checks out GitHub's merge of the pull request into | |
| # the CURRENT base branch, while base.sha can be an older base commit. | |
| # Diffed from that, every base/ row the nightly fold wrote since was | |
| # charged to this pull request. The merge commit's first parent is the | |
| # base branch it was merged into, so HEAD^1...HEAD is exactly the | |
| # pull request's own change. | |
| if git rev-parse -q --verify HEAD^2 >/dev/null; then | |
| BASE_SHA=$(git rev-parse HEAD^1) | |
| fi | |
| # Both, even when the first fails, so one comment shows every problem. | |
| status=0 | |
| if [ -n "${BASE_SHA}" ]; then | |
| python3 vm/selfhost/perf_baseline.py check --base "${BASE_SHA}" --pr "${PR_NUMBER}" || status=1 | |
| python3 scripts/flutter-bench/flutter_baseline.py check --base "${BASE_SHA}" --pr "${PR_NUMBER}" || status=1 | |
| else | |
| python3 vm/selfhost/perf_baseline.py check || status=1 | |
| python3 scripts/flutter-bench/flutter_baseline.py check || status=1 | |
| fi | |
| exit $status | |
| fold: | |
| if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: write | |
| actions: write | |
| concurrency: | |
| group: perf-baseline-fold | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: master | |
| # Full history: resolve() orders overlays by when they merged, and the size | |
| # follow checks no overlay merged after the measured commit. | |
| fetch-depth: 0 | |
| # The gate passes a pull request that beats its baseline (failing it turned every | |
| # VM win into a red master until someone rebaselined). What keeps such a win from | |
| # being given back unnoticed is this: master's recent gate results, and a row every | |
| # one of them beat clearly is lowered to the worst of them | |
| # (vm/selfhost/tighten-perf-baseline.py has the rule). | |
| - name: Collect master's recent performance gate results | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| dest="${RUNNER_TEMP}/perf-results" | |
| mkdir -p "${dest}" | |
| since="$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ)" | |
| for wf in linux-build-run.yml parparvm-tests-windows.yml scripts-macos.yml; do | |
| for run in $(gh run list --repo "${GITHUB_REPOSITORY}" --workflow "${wf}" \ | |
| --branch master --limit 30 --json databaseId,createdAt \ | |
| --jq ".[] | select(.createdAt >= \"${since}\") | .databaseId"); do | |
| for name in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run}/artifacts" \ | |
| --jq '.artifacts[] | select(.name | startswith("perf-results-")) | select(.expired | not) | .name'); do | |
| gh run download "${run}" --repo "${GITHUB_REPOSITORY}" -n "${name}" \ | |
| -D "${dest}/${run}-${name}" || echo "could not download ${name} of run ${run}" | |
| done | |
| done | |
| done | |
| find "${dest}" -name perf-results.json | wc -l | xargs echo "perf-results.json files:" | |
| # The Flutter gate's SIZE rows follow master: they are deterministic, so the latest | |
| # nightly measurement of master is what a pull request should be judged against | |
| # (flutter_baseline.py follow explains why). Ratios are not followed. | |
| - name: Collect master's latest nightly Flutter benchmark sizes | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| dest="${RUNNER_TEMP}/flutter-sizes" | |
| mkdir -p "${dest}" | |
| # A finished run, success or failure: a size regression fails the nightly run, | |
| # and that run is the signal; following it afterwards is what keeps it from | |
| # failing every later pull request. A cancelled run measured nothing complete. | |
| read -r run sha < <(gh run list --repo "${GITHUB_REPOSITORY}" --workflow flutter-bench.yml \ | |
| --branch master --event schedule --status completed --limit 10 \ | |
| --json databaseId,headSha,conclusion \ | |
| --jq '[.[] | select(.conclusion == "success" or .conclusion == "failure")][0] | "\(.databaseId) \(.headSha)"') || true | |
| if [ -z "${run:-}" ] || [ "${run}" = "null" ]; then | |
| echo "No finished nightly Flutter benchmark run; sizes stay as they are." | |
| exit 0 | |
| fi | |
| echo "${sha}" > "${dest}/measured-sha" | |
| for name in $(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run}/artifacts" \ | |
| --jq '.artifacts[] | select(.name | startswith("flutter-bench-")) | select(.expired | not) | .name'); do | |
| gh run download "${run}" --repo "${GITHUB_REPOSITORY}" -n "${name}" \ | |
| -D "${dest}/${name}" || echo "could not download ${name} of run ${run}" | |
| done | |
| find "${dest}" -name 'baseline-*.json' | wc -l | xargs echo "nightly size files:" | |
| - name: Fold merged overlays into base/ and tighten consistently beaten rows | |
| id: fold | |
| run: | | |
| set -euo pipefail | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| # A merge landing while this runs makes the push non-fast-forward. Start over | |
| # from the new master rather than rebasing: the fold is cheap, and redoing it | |
| # folds the overlay that just arrived as well. | |
| for attempt in 1 2 3; do | |
| git fetch --quiet origin master | |
| git reset --quiet --hard origin/master | |
| python3 vm/selfhost/perf_baseline.py fold | |
| python3 scripts/flutter-bench/flutter_baseline.py fold | |
| # Follow only a measurement of a master that already had every overlay merged | |
| # since: an overlay merged after the measured commit moved a row that run never | |
| # saw, and following the run would silently undo it. Skipped, the next night's | |
| # run includes it. | |
| : > "${RUNNER_TEMP}/followed.txt" | |
| sha="$(cat "${RUNNER_TEMP}/flutter-sizes/measured-sha" 2>/dev/null || true)" | |
| if [ -n "${sha}" ] && git merge-base --is-ancestor "${sha}" HEAD 2>/dev/null \ | |
| && [ -z "$(git log --format=%H "${sha}..HEAD" -- scripts/flutter-bench/baseline/pr)" ]; then | |
| find "${RUNNER_TEMP}/flutter-sizes" -name 'baseline-*.json' -print0 \ | |
| | xargs -0 python3 scripts/flutter-bench/flutter_baseline.py follow \ | |
| | tee "${RUNNER_TEMP}/followed.txt" | |
| else | |
| echo "Not following sizes: no nightly run, or an overlay merged after ${sha:-it} was measured." | |
| fi | |
| git add -A vm/selfhost/perf-baseline scripts/flutter-bench/baseline | |
| folded=true | |
| git diff --staged --quiet && folded=false | |
| python3 vm/selfhost/tighten-perf-baseline.py "${RUNNER_TEMP}/perf-results" | tee "${RUNNER_TEMP}/tightened.txt" | |
| git add -A vm/selfhost/perf-baseline | |
| if git diff --staged --quiet; then | |
| echo "No overlays to fold and no row to tighten." | |
| echo "folded=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| git diff --staged --stat | |
| if grep -q '^tightened ' "${RUNNER_TEMP}/tightened.txt"; then | |
| subject="ci: tighten performance baselines master beat consistently" | |
| [ "${folded}" = true ] && subject="ci: fold merged performance baselines and tighten consistently beaten rows" | |
| git commit --quiet -m "${subject}" -m "$(grep '^tightened ' "${RUNNER_TEMP}/tightened.txt")" \ | |
| -m "$(cat "${RUNNER_TEMP}/followed.txt")" | |
| else | |
| git commit --quiet -m "ci: fold merged baselines and follow master's Flutter benchmark sizes" \ | |
| -m "$(cat "${RUNNER_TEMP}/followed.txt")" | |
| fi | |
| if git push origin HEAD:master; then | |
| echo "folded=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "master moved during the fold (attempt ${attempt}); retrying." | |
| done | |
| echo "Could not push the fold after 3 attempts." >&2 | |
| exit 1 | |
| # One-shot on purpose. If this dispatch fails after the fold was pushed, a re-run | |
| # finds nothing to fold and skips it -- and that is acceptable: website-docs.yml | |
| # also rebuilds on its own daily schedule (and port-status-nightly dispatches it), | |
| # so the JDK 25 table, which shows gate baselines rather than live results, is at | |
| # most a day behind. Persisting a "rebuild owed" flag to close that window would | |
| # add state for no visible benefit. | |
| - name: Rebuild the Port Status page | |
| if: steps.fold.outputs.folded == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh workflow run website-docs.yml --ref master -f deploy_production=true |