Repository navigation
297 lines (283 loc) · 13.4 KB
/
Copy pathci-ready.yml
File metadata and controls
297 lines (283 loc) · 13.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
name: CI — ready / review lane
on:
pull_request:
branches: [main]
types: [opened, synchronize, reopened, ready_for_review]
concurrency:
group: ready-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
installed-package:
strategy:
fail-fast: false
matrix:
python: ["3.10", "3.11"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Build distributables
run: |
python -m pip install build
python -m build --outdir "$RUNNER_TEMP/dist"
- name: Smoke test a clean wheel installation outside the checkout
shell: bash
run: |
python -m venv "$RUNNER_TEMP/wheel-env"
wheels=("$RUNNER_TEMP"/dist/*.whl)
"$RUNNER_TEMP/wheel-env/bin/python" -m pip install "${wheels[0]}[skills]"
cp tests/check_installed.py "$RUNNER_TEMP/check_installed.py"
cd "$RUNNER_TEMP"
env -u PYTHONPATH INFERRSI_CHECKOUT_RESOURCES="$GITHUB_WORKSPACE/src/inferrsi" "$RUNNER_TEMP/wheel-env/bin/python" -I check_installed.py
full-check:
strategy:
fail-fast: false
matrix:
python: ["3.10", "3.11"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Install dev deps
run: python -m pip install -e ".[dev]"
- name: Ruff
run: ruff check src tests tools
- name: Tests (smoke + unit)
run: pytest -rs
dashboard-browser:
if: ${{ !github.event.pull_request.draft }}
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install browser test dependencies
run: |
python -m pip install -e ".[dev]" playwright
python -m playwright install --with-deps chromium
- name: Test dashboards and offline delivery reports in Chromium
run: |
python -m pytest -vv -o addopts='' \
tests/test_profile_story_checks.py::test_profile_and_trace_shortcuts_open_recorded_evidence \
tests/test_knowledge_navigation_browser.py \
tests/test_round_decisions_browser.py \
tests/test_dashboard_experiments_browser.py \
tests/test_result_delivery.py::test_offline_html_remains_readable_without_scripts_or_network \
--junitxml="$RUNNER_TEMP/dashboard-browser-results.xml"
- name: Retain browser test results
if: always()
uses: actions/upload-artifact@v4
with:
name: dashboard-browser-results
path: ${{ runner.temp }}/dashboard-browser-results.xml
retention-days: 30
pr-references:
# Reports closing syntax in PR body text at each ready-lane event; not a branch-protection gate.
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Reject closing syntax inside PR body text
env:
PR_BODY: ${{ github.event.pull_request.body }}
run: python3 tools/check_pr_references.py
host-compatibility:
name: host-compatibility (advisory, ${{ matrix.os }})
# Advisory only: failures, cancellation or missing runs do not block PR approval
# or merging. Keep these jobs out of required checks and gating job dependencies.
continue-on-error: true
# GitHub-hosted Windows and macOS CPU evidence, separate from the Ubuntu jobs above:
# a pass on Linux is not a pass on these hosts. It runs the eight plugin and DSH nodes
# that an earlier Windows run recorded as failing (63 passed, 8 failed), on Windows the
# Harness artifact-primitive and writer nodes, then an installed-wheel check. A pass
# here does not rewrite that record. No GPU, remote service or credential is used; the
# DSH node talks to the fake SSH server that its own check starts. One host at a time,
# one pytest process per step, no full suite.
strategy:
fail-fast: false
max-parallel: 1
matrix:
os: [windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- uses: actions/setup-node@v4
with:
node-version: "22"
- name: Install dev deps and the build frontend
run: python -m pip install -e ".[dev]" build
- name: Test the eight plugin and DSH host nodes
# Started from Python so both hosts receive identical node IDs; no shell rewrites
# "::" or "[...]". The checkout path is absolute and PYTHONPATH is not set.
shell: python
env:
RESULTS: ${{ runner.temp }}/host-compatibility-results.xml
run: |
import os
import shutil
import subprocess
import sys
import xml.etree.ElementTree as ET
nodes = [
"tests/test_codex_plugin.py::test_comparison_preview_execution_status_and_report[codex-fail-1-FAIL]",
"tests/test_codex_plugin.py::test_isolated_runtime_can_export_dashboard[workswarm]",
"tests/test_codex_plugin.py::test_comparison_preview_execution_status_and_report[workswarm-pass-0-PASS]",
"tests/test_codex_plugin.py::test_comparison_preview_execution_status_and_report[workswarm-fail-1-FAIL]",
"tests/test_codex_plugin.py::test_comparison_preview_execution_status_and_report[workswarm-missing-2-INCONCLUSIVE]",
"tests/test_codex_plugin.py::test_release_archive_is_installable_and_all_versions_match[codex]",
"tests/test_codex_plugin.py::test_release_archive_is_installable_and_all_versions_match[workswarm]",
"tests/test_dsh_plugin.py::test_remote_ssh_connection_and_scoped_operations",
]
# The DSH node skips without Node.js, and every node needs Git or Python.
for tool in ("node", "git"):
path = shutil.which(tool)
if path is None:
raise SystemExit(tool + " is required by the selected nodes")
subprocess.run([path, "--version"], check=True)
results = os.environ["RESULTS"]
command = [sys.executable, "-m", "pytest", "-vv", "-rs", "-o", "addopts=",
"--junitxml=" + results, *nodes]
status = subprocess.run(
command, cwd=os.environ["GITHUB_WORKSPACE"], check=False
).returncode
if status:
raise SystemExit(status)
# A skipped or deselected node is not a pass: all eight must have run.
suite = next(ET.parse(results).getroot().iter("testsuite"))
counts = {name: int(suite.get(name))
for name in ("tests", "errors", "failures", "skipped")}
print(counts, flush=True)
if counts != {"tests": len(nodes), "errors": 0, "failures": 0, "skipped": 0}:
raise SystemExit("every selected host node must run and pass")
- name: Test the Harness artifact-primitive and writer nodes on Windows
# #412 R2: Harness owns copies of the Infra digest, hash, file lock and atomic
# writer. On Windows the lock is an msvcrt byte range that even denies a read of
# the locked byte, which no Linux job can show. macOS uses the same fcntl lock as
# the Linux jobs, so only Windows runs these nodes. Runs even if the step above
# failed, so both results are recorded.
if: ${{ !cancelled() && runner.os == 'Windows' }}
shell: python
env:
RESULTS: ${{ runner.temp }}/host-primitives-results.xml
run: |
import os
import subprocess
import sys
import xml.etree.ElementTree as ET
nodes = [
"tests/test_skill_evaluation.py::test_harness_artifact_primitives_match_infra_bytes_and_semantics",
"tests/test_skill_evaluation.py::test_harness_owned_writes_and_digests_match_infra_bytes",
"tests/test_legacy_run_split.py::test_harness_writer_bytes_equal_the_infra_writer",
"tests/test_legacy_run_split.py::test_harness_writer_rejects_nan_and_leaves_no_file",
"tests/test_legacy_run_split.py::test_sampling_sha256_matches_the_infra_hasher",
"tests/test_optimization_session.py::test_runtime_identity_covers_migrated_process_domain_and_not_removed_executor",
"tests/test_agent_sessions.py",
]
results = os.environ["RESULTS"]
command = [sys.executable, "-m", "pytest", "-vv", "-rs", "-o", "addopts=",
"--junitxml=" + results, *nodes]
status = subprocess.run(
command, cwd=os.environ["GITHUB_WORKSPACE"], check=False
).returncode
if status:
raise SystemExit(status)
# pytest exits 0 with skips, and a skipped node is not a pass.
suite = next(ET.parse(results).getroot().iter("testsuite"))
counts = {name: int(suite.get(name))
for name in ("tests", "errors", "failures", "skipped")}
print(counts, flush=True)
if counts["skipped"]:
raise SystemExit("every selected primitive node must run and pass")
- name: Check a wheel built from the sdist in a fresh environment outside the checkout
# Runs even if a node above failed, so both results are recorded; either failure
# still fails the job.
if: ${{ !cancelled() }}
shell: python
run: |
import os
import shutil
import subprocess
import sys
from pathlib import Path
workspace = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
work = Path(os.environ["RUNNER_TEMP"]).resolve() / "host-wheel"
log = work.parent / "host-wheel.log"
assert work != workspace and workspace not in work.parents, work
work.mkdir()
# Children never see PYTHONPATH, and the checker compares with this absolute path.
environment = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"}
environment["INFERRSI_CHECKOUT_RESOURCES"] = str(workspace / "src" / "inferrsi")
def run(*command, cwd=work):
command = [str(part) for part in command]
result = subprocess.run(
command, cwd=cwd, env=environment, capture_output=True, text=True,
encoding="utf-8", errors="replace", check=False,
)
text = "$ " + " ".join(command) + "\n" + result.stdout + result.stderr
print(text, flush=True)
with log.open("a", encoding="utf-8") as stream:
stream.write(text)
if result.returncode:
raise SystemExit(f"exit {result.returncode}: {command[0]}")
return result.stdout
# python -m build writes the sdist and builds the wheel from that sdist.
dist = work / "dist"
run(sys.executable, "-m", "build", "--outdir", dist, cwd=workspace)
sdists, wheels = sorted(dist.glob("*.tar.gz")), sorted(dist.glob("*.whl"))
assert len(sdists) == 1 and len(wheels) == 1, (sdists, wheels)
venv = work / "env"
run(sys.executable, "-m", "venv", venv)
scripts = venv / ("Scripts" if os.name == "nt" else "bin")
python = scripts / ("python.exe" if os.name == "nt" else "python")
console = scripts / ("inferrsi.exe" if os.name == "nt" else "inferrsi")
run(python, "-m", "pip", "install", f"{wheels[0]}[skills]")
assert console.is_file(), console
shutil.copyfile(workspace / "tests" / "check_installed.py", work / "check_installed.py")
run(python, "-I", "check_installed.py")
# The package under test is the installed wheel, never the source checkout.
location = Path(
run(python, "-I", "-c", "import inferrsi; print(inferrsi.__file__)").strip()
).resolve()
assert venv.resolve() in location.parents, location
assert workspace not in location.parents, location
installed = run(
python, "-I", "-c", "from importlib.metadata import version; print(version('inferrsi'))"
)
# The console script and python -m inferrsi answer with the same bytes.
for arguments in (["--version"], ["benchmarks", "list"]):
module = run(python, "-I", "-m", "inferrsi", *arguments)
assert module.strip() and module == run(console, *arguments), arguments
if arguments == ["--version"]:
assert module == installed, (module, installed)
- name: Retain host results
if: always()
uses: actions/upload-artifact@v4
with:
name: host-compatibility-${{ matrix.os }}
path: |
${{ runner.temp }}/host-compatibility-results.xml
${{ runner.temp }}/host-primitives-results.xml
${{ runner.temp }}/host-wheel.log
retention-days: 30