diff --git a/.gitignore b/.gitignore index c18dd8d..a964611 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ __pycache__/ +.smoke/ +.pytest_cache/ +_site/ +docs/games/results_payload/ diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..cf7e2a2 --- /dev/null +++ b/conftest.py @@ -0,0 +1,13 @@ +"""Pytest bootstrap: put the src-layout package on sys.path without an editable +install, so `from agentbench_frame... import ...` resolves in tests and so the +`agentbench_frame.*` modules can be imported by helper scripts. + +Run tests with the Python 3.11+ interpreter (the framework imports `tomllib`): + py -3.13 -m pytest tests/ +""" +import sys +from pathlib import Path + +_SRC = Path(__file__).resolve().parent / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) diff --git a/docs/games/24_miracle_evaluation_protocol.v0.3.json b/docs/games/24_miracle_evaluation_protocol.v0.3.json new file mode 100644 index 0000000..1024c1d --- /dev/null +++ b/docs/games/24_miracle_evaluation_protocol.v0.3.json @@ -0,0 +1,112 @@ +{ + "protocol_version": "0.3-pre-registered", + "status": "PRE_REGISTERED_NOT_AUTHORIZED", + "_doc": "24_miracle 正式评测协议预注册草案。状态 DRAFT_NOT_AUTHORIZED：未冻结、未执行、未创建正式矩阵目录或 progress 文件。仅当局人明确授权后冻结并启动。", + "_generated": "2026-07-21", + + "matrix_meaning": { + "status": "USER_DECIDED_PLAN_A", + "chosen": "A_ifelse_vs_16（用户正式选择：if-else 对 rank01–16，每对手换阵营各 1 局，共 32 局）", + "evidence": "高翔 historical lessons 明确研究目标为 'if-else bot vs 每位人类决赛选手'（'Test against unchanged human finalists'）；战力记录均为 if-else vs rankNN。16×16 round-robin 在 SKILL.md 中被列为与 '完整16人矩阵' 不同的独立禁项。但精确协议（每对手局数/批量结构/timeout）材料未唯一确定。", + "options": [ + {"id": "A_ifelse_vs_16", "desc": "if-else Agent 分别对 16 个决赛策略，双方换 camp（最符合历史研究目标）", "estimated_games": "16 对手 × 2 camp × m 局/配置；m=1→32 局，m=5→160 局"}, + {"id": "B_round_robin_16x16", "desc": "16 策略间完整 round-robin（与 A 不同，规模大得多）", "estimated_games": "16×15=240 配对 × 换 camp × m 局；m=1→480 局，m=5→2400 局"}, + {"id": "C_other", "desc": "其他用户指定矩阵"} + ], + "do_not_conflate": true, + "total_attempts": 32, + "games_per_opponent": 2, + "camps_per_opponent": [0, 1], + "games_per_opponent_per_camp": 1 + }, + + "frozen_identities": { + "evaluated_agent": {"name": "miracle_ifelse", "source": "高翔 ifelse_bot/main.py", "sha256": "98199fae8875de63b41d2eacd92ad5c58b4d5aa95b01b05aeacedd0b55402f4b", "modifiable": false}, + "judge": {"source": "external_asset:judge_dev_logic", "main_py_sha256": "104f77bf4ec59b96b46ffc05e20482319f40fc83401a9becd785d0be98a7fc09", "modifiable": false}, + "opponents": "见 docs/games/24_miracle_roster_manifest.json（16 策略 archive_sha256）", + "build_artifacts_win64_mingw": { + "_note": "阶段9A 预检隔离编译产物 SHA256（g++ 15.2.0 MinGW, GNU Make 4.4.1，策略自带 makefile 原样编译）", + "rank01": "b451d4f99b694c4ac446749d904901b1235a1f8b7a52d4b06e738ceecc4b0347", + "rank02": "5c731a794eb8282f242defee099685fb20fbd7f2d7644046b5fba0385ad555f4", + "rank03": "84cdb1344104060e6341efea691077d0e3962ef60ad523e25126b18365d581eb", + "rank06": "b10de17f886ca930a8bad9a4136c1643780f26916018715486f6195332c3e704", + "rank08": "249b9c6f7bcea4b384089a0093eebed29fb67530a078fbf217cb7c274e8c6c4f", + "rank09": "3c7c822cec8cf613ff32210f61faa95b157225c51069e450d54ca1e39178be0f", + "rank10": "ae0c24618b1fdc3e88f64a3b5b61ab0edfd144fd5259f26aa2c40ef5d6467287", + "rank11": "6c8c169c1f23dc876969ac7f50f17aa05f3932bd51f7142d56c06f0757242037", + "rank12": "e92fe57054968b7b35f8e61a7fb2ac591e801b92f447e2043f9f0fa793c4c2ab", + "rank13": "d2e8ff65eaed3c979d1a5c2deb5a9e2cfafa7a3a8105690908bd4e08d4d647e9", + "rank14": "bb0f1b171251edc39a9b424c50207b529bb318dc0e82186551d899f24a26cc01", + "rank15": "c570d1807402b9fcfe9db841d88420c3cac8a5bbf1f9a2ec9e3a2b51cb7a6fb3", + "rank16": "ed92b36ba411cd52c1408762bbb6cedfc54a7a62ecb16f7d2d98cb95458372db", + "rank04_rank05_rank07": "Python，无需编译（main.py）", + "rank16_blocker": "已解除（阶段9B）：rank16 隔离构建副本内创建空 build/（构建环境准备，非策略修复；源码/Makefile 哈希编译前后一致），原 Makefile 编译成功。rank16 main.exe SHA256 已补入。编译警告 'control reaches end of non-void function' 保留为运行风险（不修复/不隐瞒，见 known_runtime_risks）。", + "rank03_note": "编译+启动成功，但保留历史运行时崩溃风险（invalid_now）；编译/启动成功 ≠ 比赛可用。" + } + }, + + "seed_policy": { + "requested_seed": null, + "effective_seed": null, + "deterministic_seed_supported": false, + "reproducible_from_seed": false, + "recorded_only": "realized_randomization={map_type, day_time}（Replay 头读出，不表述为 seed）", + "rationale": "Judge 用 random.randint 选 map_type/day_time，不读外部 seed；不修改 Judge" + }, + + "timeouts": { + "per_ai_operation_s": 8.0, + "wrapper_per_game_s": 120.0, + "note": "MAX_ROUND=100；最坏单局≈ wrapper_per_game_s；if-else 局可更长（smoke g2_01=630 steps 仍在限内）" + }, + + "validity": { + "valid_game": "合法 end_info + result-json 与 trace 分数一致 + raw_winner 符合 Judge 规则 + end_info 前无 ai_error/ai_timeout + Judge 未在 end_info 前崩溃 + Replay 存在且合法 + cleanup 完成", + "invalid_game": "AI crash / AI timeout / Judge crash / wrapper timeout / evidence_mismatch / replay_missing / replay_corrupt / result_json_missing / result_json_corrupt", + "rank03": "invalid_now（对手程序崩溃），不计入有效胜率" + }, + + "classification": { + "ai_crash": "AI 在 end_info 前自然异常退出，或 trace 出现 ai_error", + "ai_timeout": "trace 出现 ai_timeout，或对应 AI 在动作期限内未响应", + "runner_cleanup_nonzero": "end_info 后 runner 主动终止 AI 产生的非零 returncode —— 视为正常 cleanup，不判 AI 崩溃", + "judge_crash": "Judge 在 end_info 前异常退出，或无合法 end_info 且 Judge 非正常退出", + "wrapper_timeout": "vendor 及 Judge/AI 树未在 wrapper_per_game_s 内结束", + "infra_failure": "result-json 缺失/损坏、证据三向矛盾、Replay 损坏等基础设施层失败" + }, + + "scoring": { + "raw_winner_rule": "end_info={'0':s0,'1':s1}; winner = 0 if s0>s1 else 1", + "tie": "score0==score1 时 Judge 判 player1 获胜（judge_tiebreak_applied=true），记为 win/loss 而非 draw", + "win_rate": "wins / valid_games（分母仅 valid games）；valid_games==0 时 win_rate=null, evaluation_status=NO_VALID_GAMES（不报 0%）", + "draw": "仅用于异常输入/未来协议防御；当前 Judge 不产生 draw" + }, + + "execution": { + "no_rerun_successful": true, + "recovery": "失败恢复必须使用新 game_id 并注明 recovery_of=<原 game_id>；本轮不自动恢复", + "batching": "分批执行；每批设停止门槛（残留进程/证据矛盾/winner 映射不一致/数据损坏即停，不进入下一批）", + "camp_swap": "每对手双方各占 camp0/camp1（换边）", + "pid_createtime_residual_check": "每局后按精确 PID + psutil create_time 独立核验 judge/ai0/ai1 无残留；PID 复用不杀", + "cross_validation": "Replay 头/哈希、trace、result-json、events、summary、网页六处可相互追溯；任一权威字段冲突即 evidence_mismatch", + "failure_evidence_permanent": true, + "smoke_not_competitiveness": "4 局 smoke 仅为基础设施验证，不作为 if-else 竞争力结论" + }, + + "estimate": { + "time": "取决于 matrix_meaning 选择与 m：A 方案 32 局≈0.5–1.5h，160 局≈2.5–7h；B 方案 480 局≈4–12h，2400 局≈1–2 天（含 C++ 编译）", + "storage": "每局≈ trace+replay+result-json+stdout+stderr（KB–数十 KB 级）；A 方案<50MB，B 方案可达 GB 级", + "worst_case": "每局逼近 wrapper_per_game_s；总时间≈总局数×wrapper_per_game_s" + }, + + "known_runtime_risks": [ + {"id": "rank03_crash", "strategy": "rank03", "risk": "历史运行时崩溃（SKILL.md 记载当前环境对手程序崩溃）", "handling": "保持原策略不变；正式对局若在 end_info 前崩溃则记 invalid 并保留原始证据；不补跑；不计入 Agent 有效胜率（不进 win_rate 分母）", "not_an_infra_blocker": true}, + {"id": "rank16_compiler_warning", "strategy": "rank16", "risk": "编译警告 control reaches end of non-void function（ai-sample.cpp 等）", "handling": "属运行风险，不修复、不隐瞒、不改源码；正式对局异常按 invalid 处理并保留证据"}, + {"id": "general_cpp_warnings", "risk": "多个 C++ 策略编译产生 -Wreturn-type 等警告（非错误）", "handling": "未修改任何策略源码/Makefile/优化；警告仅记录，运行时异常按分类规则处理"} + ], + "blockers_before_matrix": [ + "FORMAL_32_GAME_MATRIX_NOT_AUTHORIZED" + ], + + "not_authorized": ["完整16人矩阵执行", "16×16 round-robin", "RL/Round81/hidden eval", "修改 if-else 策略", "push/PR/上传 Results/合并 main", "删除或重跑 smoke 证据"] +} diff --git a/docs/games/24_miracle_hl_closure_20260725/HL_CLOSURE_REPORT.md b/docs/games/24_miracle_hl_closure_20260725/HL_CLOSURE_REPORT.md new file mode 100644 index 0000000..9fd0bb0 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/HL_CLOSURE_REPORT.md @@ -0,0 +1 @@ +# Minimal HL closure`n`nResult: flow completed; strategy improvement was not demonstrated. v0 and v1 validation scores are both 0.5, so gain is 0.0. The experiment is non-deterministic and has a small sample; it is not evidence of performance improvement.`n`nThe one permitted strategy update changed the camp1 default opening from FF to SF. rank04 is train; rank09 is validation. Local raw evidence is referenced only by logical path and SHA256 in the posthoc manifest. diff --git a/docs/games/24_miracle_hl_closure_20260725/SUBMISSION_SCOPE.md b/docs/games/24_miracle_hl_closure_20260725/SUBMISSION_SCOPE.md new file mode 100644 index 0000000..272c888 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/SUBMISSION_SCOPE.md @@ -0,0 +1,5 @@ +# Submission scope + +Included: this directory only. Excluded: smoke sessions, replay, trace, +stdout, stderr, raw Provider JSONL, strategy source, Judge, opponent source, +binaries, caches, credentials, and machine-specific paths. diff --git a/docs/games/24_miracle_hl_closure_20260725/budget_summary.json b/docs/games/24_miracle_hl_closure_20260725/budget_summary.json new file mode 100644 index 0000000..dd943a8 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/budget_summary.json @@ -0,0 +1,12 @@ +{ + "coding_agent_acts": 1, + "episode_read": 2, + "step_read": 1747, + "input_tokens": 135157, + "output_tokens": 1459, + "total_tokens": 136616, + "agent_wall_clock_s": 52.88, + "tool_calls": 10, + "api_cost": null, + "gpu_cost_time": "not_used" +} diff --git a/docs/games/24_miracle_hl_closure_20260725/data_check.md b/docs/games/24_miracle_hl_closure_20260725/data_check.md new file mode 100644 index 0000000..ed029d6 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/data_check.md @@ -0,0 +1,5 @@ +# Data check + +- source: `local-only replacement session standard data directory` +- result: `4 valid, 0 invalid` +- coverage: v0/v1 train/validation, four runs total diff --git a/docs/games/24_miracle_hl_closure_20260725/failure_history.md b/docs/games/24_miracle_hl_closure_20260725/failure_history.md new file mode 100644 index 0000000..7ee7c81 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/failure_history.md @@ -0,0 +1 @@ +# Failure history`n`nThe earlier session had two infrastructure-invalid rank04 attempts caused by a relative replay path being resolved from the Judge working directory. It remains preserved. The replacement session supersedes it after the infrastructure repair; total actual attempts across both sessions: 10. diff --git a/docs/games/24_miracle_hl_closure_20260725/population_metadata.json b/docs/games/24_miracle_hl_closure_20260725/population_metadata.json new file mode 100644 index 0000000..ec352de --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/population_metadata.json @@ -0,0 +1,10 @@ +{ + "rank04": { + "role": "train", + "sha256": "024536e586604a7ce1b59ac5ba9e78e454d5c5ead0abaa8b3c1980c2ca2799d0" + }, + "rank09": { + "role": "validation", + "sha256": "1929f2e9ae6404da0b240b3b05e7cb3872cbea6d81680fcba11a34633ed349fc" + } +} diff --git a/docs/games/24_miracle_hl_closure_20260725/posthoc_manifest_note.md b/docs/games/24_miracle_hl_closure_20260725/posthoc_manifest_note.md new file mode 100644 index 0000000..2c41f57 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/posthoc_manifest_note.md @@ -0,0 +1 @@ +# Posthoc manifest notice`n`nThe runtime omitted its preregistration manifest. The local manifest is explicitly labeled POSTHOC_RECONSTRUCTED, not preregistration. Its pre-execution inventory fixes every pre-existing file hash; unavailable pre-execution repair hashes are null and marked unavailable_not_recorded_before_execution. diff --git a/docs/games/24_miracle_hl_closure_20260725/provider_local_evidence.md b/docs/games/24_miracle_hl_closure_20260725/provider_local_evidence.md new file mode 100644 index 0000000..4920a0f --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/provider_local_evidence.md @@ -0,0 +1 @@ +# Provider evidence boundary`n`nRaw Provider JSONL is local-only and excluded from this payload. Its logical reference and SHA256 appear only in the local posthoc manifest. The public audit contains no credentials, raw model stream, local workspace path, or source code. diff --git a/docs/games/24_miracle_hl_closure_20260725/replay_fields.md b/docs/games/24_miracle_hl_closure_20260725/replay_fields.md new file mode 100644 index 0000000..67343a4 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/replay_fields.md @@ -0,0 +1 @@ +# Replay field guide`n`nLocal replay evidence is excluded from this payload. Derived game records retain: game_id, version, opponent role, camp, validity, normalized result, steps, and error type. Logical references and SHA256 values in the posthoc manifest bind these fields to local evidence. diff --git a/docs/games/24_miracle_hl_closure_20260725/sanitized_events.jsonl b/docs/games/24_miracle_hl_closure_20260725/sanitized_events.jsonl new file mode 100644 index 0000000..99ad970 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/sanitized_events.jsonl @@ -0,0 +1,8 @@ +{"event":"game","game_id":"v0_rank04_00_camp0","version":"v0","opponent":"rank04","split":"train","camp":0,"valid":true,"normalized_result":"win","steps":784,"error_type":null} +{"event":"game","game_id":"v0_rank04_01_camp1","version":"v0","opponent":"rank04","split":"train","camp":1,"valid":true,"normalized_result":"loss","steps":963,"error_type":null} +{"event":"game","game_id":"v0_rank09_00_camp0","version":"v0","opponent":"rank09","split":"validation","camp":0,"valid":true,"normalized_result":"win","steps":688,"error_type":null} +{"event":"game","game_id":"v0_rank09_01_camp1","version":"v0","opponent":"rank09","split":"validation","camp":1,"valid":true,"normalized_result":"loss","steps":729,"error_type":null} +{"event":"game","game_id":"v1_rank04_00_camp0","version":"v1","opponent":"rank04","split":"train","camp":0,"valid":true,"normalized_result":"win","steps":784,"error_type":null} +{"event":"game","game_id":"v1_rank04_01_camp1","version":"v1","opponent":"rank04","split":"train","camp":1,"valid":true,"normalized_result":"loss","steps":963,"error_type":null} +{"event":"game","game_id":"v1_rank09_00_camp0","version":"v1","opponent":"rank09","split":"validation","camp":0,"valid":true,"normalized_result":"win","steps":688,"error_type":null} +{"event":"game","game_id":"v1_rank09_01_camp1","version":"v1","opponent":"rank09","split":"validation","camp":1,"valid":true,"normalized_result":"loss","steps":729,"error_type":null} diff --git a/docs/games/24_miracle_hl_closure_20260725/sanitized_summary.json b/docs/games/24_miracle_hl_closure_20260725/sanitized_summary.json new file mode 100644 index 0000000..3dd9b8b --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/sanitized_summary.json @@ -0,0 +1,49 @@ +{ + "schema": "derived_public_audit_v1", + "attempts": 8, + "valid": 8, + "invalid": 0, + "versions": { + "v0": { + "wins": 2, + "losses": 2, + "draws": 0, + "train": { + "wins": 1, + "losses": 1, + "steps": 1747 + }, + "validation": { + "wins": 1, + "losses": 1, + "steps": 1417 + }, + "total_steps": 3164 + }, + "v1": { + "wins": 2, + "losses": 2, + "draws": 0, + "train": { + "wins": 1, + "losses": 1, + "steps": 1747 + }, + "validation": { + "wins": 1, + "losses": 1, + "steps": 1417 + }, + "total_steps": 3164 + } + }, + "raw": 0.5, + "evo": 0.5, + "gain": 0, + "information_gain": null, + "information_gain_status": "pending_authoritative_definition", + "evidence": { + "session_logical_reference": "iteration/20260725-063826_764716", + "posthoc_manifest_sha256": "E06CCE2D7DE82A435185460E05AB24A6323392DEF9CCB25BB9E2FB92DFD5A034" + } +} diff --git a/docs/games/24_miracle_hl_closure_20260725/version_metadata.json b/docs/games/24_miracle_hl_closure_20260725/version_metadata.json new file mode 100644 index 0000000..46541c4 --- /dev/null +++ b/docs/games/24_miracle_hl_closure_20260725/version_metadata.json @@ -0,0 +1,18 @@ +{ + "v0": { + "version_id": "miracle_ifelse/v0", + "parent": null, + "source_sha256": "209f182637e1abaee4ff50de6b1a37777fbba9fbab5f611aae58a06a109c0a3b", + "workspace_sha256": "209f182637e1abaee4ff50de6b1a37777fbba9fbab5f611aae58a06a109c0a3b", + "immutable": true + }, + "v1": { + "version_id": "miracle_ifelse/v1", + "parent": "miracle_ifelse/v0", + "source_sha256": "64d8496a3d20b3357bec86d831c4511306c4e5421ac16a3fa8e0c744897a7087", + "workspace_sha256": "64d8496a3d20b3357bec86d831c4511306c4e5421ac16a3fa8e0c744897a7087", + "diff_sha256": "a2be0cfd65a6dd86072fa3e2cc391834a94ac5c2df927326581191fb89577a58", + "immutable": true, + "change": "camp1 default opening FF to SF" + } +} diff --git a/docs/games/24_miracle_roster_manifest.json b/docs/games/24_miracle_roster_manifest.json new file mode 100644 index 0000000..3001b26 --- /dev/null +++ b/docs/games/24_miracle_roster_manifest.json @@ -0,0 +1,39 @@ +{ + "_doc": "24_miracle 16 决赛策略运行时兼容性静态盘点。仅静态检查与轻量入口审计；未编译、未启动任何 Judge/AI 比赛/完整策略。每个策略的 verification_status 标注为 static_only，绝不写成已成功运行。", + "_generated": "2026-07-21", + "_source_extracted": "external_asset:24_miracle_final/extracted", + "_source_archives": "external_asset:24_miracle_final/archives", + "_toolchain_observed": { + "g++": "MinGW 15.2.0 (x86_64-win32-seh) available at /c/Program Files/mingw64/bin/g++", + "make": "GNU Make 4.4.1 available", + "note": "本机有 g++/make，可编译 C++ 策略；但 MinGW `g++ -o main` 产出 main.exe，而 vendor/上游 run_match.resolve_ai_command 只探测 ./main（无扩展名），故 Windows 下编译后仍无法按现 runner 协议启动 C++ 策略。" + }, + "_runner_launch_gap": "vendor/miracle_local/run_match.py::resolve_ai_command 仅识别 ./main 与 main.py；MinGW 产出的 main.exe 不被识别。C++ 策略需显式 --p0-cmd/--p1-cmd main.exe、或适配层补 main.exe 探测、或在 Linux/WSL 运行。", + "strategies": [ + {"rank": 1, "username": "Bruce", "display_name": "朱昱熹", "entity": "Maiev", "language": "make", "version": 10, "type": "cpp_source", "entry": "makefile -> main (g++ -std=c++11 gameunit.cpp calculator.cpp ai_client.cpp ai.cpp -o main)", "archive_sha256": "b4603d7197447a3df9bfff126da9340659ea6977d320814febb79a5df0df4fed", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 2, "username": "wiku30", "display_name": "赵梓硕", "entity": "碧海潮生曲", "language": "make", "version": 18, "type": "cpp_source", "entry": "makefile -> main (含 backup.bat，仅为 git 操作非构建)", "archive_sha256": "6b440894709982433cddbe047bae211e7b45988c3d1cfd2dd0bf4e91c38b7708", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 3, "username": "nzhtl1477", "display_name": "李欣隆", "entity": "深浅值藏的第六分块", "language": "make", "version": 1, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "6c526e491ccc5090ece014ababde0c5c021acc3c789981cf50cd204262e7547e", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only", "invalid_now": true, "invalid_reason": "SKILL.md 记载当前环境对手程序崩溃，不能算 Agent 胜利"}, + {"rank": 4, "username": "SuperJasper", "display_name": "宋子萌", "entity": "K", "language": "python_zip", "version": 1, "type": "python_script", "entry": "main.py", "runnable_sha256": "1dfe69738141f80f7d2fb5ce1a8321126141360d86d515caf66d3f4010d86f88", "archive_sha256": "ec7c574c21625736ae38dede760e0cdb796b9cb375216f9086b28c00fe07d2e2", "windows_native_runnable": true, "needs_compile": false, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": true, "verification_status": "static_only", "skill_note": "有效战胜（SKILL.md）"}, + {"rank": 5, "username": "zex18", "display_name": "周恩贤", "entity": "zex的人工智障", "language": "python_zip", "version": 56, "type": "python_script", "entry": "main.py", "runnable_sha256": "4166a54d042e9e2e5b37cf172705dbf7617872ffb1184adb3ceb697bcc6b9707", "archive_sha256": "363d68c5fbdf1e6d869ca3a3612fd67b76fae8abc051b3ad164a8942bed0ec87", "windows_native_runnable": true, "needs_compile": false, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": true, "verification_status": "static_only", "skill_note": "较接近的强对手"}, + {"rank": 6, "username": "robinliu", "display_name": "刘子奇", "entity": "Mooncell", "language": "make", "version": 78, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "dc25d4357ebee7aa6274f8b54cee885c5387a62e1ec62dcc71446f4309cd4806", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 7, "username": "blazingBonfire", "display_name": "陈英发", "entity": "守护者", "language": "python_zip", "version": 22, "type": "python_script", "entry": "main.py", "runnable_sha256": "eab339ef43fcc521ec8eb7bcb39859e35e8ea4681bb00661a8faf91c5d63816c", "archive_sha256": "410269e658407193786328505fb0bdf8b585c295ed5322315d6ac35a8c755ba1", "windows_native_runnable": true, "needs_compile": false, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": true, "verification_status": "static_only", "skill_note": "接近但未解决"}, + {"rank": 8, "username": "wenyi", "display_name": "洪文逸", "entity": "yyy", "language": "make", "version": 1, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "81e4d6a21a5790d7384be8cf9c8fe172b7fdc97933186204a05fe28e52914461", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 9, "username": "hsiachi", "display_name": "夏奇", "entity": "タチコマ(na)", "language": "make", "version": 8, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "48e2db35e6804ba4e0b8717fd2fd4b50d9ea931dd33a158fbe914838b69a2701", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 10, "username": "SD_le", "display_name": "杨卓毅", "entity": "AK", "language": "make", "version": 8, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "efab49207a2f89509ebf64dc286b31f1147e1af65a4fd0bedca28d66d1539f50", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 11, "username": "jasonvictoryan", "display_name": "颜杰龙", "entity": "Halcyon", "language": "make", "version": 14, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "656d009766064a096f135a8aca6b6a7d2f695e5c0eed2f3b85aba7b6d3ed812e", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 12, "username": "omegafantasy", "display_name": "刘家宏", "entity": "星之梦", "language": "make", "version": 27, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "ea7be1c136d39404e9e788d609a16f0563762acb1cb542645049d05d1352dc67", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only", "skill_note": "接近但未解决"}, + {"rank": 13, "username": "ZarkLngeW", "display_name": "张龙文", "entity": "啥都没改", "language": "make", "version": 1, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "3f0e72b410e1d065414327e1018a973aa98992240442d41466b0adea41c1a976", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only", "skill_note": "Archer lattice 研究断点之一"}, + {"rank": 14, "username": "ccz181078", "display_name": "蔡承泽", "entity": "sample", "language": "make", "version": 21, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "1b1abdc0c8e619f20688627b6b49d41862dbac72c6ffd1a0671a9acb19abf256", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 15, "username": "tshoigyr", "display_name": "高宇睿", "entity": "a_idoit", "language": "make", "version": 21, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "5170538ce62fbeacd7d837c2b2f3c6f8fe41a373c46ef1f497fa1bb2ac6773de", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only"}, + {"rank": 16, "username": "MoebiusMeow", "display_name": "郭佳琪", "entity": "起床大失败", "language": "make", "version": 1, "type": "cpp_source", "entry": "makefile -> main", "archive_sha256": "5f606babd6f99c8f1fe6b62ad959c233af01b44d6cc4e063510684db8785268a", "windows_native_runnable": "not_without_compile_and_mainexe_fix", "needs_compile": true, "needs_wsl_or_linux": false, "compiled_artifacts_present": false, "missing_files": false, "runner_launchable_via_current_protocol": false, "verification_status": "static_only", "skill_note": "Archer lattice 研究断点之一；由大败改善到接近平局但仍未获胜"} + ], + "_summary": { + "total": 16, + "python_script": ["rank04", "rank05", "rank07"], + "cpp_source": ["rank01", "rank02", "rank03", "rank06", "rank08", "rank09", "rank10", "rank11", "rank12", "rank13", "rank14", "rank15", "rank16"], + "windows_native_runnable_now": ["rank04", "rank05", "rank07"], + "needs_compile_plus_mainexe_fix_or_wsl": "13 (all cpp_source)", + "invalid_now": ["rank03"], + "runner_launchable_via_current_protocol": ["rank04", "rank05", "rank07"] + } +} diff --git a/docs/superpowers/plans/2026-07-25-miracle-evaluation-reproducibility.md b/docs/superpowers/plans/2026-07-25-miracle-evaluation-reproducibility.md new file mode 100644 index 0000000..a669696 --- /dev/null +++ b/docs/superpowers/plans/2026-07-25-miracle-evaluation-reproducibility.md @@ -0,0 +1,322 @@ +# 24 Miracle Evaluation Reproducibility Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make PR #3 runnable from its own checkout with explicit control-plane inputs, reproducible resume checks, and deterministic Python opponent identity verification. + +**Architecture:** Keep `tools/miracle_matrix.py` as the game-specific CLI boundary, but move input loading and preflight ahead of session creation. Store protocol/roster hashes in the existing `MatrixRunner` manifest and validate them read-only before resume. Resolve opponent directories deterministically and verify Python entry-point hashes in addition to the existing C++ artifact checks. + +**Tech Stack:** Python 3.11+, `argparse`, `pathlib`, `hashlib`, JSON, pytest, existing AgentBench tracking and Miracle matrix runner. + +## Global Constraints + +- Do not add the complete evaluation set, replay corpus, or large binaries. +- Do not define or calculate information gain, raw/evo/gain/AUC, or RL statistics in the game adapter. +- Preserve existing event and manifest fields; additions must be forward-compatible. +- Resume verification must be read-only and must not change the session on failure. +- Local `main` is never merged; changes stay on the independent PR #3 repair branch. +- CLI defaults must preserve existing invocation paths while allowing explicit path overrides. + +--- + +### Task 1: Bundle control-plane metadata and add preflight loading + +**Files:** +- Create: `docs/games/24_miracle_evaluation_protocol.v0.3.json` +- Create: `docs/games/24_miracle_roster_manifest.json` +- Modify: `tools/miracle_matrix.py:1-80` +- Create: `tests/miracle/test_matrix_cli_inputs.py` + +**Interfaces:** +- Produces `PreflightError(RuntimeError)` for user-facing input failures. +- Produces `ControlInputs` with fields `protocol: dict`, `roster: dict`, and `hashes: dict[str, str]`. +- Produces `load_control_inputs(protocol_path: Path, roster_path: Path, expected_protocol_sha: str | None = None) -> ControlInputs`. +- Produces `parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace` with `--dry-run`, `--resume`, `--protocol`, `--roster`, and `--protocol-sha`. + +- [ ] **Step 1: Restore the two small metadata files from the canonical PR #1 protocol.** + +Use the v0.3 JSON and roster manifest already present in `origin/pr-1`; keep their metadata and frozen hashes unchanged, and add `runnable_sha256` only to the three Python strategy entries using the hashes of their current `main.py` entry points: + +```text +rank04: 1dfe69738141f80f7d2fb5ce1a8321126141360d86d515caf66d3f4010d86f88 +rank05: 4166a54d042e9e2e5b37cf172705dbf7617872ffb1184adb3ceb697bcc6b9707 +rank07: eab339ef43fcc521ec8eb7bcb39859e35e8ea4681bb00661a8faf91c5d63816c +``` + +- [ ] **Step 2: Write failing tests for clear control-input failures and argument parsing.** + +```python +def test_load_control_inputs_reports_missing_file(tmp_path): + with pytest.raises(PreflightError, match="protocol file missing"): + load_control_inputs(tmp_path / "missing.json", tmp_path / "roster.json") + + +def test_parse_args_accepts_explicit_control_paths(tmp_path): + args = parse_args(["--dry-run", "--protocol", str(tmp_path / "p.json"), + "--roster", str(tmp_path / "r.json")]) + assert args.dry_run is True + assert args.protocol == tmp_path / "p.json" + assert args.roster == tmp_path / "r.json" +``` + +- [ ] **Step 3: Run the focused tests and confirm they fail for the missing interfaces.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_cli_inputs.py` + +Expected: FAIL because `PreflightError`, `ControlInputs`, `load_control_inputs`, and `parse_args` are not implemented. + +- [ ] **Step 4: Implement minimal preflight loading and parser defaults.** + +The loader must check existence, JSON decoding, object type, protocol SHA when supplied, and roster strategy ranks exactly equal to `1..16`; every failure raises `PreflightError` with the input label. The parser must preserve the current defaults and return `Path` objects for path options. + +- [ ] **Step 5: Run the focused tests and commit the control-plane change.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_cli_inputs.py` + +Expected: PASS. + +```bash +git add docs/games/24_miracle_evaluation_protocol.v0.3.json \ + docs/games/24_miracle_roster_manifest.json \ + tools/miracle_matrix.py tests/miracle/test_matrix_cli_inputs.py +git commit -m "fix(24_miracle): add control-plane preflight inputs" +``` + +### Task 2: Persist and verify control-input hashes in sessions + +**Files:** +- Modify: `src/agentbench_frame/games/miracle/matrix_runner.py:104-230,440-466` +- Modify: `tools/miracle_matrix.py:100-140,165-205` +- Modify: `tests/miracle/test_matrix_runner.py:82-100` +- Modify: `tests/miracle/test_section2_strictness.py:341-433` + +**Interfaces:** +- Extends `MatrixRunner.record_manifest(..., control_inputs: dict[str, dict[str, str]] | None = None) -> Path`. +- Extends `verify_session_for_resume(..., expected_control_inputs: dict[str, str] | None = None) -> tuple[bool, list[str]]`. +- Stores `manifest["control_inputs"]` as `{name: {"path": str, "sha256": str}}`. +- `_run_resume(r, resume_sid, *, control_inputs=None, ...)` remains callable with its existing two positional arguments for compatibility. + +- [ ] **Step 1: Write failing tests for manifest persistence and mismatch rejection.** + +```python +def test_manifest_records_control_input_hashes(tmp_path): + r = _runner(tmp_path, lambda **k: None) + r.prepare_session() + r.record_manifest( + opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}, + control_inputs={"protocol": {"path": "/p.json", "sha256": "p"}, + "roster": {"path": "/r.json", "sha256": "r"}}, + ) + manifest = json.loads((r.session_dir / "manifest.json").read_text()) + assert manifest["control_inputs"]["protocol"]["sha256"] == "p" + + +def test_resume_rejects_control_input_hash_change(tmp_path): + manifest = _valid_manifest(tmp_path) + manifest["control_inputs"] = { + "protocol": {"path": "/p.json", "sha256": "old"}, + "roster": {"path": "/r.json", "sha256": "same"}, + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + ok, errors = verify_session_for_resume( + tmp_path, + expected_control_inputs={"protocol": "new", "roster": "same"}, + ) + assert not ok and any("control input hash mismatch" in error for error in errors) +``` + +- [ ] **Step 2: Run the focused tests and confirm the new assertions fail.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_runner.py::test_manifest_records_control_input_hashes tests/miracle/test_section2_strictness.py::test_resume_rejects_control_input_hash_change` + +Expected: FAIL because the manifest does not store or verify `control_inputs`. + +- [ ] **Step 3: Implement forward-compatible manifest storage and read-only verification.** + +Store `control_inputs` only when supplied so old callers remain valid. When `expected_control_inputs` is supplied, reject missing entries, unexpected entries, or mismatched SHA-256 values without writing any file. + +- [ ] **Step 4: Run the focused and existing strictness tests.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_runner.py tests/miracle/test_section2_strictness.py` + +Expected: PASS. + +- [ ] **Step 5: Commit the session identity change.** + +```bash +git add src/agentbench_frame/games/miracle/matrix_runner.py \ + tools/miracle_matrix.py tests/miracle/test_matrix_runner.py \ + tests/miracle/test_section2_strictness.py +git commit -m "fix(24_miracle): persist control input identities" +``` + +### Task 3: Make opponent resolution deterministic and verify Python entries + +**Files:** +- Modify: `tools/miracle_matrix.py:55-98` +- Create: `tests/miracle/test_matrix_identity.py` + +**Interfaces:** +- Produces `resolve_unique_dir(root: Path, pattern: str) -> Path`. +- Produces `resolve_opponent_dir(rank: int, roster: dict, *, extracted_root: Path, precheck_root: Path, rank16_build_root: Path) -> Path`. +- Produces `verify_python_strategy_hashes(strategy: dict, extracted_root: Path, archives_root: Path | None = None) -> list[str]`. +- Extends `verify_hashes(v3, roster, *, extracted_root=EXTRACTED, archives_root=ARCHIVES, precheck_root=PRECHECK_9A, rank16_build_root=RANK16_BUILD) -> list[str]`. + +- [ ] **Step 1: Write failing tests for zero/multiple Python directories and modified `main.py`.** + +```python +def test_resolve_unique_dir_rejects_ambiguous_matches(tmp_path): + (tmp_path / "rank04__a").mkdir() + (tmp_path / "rank04__b").mkdir() + with pytest.raises(RuntimeError, match="multiple opponent directories"): + resolve_unique_dir(tmp_path, "rank04__*") + + +def test_verify_hashes_detects_modified_python_entry(tmp_path): + python_dir = tmp_path / "rank04__fixture" + python_dir.mkdir() + (python_dir / "main.py").write_text("modified\n") + roster = {"strategies": [{"rank": 4, "type": "python_script", + "entry": "main.py", + "runnable_sha256": "expected"}]} + errors = verify_python_strategy_hashes(roster["strategies"][0], tmp_path) + assert any("rank04" in error and "runnable sha" in error for error in errors) +``` + +- [ ] **Step 2: Run the identity tests and confirm they fail.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_identity.py` + +Expected: FAIL because the resolver accepts `next(glob)` semantics and Python entry hashes are not checked. + +- [ ] **Step 3: Implement deterministic resolution and Python hash checks.** + +Use sorted matches and require exactly one extracted directory. For Python strategies require the declared `entry`, require its file to exist, and compare its SHA-256 with `runnable_sha256`. Also verify the uniquely matched archive bytes against `archive_sha256` when `archives_root` is available; report missing or ambiguous archive matches as identity errors. Keep the existing C++ `main.exe` hash behavior unchanged. + +- [ ] **Step 4: Run the identity and existing matrix tests.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_identity.py tests/miracle/test_matrix_runner.py tests/miracle/test_pr1_expanded.py` + +Expected: PASS. + +- [ ] **Step 5: Commit the identity verification change.** + +```bash +git add tools/miracle_matrix.py tests/miracle/test_matrix_identity.py +git commit -m "fix(24_miracle): verify deterministic opponent identities" +``` + +### Task 4: Integrate the CLI, preserve compatibility, and verify the complete flow + +**Files:** +- Modify: `tools/miracle_matrix.py:100-215` +- Modify: `tests/miracle/test_section2_strictness.py:319-433` +- Create: `tests/miracle/test_matrix_cli_flow.py` + +**Interfaces:** +- `main(argv: Sequence[str] | None = None) -> int` consumes parsed paths and modes. +- CLI path options are `--protocol`, `--roster`, `--session-root`, `--judge-dir`, `--ifelse-dir`, `--extracted-root`, `--archives-root`, `--precheck-root`, and `--rank16-build-root`. +- New sessions perform control-input preflight before `MatrixRunner.prepare_session()`. +- Resume passes current control-input hashes to `verify_session_for_resume()` before opening `matrix.full.log`. + +- [ ] **Step 1: Write failing CLI flow tests.** + +```python +def test_main_missing_protocol_returns_preflight_error_without_session(tmp_path, monkeypatch, capsys): + rc = main(["--dry-run", "--protocol", str(tmp_path / "missing.json"), + "--roster", str(tmp_path / "roster.json"), + "--session-root", str(tmp_path / "sessions")]) + assert rc == 2 + assert "protocol file missing" in capsys.readouterr().err + assert not (tmp_path / "sessions").exists() + + +def test_cli_resume_verifier_receives_control_hashes(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + sid = "sid" + session_dir = tmp_path / sid + session_dir.mkdir() + monkeypatch.setattr(mm, "SESSION_ROOT", tmp_path) + observed = {} + + def fake_verify(_session_dir, **kwargs): + observed.update(kwargs) + return False, ["stop"] + + monkeypatch.setattr(mm, "verify_session_for_resume", fake_verify) + + class FakeRunner: + def __init__(self): + self.session_dir = session_dir + + rc = mm._run_resume( + FakeRunner(), sid, + control_inputs={"protocol": {"path": "/p.json", "sha256": "p"}, + "roster": {"path": "/r.json", "sha256": "r"}}, + ) + assert rc == 2 + assert observed["expected_control_inputs"] == {"protocol": "p", "roster": "r"} +``` + +- [ ] **Step 2: Run the CLI flow tests and confirm they fail.** + +Run: `PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_cli_flow.py tests/miracle/test_section2_strictness.py` + +Expected: FAIL because `main()` only scans `sys.argv`, reads control files before validation, creates sessions before preflight, and does not pass control hashes to resume verification. + +- [ ] **Step 3: Refactor `main()` to use parsed arguments and preflight before side effects.** + +Pass explicit paths into the resolver closure and `MatrixRunner`. For new sessions, call `load_control_inputs()` before `prepare_session()`, then store protocol and roster paths/hashes in `record_manifest()`. For resume, load and validate control inputs before `verify_session_for_resume()`, pass their hashes, and keep the existing verify-before-write ordering. Convert `PreflightError` into a single stderr line and return code `2`. + +- [ ] **Step 4: Run focused flow tests and the complete test suite.** + +Run: + +```bash +PYTHONPATH=src python -m pytest -q tests/miracle/test_matrix_cli_inputs.py \ + tests/miracle/test_matrix_identity.py tests/miracle/test_matrix_cli_flow.py \ + tests/miracle/test_matrix_runner.py tests/miracle/test_section2_strictness.py +PYTHONPATH=src python -m pytest -q +python -m compileall -q src tests tools vendor +git diff --check +``` + +Expected: all focused and full tests pass; compileall exits `0`; `git diff --check` has no output. + +- [ ] **Step 5: Commit the integrated CLI change.** + +```bash +git add tools/miracle_matrix.py tests/miracle/test_section2_strictness.py \ + tests/miracle/test_matrix_cli_flow.py +git commit -m "fix(24_miracle): make matrix CLI reproducible" +``` + +### Task 5: Final review and publish the PR branch + +**Files:** +- Review only: all files changed by Tasks 1-4 +- No changes: `/home/wkj/projects/AgentBenchFrame` local `main` + +- [ ] **Step 1: Inspect the final diff and verify only the intended worktree changed.** + +Run: `git status --short --branch && git diff --stat origin/pr-3...HEAD && git diff --check origin/pr-3...HEAD` + +Expected: only the PR #3 repair branch has changes, with no untracked or unrelated files. + +- [ ] **Step 2: Run the clean-checkout-equivalent verification.** + +Run: `PYTHONPATH=src python -m pytest -q` from the PR worktree, followed by `python -m compileall -q src tests tools vendor`. + +Expected: zero test failures and zero compilation errors. + +- [ ] **Step 3: Push the repair branch to PR #3's head branch without updating local main.** + +```bash +git push origin HEAD:gongheng/24-miracle-review1-framework +``` + +- [ ] **Step 4: Report exact verification counts and the PR update.** + +Do not claim completion unless the fresh commands above show zero failures; report any environment-only skips separately. diff --git a/docs/superpowers/specs/2026-07-25-miracle-evaluation-reproducibility-design.md b/docs/superpowers/specs/2026-07-25-miracle-evaluation-reproducibility-design.md new file mode 100644 index 0000000..52fece3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-miracle-evaluation-reproducibility-design.md @@ -0,0 +1,64 @@ +# 24 Miracle 评测可复现性修复设计 + +## 背景 + +PR #3 为 `24_miracle` 提供外部 Judge 评测、矩阵运行、可恢复 session 和 replay-driven HL 迭代。当前实现依赖仓库外或未随提交提供的 protocol/roster 文件，导致从 PR 的干净提交树运行时在参数校验前直接抛出 `FileNotFoundError`。此外，Python 对手的冻结身份只检查目录存在，没有验证其实际可执行内容。 + +本修复只完善 PR #3 的执行与原始数据边界，不把评测指标或信息增益计算提前塞入 game adapter。 + +## 目标 + +1. 让 PR #3 的干净 checkout 具备可验证的控制面输入。 +2. 让新建 session 和 resume 使用明确、可追溯的 protocol/roster 输入。 +3. 让缺少外部运行资产时返回稳定、可读的 preflight 错误。 +4. 对 Python 与 C++ 对手采用一致的冻结身份校验，并消除不确定的目录匹配。 +5. 保持 framework 保存一手数据、CI 计算派生指标的职责边界。 + +## 非目标 + +- 不提交完整评测集、对局 replay 集或大型二进制。 +- 不定义或计算信息增益、raw/evo/gain/AUC。 +- 不实现 RL 训练或 RL/HL 的统一统计聚合。 +- 不修改本地 `main`，也不改变 CI 页面职责。 + +## 设计 + +### 1. 控制面输入 + +将 `24_miracle_evaluation_protocol.v0.3.json` 与 `24_miracle_roster_manifest.json` 作为小型、版本化的控制面元数据纳入 game adapter。完整运行资产继续由运行环境提供。 + +`miracle_matrix.py` 为 protocol、roster 和运行资产提供 CLI 路径参数；默认值保持仓库内约定路径，以保留现有调用方式。程序启动后先执行 preflight，检查输入文件存在、JSON 可解析、协议 hash 与 manifest 结构满足要求；失败时打印稳定的错误信息并返回非零状态，不让底层 `Path.read_text()` 异常成为用户接口。 + +### 2. Session 与 resume + +新建 session 时，在 session manifest 中记录 protocol/roster 的路径、SHA-256 和结构版本；保留现有事件和结果文件格式。resume 在任何写入前读取 session manifest，验证当前输入与原 session 身份一致，并继续执行既有的只读验证流程。验证失败时 session 目录保持不变。 + +本修复不改变历史事件字段含义；新增字段使用现有 envelope/前向兼容策略。 + +### 3. 对手身份 + +manifest 为每个 Python 对手提供唯一的冻结身份来源：优先校验归档文件 hash，并在解压后校验 manifest 指定的 runnable entry hash；C++ 继续校验 `main.exe` hash。对手目录解析改为 manifest 驱动的确定性路径，不使用 `next(glob)` 隐式选择。缺少匹配项或存在多个候选项时，preflight 直接失败。 + +### 4. 原始数据边界 + +每个 event 继续保存 `run_id`、session 标识、事件类型、时间、结果状态和 artifact 引用；usage 与过程退出信息保留在原始记录中。closure report 仅作为派生汇总，不作为 CI 的唯一数据源。后续 CI 可从这些记录计算 episode 曲线、性能分和信息增益。 + +## 测试策略 + +先添加回归测试并确认其在修复前失败，再实现代码。测试覆盖： + +- 干净路径下 protocol/roster 的 preflight 成功与缺失报错； +- CLI 自定义 protocol/roster 路径； +- 新 session 保存输入 hash； +- resume 成功、输入被替换时拒绝且不写入 session； +- Python 对手 hash 正确、被修改、缺失和重复目录时的结果； +- `--dry-run` 不依赖完整评测集且不产生未声明的持久化副作用。 + +实现后运行相关测试、全量 pytest、`compileall` 和 `git diff --check`。只有从 PR 分支的实际 checkout 得到全量测试通过，才报告修复完成。 + +## 验收标准 + +1. PR #3 的实际提交树中全量测试无失败。 +2. 无外部完整测试集时，CLI 能完成 preflight/dry-run 或给出明确的资产缺失说明。 +3. resume 对 protocol、roster 或 Python runnable identity 的任何不一致都拒绝继续，并保持 session 不变。 +4. 原始 event 和 session manifest 可被 CI 独立读取，adapter 不计算科研派生指标。 diff --git a/docs/superpowers/specs/2026-07-25-miracle-public-control-copy-repair.md b/docs/superpowers/specs/2026-07-25-miracle-public-control-copy-repair.md new file mode 100644 index 0000000..5fe22ea --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-miracle-public-control-copy-repair.md @@ -0,0 +1,10 @@ +# Public control-copy reproducibility repair + +The historical execution protocol SHA was +`866696fd9e094da85e3f2c04dc5ba20d0500faf8461531a242362323c4efe0b3`. + +The tracked protocol is a public, sanitized control copy. Its normalized SHA +identifies only that public copy and does not rewrite historical matrix +provenance. Protocol and roster control hashes normalize UTF-8 line endings to +LF. Strategy, Judge, executable, and archive assets continue to use raw-byte +SHA256 hashing. diff --git a/pyproject.toml b/pyproject.toml index d18a089..7962bcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [] [project.optional-dependencies] tracking = ["psutil"] +miracle = ["psutil"] rl = ["torch", "numpy"] report = ["jinja2"] all = ["psutil", "torch", "numpy", "jinja2"] diff --git a/src/agentbench_frame/cli.py b/src/agentbench_frame/cli.py index 4554347..ed6589c 100644 --- a/src/agentbench_frame/cli.py +++ b/src/agentbench_frame/cli.py @@ -131,7 +131,7 @@ def _cmd_data_check(args): errs = [] # Check run.toml try: - meta = tomllib.loads(run_toml_path.read_text()) + meta = tomllib.loads(run_toml_path.read_text(encoding="utf-8")) for field in ["run_id", "game", "agent", "type", "created"]: val = meta.get("run", {}).get(field, "") if not val: @@ -145,7 +145,7 @@ def _cmd_data_check(args): errs.append("missing summary.json") else: try: - s = json.loads(summary_path.read_text()) + s = json.loads(summary_path.read_text(encoding="utf-8")) for field in ["run_id", "game", "agent", "wall_hours", "total_steps", "win_rate"]: if field not in s: errs.append(f"summary.json: missing '{field}'") @@ -176,7 +176,7 @@ def _cmd_data_list(args): for run_dir in sorted(runs_root.rglob("summary.json")): rel = run_dir.parent.relative_to(data_dir) try: - s = json.loads(run_dir.read_text()) + s = json.loads(run_dir.read_text(encoding="utf-8")) print(f" {s.get('game','?')}/{s.get('agent','?')} " f"type={s.get('run_type','?')} " f"Elo={s.get('best_elo','-')} " diff --git a/src/agentbench_frame/games/__init__.py b/src/agentbench_frame/games/__init__.py new file mode 100644 index 0000000..78d05ff --- /dev/null +++ b/src/agentbench_frame/games/__init__.py @@ -0,0 +1,4 @@ +"""Game-specific adapters under the AgentBench framework. + +Each subpackage adapts one Saiblo game. `miracle` adapts the 24th Miracle game +via the original external Judge (no in-process reimplementation).""" diff --git a/src/agentbench_frame/games/miracle/__init__.py b/src/agentbench_frame/games/miracle/__init__.py new file mode 100644 index 0000000..b152f91 --- /dev/null +++ b/src/agentbench_frame/games/miracle/__init__.py @@ -0,0 +1,31 @@ +"""24_miracle adapter — wraps the original external Judge + AI subprocesses +(高翔's run_match.py) and feeds normalized results into the framework's `Run`, +bypassing the framework's `Match`/`BaseRunner` whose win attribution is unsafe +under side-swapping. See docs/games/24_miracle_adapter_status.md.""" + +from agentbench_frame.games.miracle.result import ( + GameOutcome, + WIN, + LOSS, + DRAW, + ERROR, + VALID_RESULTS, + derive_raw_winner, + finalize, + normalize, + compute_win_rate, + compute_h2h, + outcome_counts, + read_replay_header, + sha256_file, + select_games_to_run, + would_rerun_successful, + to_event_record, +) + +__all__ = [ + "GameOutcome", "WIN", "LOSS", "DRAW", "ERROR", "VALID_RESULTS", + "derive_raw_winner", "finalize", "normalize", "compute_win_rate", + "compute_h2h", "outcome_counts", "read_replay_header", "sha256_file", + "select_games_to_run", "would_rerun_successful", "to_event_record", +] diff --git a/src/agentbench_frame/games/miracle/atomicio.py b/src/agentbench_frame/games/miracle/atomicio.py new file mode 100644 index 0000000..07f05ee --- /dev/null +++ b/src/agentbench_frame/games/miracle/atomicio.py @@ -0,0 +1,57 @@ +"""Atomic JSON file writer: temp file + fsync + os.replace. + +Guarantees the final path is never partially written. The new content is +written to a same-directory temp file, fsync'd, then atomically renamed over +the target with ``os.replace``. A crash anywhere before the rename leaves the +target with its prior complete content (or absent); a leftover temp file +(``..*.tmp``) is the only audit trace. Used for vendor result-json so a +force-killed vendor can never leave a half-written result. +""" +from __future__ import annotations + +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Any + +_SENTINEL = object() +_WINDOWS_SHARING_WINERRORS = {5, 32} + + +def _is_windows() -> bool: + """Return the host platform without making tests mutate global ``os``.""" + return os.name == "nt" + + +def _replace_with_windows_retry(tmp: str, target: str) -> None: + """Retry only transient Windows sharing/access failures within 150ms.""" + for i, delay in enumerate((0.01, 0.02, 0.04, 0.08), start=1): + try: + os.replace(tmp, target) + return + except PermissionError as exc: + if not _is_windows() or getattr(exc, "winerror", None) not in _WINDOWS_SHARING_WINERRORS or i == 4: + raise + time.sleep(delay) + + +def atomic_write_json(path, obj: Any, *, encoding: str = "utf-8", + indent: int = 2, default=_SENTINEL) -> Path: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=f".{p.name}.", suffix=".tmp", dir=str(p.parent)) + try: + with os.fdopen(fd, "w", encoding=encoding) as f: + if default is _SENTINEL: + json.dump(obj, f, ensure_ascii=False, indent=indent) + else: + json.dump(obj, f, ensure_ascii=False, indent=indent, default=default) + f.flush() + os.fsync(f.fileno()) + _replace_with_windows_retry(tmp, str(p)) + except BaseException: + # never leave the target half-written; leave the temp for audit (do NOT unlink) + raise + return p diff --git a/src/agentbench_frame/games/miracle/driver.py b/src/agentbench_frame/games/miracle/driver.py new file mode 100644 index 0000000..0e6dfde --- /dev/null +++ b/src/agentbench_frame/games/miracle/driver.py @@ -0,0 +1,65 @@ +"""Glue between finalized Miracle :class:`GameOutcome` instances and the +framework's :class:`Run`. + +This module is the single place that defeats framework risks #2 / #3 / #5 +(see docs/games/24_miracle_adapter_status.md): + +* #2 / #3 — ``Match``/``Run`` count ``raw_winner == 0`` as a win. We feed + ``Run.log_episode`` the **normalized** winner (0 = evaluated agent won), + so ``Run._build_summary`` persists a win_rate that is correct even after + side-swapping. +* #5 — ``BaseRunner.run()`` writes ``summary.json`` before merging the + subclass result. We never use ``BaseRunner``; the caller drives ``Run`` + directly and everything is injected *before* ``finish()``. + +Error games are recorded as ``game`` audit events but are NOT logged as +episodes, so they fall out of ``total_episodes`` and ``win_rate`` (有效对局). +""" +from __future__ import annotations + +from typing import Dict, Sequence + +from agentbench_frame.games.miracle.result import ( + DRAW, + LOSS, + WIN, + GameOutcome, + compute_h2h, + compute_win_rate, + to_event_record, +) +from agentbench_frame.tracking.run import Run + +#: map normalized result -> the integer winner Run.log_episode expects. +#: Run counts ``winner == 0`` as a win, so 0 = evaluated-agent win. +_EPISODE_WINNER = {WIN: 0, LOSS: 1, DRAW: -1} + + +def feed_outcomes_to_run(run: Run, outcomes: Sequence[GameOutcome]) -> Dict: + """Write every outcome as a ``game`` audit event and log the valid ones as + episodes with normalized winners + auditable step counts. + + Does **not** call ``run.finish()`` — the caller owns the run lifecycle so it + can set ``run_type`` / ``data_dir`` correctly (risk #4 / #6) before any + summary is written. + """ + for o in outcomes: + run.write(**to_event_record(o)) + if o.valid: + evaluated_score = o.score0 if o.evaluated_agent_camp == 0 else o.score1 + run.log_episode( + reward=float(evaluated_score or 0.0), + steps=int(o.steps), + winner=_EPISODE_WINNER[o.normalized_result], + info={ + "game_id": o.game_id, + "opponent": o.opponent, + "evaluated_agent_camp": o.evaluated_agent_camp, + "raw_winner": o.raw_winner, + }, + ) + run.log_h2h(compute_h2h(outcomes)) + return { + "win_rate": compute_win_rate(outcomes), + "valid_games": sum(1 for o in outcomes if o.valid), + } diff --git a/src/agentbench_frame/games/miracle/entry.py b/src/agentbench_frame/games/miracle/entry.py new file mode 100644 index 0000000..f2164c0 --- /dev/null +++ b/src/agentbench_frame/games/miracle/entry.py @@ -0,0 +1,72 @@ +"""Cross-platform AI entry resolution for the 24_miracle runner. + +Resolves the argv list used to launch an AI subprocess via ``subprocess.Popen`` +(``shell=False``). The executable is returned as an **ABSOLUTE** path because +Windows ``CreateProcess`` does NOT search the ``cwd=`` argument's directory for a +bare executable name (``Popen(["main.exe"], cwd=dir)`` raises FileNotFoundError); +an absolute path is found regardless of the child's working directory. + +Precedence: + 1. ``explicit`` (a list is kept as-is; a string is kept WHOLE as a single + executable path — never whitespace-split, so paths with spaces survive) + takes priority over auto-detection. + 2. Auto-detection (deterministic and documented): + Windows (``os.name == 'nt'``): ``main.exe`` > ``main.py`` > ``main`` + POSIX: ``main`` > ``main.py`` > ``main.exe`` + When ``main.exe`` and ``main`` coexist, Windows picks ``main.exe`` and + POSIX picks ``main`` (recorded behaviour). + ``main.py`` is launched via ``sys.executable`` + absolute script path. + +This never modifies the strategy directory and never reads strategy source. +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import List, Optional, Union + +#: platform flag read at call time (not the global ``os.name``), so tests can +#: patch it via ``monkeypatch.setattr(entry, "_IS_NT", ...)`` without disturbing +#: pathlib or other stdlib code that reads the real ``os.name``. +_IS_NT = os.name == "nt" + + +def resolve_ai_command( + ai_dir: Union[str, Path], + explicit: Optional[Union[str, List[str]]] = None, +) -> List[str]: + """Return the argv list for the AI subprocess (executable as an ABSOLUTE path). + + Raises ``FileNotFoundError`` (mentioning ``ai_dir``) when no entry exists. + """ + # 1. explicit takes priority; a string is kept whole (paths with spaces!) + if explicit is not None and explicit != "" and not ( + isinstance(explicit, (list, tuple)) and len(explicit) == 0 + ): + if isinstance(explicit, (list, tuple)): + return [str(x) for x in explicit] + return [str(explicit)] + + ai_dir = Path(ai_dir).resolve() # absolute, so Popen(cwd=...) finds the exe on Windows + is_nt = _IS_NT + + # 2. auto-detection — absolute path to the entry + if is_nt: + if (ai_dir / "main.exe").exists(): + return [str(ai_dir / "main.exe")] + if (ai_dir / "main.py").exists(): + return [sys.executable, str(ai_dir / "main.py")] + if (ai_dir / "main").exists(): # unusual on Windows (no extension) + return [str(ai_dir / "main")] + else: + if (ai_dir / "main").exists(): + return [str(ai_dir / "main")] + if (ai_dir / "main.py").exists(): + return [sys.executable, str(ai_dir / "main.py")] + if (ai_dir / "main.exe").exists(): # cross-mounted POSIX + return [str(ai_dir / "main.exe")] + + raise FileNotFoundError( + f"no AI entry found (main.exe on Windows / main on POSIX / main.py) in {ai_dir}" + ) diff --git a/src/agentbench_frame/games/miracle/match_runner.py b/src/agentbench_frame/games/miracle/match_runner.py new file mode 100644 index 0000000..b2437b4 --- /dev/null +++ b/src/agentbench_frame/games/miracle/match_runner.py @@ -0,0 +1,595 @@ +"""Miracle match wrapper: run one game through the (vendored) run_match runner, +then independently cross-verify result-json + trace + Replay and classify the +outcome on the event timeline (not just the final returncode). + +Design points (阶段4b-4 spec): + * Invokes the vendor runner with ``sys.executable -u``, ``shell=False``, an + explicit arg array, and explicit ``MIRACLE_JUDGE_DIR`` / paths / identities. + * stdout/stderr are redirected to FILES (never pipes) so huge output cannot + deadlock or grow without bound in memory. + * A wrapper-level overall timeout; on expiry the vendor subtree (and its AI + descendants) is cleaned by exact-PID process-tree kill (proctree). + * All evidence (stdout/stderr/trace/result-json) is preserved on disk. + * The vendor runner writes result-json atomically (temp + replace). If it is + missing (e.g. force-killed mid-write) the attempt is classified + ``result_json_missing`` — never guessed. + * Three-way cross-validation: result-json vs trace (streamed) vs Replay. Any + authoritative conflict -> ``evidence_mismatch``; nothing is silently picked. + * Returns a ``MatchAttempt``. It does NOT mutate any aggregate; the runner + (MiracleEvalRunner) decides what to log. + +This module is psutil-backed (via proctree); see the ``miracle`` extra. +""" +from __future__ import annotations + +import json +import os +import struct +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from agentbench_frame.games.miracle.proctree import ProcessTreeManager +from agentbench_frame.games.miracle.result import sha256_file + +#: replay header = 7 big-endian signed int32: [0,0,0,map_type,day_time,0,0] +REPLAY_HEADER_BYTES = 28 +#: the Judge draws map_type/day_time via random.randint(0,1); anything outside +#: {0,1} means the header is not a valid Miracle replay. +_VALID_MAP_VALUES = {0, 1} + + +# --------------------------------------------------------------------------- # +# streaming trace stats +# --------------------------------------------------------------------------- # +@dataclass +class TraceStats: + n_ai_operation: int = 0 + ai_error_players: List[int] = field(default_factory=list) + ai_timeout_players: List[int] = field(default_factory=list) + end_info_seen: bool = False + end_info: Optional[Dict[str, int]] = None + error_before_end: bool = False + + +def stream_trace(path) -> TraceStats: + """Stream a trace JSONL line by line (never load it whole).""" + ts = TraceStats() + p = Path(path) + if not p.exists(): + return ts + with p.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + kind = e.get("kind") + if kind == "ai_operation": + ts.n_ai_operation += 1 + elif kind == "ai_error": + pl = e.get("player") + if pl is not None: + ts.ai_error_players.append(pl) + if not ts.end_info_seen: + ts.error_before_end = True + elif kind == "ai_timeout": + pl = e.get("player") + if pl is not None: + ts.ai_timeout_players.append(pl) + if not ts.end_info_seen: + ts.error_before_end = True + elif kind == "match_end": + ts.end_info_seen = True + ei = e.get("end_info") + if isinstance(ei, str): + try: + ts.end_info = json.loads(ei) + except json.JSONDecodeError: + ts.end_info = None + elif isinstance(ei, dict): + ts.end_info = ei + return ts + + +# --------------------------------------------------------------------------- # +# result-json loader +# --------------------------------------------------------------------------- # +def load_result_json(path) -> Tuple[str, Optional[dict]]: + """Return (status, data) where status is 'ok' | 'missing' | 'corrupt'.""" + p = Path(path) + if not p.exists(): + return ("missing", None) + try: + return ("ok", json.loads(p.read_text(encoding="utf-8"))) + except (json.JSONDecodeError, ValueError, UnicodeDecodeError): + return ("corrupt", None) + + +# --------------------------------------------------------------------------- # +# replay info +# --------------------------------------------------------------------------- # +@dataclass +class ReplayInfo: + exists: bool = False + length_ok: bool = False + header_valid: bool = False + map_type: Optional[int] = None + day_time: Optional[int] = None + sha256: Optional[str] = None + + +def read_replay_info(path) -> ReplayInfo: + p = Path(path) + info = ReplayInfo(exists=p.exists()) + if not info.exists: + return info + info.sha256 = sha256_file(p) + data = p.read_bytes() + info.length_ok = len(data) >= REPLAY_HEADER_BYTES + if info.length_ok: + try: + vals = struct.unpack(">7i", data[:REPLAY_HEADER_BYTES]) + info.map_type, info.day_time = int(vals[3]), int(vals[4]) + info.header_valid = ( + info.map_type in _VALID_MAP_VALUES and info.day_time in _VALID_MAP_VALUES + ) + except struct.error: + info.header_valid = False + return info + + +# --------------------------------------------------------------------------- # +# cross-validation (result-json vs trace vs scores) +# --------------------------------------------------------------------------- # +def cross_validate(rj: Optional[dict], ts: TraceStats, ri: ReplayInfo) -> List[str]: + """Return a list of authoritative-conflict descriptions. Only CONFLICTS are + reported here (both sides present and disagreeing); missing end_info in the + trace is handled by the classifier, not treated as a field conflict.""" + discs: List[str] = [] + if rj is None: + return discs + if rj.get("schema_version") != 1: + discs.append("schema_version_unexpected") + rj_end = rj.get("end_info") if rj.get("end_info_received") else None + if rj_end is None: + return discs + # end_info conflict (only when trace also has one) + if ts.end_info is not None and ts.end_info != rj_end: + discs.append("end_info_scores_mismatch_between_result_json_and_trace") + # scores field vs end_info + rj_scores = rj.get("scores") + if rj_scores is not None: + try: + if (int(rj_scores.get("0")) != int(rj_end.get("0")) + or int(rj_scores.get("1")) != int(rj_end.get("1"))): + discs.append("scores_field_conflicts_end_info") + except (TypeError, ValueError, AttributeError): + discs.append("scores_or_end_info_unparseable") + # raw_winner vs score rule (0 if s0>s1 else 1; ties -> 1) + try: + s0, s1 = int(rj_end.get("0")), int(rj_end.get("1")) + expected = 0 if s0 > s1 else 1 + if rj.get("raw_winner") != expected: + discs.append(f"raw_winner_mismatch_expected_{expected}_got_{rj.get('raw_winner')}") + except (TypeError, ValueError, AttributeError): + discs.append("end_info_unparseable") + return discs + + +# --------------------------------------------------------------------------- # +# timeline classification +# --------------------------------------------------------------------------- # +@dataclass +class Classification: + normalized_result: str = "error" + error_type: Optional[str] = None + valid: bool = False + reason: str = "" + ai_crash_player: Optional[int] = None + ai_timeout_player: Optional[int] = None + judge_crash: bool = False + wrapper_timeout: bool = False + normal_cleanup_nonzero: bool = False + raw_winner: Optional[int] = None + winner_agent: Optional[str] = None + + +def classify(*, rj_status: str, rj: Optional[dict], ts: TraceStats, ri: ReplayInfo, + discrepancies: List[str], vendor_returncode: int, wrapper_timeout: bool, + evaluated_agent_camp: int, evaluated_agent: str = "eval", + opponent: str = "opp") -> Classification: + """Classify a game by event timeline + end state. Precedence (most severe + first): wrapper_timeout > result_json missing/corrupt > evidence_mismatch > + ai_crash > ai_timeout > judge_crash > replay_missing > replay_corrupt > valid.""" + c = Classification() + agent_at = lambda camp: evaluated_agent if camp == evaluated_agent_camp else opponent + + if wrapper_timeout: + c.wrapper_timeout = True + c.error_type = "wrapper_timeout" + c.reason = f"vendor tree exceeded wrapper timeout (vendor rc={vendor_returncode})" + return c + if rj_status == "missing": + c.error_type = "result_json_missing" + c.reason = f"no result-json produced (vendor rc={vendor_returncode})" + return c + if rj_status == "corrupt": + c.error_type = "result_json_corrupt" + c.reason = "result-json unparseable" + return c + if discrepancies: + c.error_type = "evidence_mismatch" + c.reason = "; ".join(discrepancies) + return c + + # Infrastructure failures (review#6): cleanup failure and vendor exception. + # These are more severe than game-level AI failures (which allow continue). + rj_dict = rj or {} + if rj_dict.get("cleanup_all_succeeded") is False: + c.error_type = "cleanup_failure" + c.reason = "process cleanup did not succeed (possible residual/orphan)" + return c + _rm_rc = rj_dict.get("run_match_returncode") + _vendor_exc = str(rj_dict.get("exception") or "") + if ("FileNotFoundError" in _vendor_exc and not rj_dict.get("end_info_received")): + c.error_type = "judge_start_path_error" + c.reason = f"judge startup path failure: {_vendor_exc}" + return c + if vendor_returncode != 0 or rj_dict.get("exception") or (_rm_rc is not None and _rm_rc != 0): + c.error_type = "vendor_exception" + c.reason = f"vendor runner exception (rc={vendor_returncode}, rj_rc={_rm_rc}, exc={rj_dict.get('exception')})" + return c + + end_received = bool(rj and rj.get("end_info_received")) + + # AI crash: trace ai_error, OR an AI natural-exited before end_info + if ts.ai_error_players: + c.error_type = "ai_crash" + c.ai_crash_player = ts.ai_error_players[0] + c.reason = f"trace ai_error (player {c.ai_crash_player})" + return c + if not end_received: + for role, idx in (("ai0", 0), ("ai1", 1)): + p = (rj or {}).get(role) or {} + if p.get("natural_exit") and not p.get("termination_requested"): + c.error_type = "ai_crash" + c.ai_crash_player = idx + c.reason = f"{role} exited naturally before end_info" + return c + + if ts.ai_timeout_players: + c.error_type = "ai_timeout" + c.ai_timeout_player = ts.ai_timeout_players[0] + c.reason = f"trace ai_timeout (player {c.ai_timeout_player})" + return c + + if not end_received: + c.error_type = "judge_crash" + c.judge_crash = True + c.reason = "no legal end_info produced" + return c + + if not ri.exists: + c.error_type = "replay_missing" + c.reason = "replay file absent" + return c + if not ri.header_valid: + c.error_type = "replay_corrupt" + c.reason = "replay header invalid (length/format/range)" + return c + + # valid candidate + raw = rj.get("raw_winner") + c.raw_winner = raw + c.winner_agent = agent_at(raw) if raw in (0, 1) else None + c.valid = True + c.error_type = None + if raw == evaluated_agent_camp: + c.normalized_result = "win" + elif raw in (0, 1): + c.normalized_result = "loss" + else: + c.normalized_result = "draw" + # post-end_info cleanup of idling AIs may yield nonzero returncodes; that's normal + for role in ("ai0", "ai1"): + p = (rj or {}).get(role) or {} + if p.get("termination_requested") and p.get("final_returncode") not in (0, None): + c.normal_cleanup_nonzero = True + return c + + +# --------------------------------------------------------------------------- # +# per-game attempt +# --------------------------------------------------------------------------- # +@dataclass +class MatchAttempt: + game_id: str + evaluated_agent: str + opponent: str + evaluated_agent_camp: int + valid: bool + normalized_result: str + error_type: Optional[str] + reason: str + raw_winner: Optional[int] + winner_agent: Optional[str] + scores: Optional[Dict[str, int]] + steps: int + realized_randomization: Optional[Dict[str, int]] + result_json_status: str + discrepancies: List[str] + ai_crash_player: Optional[int] + ai_timeout_player: Optional[int] + judge_crash: bool + wrapper_timeout: bool + normal_cleanup_nonzero: bool + evidence_paths: Dict[str, str] + collision_detected: bool + process_cleanup: List[dict] + vendor_returncode: int + started_at: float + finished_at: float + duration_s: float + exception: Optional[str] + judge_exit: Optional[int] = None + ai0_exit: Optional[int] = None + ai1_exit: Optional[int] = None + replay_sha256: Optional[str] = None + # ---- Section 4 first-hand process signals (one source of truth each) ---- # + #: raw ``timeout`` field from the result-json, verbatim. Vendor emits this as + #: ``{"ai0": bool, "ai1": bool}`` per-player flags, but we preserve whatever + #: the vendor wrote (never re-shape/flatten). + timeout: Optional[Any] = None + #: per-step Judge timeout actually passed to ``run_match_attempt`` (the + #: ``timeout`` kwarg). Distinct from ``timeout`` (the vendor's view). + timeout_s: Optional[float] = None + #: outer Popen / wrapper-layer exception (``repr(exc)`` or None); separate + #: from the vendor exception reported inside result-json. + wrapper_exception: Optional[str] = None + #: verbatim ``exception`` string from the result-json's own view; the vendor + #: subprocess saw an exception (or none) and recorded it. + vendor_exception: Optional[str] = None + #: internal ``run_match_returncode`` from the result-json (vendor side), + #: separate from the outer ``vendor_returncode`` (Popen.returncode of the + #: vendor script). + run_match_returncode: Optional[int] = None + + +def _merge_process_cleanup(vendor_manager_status: List[dict], + rj: Optional[dict]) -> List[dict]: + """Compose the unified ``process_cleanup`` list. Four sources, labelled: + + * vendor → ``source="process_tree_manager"`` (outer Popen we owned). + * judge → ``source="result_json"`` (vendor's inner Judge). + * ai0 → ``source="result_json"`` (vendor's inner AI #0). + * ai1 → ``source="result_json"`` (vendor's inner AI #1). + + The two exception sources must NOT overwrite each other; here they're just + appended in role-sorted order so downstream code reads both rows verbatim. + """ + rows: List[dict] = [] + for vrow in vendor_manager_status or []: + item = dict(vrow) + item.setdefault("role", "vendor") + item["source"] = "process_tree_manager" + rows.append(item) + _rj = rj or {} + for role in ("judge", "ai0", "ai1"): + v = _rj.get(role) + if isinstance(v, dict): + item = dict(v) + item.setdefault("role", role) + item["source"] = "result_json" + rows.append(item) + return rows + + +def _build_attempt_from_files(*, game_id: str, evaluated_agent: str, opponent: str, + evaluated_agent_camp: int, + work_dir, tag: str, + vendor_returncode: int, + wrapper_timeout: bool, + wrapper_exception: Optional[str], + vendor_manager_status: List[dict], + timeout_s: float, + started: float, finished: float, + collision_detected: bool = False) -> MatchAttempt: + """Build a fully classified MatchAttempt from on-disk result-json/trace/ + replay artifacts plus the wrapper-level signals (vendor_returncode, + wrapper_timeout, wrapper_exception, vendor_manager_status, ``timeout_s``). + + Pure post-subprocess construction: NO subprocess, NO Judge/AI invocations. + Used by ``run_match_attempt`` after the vendor subprocess has exited and by + end-to-end tests that drive a fake result-json through the full pipeline + (MatchAttempt → GameOutcome → to_event_record → JSON落盘重读). + """ + work_dir = Path(work_dir).expanduser().resolve() + result_json = work_dir / f"{tag}.result.json" + trace = work_dir / f"{tag}.jsonl" + replay = work_dir / f"{tag}.replay" + stdout_file = work_dir / f"{tag}.stdout" + stderr_file = work_dir / f"{tag}.stderr" + + rj_status, rj = load_result_json(result_json) + ts = stream_trace(trace) + ri = read_replay_info(replay) + discs = cross_validate(rj, ts, ri) if rj is not None else [] + c = classify(rj_status=rj_status, rj=rj, ts=ts, ri=ri, discrepancies=discs, + vendor_returncode=(vendor_returncode + if isinstance(vendor_returncode, int) else -1), + wrapper_timeout=wrapper_timeout, + evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent=evaluated_agent, opponent=opponent) + + _rj = rj or {} + judge_exit = (_rj.get("judge") or {}).get("final_returncode") + ai0_exit = (_rj.get("ai0") or {}).get("final_returncode") + ai1_exit = (_rj.get("ai1") or {}).get("final_returncode") + replay_sha = sha256_file(replay) if replay.exists() else None + # vendor_exception = verbatim result-json ``exception`` field + vendor_exception_field = _rj.get("exception") + # timeout = raw result-json ``timeout`` field, verbatim (may be dict) + timeout_field = _rj.get("timeout") + # internal run_match_returncode from the result-json + internal_rc = _rj.get("run_match_returncode") + + # ---- compat ``exception`` COMES FROM REAL EXCEPTIONS ONLY ---- + # Rule (deterministic, test-covered): + # wrapper_exception (outer Popen/包装异常) wins; + # else vendor_exception (verbatim result-json exception); + # else None. + # ``reason`` is NEVER used to fill ``exception``. + if wrapper_exception is not None: + compat_exception = wrapper_exception + elif vendor_exception_field is not None: + compat_exception = vendor_exception_field + else: + compat_exception = None + + process_cleanup = _merge_process_cleanup(vendor_manager_status, rj) + + return MatchAttempt( + game_id=game_id, evaluated_agent=evaluated_agent, opponent=opponent, + evaluated_agent_camp=evaluated_agent_camp, + valid=c.valid, normalized_result=c.normalized_result, + error_type=c.error_type, reason=c.reason, + raw_winner=c.raw_winner, winner_agent=c.winner_agent, + scores=(rj.get("scores") if rj else None), + steps=ts.n_ai_operation, + realized_randomization=( + {"map_type": ri.map_type, "day_time": ri.day_time} + if ri.header_valid else None + ), + result_json_status=rj_status, discrepancies=discs, + ai_crash_player=c.ai_crash_player, ai_timeout_player=c.ai_timeout_player, + judge_crash=c.judge_crash, wrapper_timeout=c.wrapper_timeout, + normal_cleanup_nonzero=c.normal_cleanup_nonzero, + evidence_paths={ + "stdout": str(stdout_file), "stderr": str(stderr_file), + "trace": str(trace), "replay": str(replay), + "result_json": str(result_json), + }, + collision_detected=collision_detected, + process_cleanup=process_cleanup, + vendor_returncode=(vendor_returncode if isinstance(vendor_returncode, int) else -1), + judge_exit=judge_exit, ai0_exit=ai0_exit, ai1_exit=ai1_exit, + replay_sha256=replay_sha, + started_at=started, finished_at=finished, + duration_s=round(finished - started, 3), + exception=compat_exception, + timeout=timeout_field, + timeout_s=timeout_s, + wrapper_exception=wrapper_exception, + vendor_exception=vendor_exception_field, + run_match_returncode=(internal_rc if isinstance(internal_rc, int) else None), + ) + + +def run_match_attempt(*, game_id: str, p0_dir, p1_dir, p0_name: str, p1_name: str, + judge_dir, work_dir, vendor_script, framework_src, + timeout: float = 12.0, wrapper_timeout_s: float = 60.0, + evaluated_agent_camp: int = 0, + evaluated_agent: str = "ifelse", opponent: str = "rank04", + extra_vendor_args: Optional[List[str]] = None, + extra_env: Optional[Dict[str, str]] = None, + python: Optional[str] = None) -> MatchAttempt: + """Run ONE game via the vendor runner and return a fully classified attempt. + + Does not mutate any aggregate. All evidence is preserved under work_dir. + The classification / field-packing logic lives in + :func:`_build_attempt_from_files`; this function owns the subprocess + invocation (vendor tree cleanup, wrapper timeout) and forwards the + wrapper-level signals (vendor_returncode, wrapper_timeout, + wrapper_exception, ProcessTreeManager status, per-step ``timeout``). + """ + # The vendor launches the Judge with ``cwd=judge_dir``. Every execution + # artifact therefore must be absolute before crossing this boundary: a + # relative replay path would otherwise be interpreted under Judge cwd. + work_dir = Path(work_dir).expanduser().resolve() + work_dir.mkdir(parents=True, exist_ok=True) + judge_dir = Path(judge_dir).expanduser().resolve() + p0_dir = Path(p0_dir).expanduser().resolve() + p1_dir = Path(p1_dir).expanduser().resolve() + vendor_script = Path(vendor_script).expanduser().resolve() + framework_src = Path(framework_src).expanduser().resolve() + python = python or sys.executable + + # collision handling: never clobber a prior attempt's evidence for this game_id + result_json = work_dir / f"{game_id}.result.json" + collision = result_json.exists() + tag = game_id + if collision: + i = 2 + while (work_dir / f"{game_id}__{i}.result.json").exists(): + i += 1 + tag = f"{game_id}__{i}" + result_json = work_dir / f"{tag}.result.json" + trace = work_dir / f"{tag}.jsonl" + replay = work_dir / f"{tag}.replay" + stdout_file = work_dir / f"{tag}.stdout" + stderr_file = work_dir / f"{tag}.stderr" + + cmd: List[str] = [ + python, "-u", str(vendor_script), + "--p0-dir", str(p0_dir), "--p1-dir", str(p1_dir), + "--p0-name", str(p0_name), "--p1-name", str(p1_name), + "--timeout", str(timeout), + "--out", str(work_dir), "--tag", tag, + "--result-json", str(result_json), + ] + if extra_vendor_args: + cmd.extend(extra_vendor_args) + + env = dict(os.environ) + env["MIRACLE_JUDGE_DIR"] = str(judge_dir) + env["MIRACLE_FRAMEWORK_SRC"] = str(framework_src) + if extra_env: + env.update(extra_env) + + started = time.time() + mgr = ProcessTreeManager() + out_fh = open(stdout_file, "wb") + err_fh = open(stderr_file, "wb") + wrapper_timeout = False + try: + proc = subprocess.Popen(cmd, shell=False, stdout=out_fh, stderr=err_fh, env=env) + mgr.register_popen(proc, "vendor") + try: + proc.communicate(timeout=wrapper_timeout_s) + rc = proc.returncode + except subprocess.TimeoutExpired: + wrapper_timeout = True + mgr.cleanup_all("wrapper-timeout") + try: + rc = proc.wait(timeout=5) + except Exception: + rc = proc.returncode + except Exception as exc: # noqa: BLE001 + rc = -1 + _exc = repr(exc) + else: + _exc = None + finally: + out_fh.close() + err_fh.close() + finished = time.time() + + return _build_attempt_from_files( + game_id=game_id, evaluated_agent=evaluated_agent, opponent=opponent, + evaluated_agent_camp=evaluated_agent_camp, + work_dir=work_dir, tag=tag, + vendor_returncode=(rc if isinstance(rc, int) else -1), + wrapper_timeout=wrapper_timeout, + wrapper_exception=_exc, + vendor_manager_status=mgr.status(), + timeout_s=timeout, + started=started, finished=finished, + collision_detected=collision, + ) diff --git a/src/agentbench_frame/games/miracle/matrix.py b/src/agentbench_frame/games/miracle/matrix.py new file mode 100644 index 0000000..851fb9f --- /dev/null +++ b/src/agentbench_frame/games/miracle/matrix.py @@ -0,0 +1,316 @@ +"""32-game matrix orchestrator logic for 24_miracle (Plan A). + +Pure, unit-tested logic (no Judge / no real match in this module): the fixed +32-attempt plan, atomic+resumable progress, per-game classification wiring, +per-rank audit gate, infrastructure-stop decision, independent PID+create_time +residual check, and final aggregation. The runner that drives real games lives +in ``tools/miracle_matrix.py`` and reuses these functions. + +Plan A (frozen): if-else Agent vs rank01–16, each opponent camp0 + camp1 once, +total attempts = 32. game_id = ``m_rank{NN}_camp{C}``; camp = the camp the +if-else Agent plays. +""" +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from agentbench_frame.games.miracle.result import build_seed_provenance +from agentbench_frame.games.miracle.smoke_audit import ( + check_residual_procs, load_managed_procs_from_result_json, make_session_id, +) + +#: error_types that STOP the whole matrix (infrastructure failures). +#: ai_crash / ai_timeout are recorded invalid but do NOT stop (continue). +INFRA_STOP_ERRORS = { + "wrapper_timeout", "result_json_missing", "result_json_corrupt", + "evidence_mismatch", "replay_missing", "replay_corrupt", "judge_crash", + "cleanup_failure", "vendor_exception", +} + +_GAMEID_RE = re.compile(r"^m_rank(\d{2})_camp([01])$") + + +# --------------------------------------------------------------------------- # +# plan +# --------------------------------------------------------------------------- # +def make_attempt_plan() -> List[Dict[str, Any]]: + """32 attempts: rank01 camp0, rank01 camp1, ..., rank16 camp1.""" + plan = [] + for rank in range(1, 17): + for camp in (0, 1): + plan.append({"rank": rank, "camp": camp, + "game_id": f"m_rank{rank:02d}_camp{camp}"}) + return plan + + +def parse_game_id(game_id: str) -> Optional[Tuple[int, int]]: + m = _GAMEID_RE.match(game_id or "") + if not m: + return None + return int(m.group(1)), int(m.group(2)) + + +# --------------------------------------------------------------------------- # +# progress (atomic + resumable) +# --------------------------------------------------------------------------- # +def load_progress(path) -> Dict[str, Any]: + p = Path(path) + if not p.exists(): + return {"attempts": {}} + try: + return json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return {"attempts": {}} + + +def write_progress_atomic(path, progress: Dict[str, Any]) -> Path: + from agentbench_frame.games.miracle.atomicio import atomic_write_json + return atomic_write_json(path, progress) + + +def state_of(progress: Dict[str, Any], game_id: str) -> str: + return progress.get("attempts", {}).get(game_id, {}).get("state", "not_started") + + +def mark_running(progress: Dict[str, Any], game_id: str) -> None: + progress.setdefault("attempts", {})[game_id] = {"state": "running", "game_id": game_id} + + +def mark_done(progress: Dict[str, Any], game_id: str, result: Dict[str, Any]) -> None: + entry = {"state": "done", "game_id": game_id} + entry.update(result) + progress.setdefault("attempts", {})[game_id] = entry + + +def is_done(progress: Dict[str, Any], game_id: str) -> bool: + return state_of(progress, game_id) == "done" + + +def next_incomplete(progress: Dict[str, Any], plan: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + for a in plan: + if not is_done(progress, a["game_id"]): + return a + return None + + +# --------------------------------------------------------------------------- # +# events append (no duplicate game_id) +# --------------------------------------------------------------------------- # +def append_event_atomic(path, event: Dict[str, Any]) -> bool: + """Append a JSON line; skip if its game_id already present (no rerun/dup).""" + p = Path(path) + gid = event.get("game_id") + existing = set() + if p.exists(): + for line in p.read_text(encoding="utf-8").splitlines(): + if line.strip(): + try: + e = json.loads(line) + if "game_id" in e: + existing.add(e["game_id"]) + except json.JSONDecodeError: + pass + if gid in existing: + return False + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("a", encoding="utf-8") as f: + f.write(json.dumps(event, ensure_ascii=False) + "\n") + return True + + +# --------------------------------------------------------------------------- # +# per-game classification +# --------------------------------------------------------------------------- # +def classify_game(att) -> Dict[str, Any]: + """Map a MatchAttempt (with rank/camp or parseable game_id) to a per-game record.""" + rc = getattr(att, "rank", None), getattr(att, "camp", None) + if rc[0] is None or rc[1] is None: + parsed = parse_game_id(getattr(att, "game_id", "")) + rank = rc[0] if rc[0] is not None else (parsed[0] if parsed else None) + camp = rc[1] if rc[1] is not None else (parsed[1] if parsed else None) + else: + rank, camp = rc + rr = getattr(att, "realized_randomization", None) or {} + scores = getattr(att, "scores", None) or {} + return { + "game_id": att.game_id, + "rank": rank, + "camp": camp, + "ifelse_camp": camp, + "valid": bool(att.valid), + "normalized_result": att.normalized_result, + "raw_winner": getattr(att, "raw_winner", None), + "winner_agent": getattr(att, "winner_agent", None), + "scores": {"0": scores.get("0"), "1": scores.get("1")} if isinstance(scores, dict) else None, + "steps": int(getattr(att, "steps", 0) or 0), + "realized_randomization": rr, + "seed": build_seed_provenance(rr if rr else None), + "error_type": getattr(att, "error_type", None), + "reason": getattr(att, "reason", None), + "wrapper_timeout": bool(getattr(att, "wrapper_timeout", False)), + "ai_crash_player": getattr(att, "ai_crash_player", None), + "judge_crash": bool(getattr(att, "judge_crash", False)), + "normal_cleanup_nonzero": bool(getattr(att, "normal_cleanup_nonzero", False)), + "result_json_status": getattr(att, "result_json_status", "ok"), + "evidence_paths": getattr(att, "evidence_paths", {}), + } + + +# --------------------------------------------------------------------------- # +# stop decision +# --------------------------------------------------------------------------- # +def should_stop(att) -> Tuple[str, Optional[str]]: + """Infrastructure failures stop the matrix; AI crash/timeout (invalid) do NOT.""" + if getattr(att, "wrapper_timeout", False): + return ("stop", "wrapper_timeout") + et = getattr(att, "error_type", None) + if et in INFRA_STOP_ERRORS: + return ("stop", et) + rjs = getattr(att, "result_json_status", "ok") + if rjs in ("missing", "corrupt"): + return ("stop", f"result_json_{rjs}") + if getattr(att, "discrepancies", None): + return ("stop", "evidence_mismatch") + return ("continue", None) + + +def should_stop_with_residual(att) -> Tuple[str, str]: + """Like should_stop, plus an INDEPENDENT PID+create_time residual check on + the result-json's managed processes.""" + decision, reason = should_stop(att) + if decision == "stop": + return ("stop", reason) + rj_path = getattr(att, "evidence_paths", {}).get("result_json") + procs = load_managed_procs_from_result_json(rj_path) if rj_path else [] + if not procs: + return ("continue", "no_managed_procs_identity") + res = check_residual_procs(procs) + if res["residual"]: + return ("stop", f"residual_pids_{[(p.pid, p.role) for p in res['residual']]}") + return ("continue", "clean") + + +# --------------------------------------------------------------------------- # +# per-rank audit +# --------------------------------------------------------------------------- # +def rank_audit(rank: int, games) -> Tuple[bool, List[str]]: + reasons: List[str] = [] + recs = [g if isinstance(g, dict) else classify_game(g) for g in games] + if len(recs) != 2: + reasons.append(f"rank{rank}: expected 2 games, got {len(recs)}") + camps = sorted(r.get("camp") for r in recs if r.get("camp") is not None) + if camps != [0, 1]: + reasons.append(f"rank{rank}: camps={camps} != [0,1]") + ids = [r.get("game_id") for r in recs] + if len(set(ids)) != len(ids): + reasons.append(f"rank{rank}: duplicate game_id {ids}") + for r in recs: + if r.get("valid") and r.get("normalized_result") not in ("win", "loss", "draw"): + reasons.append(f"{r.get('game_id')}: valid but normalized_result={r.get('normalized_result')}") + if not r.get("valid") and not r.get("error_type"): + reasons.append(f"{r.get('game_id')}: invalid without error_type") + return (len(reasons) == 0, reasons) + + +# --------------------------------------------------------------------------- # +# aggregation +# --------------------------------------------------------------------------- # +def _safe_num(x): + try: + return float(x) + except (TypeError, ValueError): + return None + + +def aggregate(records: List[Dict[str, Any]]) -> Dict[str, Any]: + """Final matrix aggregate. win_rate = valid_wins / valid_games (null if 0).""" + valid = [r for r in records if r.get("valid")] + wins = [r for r in valid if r.get("normalized_result") == "win"] + losses = [r for r in valid if r.get("normalized_result") == "loss"] + invalid = [r for r in records if not r.get("valid")] + valid_games = len(valid) + + # per-rank + per_rank: Dict[int, Dict[str, Any]] = {} + for rank in range(1, 17): + rs = [r for r in records if r.get("rank") == rank] + rv = [r for r in rs if r.get("valid")] + rw = [r for r in rv if r.get("normalized_result") == "win"] + rl = [r for r in rv if r.get("normalized_result") == "loss"] + per_rank[rank] = { + "attempts": len(rs), "valid": len(rv), "invalid": len(rs) - len(rv), + "wins": len(rw), "losses": len(rl), + "win_rate": (len(rw) / len(rv)) if rv else None, + } + + # camp split + camp_split = {} + for camp in (0, 1): + cv = [r for r in valid if r.get("camp") == camp] + cw = [r for r in cv if r.get("normalized_result") == "win"] + camp_split[camp] = {"valid": len(cv), "wins": len(cw), + "win_rate": (len(cw) / len(cv)) if cv else None} + + # score stats (valid games, ifelse score minus opponent score) + diffs = [] + for r in valid: + s = r.get("scores") or {} + s0, s1 = _safe_num(s.get("0")), _safe_num(s.get("1")) + if s0 is not None and s1 is not None: + camp = r.get("camp") + diffs.append(s0 - s1 if camp == 0 else s1 - s0) # from ifelse perspective + diffs_sorted = sorted(diffs) + def median(xs): + n = len(xs) + if n == 0: + return None + return xs[n // 2] if n % 2 else (xs[n // 2 - 1] + xs[n // 2]) / 2 + score_stats = { + "n": len(diffs), + "mean_diff": (sum(diffs) / len(diffs)) if diffs else None, + "median_diff": median(diffs_sorted), + } + + # steps stats (valid) + vsteps = [int(r.get("steps", 0) or 0) for r in valid] + steps_stats = { + "n": len(vsteps), + "total": sum(vsteps), + "mean": (sum(vsteps) / len(vsteps)) if vsteps else None, + "median": median(sorted(vsteps)), + } + + # realized randomization distribution (valid games) + map_dist = {"map_type": {}, "day_time": {}} + for r in valid: + rr = r.get("realized_randomization") or {} + for k in ("map_type", "day_time"): + v = rr.get(k) + if v is not None: + map_dist[k][v] = map_dist[k].get(v, 0) + 1 + + # invalid reason distribution + invalid_reasons: Dict[str, int] = {} + for r in invalid: + et = r.get("error_type") or "unknown" + invalid_reasons[et] = invalid_reasons.get(et, 0) + 1 + + return { + "total_attempts": len(records), + "valid_games": valid_games, + "invalid_games": len(invalid), + "wins": len(wins), + "losses": len(losses), + "win_rate": (len(wins) / valid_games) if valid_games else None, + "per_rank": per_rank, + "camp_split": camp_split, + "score_stats": score_stats, + "steps_stats": steps_stats, + "realized_randomization_distribution": map_dist, + "invalid_reason_distribution": invalid_reasons, + "small_sample_note": "32 局为小样本；含 invalid；不把对手崩溃包装为 Agent 胜利；win_rate 分母仅 valid games。", + } diff --git a/src/agentbench_frame/games/miracle/matrix_runner.py b/src/agentbench_frame/games/miracle/matrix_runner.py new file mode 100644 index 0000000..2b41a90 --- /dev/null +++ b/src/agentbench_frame/games/miracle/matrix_runner.py @@ -0,0 +1,945 @@ +"""Matrix runner: drives the 32-game Plan A using the tested matrix.py logic. + +Designed for ONE long-running process (single background runner). Safety: + * ``--dry-run`` / ``dry_run()`` never calls attempt_fn (no Judge/AI). + * Unique session; existing session refused (no overwrite/delete). + * Atomic manifest written before game 1. + * ``running`` entries are UNCERTAIN_IN_FLIGHT on restart → STOP, never auto-rerun. + * ``done`` written only after the attempt returned + event landed. + * valid / AI-invalid (continue, not in win-rate denom) / infra-failure (halt) distinct. + * Per-rank audit after both camps; PID+create_time residual halts. + * events.jsonl is the source of truth → summary independently re-computable. + +The real attempt_fn wraps ``match_runner.run_match_attempt``; tests inject a fake. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from agentbench_frame.games.miracle import matrix +from agentbench_frame.games.miracle.matrix import ( + aggregate, + append_event_atomic, + classify_game, + is_done, + load_progress, + make_attempt_plan, + make_session_id, + mark_done, + mark_running, + parse_game_id, + rank_audit, + should_stop_with_residual, + write_progress_atomic, +) +from agentbench_frame.games.miracle.smoke_audit import ensure_fresh_session + +# re-export for tests +__all__ = ["MatrixRunner", "make_attempt_plan", "parse_game_id", "mark_running", + "write_progress_atomic", "has_uncertain"] + + +def has_uncertain(progress: Dict[str, Any]) -> List[str]: + """game_ids left in 'running' state — uncertain whether they played.""" + return [gid for gid, e in progress.get("attempts", {}).items() if e.get("state") == "running"] + + +# --------------------------------------------------------------------------- # +# strict manifest / progress loaders (review#2 严格性收口) +# --------------------------------------------------------------------------- # +# The 32 standard Plan A game_ids. plan_in_manifest must overwhelmingly equal +# this set; used to reject unknown progress entries when no expected_plan is +# available and to derive the known-gid set. +_LEGAL_STATES = ("not_started", "running", "done") + + +def _known_gids_from_plan(*plans) -> set: + """Derive the authoritative set of game_ids from one or more plan lists.""" + out: set = set() + for p in plans: + if p: + for a in p: + if isinstance(a, dict) and a.get("game_id") is not None: + out.add(a["game_id"]) + return out + + +def _load_progress_strict(path) -> Dict[str, Any]: + """Strict progress.json parse — NEVER swallow errors. Raises if the file is + present but corrupt/non-dict so resume() cannot accidentally proceed on a + half-broken progress that the prior verify_session_for_resume already + flagged (review#2 gap 9: no lenient re-parse after strict verify).""" + p = Path(path) + if not p.exists(): + return {"attempts": {}} + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, ValueError, UnicodeDecodeError) as e: + raise RuntimeError(f"progress.json corrupt/unparseable: {e}") from e + if not isinstance(data, dict): + raise RuntimeError(f"progress.json not a JSON object: got {type(data).__name__}") + if not isinstance(data.get("attempts", {}), dict): + raise RuntimeError("progress.attempts is not a dict") + return data + + +def _load_manifest_strict(path) -> Dict[str, Any]: + """Strict manifest.json parse for resume(): raises on corrupt/non-dict so + a broken manifest cannot silently let a wrong run_id through.""" + p = Path(path) + if not p.exists(): + return {} + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, ValueError, UnicodeDecodeError) as e: + raise RuntimeError(f"manifest.json corrupt/unparseable: {e}") from e + if not isinstance(data, dict): + raise RuntimeError(f"manifest.json not a JSON object: got {type(data).__name__}") + return data + + +def verify_session_for_resume(session_dir, *, + protocol_sha: Optional[str] = None, + code_files: Optional[Dict[str, str]] = None, + expected_plan: Optional[List[Dict]] = None, + expected_ifelse_sha: Optional[str] = None, + expected_judge_sha: Optional[str] = None, + expected_opponent_shas: Optional[Dict[int, str]] = None, + expected_build_shas: Optional[Dict[int, str]] = None, + expected_python: Optional[str] = None, + expected_platform: Optional[str] = None, + expected_control_inputs: Optional[Dict[str, str]] = None) -> Tuple[bool, List[str]]: + """Full READ-ONLY session verification before resume. Returns (ok, errors). + Does NOT write or modify anything. If ANY check fails, the session must be + left byte-for-byte unchanged.""" + import hashlib + sd = Path(session_dir) + errs: List[str] = [] + + # --- manifest --- + mp = sd / "manifest.json" + if not mp.exists(): + return (False, ["manifest missing"]) + try: + m = json.loads(mp.read_text(encoding="utf-8")) + except Exception: + return (False, ["manifest corrupt/unparseable"]) + if expected_control_inputs is not None: + stored_inputs = m.get("control_inputs") + if not isinstance(stored_inputs, dict): + errs.append("control input hash mismatch: control_inputs missing") + else: + expected_names = set(expected_control_inputs) + stored_names = set(stored_inputs) + for name in sorted(expected_names - stored_names): + errs.append(f"control input hash mismatch: {name} missing") + for name in sorted(stored_names - expected_names): + errs.append(f"control input hash mismatch: {name} unexpected") + for name in sorted(expected_names & stored_names): + item = stored_inputs[name] + stored_sha = item.get("sha256") if isinstance(item, dict) else None + if stored_sha != expected_control_inputs[name]: + errs.append(f"control input hash mismatch: {name}") + if protocol_sha and m.get("protocol_sha256") != protocol_sha: + errs.append(f"protocol sha mismatch") + if code_files: + stored = m.get("code_hashes", {}) + for name, p in code_files.items(): + p = Path(p) + if not p.exists(): + errs.append(f"code file missing: {name}") + elif stored.get(name) != hashlib.sha256(p.read_bytes()).hexdigest(): + errs.append(f"code hash mismatch: {name}") + if m.get("timeout") != 8.0: + errs.append(f"timeout mismatch: {m.get('timeout')}") + _wts = m.get("wrapper_timeout_s") + if _wts is None: + errs.append("wrapper_timeout_s missing or null") + elif _wts != 180.0: + errs.append(f"wrapper_timeout_s mismatch: {_wts}") + # --- asset hashes (identity frozen at session creation) --- + if expected_ifelse_sha and m.get("ifelse_sha256") != expected_ifelse_sha: + errs.append("if-else sha mismatch") + if expected_judge_sha and m.get("judge_sha256") != expected_judge_sha: + errs.append("judge sha mismatch") + if expected_opponent_shas: + stored_opp = m.get("opponent_archive_sha256", {}) + for rk, v in expected_opponent_shas.items(): + key = f"rank{rk:02d}" if isinstance(rk, int) else str(rk) + if stored_opp.get(key) != v: + errs.append(f"opponent archive sha mismatch: {key}") + if expected_build_shas: + stored_build = m.get("cpp_build_sha256", {}) + for rk, v in expected_build_shas.items(): + key = f"rank{rk:02d}" if isinstance(rk, int) else str(rk) + if stored_build.get(key) != v: + errs.append(f"build sha mismatch: {key}") + # --- plan_count must equal len(plan) AND (when len-plan fallback) 32 --- + plan_in_manifest = m.get("plan", []) + if not isinstance(plan_in_manifest, list): + errs.append("plan field is not a list") + plan_in_manifest = [] + plan_len = len(plan_in_manifest) + declared_count = m.get("plan_count") + if declared_count is None: + errs.append("plan_count field missing") + elif not isinstance(declared_count, int) or declared_count != plan_len: + errs.append(f"plan_count={declared_count!r} != len(plan)={plan_len}") + # --- plan: full 32 content + order --- + if expected_plan: + if plan_len != len(expected_plan): + errs.append(f"plan length {plan_len} != {len(expected_plan)}") + else: + for i, (got, want) in enumerate(zip(plan_in_manifest, expected_plan)): + if not isinstance(got, dict): + errs.append(f"plan[{i}] not a dict: {got!r}") + continue + if got.get("game_id") != want.get("game_id"): + errs.append(f"plan[{i}] game_id {got.get('game_id')} != {want.get('game_id')}") + if got.get("rank") != want.get("rank"): + errs.append(f"plan[{i}] rank {got.get('rank')} != {want.get('rank')}") + if got.get("camp") != want.get("camp"): + errs.append(f"plan[{i}] camp {got.get('camp')} != {want.get('camp')}") + # reject extra/unexpected keys in the per-attempt plan entry + _want_keys = set(want.keys()) + _got_keys = set(got.keys()) + _extra = _got_keys - _want_keys + if _extra: + errs.append(f"plan[{i}] unexpected keys: {sorted(_extra)}") + elif plan_len != 32: + errs.append(f"plan_count={plan_len} != 32 (len(plan) fallback)") + # --- session_id MUST exist (review#2 gap 4: missing → reject) --- + m_sid = m.get("session_id") + if not m_sid: + errs.append("session_id missing or empty") + elif m_sid != sd.name: + errs.append(f"session_id mismatch: manifest={m_sid} != dir={sd.name}") + # --- run_id MUST exist --- + if not m.get("run_id"): + errs.append("run_id missing") + # --- runtime identity --- + if expected_python and m.get("python_version") and m.get("python_version") != expected_python: + errs.append(f"python version mismatch: {m.get('python_version')} != {expected_python}") + if expected_platform and m.get("platform") and m.get("platform") != expected_platform: + errs.append(f"platform mismatch: {m.get('platform')} != {expected_platform}") + # --- opponent / build / code_hashes: EXACT key-set match (review#2 gap 3) --- + if expected_opponent_shas is not None: + stored_opp = m.get("opponent_archive_sha256", {}) + if not isinstance(stored_opp, dict): + errs.append("opponent_archive_sha256 not a dict") + else: + expected_keys = {f"rank{rk:02d}" if isinstance(rk, int) else str(rk) for rk in expected_opponent_shas} + extra = set(stored_opp.keys()) - expected_keys + missing = expected_keys - set(stored_opp.keys()) + if extra: + errs.append(f"opponent_archive_sha256 unexpected/extra keys: {sorted(extra)}") + if missing: + errs.append(f"opponent_archive_sha256 missing keys: {sorted(missing)}") + if expected_build_shas is not None: + stored_build = m.get("cpp_build_sha256", {}) + if not isinstance(stored_build, dict): + errs.append("cpp_build_sha256 not a dict") + else: + expected_keys = {f"rank{rk:02d}" if isinstance(rk, int) else str(rk) for rk in expected_build_shas} + extra = set(stored_build.keys()) - expected_keys + missing = expected_keys - set(stored_build.keys()) + if extra: + errs.append(f"cpp_build_sha256 unexpected/extra keys: {sorted(extra)}") + if missing: + errs.append(f"cpp_build_sha256 missing keys: {sorted(missing)}") + if code_files is not None: + stored_code = m.get("code_hashes", {}) + if not isinstance(stored_code, dict): + errs.append("code_hashes not a dict") + else: + expected_keys = set(code_files.keys()) + extra = set(stored_code.keys()) - expected_keys + missing = expected_keys - set(stored_code.keys()) + if extra: + errs.append(f"code_hashes unexpected/extra keys: {sorted(extra)}") + if missing: + errs.append(f"code_hashes missing keys: {sorted(missing)}") + + # --- progress / events / audit consistency --- + pp = sd / "progress.json" + ep = sd / "events.jsonl" + ad = sd / "audit" + # STRICT progress parsing (do NOT use lenient load_progress which swallows errors) + if not pp.exists(): + errs.append("progress.json missing") + attempts = {} + else: + try: + progress = json.loads(pp.read_text(encoding="utf-8")) + except Exception: + errs.append("progress.json corrupt") + progress = {} + if not isinstance(progress, dict): + errs.append("progress.json not a dict") + progress = {} + attempts = progress.get("attempts", {}) + if not isinstance(attempts, dict): + errs.append("progress attempts not a dict") + attempts = {} + # events.jsonl — STRICT line-by-line: each non-empty line must be a JSON OBJECT + event_gids: List[str] = [] + if ep.exists(): + for line_no, line in enumerate(ep.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + e = json.loads(line) + except json.JSONDecodeError: + errs.append(f"events.jsonl line {line_no}: corrupt (not JSON)") + continue + if not isinstance(e, dict): + errs.append(f"events.jsonl line {line_no}: top-level not a JSON object") + continue + if "game_id" in e: + event_gids.append(e["game_id"]) + # --- progress attempts: known gids + legal state (review#2 gaps 1, 2) --- + known_gids = _known_gids_from_plan(expected_plan, plan_in_manifest) + if not known_gids: + # expected_plan not provided AND plan was rejected; use the std 32. + known_gids = {a["game_id"] for a in make_attempt_plan()} + for gid, entry in attempts.items(): + if not isinstance(entry, dict): + errs.append(f"progress attempt {gid!r}: entry not a dict") + continue + if gid not in known_gids: + errs.append(f"progress entry unknown game_id: {gid}") + continue + state = entry.get("state", "not_started") + if state not in _LEGAL_STATES: + errs.append(f"progress {gid}: illegal state {state!r}") + # running → UNCERTAIN + running = [gid for gid, e in attempts.items() if isinstance(e, dict) and e.get("state") == "running"] + if running: + errs.append(f"UNCERTAIN_IN_FLIGHT: {running}") + # done → exactly one event + done_gids = {gid for gid, e in attempts.items() if isinstance(e, dict) and e.get("state") == "done"} + from collections import Counter + ev_counts = Counter(event_gids) + dups = {gid: c for gid, c in ev_counts.items() if c > 1} + if dups: + errs.append(f"duplicate events: {dups}") + for gid in done_gids: + if ev_counts.get(gid, 0) != 1: + errs.append(f"done {gid} has {ev_counts.get(gid, 0)} events (expected 1)") + # not_started must not have events + not_started_with_ev = [gid for gid in ev_counts if gid not in done_gids and gid not in running] + if not_started_with_ev: + errs.append(f"not_started with events: {not_started_with_ev}") + # --- per-rank audit (review#2 gaps 6, 7) --- + for rank in range(1, 17): + c0 = f"m_rank{rank:02d}_camp0" + c1 = f"m_rank{rank:02d}_camp1" + both_done = c0 in done_gids and c1 in done_gids + af = ad / f"rank{rank:02d}.json" + if af.exists() and not both_done: + # premature/partial audit: audit present while neither OR only one camp done + only_one = (c0 in done_gids) ^ (c1 in done_gids) + errs.append(f"rank{rank:02d} premature/partial complete audit present " + f"(both_done={both_done} only_one_done={only_one})") + continue + if not af.exists(): + if both_done: + errs.append(f"complete rank{rank:02d} missing audit file") + continue + # audit present AND both done → deep validation (review#2 gap 6) + try: + audit = json.loads(af.read_text(encoding="utf-8")) + except Exception: + errs.append(f"rank{rank:02d} audit corrupt/unparseable") + continue + if not isinstance(audit, dict): + errs.append(f"rank{rank:02d} audit not a JSON object") + continue + if audit.get("rank") != rank: + errs.append(f"rank{rank:02d} audit rank mismatch: {audit.get('rank')!r}") + if audit.get("ok") is not True: + errs.append(f"rank{rank:02d} audit ok is not True (got {audit.get('ok')!r})") + audit_games = audit.get("games") + audit_pairs: List[Tuple] = [] + if isinstance(audit_games, list): + for g in audit_games: + if isinstance(g, dict): + audit_pairs.append((g.get("game_id"), g.get("camp"))) + expected_pairs = {(c0, 0), (c1, 1)} + actual_set = set(audit_pairs) + if actual_set != expected_pairs: + errs.append(f"rank{rank:02d} audit game_id/camp mismatch: " + f"{sorted(map(str, actual_set))} != {sorted(map(str, expected_pairs))}") + + return (len(errs) == 0, errs) + + +class MatrixRunner: + def __init__(self, *, session_root, judge_dir, ifelse_dir, opponent_dir_of, + vendor_script, framework_src, timeout: float = 8.0, + wrapper_timeout_s: float = 60.0, attempt_fn: Optional[Callable] = None, + protocol_sha: str = "", run_id: Optional[str] = None, + python: Optional[str] = None, evaluated_agent: str = "miracle_ifelse", + auth_text: str = ""): + self.session_root = Path(session_root) + self.judge_dir = Path(judge_dir) + self.ifelse_dir = Path(ifelse_dir) + self.opponent_dir_of = opponent_dir_of + self.vendor_script = Path(vendor_script) + self.framework_src = Path(framework_src) + self.timeout = timeout + self.wrapper_timeout_s = wrapper_timeout_s + self.attempt_fn = attempt_fn or self._default_attempt_fn + self.protocol_sha = protocol_sha + self.run_id = run_id or make_session_id() + self.python = python + self.evaluated_agent = evaluated_agent + self.auth_text = auth_text + self.session_id: Optional[str] = None + self.session_dir: Optional[Path] = None + self.progress: Dict[str, Any] = {"attempts": {}} + self.plan = make_attempt_plan() + + # ---- session / paths ---- # + def _setup_paths(self): + self.work_dir = self.session_dir / "work" + self.events_path = self.session_dir / "events.jsonl" + self.progress_path = self.session_dir / "progress.json" + self.data_dir = self.session_dir / "data" + self.run_dir = self.data_dir / "runs" / "24_miracle" / self.evaluated_agent / self.run_id + self.work_dir.mkdir(parents=True, exist_ok=True) + self.data_dir.mkdir(parents=True, exist_ok=True) + + def prepare_session(self) -> Path: + self.session_id = make_session_id() + self.session_dir = ensure_fresh_session(self.session_root, self.session_id) + self._setup_paths() + write_progress_atomic(self.progress_path, self.progress) + return self.session_dir + + def prepare_session_for_existing(self, session_id: str) -> Path: + sd = self.session_root / session_id + if sd.exists(): + raise FileExistsError(f"session already exists; refusing to overwrite: {sd}") + self.session_id = session_id + self.session_dir = ensure_fresh_session(self.session_root, session_id) + self._setup_paths() + return self.session_dir + + def resume(self, session_id: str) -> Path: + """Open an EXISTING session for resumption (no new session created). + Restores the original run_id from manifest BEFORE _setup_paths so the + run_dir matches the original session, not a new run_id. + + Uses STRICT manifest+progress parse (review#2 gap 9): a corrupt file + must RAISE, never silently swallow on lenient re-parse after a strict + verify_session_for_resume already approved the session. + """ + sd = self.session_root / session_id + if not sd.exists(): + raise FileNotFoundError(f"cannot resume: session not found: {sd}") + # restore run_id from manifest BEFORE _setup_paths (which builds run_dir from run_id) + _m = _load_manifest_strict(sd / "manifest.json") + _rid = _m.get("run_id") if _m else None + if _rid: + self.run_id = _rid + self.session_id = session_id + self.session_dir = sd + self._setup_paths() + # STRICT progress parse — never swallow errors after a successful verify + self.progress = _load_progress_strict(self.progress_path) + return self.session_dir + + # ---- manifest ---- # + def record_manifest(self, *, opponent_hashes: Dict[int, str], build_hashes: Dict[int, str], + ifelse_sha: str, judge_sha: str, code_hashes: Dict[str, str], + control_inputs: Optional[Dict[str, Dict[str, str]]] = None, + ) -> Path: + import platform, time + m = { + "session_id": self.session_id, "run_id": self.run_id, + "created_unix": time.time(), + "protocol_sha256": self.protocol_sha, "auth_text": self.auth_text, + "plan_count": len(self.plan), "plan": self.plan, + "timeout": self.timeout, "wrapper_timeout_s": self.wrapper_timeout_s, + "evaluated_agent": self.evaluated_agent, + "ifelse_sha256": ifelse_sha, "judge_sha256": judge_sha, + "opponent_archive_sha256": {f"rank{k:02d}": v for k, v in opponent_hashes.items()}, + "cpp_build_sha256": {f"rank{k:02d}": v for k, v in build_hashes.items()}, + "code_hashes": code_hashes, + "platform": platform.platform(), + } + if control_inputs is not None: + m["control_inputs"] = control_inputs + mp = self.session_dir / "manifest.json" + tmp = mp.with_suffix(".json.tmp") + tmp.write_text(json.dumps(m, ensure_ascii=False, indent=2), encoding="utf-8") + os.replace(tmp, mp) + return mp + + # ---- dry run (no subprocess) ---- # + def dry_run(self) -> Dict[str, Any]: + return {"session_id": self.session_id, "run_id": self.run_id, + "plan_count": len(self.plan), "plan": self.plan, + "judge_dir": str(self.judge_dir), "ifelse_dir": str(self.ifelse_dir), + "evaluated_agent": self.evaluated_agent, "timeout": self.timeout, + "wrapper_timeout_s": self.wrapper_timeout_s} + + # ---- per-game ---- # + def _attempt_kwargs(self, attempt: Dict[str, Any]) -> Dict[str, Any]: + rank, camp = attempt["rank"], attempt["camp"] + opp_dir = self.opponent_dir_of(rank) + opp_name = f"rank{rank:02d}" + if camp == 0: + p0_dir, p1_dir = self.ifelse_dir, opp_dir + p0_name, p1_name = self.evaluated_agent, opp_name + else: + p0_dir, p1_dir = opp_dir, self.ifelse_dir + p0_name, p1_name = opp_name, self.evaluated_agent + return dict(game_id=attempt["game_id"], p0_dir=p0_dir, p1_dir=p1_dir, + p0_name=p0_name, p1_name=p1_name, evaluated_agent_camp=camp, + evaluated_agent=self.evaluated_agent, opponent=opp_name, + work_dir=self.work_dir, judge_dir=self.judge_dir, + vendor_script=self.vendor_script, framework_src=self.framework_src, + timeout=self.timeout, wrapper_timeout_s=self.wrapper_timeout_s, + python=self.python) + + def _default_attempt_fn(self, **kw): + from agentbench_frame.games.miracle.match_runner import run_match_attempt + return run_match_attempt(**kw) + + def _run_one(self, attempt: Dict[str, Any]) -> Dict[str, Any]: + gid = attempt["game_id"] + self.progress = load_progress(self.progress_path) # resumable read + if is_done(self.progress, gid): + return {"halted": False, "skipped": True, "game_id": gid} + uncertain = has_uncertain(self.progress) + if uncertain: + return {"halted": True, "state": "HALTED_INFRA_FAILURE", + "reason": f"UNCERTAIN_IN_FLIGHT: {uncertain}"} + mark_running(self.progress, gid) + write_progress_atomic(self.progress_path, self.progress) + att = self.attempt_fn(**self._attempt_kwargs(attempt)) + rec = classify_game(att) + append_event_atomic(self.events_path, rec) + decision, reason = should_stop_with_residual(att) + if decision == "stop": + rec["halt_reason"] = reason + mark_done(self.progress, gid, rec) + write_progress_atomic(self.progress_path, self.progress) + return {"halted": True, "state": "HALTED_INFRA_FAILURE", + "reason": reason, "record": rec, "game_id": gid} + mark_done(self.progress, gid, rec) + write_progress_atomic(self.progress_path, self.progress) + return {"halted": False, "record": rec, "game_id": gid} + + # ---- per-rank ---- # + def execute_rank(self, rank: int) -> Dict[str, Any]: + attempts = [a for a in self.plan if a["rank"] == rank] + audit_dir = self.session_dir / "audit" + audit_dir.mkdir(parents=True, exist_ok=True) + for att in attempts: + res = self._run_one(att) + if res.get("halted"): + return res + # rank audit + self.progress = load_progress(self.progress_path) + games = [self.progress["attempts"][a["game_id"]] for a in attempts] + ok, reasons = rank_audit(rank, games) + (audit_dir / f"rank{rank:02d}.json").write_text( + json.dumps({"rank": rank, "ok": ok, "reasons": reasons, + "games": games}, ensure_ascii=False, indent=2), encoding="utf-8") + if not ok: + return {"halted": True, "state": "HALTED_INFRA_FAILURE", + "reason": f"rank{rank:02d} audit failed: {reasons}"} + return {"halted": False, "rank": rank} + + # ---- whole matrix ---- # + def execute(self) -> Dict[str, Any]: + self.progress = load_progress(self.progress_path) + uncertain = has_uncertain(self.progress) + if uncertain: + return {"halted": True, "state": "HALTED_INFRA_FAILURE", + "reason": f"UNCERTAIN_IN_FLIGHT: {uncertain}"} + for rank in range(1, 17): + rank_atts = [a for a in self.plan if a["rank"] == rank] + if all(is_done(self.progress, a["game_id"]) for a in rank_atts): + continue + res = self.execute_rank(rank) + if res.get("halted"): + return res + return {"halted": False, "completed": True, "state": "COMPLETE"} + + # ---- aggregate from events (independently re-computable) ---- # + def aggregate_from_events(self) -> Dict[str, Any]: + records = [] + if self.events_path.exists(): + for line in self.events_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + pass + return aggregate(records) + + # ---- Run-compatible output for Results pipeline (review #1) ---- # + def write_run_compatible_output(self) -> Path: + """Drive the framework ``Run`` lifecycle to produce CI-compatible + run output. + + Review #1 §5 + §6 收口：使用 staging → validation → atomic promotion + 模式。**绝不破坏性覆写**既有 run 目录。 + + 步骤： + 1. 在 *同一文件系统* 的 staging 目录（``/.staging/``) + 下生成完整候选 run（events.jsonl + summary.json + run.toml）。 + 2. 对候选 run 完整校验：32 unique game_ids（计划数）、event quality + 合法、run_id 一致性、events↔summary 重算一致、totals 通过 + ``Run.recompute_totals_from_events()`` 来自真实事件计数。 + 3. 校验全部通过后，对既有 live run 若存在则备份到 + ``/.backup_``（同文件系统），然后原子 os.replace + 交换 staging ↔ live。 + 4. promotion 失败时从备份回滚，不留下半截 events。 + 5. 最终 live 目录正好含一个 run_id 目录（不生成第二个）。 + + framework envelope 由 ``Run.write("game", **rec)`` 自动填，Miracle + 一手字段逐字透传。 + """ + from agentbench_frame.tracking.run import Run + from agentbench_frame.tracking.quality import inspect_event_file + + agg = self.aggregate_from_events() + + # 1) staging directory: /.staging/ + # same fs as data_dir so os.replace is atomic. + staging_root = self.data_dir / ".staging" + staging_root.mkdir(parents=True, exist_ok=True) + staging = staging_root / self.run_id + live_run_dir = self.run_dir + # A previous writer may have been terminated between the two directory + # renames. Recover before deleting/reusing any staging path. + self._recover_promotion_transaction( + live_run_dir, + live_run_dir.with_name(live_run_dir.name + ".backup_promote"), + live_run_dir.with_name(live_run_dir.name + ".promote_marker.json"), + ) + # always start staging clean (this is a scratch path, not run storage) + if staging.exists(): + import shutil as _sh + _sh.rmtree(staging) + # build a Run inside staging (NOT in the final runs/// + # path): we point data_dir at staging so Run writes its files there. + run = Run.start(game="24_miracle", agent=self.evaluated_agent, + run_type="eval", data_dir=str(staging), + run_id=self.run_id, append=False, + config={"matrix": "plan_a_32", + "total_attempts": agg["total_attempts"]}) + + # 2) re-emit every matrix session event through framework envelope + if self.events_path.exists(): + for line in self.events_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(rec, dict): + continue + run.write("game", **rec) + + # 3) recompute totals from valid game events on disk + recomputed = run.recompute_totals_from_events(game_event_type="game") + # log h2h (matrix synth-extracted) for summary wpis + run.log_h2h(agg.get("h2h", {}) if isinstance(agg, dict) else {}) + + # 4) finish in staging + run.finish(extra_summary={ + "win_rate": agg["win_rate"], + "win_rate_available": (agg["valid_games"] > 0), + "total_episodes": recomputed["episodes"], + "total_steps": recomputed["total_steps"], + "matrix_aggregate": agg, + "wins": recomputed["wins"], + "losses": recomputed["losses"], + "evaluation_status": + ("COMPLETE" if agg["valid_games"] > 0 else "NO_VALID_GAMES"), + }) + + # 5) validate candidate BEFORE promotion + candidate_run_dir = staging / "runs" / "24_miracle" / self.evaluated_agent / self.run_id + errs = self._validate_candidate_run(candidate_run_dir, expected_count=agg["total_attempts"]) + if errs: + # leave staging in place for inspection (it's under .staging/, not + # the real runs/ path), but do NOT promote. Caller-visible live run + # (if any) is byte-identical to its prior state. + raise RuntimeError(f"candidate run validation failed: {errs}") + + # 6) atomic promotion: staging → live, with backup-and-rollback. + self._atomic_promote(candidate_run_dir, live_run_dir) + self._cleanup_staging(staging_root, self.run_id) + return live_run_dir + + # ---- promotion helpers ------------------------------------------------ # + @staticmethod + def _validate_candidate_run(run_dir: Path, *, + expected_count: int) -> List[str]: + """Validate the candidate run directory prior to promotion. Returns + a list of failure descriptions (empty list = candidate valid).""" + errs: List[str] = [] + if not run_dir.exists(): + return [f"candidate run dir absent: {run_dir}"] + for fname in ("events.jsonl", "summary.json", "run.toml"): + if not (run_dir / fname).exists(): + errs.append(f"candidate missing: {fname}") + if errs: + return errs + # event quality + from agentbench_frame.tracking.quality import inspect_event_file + rep = inspect_event_file(run_dir / "events.jsonl") + if rep.malformed_lines: + errs.append(f"candidate event_quality: malformed_lines={rep.malformed_lines}") + if rep.duplicate_event_ids: + errs.append(f"candidate event_quality: duplicates={rep.duplicate_event_ids}") + if rep.missing_event_ids: + errs.append(f"candidate event_quality: missing event_ids={rep.missing_event_ids}") + if rep.missing_run_ids: + errs.append(f"candidate event_quality: missing run_ids={rep.missing_run_ids}") + # 32 (or expected_count) unique game_ids + ev = [] + for line in (run_dir / "events.jsonl").read_text(encoding="utf-8").splitlines(): + if line.strip(): + try: + ev.append(json.loads(line)) + except json.JSONDecodeError: + pass + game_events = [e for e in ev + if isinstance(e, dict) + and (e.get("event_type") == "game" or e.get("event") == "game")] + gids = [e.get("game_id") for e in game_events if e.get("game_id") is not None] + unique = set(gids) + if len(unique) != expected_count: + errs.append(f"candidate unique game_ids={len(unique)} != expected {expected_count}") + if len(gids) != len(unique): + errs.append(f"candidate duplicate game_ids in events.jsonl={len(gids)-len(unique)}") + # summary.json wins/losses/total_episodes == recomputed from events + s = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + valid = [e for e in game_events if e.get("valid")] + wins_ev = sum(1 for e in valid if e.get("normalized_result") == "win") + loss_ev = sum(1 for e in valid if e.get("normalized_result") == "loss") + steps_ev = sum(int(e.get("steps") or 0) for e in valid) + if s.get("total_episodes") != len(valid): + errs.append(f"summary.total_episodes={s.get('total_episodes')} != valid_events={len(valid)}") + if s.get("total_steps") != steps_ev: + errs.append(f"summary.total_steps={s.get('total_steps')} != events_steps_sum={steps_ev}") + if s.get("wins") != wins_ev: + errs.append(f"summary.wins={s.get('wins')} != events_wins={wins_ev}") + if s.get("losses") != loss_ev: + errs.append(f"summary.losses={s.get('losses')} != events_losses={loss_ev}") + # run.toml totals + try: + import tomllib + with open(run_dir / "run.toml", "rb") as f: + t = tomllib.load(f) + if t.get("run", {}).get("total_episodes") != len(valid): + errs.append(f"run.toml.total_episodes != {len(valid)}") + if t.get("run", {}).get("total_steps") != steps_ev: + errs.append(f"run.toml.total_steps != {steps_ev}") + except Exception as e: + errs.append(f"run.toml parse fail: {e!r}") + return errs + + @staticmethod + def _atomic_promote(candidate_dir: Path, live_dir: Path) -> None: + """Directory-level atomic promotion: candidate → live. + + Two-rename transaction on the SAME filesystem (guaranteed by staging + under ``/.staging/``): + + 1. (recovery) If backup exists but live doesn't → crash interrupted + between step 3 and 4 → restore backup → live. + 2. (recovery) If both live and backup exist → crash interrupted + after step 4 but before cleanup → promotion succeeded, delete backup. + 3. If live exists: ``os.rename(live, backup)`` → live now absent. + 4. ``os.rename(candidate, live)`` → candidate now at live path. + 5. If step 4 raised: ``os.rename(backup, live)`` → restore old. + 6. If step 4 succeeded: ``shutil.rmtree(backup)``. + + Uses ``os.rename`` (not ``os.replace``) because on Windows renaming + to a NON-existent destination works for directories, whereas + ``os.replace`` on a non-empty dir raises WinError 5. We guarantee + the destination doesn't exist by renaming live→backup first. + """ + # The implementation below is deliberately directory-granular: the + # three publishable files move together, never one at a time. Keep + # the legacy implementation below unreachable for compatibility with + # older patches; all callers return through this transaction path. + import shutil as _sh + from agentbench_frame.games.miracle.atomicio import atomic_write_json + if not candidate_dir.exists(): + raise FileNotFoundError(f"candidate missing: {candidate_dir}") + backup_dir = live_dir.with_name(live_dir.name + ".backup_promote") + marker_path = live_dir.with_name(live_dir.name + ".promote_marker.json") + MatrixRunner._recover_promotion_transaction(live_dir, backup_dir, marker_path) + live_dir.parent.mkdir(parents=True, exist_ok=True) + atomic_write_json(marker_path, { + "schema_version": 1, + "state": "prepared", + "live_name": live_dir.name, + "backup_name": backup_dir.name, + "candidate_name": candidate_dir.name, + }) + prior_live = live_dir.exists() + if prior_live: + if backup_dir.exists(): + raise RuntimeError(f"refusing to overwrite promotion backup: {backup_dir}") + os.rename(str(live_dir), str(backup_dir)) + # Test-only seam: a real child process exits after the dangerous + # first rename. It is never enabled by normal callers. + if os.environ.get("MIRACLE_TEST_KILL_AFTER_LIVE_RENAME") == "1": + os._exit(86) + try: + os.rename(str(candidate_dir), str(live_dir)) + except Exception: + if prior_live and backup_dir.exists(): + if live_dir.exists(): + preserved = live_dir.with_name(live_dir.name + ".failed_candidate") + if preserved.exists(): + raise RuntimeError(f"refusing to overwrite {preserved}") + os.rename(str(live_dir), str(preserved)) + os.rename(str(backup_dir), str(live_dir)) + raise + if prior_live and backup_dir.exists(): + _sh.rmtree(backup_dir) + try: + marker_path.unlink() + except FileNotFoundError: + pass + return + + import shutil as _sh + if not candidate_dir.exists(): + raise FileNotFoundError(f"candidate missing: {candidate_dir}") + backup_dir = live_dir.with_name(live_dir.name + ".backup_promote") + + # --- crash recovery --- + MatrixRunner._recover_interrupted_promote(live_dir, backup_dir) + + live_parent = live_dir.parent + live_parent.mkdir(parents=True, exist_ok=True) + + had_existing = live_dir.exists() and any(live_dir.iterdir()) + + if had_existing: + # Step 3: rename live → backup (live now absent) + if backup_dir.exists(): + _sh.rmtree(backup_dir) + os.rename(str(live_dir), str(backup_dir)) + else: + # live doesn't exist or is empty — remove if empty + if live_dir.exists(): + _sh.rmtree(live_dir) + backup_dir.mkdir(exist_ok=True) # empty placeholder for rollback safety + + try: + # Step 4: rename candidate → live (candidate now at live path) + os.rename(str(candidate_dir), str(live_dir)) + except Exception: + # Step 5: restore backup → live + if backup_dir.exists() and backup_dir != live_dir: + try: + if live_dir.exists(): + _sh.rmtree(live_dir) + os.rename(str(backup_dir), str(live_dir)) + except Exception: + pass # best-effort; caller sees the original exception + raise + + # Step 6: success — clean backup + if backup_dir.exists(): + try: + _sh.rmtree(backup_dir) + except Exception: + pass # non-critical; leftover is detected on next run + + @staticmethod + def _recover_promotion_transaction(live_dir: Path, backup_dir: Path, + marker_path: Path) -> None: + """Conservatively recover a directory promotion interrupted by a kill. + + A marker denotes an incomplete transaction. When both the old backup + and a candidate at the live path exist, the old live bytes win: the + candidate is moved aside for inspection and is never published by + inference. This is intentionally stricter than treating both paths + as a successful promotion. + """ + backup_exists = backup_dir.exists() + live_exists = live_dir.exists() + if not marker_path.exists(): + if backup_exists: + raise RuntimeError( + f"refusing ambiguous promotion state without marker: {backup_dir}" + ) + return + if backup_exists and not live_exists: + os.rename(str(backup_dir), str(live_dir)) + elif backup_exists and live_exists: + preserved = live_dir.with_name(live_dir.name + ".interrupted_candidate") + if preserved.exists(): + raise RuntimeError(f"refusing to overwrite {preserved}") + os.rename(str(live_dir), str(preserved)) + os.rename(str(backup_dir), str(live_dir)) + # A marker without a backup is a first-write transaction. The live + # directory is its only complete copy; leave it untouched. + if live_dir.exists(): + try: + marker_path.unlink() + except FileNotFoundError: + pass + + @staticmethod + def _recover_interrupted_promote(live_dir: Path, backup_dir: Path) -> None: + """Detect and recover from a crash during a previous _atomic_promote. + + Invariants after a clean run: live exists, backup does NOT exist. + Crash states: + - backup exists, live absent: interrupted between step 3 and 4 + → restore backup → live. + - both exist: interrupted after step 4 but before cleanup + → promotion succeeded; delete backup. + """ + import shutil as _sh + b_exists = backup_dir.exists() + l_exists = live_dir.exists() + if b_exists and not l_exists: + # crash between rename(live→backup) and rename(candidate→live) + try: + os.rename(str(backup_dir), str(live_dir)) + except OSError: + pass # best-effort + elif b_exists and l_exists: + # crash after successful promotion, before cleanup + try: + _sh.rmtree(backup_dir) + except Exception: + pass + + @staticmethod + def _cleanup_staging(staging_root: Path, run_id: str) -> None: + """Clean any leftover staging entries for this run_id.""" + import shutil as _sh + candidate = staging_root / run_id + if candidate.exists(): + try: + _sh.rmtree(candidate) + except Exception: + pass + backup = staging_root / (run_id + ".backup_promote") + if backup.exists(): + try: + _sh.rmtree(backup) + except Exception: + pass diff --git a/src/agentbench_frame/games/miracle/paths.py b/src/agentbench_frame/games/miracle/paths.py new file mode 100644 index 0000000..707b50d --- /dev/null +++ b/src/agentbench_frame/games/miracle/paths.py @@ -0,0 +1,49 @@ +"""Environment-driven path resolution for the 24_miracle adapter tools/tests. + +No machine-specific absolute paths live in source. Callers set: + AGENTBENCH_ROOT - AgentBench corpus root (has backend_sources/ + top_algorithms/) + MIRACLE_IFELSE_DIR - 高翔 if-else bot directory (ifelse_bot/) + AGENTBENCH_RESULTS - AgentBenchResults repo root (optional; pipeline tests skip if unset) +Falls back with a clear error / None rather than embedding any local path. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Optional + + +def _required(env: str, what: str) -> Path: + v = os.environ.get(env) + if not v: + raise RuntimeError(f"env {env} not set; point it at the {what}") + return Path(v) + + +def agentbench_root() -> Path: + return _required("AGENTBENCH_ROOT", "AgentBench corpus root (backend_sources/ + top_algorithms/)") + + +def judge_dir() -> Path: + return agentbench_root() / "backend_sources" / "corpus" / "24_miracle" / "logic" / "judge_dev_logic" + + +def sample_ai_dir() -> Path: + return agentbench_root() / "backend_sources" / "corpus" / "24_miracle" / "logic" / "judge_dev_sample_ai" + + +def extracted_dir() -> Path: + return agentbench_root() / "top_algorithms" / "corpus" / "24_miracle_final" / "extracted" + + +def archives_dir() -> Path: + return agentbench_root() / "top_algorithms" / "corpus" / "24_miracle_final" / "archives" + + +def ifelse_dir() -> Path: + return _required("MIRACLE_IFELSE_DIR", "高翔 if-else bot directory (contains main.py)") + + +def results_repo() -> Optional[Path]: + v = os.environ.get("AGENTBENCH_RESULTS") + return Path(v) if v else None diff --git a/src/agentbench_frame/games/miracle/proctree.py b/src/agentbench_frame/games/miracle/proctree.py new file mode 100644 index 0000000..2510466 --- /dev/null +++ b/src/agentbench_frame/games/miracle/proctree.py @@ -0,0 +1,226 @@ +"""Cross-platform process-tree management for the Miracle subprocess wrapper. + +Safety contract (阶段4b spec): + * Only exact PIDs that this manager registered are ever signalled. + * Before any signal, the PID's psutil create_time must match the value + recorded at registration. A reused / stale PID is NEVER killed. + * Graceful terminate first, then a short grace window (polled, not slept), + then force-kill the tree (descendants included) only if still alive. + * No name-based / fuzzy kill: a process is never selected by its image + name, and signalling is never issued in bulk by interpreter or script + name. All signalling targets exact registered PIDs via psutil; descendant + discovery uses psutil's own process parentage, which is identity-confirming. + * Idempotent: cleaning an already-dead PID is a silent no-op. + +Per-process bookkeeping distinguishes a natural exit from a runner-requested +termination, so the adapter never mistakes a post-end_info cleanup-kill of an +idling AI client for a strategy crash (see match_runner.py classification). +""" +from __future__ import annotations + +import subprocess +import time +from dataclasses import dataclass +from typing import Dict, List, Optional + +try: + import psutil +except ImportError as _exc: # pragma: no cover - exercised via subprocess in tests + raise ImportError( + "Miracle process-tree cleanup requires psutil.\n" + "Install with: uv sync --extra miracle (or: pip install psutil)" + ) from _exc + +#: tolerance (seconds) for create_time comparison when confirming PID identity. +IDENTITY_TOL_S = 1.0 + + +@dataclass +class ManagedProcess: + pid: int + role: str + started_at: float # psutil create_time captured at registration + popen: Optional[object] = None # subprocess.Popen when we own the process + natural_exit: bool = False + natural_returncode: Optional[int] = None + termination_requested: bool = False + termination_reason: Optional[str] = None + final_returncode: Optional[int] = None + forced_kill: bool = False + cleanup_succeeded: bool = False + identity_confirmed: bool = True + wait_error: Optional[str] = None # create_time matched at the moment we acted + + +# --------------------------------------------------------------------------- # +# low-level helpers (all exact-PID, psutil-based) +# --------------------------------------------------------------------------- # +def _create_time(pid: int) -> Optional[float]: + try: + return psutil.Process(pid).create_time() + except (psutil.NoSuchProcess, psutil.AccessDenied): + return None + + +def _identity_ok(pid: int, started_at: float) -> bool: + """True if `pid` currently belongs to the same process recorded at `started_at`.""" + ct = _create_time(pid) + if ct is None: + return False + return abs(ct - started_at) < IDENTITY_TOL_S + + +def _exists(pid: int) -> bool: + return psutil.pid_exists(pid) + + +def _wait_for_exit(pid: int, timeout: float, step: float = 0.03) -> bool: + """Poll until `pid` is gone or `timeout` elapses. Returns True if it died.""" + deadline = time.time() + timeout + while time.time() < deadline: + if not _exists(pid): + return True + time.sleep(step) + return not _exists(pid) + + +def _graceful_terminate(pid: int) -> None: + try: + psutil.Process(pid).terminate() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +def _collect_tree(root_pid: int): + """Return [root_pid, *descendant_pids] while the root is alive, so cleanup can + reach orphaned descendants even after the root exits. Descendants come from + psutil parentage (identity-confirming); selection is never by image name. + If the root is already gone, returns [root_pid] only.""" + try: + root = psutil.Process(root_pid) + kids = root.children(recursive=True) + return [root_pid] + [k.pid for k in kids] + except psutil.NoSuchProcess: + return [root_pid] + + +def _force_kill_one(pid: int) -> None: + """Force-kill a single EXACT pid (no descendant walk, no name match).""" + try: + psutil.Process(pid).kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +# --------------------------------------------------------------------------- # +# manager +# --------------------------------------------------------------------------- # +class ProcessTreeManager: + """Tracks exact PIDs and cleans them up with graceful -> force semantics. + + Use :meth:`register_popen` for processes you spawned directly and + :meth:`register_pid` for descendant PIDs you only learned about (e.g. read + from a child's stdout). Only registered PIDs are ever touched. + """ + + def __init__(self, grace_s: float = 2.0): + self.grace_s = grace_s + self._procs: Dict[int, ManagedProcess] = {} + + # ---- registration ---- + def register_popen(self, popen, role: str) -> ManagedProcess: + mp = ManagedProcess( + pid=popen.pid, role=role, + started_at=_create_time(popen.pid) or 0.0, popen=popen, + ) + self._procs[mp.pid] = mp + return mp + + def register_pid(self, pid: int, role: str) -> ManagedProcess: + mp = ManagedProcess(pid=pid, role=role, started_at=_create_time(pid) or 0.0) + self._procs[mp.pid] = mp + return mp + + # ---- liveness / refresh ---- + def is_alive(self, mp: ManagedProcess) -> bool: + return _identity_ok(mp.pid, mp.started_at) + + def poll(self) -> None: + for mp in self._procs.values(): + self._refresh(mp) + + def _refresh(self, mp: ManagedProcess) -> None: + if mp.natural_exit: + return + if mp.popen is not None: + rc = mp.popen.poll() + if rc is not None: + mp.natural_exit = True + mp.natural_returncode = rc + if mp.final_returncode is None: + mp.final_returncode = rc + elif not _identity_ok(mp.pid, mp.started_at): + # descendant we don't own has disappeared + mp.natural_exit = True + mp.natural_returncode = None + + # ---- cleanup ---- + def cleanup_one(self, mp: ManagedProcess, reason: str) -> ManagedProcess: + self._refresh(mp) + if mp.natural_exit: + mp.cleanup_succeeded = True + return mp + if not _identity_ok(mp.pid, mp.started_at): + # PID gone OR reused (create_time mismatch): never kill an unconfirmed PID + mp.identity_confirmed = False + mp.cleanup_succeeded = True + return mp + # live + identity-confirmed: request termination of the WHOLE tree. + # Collect descendants now, while the root is alive, so we can still reach + # them after the root exits (orphaned descendants would otherwise survive). + mp.identity_confirmed = True + mp.termination_requested = True + mp.termination_reason = reason + tree_pids = _collect_tree(mp.pid) + for p in tree_pids: + _graceful_terminate(p) + _wait_for_exit(mp.pid, self.grace_s) + survivors = [p for p in tree_pids if _exists(p)] + if survivors: + for p in survivors: + _force_kill_one(p) + mp.forced_kill = True + _wait_for_exit(mp.pid, max(self.grace_s, 1.0)) + # ALWAYS reap owned Popen BEFORE identity check (POSIX zombie stays in + # the process table until parent waits; without this, _identity_ok sees + # the zombie as alive → wrongly reports cleanup_failed) + if mp.popen is not None and mp.final_returncode is None: + try: + mp.final_returncode = mp.popen.wait(timeout=max(self.grace_s, 1.0)) + except subprocess.TimeoutExpired: + mp.wait_error = "TimeoutExpired (process did not exit after wait)" + except Exception as e: + mp.wait_error = repr(e) + # THEN check identity (after reap, zombie gone → PID freed → clean) + mp.cleanup_succeeded = not _identity_ok(mp.pid, mp.started_at) + return mp + + def cleanup_all(self, reason: str) -> List[ManagedProcess]: + return [self.cleanup_one(mp, reason) for mp in list(self._procs.values())] + + def status(self) -> List[dict]: + self.poll() + return [ + { + "pid": mp.pid, "role": mp.role, "started_at": mp.started_at, + "natural_exit": mp.natural_exit, + "natural_returncode": mp.natural_returncode, + "termination_requested": mp.termination_requested, + "termination_reason": mp.termination_reason, + "final_returncode": mp.final_returncode, + "forced_kill": mp.forced_kill, + "cleanup_succeeded": mp.cleanup_succeeded, + "identity_confirmed": mp.identity_confirmed, + } + for mp in self._procs.values() + ] diff --git a/src/agentbench_frame/games/miracle/result.py b/src/agentbench_frame/games/miracle/result.py new file mode 100644 index 0000000..c96f560 --- /dev/null +++ b/src/agentbench_frame/games/miracle/result.py @@ -0,0 +1,397 @@ +"""Pure result-normalization logic for the 24_miracle adapter. + +No subprocess, no framework imports — only the standard library. Fully +unit-testable. Semantics are pinned to: + +* Judge ``main.py`` (judge_dev_logic): ``end_info`` format, winner derivation, + crash/timeout handling, replay header layout. +* ``SKILL.md`` sections: 胜负归一化, h2h方向, total_steps定义, + AgentBenchResults数据契约. + +Keeping this module dependency-free means the contract tests run without the +Judge, the framework, or any external process. +""" +from __future__ import annotations + +import hashlib +import struct +from collections import Counter +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Sequence, Tuple + +# ---- normalized-result vocabulary (SKILL.md 胜负归一化) ---- +WIN = "win" +LOSS = "loss" +DRAW = "draw" +ERROR = "error" +#: results that count as 有效对局 (valid games). Errors are excluded. +VALID_RESULTS: Tuple[str, ...] = (WIN, LOSS, DRAW) + + +# --------------------------------------------------------------------------- # +# end_info parsing (Judge main.py:440-459) +# --------------------------------------------------------------------------- # +def derive_raw_winner(end_info: Optional[dict]) -> Optional[int]: + """Map the Judge's terminal ``end_info`` to a camp index. + + ``end_info = {"0": player0_score, "1": player1_score}`` (main.py:457) and + ``winner = 0 if score0 > score1 else 1`` (main.py:450). Ties are broken + toward player1 (main.py:448-449), so a decisive 0/1 is always produced when + the Judge finishes normally. Returns ``None`` when ``end_info`` is absent or + malformed — i.e. the Judge crashed or the match ended catastrophically. + """ + if not isinstance(end_info, dict) or "0" not in end_info or "1" not in end_info: + return None + try: + s0, s1 = int(end_info["0"]), int(end_info["1"]) + except (TypeError, ValueError): + return None + return 0 if s0 > s1 else 1 + + +def scores_from_end_info(end_info: Optional[dict]) -> Tuple[Optional[int], Optional[int]]: + """Return (player0_score, player1_score) from end_info, (None, None) if absent.""" + if not isinstance(end_info, dict): + return None, None + try: + return int(end_info.get("0")), int(end_info.get("1")) + except (TypeError, ValueError): + return None, None + + +# --------------------------------------------------------------------------- # +# per-game outcome +# --------------------------------------------------------------------------- # +@dataclass +class GameOutcome: + """Everything the adapter knows about one played game. + + ``evaluated_agent_camp`` is the player index (0/1) the evaluated agent sat + at for THIS game; side-swapping is expressed by flipping it across games. + """ + + game_id: str + evaluated_agent: str + opponent: str + evaluated_agent_camp: int + + # verdict inputs + raw_winner: Optional[int] = None # camp index 0/1, or None + score0: Optional[int] = None + score1: Optional[int] = None + ai_error_player: Optional[int] = None # camp index of an AI that crashed + ai_timeout_player: Optional[int] = None # camp index of an AI that timed out + judge_ok: bool = True # end_info present AND run_match exited 0 + replay_ok: bool = True + + # process-level signals (filled by the subprocess wrapper) + judge_exit: Optional[int] = None + ai0_exit: Optional[int] = None + ai1_exit: Optional[int] = None + process_cleanup: List[Dict[str, Any]] = field(default_factory=list) + exception: Optional[str] = None + timeout_s: Optional[float] = None + #: raw ``timeout`` from the result-json (vendor per-player flags shape). + timeout: Optional[Dict[str, Any]] = None + #: outer Popen / wrapper-layer exception; separate from vendor_exception. + wrapper_exception: Optional[str] = None + #: verbatim ``exception`` reported by the vendor inside the result-json. + vendor_exception: Optional[str] = None + #: internal ``run_match_returncode`` from the result-json; distinct from + #: outer ``vendor_returncode`` (which the wrapper observed). + run_match_returncode: Optional[int] = None + #: classification-flag: wrapper-level timeout drained the vendor tree. + wrapper_timeout: Optional[bool] = None + + # provenance / budget + realized_randomization: Optional[Dict[str, int]] = None # {map_type, day_time} from replay + steps: int = 0 # ai_operation count == environment steps + score_tie: bool = False # score0 == score1 (Judge resolves to player1) + judge_tiebreak_applied: bool = False # a tie was resolved to player1 by the Judge + reason: Optional[str] = None # classification reason (separate from raw exception) + error_type: Optional[str] = None # machine-readable error type (ai_crash, cleanup_failure, ...) + result_json_status: str = "ok" + vendor_returncode: Optional[int] = None + evidence_paths: Dict[str, str] = field(default_factory=dict) + duration_s: Optional[float] = None + started_at: Optional[float] = None + finished_at: Optional[float] = None + replay_path: Optional[str] = None + replay_sha256: Optional[str] = None + evaluated_source_sha256: Optional[str] = None + opponent_source_sha256: Optional[str] = None + is_resume: bool = False + is_rerun: bool = False + + # derived — filled by finalize() + winner_agent: Optional[str] = None + normalized_result: str = ERROR + valid: bool = False + draw: bool = False + + @property + def opponent_camp(self) -> int: + return 1 - self.evaluated_agent_camp + + def agent_at_camp(self, camp: int) -> str: + return self.evaluated_agent if camp == self.evaluated_agent_camp else self.opponent + + +def normalize(outcome: GameOutcome) -> str: + """Classify a game into ``{win, loss, draw, error}`` (SKILL.md 胜负归一化). + + Any AI crash, Judge crash, timeout, or missing replay => ``error`` and is + NOT counted as a capability win (rank03's opponent crash is exactly this). + The Judge never produces a draw (ties -> player1, main.py:448-449); DRAW is + retained only for protocol safety. + """ + if (not outcome.judge_ok) or (not outcome.replay_ok) or outcome.raw_winner is None: + return ERROR + if outcome.ai_error_player is not None or outcome.ai_timeout_player is not None: + return ERROR + if outcome.raw_winner not in (0, 1): + return DRAW + return WIN if outcome.raw_winner == outcome.evaluated_agent_camp else LOSS + + +def finalize(outcome: GameOutcome) -> GameOutcome: + """Fill derived fields in place and return the outcome.""" + outcome.normalized_result = normalize(outcome) + outcome.valid = outcome.normalized_result in VALID_RESULTS + outcome.draw = outcome.normalized_result == DRAW + outcome.winner_agent = ( + outcome.agent_at_camp(outcome.raw_winner) if outcome.raw_winner in (0, 1) else None + ) + outcome.score_tie = (outcome.score0 is not None and outcome.score1 is not None + and outcome.score0 == outcome.score1) + outcome.judge_tiebreak_applied = outcome.score_tie + return outcome + + +# --------------------------------------------------------------------------- # +# aggregates +# --------------------------------------------------------------------------- # +def compute_win_rate(outcomes: Sequence[GameOutcome]) -> float: + """``win_rate = 有效胜局数 / 有效对局数`` (SKILL.md). + + Draws are valid games but not wins, so they stay in the denominator; error + games are excluded entirely. Returns 0.0 when there are no valid games + (mirrors the framework Run's ``n = max(1, 0)`` behaviour). + """ + valid = [o for o in outcomes if o.normalized_result in VALID_RESULTS] + if not valid: + return 0.0 + wins = sum(1 for o in valid if o.normalized_result == WIN) + return wins / len(valid) + + +def compute_h2h(outcomes: Sequence[GameOutcome]) -> Dict[str, Dict[str, float]]: + """``h2h[row][col]`` = fraction of valid games between row and col that row + won (SKILL.md h2h方向: 行策略战胜列策略的胜率). + + Error games and games without a decisive winner are excluded. With draws, + ``h2h[row][col] + h2h[col][row]`` need not equal 1 (draws in denominator). + For Miracle (Judge never draws) the two entries over a pair sum to 1. + """ + games: Dict[Tuple[str, str], int] = {} + row_wins: Dict[Tuple[str, str], int] = {} + for o in outcomes: + if o.normalized_result == ERROR: + continue + a0, a1 = o.agent_at_camp(0), o.agent_at_camp(1) + # every valid game (win/loss/draw) counts toward the denominator + for pair in ((a0, a1), (a1, a0)): + games[pair] = games.get(pair, 0) + 1 + if o.raw_winner in (0, 1): # draws add to denominator but not to wins + winner = o.agent_at_camp(o.raw_winner) + loser = a1 if winner == a0 else a0 + row_wins[(winner, loser)] = row_wins.get((winner, loser), 0) + 1 + h2h: Dict[str, Dict[str, float]] = {} + for (row, col), n in games.items(): + h2h.setdefault(row, {})[col] = row_wins.get((row, col), 0) / n + return h2h + + +def outcome_counts(outcomes: Sequence[GameOutcome]) -> Dict[str, int]: + """Tally {win, loss, draw, error} counts (every key always present).""" + c = Counter(o.normalized_result for o in outcomes) + return {k: int(c.get(k, 0)) for k in (WIN, LOSS, DRAW, ERROR)} + + +# --------------------------------------------------------------------------- # +# run-level statistics (written into summary.json by MiracleEvalRunner) +# --------------------------------------------------------------------------- # +#: No deterministic seed support this round: the Judge draws map_type/day_time +#: via random.randint and reads no external seed. map_type/day_time are the +#: realized random environment parameters, NOT a seed. +DETERMINISTIC_SEED_SUPPORTED = False + + +def build_seed_provenance(realized_randomization: Optional[Dict[str, int]]) -> Dict[str, Any]: + """Build the per-game seed-provenance record. The Judge supports no seed, so + requested/effective seed are null; realized_randomization records the actual + random environment parameters (map_type/day_time) from the replay.""" + return { + "requested_seed": None, + "effective_seed": None, + "deterministic_seed_supported": DETERMINISTIC_SEED_SUPPORTED, + "reproducible_from_seed": False, + "realized_randomization": realized_randomization, + } + + +def compute_run_stats(outcomes: Sequence[GameOutcome]) -> Dict[str, Any]: + """Run-level statistics for summary.json. + + * attempted_games : every game whose attempt was recorded + * valid_games : games with a decisive, evidence-consistent result + * win_rate_denominator == valid_games; win_rate = wins / valid_games + * total_steps counts only valid games; attempted_steps counts all attempts + * win_rate is None and evaluation_status is NO_VALID_GAMES when valid_games == 0 + (we never report a fabricated 0% strength conclusion). + """ + attempted = len(outcomes) + valid = [o for o in outcomes if o.normalized_result in VALID_RESULTS] + valid_games = len(valid) + wins = sum(1 for o in valid if o.normalized_result == WIN) + losses = sum(1 for o in valid if o.normalized_result == LOSS) + draws = sum(1 for o in valid if o.normalized_result == DRAW) + attempted_steps = sum(int(getattr(o, "steps", 0) or 0) for o in outcomes) + total_steps = sum(int(getattr(o, "steps", 0) or 0) for o in valid) + return { + "attempted_games": attempted, + "valid_games": valid_games, + "invalid_games": attempted - valid_games, + "wins": wins, + "losses": losses, + "draws": draws, + "win_rate_denominator": valid_games, + "attempted_steps": attempted_steps, + "total_steps": total_steps, + "win_rate": (wins / valid_games) if valid_games > 0 else None, + "evaluation_status": ("COMPLETE" if valid_games > 0 else "NO_VALID_GAMES"), + } + + +# --------------------------------------------------------------------------- # +# replay / file hashing +# --------------------------------------------------------------------------- # +def read_replay_header(path) -> Optional[Dict[str, int]]: + """Read the actual random environment parameters from a Miracle replay file. + + The Judge writes the replay as big-endian signed int32s; the first 7 ints + are ``[0, 0, 0, map_type, day_time, 0, 0]`` (main.py:310-313). The Judge + draws ``map_type``/``day_time`` via ``random.randint`` (main.py:85-86) and + reads no external seed, so these are NOT a seed — they are the realized + random environment parameters recorded in the replay. Returns None if the + file is missing or too short. + """ + try: + with open(path, "rb") as f: + head = f.read(28) + except OSError: + return None + if len(head) < 28: + return None + try: + vals = struct.unpack(">7i", head) + except struct.error: + return None + return {"map_type": int(vals[3]), "day_time": int(vals[4])} + + +def sha256_file(path, chunk: int = 1 << 20) -> Optional[str]: + """SHA-256 of a file, or None if unreadable.""" + try: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(chunk), b""): + h.update(block) + return h.hexdigest() + except OSError: + return None + + +# --------------------------------------------------------------------------- # +# resumability (SKILL.md: 可恢复但不重复成功对局) +# --------------------------------------------------------------------------- # +def select_games_to_run(planned_ids: Sequence[str], + completed_valid_ids: Sequence[str]) -> List[str]: + """Return planned game ids that still need to run, i.e. those without an + existing valid (successful) recorded result. Order preserved.""" + done = set(completed_valid_ids) + return [gid for gid in planned_ids if gid not in done] + + +def would_rerun_successful(game_id: str, completed_valid_ids: Sequence[str]) -> bool: + """True if ``game_id`` already has a valid result and would be re-run.""" + return game_id in set(completed_valid_ids) + + +# --------------------------------------------------------------------------- # +# event-record builder (SKILL.md events.jsonl per-game fields) +# --------------------------------------------------------------------------- # +#: fields SKILL.md requires in each per-game event. Used by tests to assert +#: the record is contract-complete. +REQUIRED_EVENT_FIELDS = ( + "game_id", "seed", "policy_ids", "policy_source_sha256", "camps", + "raw_winner", "winner_agent", "normalized_result", "scores", "draw", + "started_at", "finished_at", "duration", "judge_exit", "ai0_exit", "ai1_exit", + "process_cleanup", "timeout_s", "exception", "replay_path", "replay_sha256", + "valid", "is_resume", "is_rerun", +) + + +def to_event_record(o: GameOutcome) -> Dict: + """Build the per-game event dict for ``Run.write("game", **...)``. + + Covers every SKILL.md events.jsonl per-game field; ``REQUIRED_EVENT_FIELDS`` + lists the contract keys so tests can assert completeness. + """ + return { + "event": "game", + "game_id": o.game_id, + "seed": build_seed_provenance(o.realized_randomization), + "score_tie": o.score_tie, + "judge_tiebreak_applied": o.judge_tiebreak_applied, + "evaluated_agent": o.evaluated_agent, + "opponent": o.opponent, + "evaluated_agent_camp": o.evaluated_agent_camp, + "policy_ids": [o.evaluated_agent, o.opponent], + "policy_source_sha256": [o.evaluated_source_sha256, o.opponent_source_sha256], + "camps": [0, 1], + "raw_winner": o.raw_winner, + "winner_agent": o.winner_agent, + "normalized_result": o.normalized_result, + "scores": {"0": o.score0, "1": o.score1}, + "draw": o.draw, + "valid": o.valid, + "started_at": o.started_at, + "finished_at": o.finished_at, + "duration": o.duration_s, + "judge_exit": o.judge_exit, + "ai0_exit": o.ai0_exit, + "ai1_exit": o.ai1_exit, + "process_cleanup": o.process_cleanup, + "timeout_s": o.timeout_s, + "timeout": o.timeout, + "wrapper_exception": o.wrapper_exception, + "vendor_exception": o.vendor_exception, + "run_match_returncode": o.run_match_returncode, + "wrapper_timeout": o.wrapper_timeout, + "exception": o.exception, + "ai_error_player": o.ai_error_player, + "ai_timeout_player": o.ai_timeout_player, + "judge_ok": o.judge_ok, + "replay_ok": o.replay_ok, + "steps": o.steps, + "replay_path": o.replay_path, + "replay_sha256": o.replay_sha256, + "reason": o.reason, + "error_type": o.error_type, + "result_json_status": o.result_json_status, + "vendor_returncode": o.vendor_returncode, + "evidence_paths": o.evidence_paths, + "is_resume": o.is_resume, + "is_rerun": o.is_rerun, + } diff --git a/src/agentbench_frame/games/miracle/runner.py b/src/agentbench_frame/games/miracle/runner.py new file mode 100644 index 0000000..ef9e8e8 --- /dev/null +++ b/src/agentbench_frame/games/miracle/runner.py @@ -0,0 +1,214 @@ +"""MiracleEvalRunner — top-level evaluation driver for 24_miracle. + +Does NOT inherit ``BaseRunner`` (that path has the win-attribution / run_type / +summary-ordering bugs — risks #2/#3/#4/#5/#6). Instead it drives ``Run`` directly: + + Run.start(game="24_miracle", agent=…, run_type="eval", data_dir=) + → run each game via match_runner.run_match_attempt (side-swapped) + → map each MatchAttempt to a GameOutcome (match_runner's classification wins) + → feed_outcomes_to_run (events for ALL attempts; log_episode for valid only) + → log_h2h + → Run.finish() + → atomically enrich summary.json with run-level statistics + → re-read disk summary + independently recompute from events.jsonl, assert equal + +When ``valid_games == 0`` the persisted ``win_rate`` is ``null`` and +``evaluation_status`` is ``NO_VALID_GAMES`` (never a fabricated 0% conclusion). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from agentbench_frame.games.miracle.driver import feed_outcomes_to_run +from agentbench_frame.games.miracle.match_runner import MatchAttempt, run_match_attempt +from agentbench_frame.games.miracle.result import ( + DRAW, + VALID_RESULTS, + GameOutcome, + compute_h2h, + compute_run_stats, +) +from agentbench_frame.tracking.run import Run + +GAME_ID = "24_miracle" + + +def attempt_to_outcome(att: MatchAttempt) -> GameOutcome: + """Carry a MatchAttempt into a GameOutcome. match_runner.classify sees the + full timeline (end_info order, cleanup vs crash, evidence consistency) so its + normalized_result/valid are authoritative here — we do not re-run normalize.""" + scores = att.scores or {} + o = GameOutcome( + game_id=att.game_id, + evaluated_agent=att.evaluated_agent, + opponent=att.opponent, + evaluated_agent_camp=att.evaluated_agent_camp, + raw_winner=att.raw_winner, + score0=scores.get("0") if isinstance(scores, dict) else None, + score1=scores.get("1") if isinstance(scores, dict) else None, + ai_error_player=att.ai_crash_player, + ai_timeout_player=att.ai_timeout_player, + judge_ok=(att.result_json_status == "ok" and not att.judge_crash + and not att.wrapper_timeout), + replay_ok=(att.realized_randomization is not None), + realized_randomization=att.realized_randomization, + steps=att.steps, + duration_s=att.duration_s, + started_at=att.started_at, + finished_at=att.finished_at, + replay_path=att.evidence_paths.get("replay"), + is_resume=att.collision_detected, + judge_exit=att.judge_exit, + ai0_exit=att.ai0_exit, + ai1_exit=att.ai1_exit, + process_cleanup=att.process_cleanup, + # Section 4: compat ``exception`` carries REAL exceptions only + # (wrapper_exception OR vendor_exception), NEVER the classification + # ``reason``. The rule lives in match_runner._build_attempt_from_files; + # here we just forward the deterministic value verbatim. + exception=att.exception, + timeout=getattr(att, "timeout", None), + timeout_s=getattr(att, "timeout_s", None), + wrapper_exception=getattr(att, "wrapper_exception", None), + vendor_exception=getattr(att, "vendor_exception", None), + run_match_returncode=getattr(att, "run_match_returncode", None), + wrapper_timeout=getattr(att, "wrapper_timeout", None), + ) + o.normalized_result = att.normalized_result + o.valid = att.valid + o.draw = (att.normalized_result == DRAW) + o.winner_agent = att.winner_agent + o.reason = att.reason + o.error_type = att.error_type + o.result_json_status = att.result_json_status + o.vendor_returncode = att.vendor_returncode + o.evidence_paths = att.evidence_paths + o.replay_sha256 = getattr(att, "replay_sha256", None) + o.score_tie = (o.score0 is not None and o.score1 is not None and o.score0 == o.score1) + o.judge_tiebreak_applied = o.score_tie + return o + + +def enrich_summary_atomically(run_dir: Path, outcomes: List[GameOutcome]) -> Dict[str, Any]: + """Merge run-level statistics into summary.json with an atomic temp+replace.""" + summary_path = run_dir / "summary.json" + summary = json.loads(summary_path.read_text(encoding="utf-8")) + stats = compute_run_stats(outcomes) + summary.update(stats) + summary["win_rate"] = stats["win_rate"] # None when valid_games == 0 + summary["win_rate_available"] = (stats["valid_games"] > 0) + summary["h2h"] = compute_h2h(outcomes) + from agentbench_frame.games.miracle.atomicio import atomic_write_json + atomic_write_json(summary_path, summary) + return summary + + +def recompute_from_events(run_dir: Path) -> Dict[str, Any]: + """Independently recompute win_rate / counts straight from events.jsonl.""" + games: List[dict] = [] + p = run_dir / "events.jsonl" + if p.exists(): + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + if e.get("event") == "game": + games.append(e) + valid = [g for g in games if g.get("normalized_result") in VALID_RESULTS] + wins = sum(1 for g in valid if g.get("normalized_result") == "win") + return { + "attempted_games": len(games), + "valid_games": len(valid), + "win_rate": (wins / len(valid)) if valid else None, + } + + +class MiracleEvalRunner: + def __init__(self, *, agent: str, data_dir: str, judge_dir, vendor_script, + framework_src, evaluated_dir, opponent_dir, n_games: int, + opponent: str = "opponent", + timeout: float = 12.0, wrapper_timeout_s: float = 60.0, work_dir, + config: Optional[Dict[str, Any]] = None, + attempt_fn: Optional[Callable] = None, prefix: str = "smoke"): + self.agent = agent + self.data_dir = data_dir + self.judge_dir = judge_dir + self.vendor_script = vendor_script + self.framework_src = framework_src + self.evaluated_dir = evaluated_dir + self.opponent_dir = opponent_dir + self.n_games = n_games + self.opponent = opponent + self.timeout = timeout + self.wrapper_timeout_s = wrapper_timeout_s + self.work_dir = work_dir + self.config = config or {} + self.prefix = prefix + self.attempt_fn = attempt_fn or self._default_attempt_fn + + def _default_attempt_fn(self, *, game_id, evaluated_agent_camp, + evaluated_agent, opponent, **_): + # TRUE side-swap: the evaluated agent is player0 on camp0 games and + # player1 on camp1 games, so the Judge sees both camp assignments. + if evaluated_agent_camp == 0: + p0_dir, p1_dir = self.evaluated_dir, self.opponent_dir + p0_name, p1_name = evaluated_agent, opponent + else: + p0_dir, p1_dir = self.opponent_dir, self.evaluated_dir + p0_name, p1_name = opponent, evaluated_agent + return run_match_attempt( + game_id=game_id, p0_dir=p0_dir, p1_dir=p1_dir, + p0_name=p0_name, p1_name=p1_name, + judge_dir=self.judge_dir, work_dir=self.work_dir, + vendor_script=self.vendor_script, framework_src=self.framework_src, + timeout=self.timeout, wrapper_timeout_s=self.wrapper_timeout_s, + evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent=evaluated_agent, opponent=opponent, + ) + + def run(self) -> Dict[str, Any]: + run = Run.start(game=GAME_ID, agent=self.agent, run_type="eval", + data_dir=self.data_dir, config=self.config) + outcomes: List[GameOutcome] = [] + attempts: List[MatchAttempt] = [] + for i in range(self.n_games): + eval_camp = i % 2 # alternate sides + game_id = f"{self.prefix}_{i:02d}_camp{eval_camp}" + att = self.attempt_fn( + game_id=game_id, evaluated_agent_camp=eval_camp, + evaluated_agent=self.agent, opponent=self.opponent, + ) + attempts.append(att) + outcomes.append(attempt_to_outcome(att)) + self.attempts = attempts + + feed_outcomes_to_run(run, outcomes) + run.finish() + + run_dir = Path(run.run_dir) + summary = enrich_summary_atomically(run_dir, outcomes) + + # cross-check: re-read disk summary and recompute from events independently + disk = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + recompute = recompute_from_events(run_dir) + wr_disk, wr_recompute = disk.get("win_rate"), recompute["win_rate"] + if wr_disk is None or wr_recompute is None: + assert wr_disk is wr_recompute, f"win_rate None-mismatch: {wr_disk} vs {wr_recompute}" + else: + assert abs(wr_disk - wr_recompute) < 1e-9, \ + f"win_rate mismatch: disk={wr_disk} recompute={wr_recompute}" + assert disk["total_episodes"] == recompute["valid_games"] + # guard against the double-runs path bug (risk #6): run_dir must be + # /runs/24_miracle//, never .../runs/runs/... + rel = run_dir.relative_to(self.data_dir) + assert rel.parts == ("runs", GAME_ID, self.agent, run.run_id), \ + f"unexpected run path shape (risk #6): {rel}" + summary["_recompute_check"] = recompute + return summary diff --git a/src/agentbench_frame/games/miracle/smoke_audit.py b/src/agentbench_frame/games/miracle/smoke_audit.py new file mode 100644 index 0000000..13dcd20 --- /dev/null +++ b/src/agentbench_frame/games/miracle/smoke_audit.py @@ -0,0 +1,197 @@ +"""Evidence-safety helpers for the 24_miracle smoke driver. + +Pure + unit-testable: session identity (never delete/overwrite), a STRICT +Group-1 gate, an INDEPENDENT residual-process check (exact PID + psutil +create_time — never trusts ``cleanup_succeeded``, never kills by name, never +touches unrelated processes), and manifest building. No subprocess execution +lives here; that stays in ``tools/miracle_smoke.py``. +""" +from __future__ import annotations + +import hashlib +import json +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +try: + import psutil +except ImportError as _exc: # pragma: no cover + raise ImportError( + "Miracle smoke residual check requires psutil.\n" + "Install with: uv sync --extra miracle (or: pip install psutil)" + ) from _exc + +IDENTITY_TOL_S = 1.0 + + +@dataclass +class ManagedProc: + pid: int + started_at: float # psutil create_time captured when the process was spawned + role: str + + +# --------------------------------------------------------------------------- # +# session identity +# --------------------------------------------------------------------------- # +def make_session_id() -> str: + """A fresh, effectively-unique session id (timestamp + token).""" + import datetime + import secrets + return datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "_" + secrets.token_hex(3) + + +def session_exists(root, session_id: str) -> bool: + return (Path(root) / session_id).exists() + + +def ensure_fresh_session(root, session_id: str) -> Path: + """Create and return a brand-new session dir. REFUSE (raise) if it already + exists — this module NEVER deletes or overwrites a prior session.""" + root = Path(root) + sd = root / session_id + if sd.exists(): + raise FileExistsError(f"session already exists; refusing to overwrite: {sd}") + root.mkdir(parents=True, exist_ok=True) + sd.mkdir(parents=True) + return sd + + +# --------------------------------------------------------------------------- # +# hashing + manifest +# --------------------------------------------------------------------------- # +def sha256_file(path) -> Optional[str]: + try: + h = hashlib.sha256() + with open(path, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + except OSError: + return None + + +def build_manifest(*, session_id: str, auth_cap: int, python_executable: str, + python_version: str, code_files, asset_files, + groups_planned: List[Dict[str, Any]], + notes: Optional[List[str]] = None) -> Dict[str, Any]: + return { + "session_id": session_id, + "created_unix": time.time(), + "auth_cap_games": auth_cap, + "python_executable": python_executable, + "python_version": python_version, + "code_hashes": {str(p): sha256_file(p) for p in code_files}, + "asset_hashes": {str(p): sha256_file(p) for p in asset_files}, + "groups_planned": groups_planned, + "notes": notes or [], + } + + +def write_manifest_atomic(session_dir, manifest: Dict[str, Any]) -> Path: + p = Path(session_dir) / "manifest.json" + from agentbench_frame.games.miracle.atomicio import atomic_write_json + return atomic_write_json(p, manifest) + + +# --------------------------------------------------------------------------- # +# INDEPENDENT residual-process check (exact PID + create_time identity) +# --------------------------------------------------------------------------- # +def load_managed_procs_from_result_json(path) -> List[ManagedProc]: + """Independently read a result-json and extract Judge/AI0/AI1 with their + PID + create_time identity. Returns [] if the file is missing/corrupt or has + no usable identity (callers must treat [] as 'cannot verify', never 'clean').""" + try: + rj = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return [] + out: List[ManagedProc] = [] + for role in ("judge", "ai0", "ai1"): + p = rj.get(role) or {} + pid = p.get("pid") + started = p.get("started_at") + if isinstance(pid, int) and pid > 0 and isinstance(started, (int, float)): + out.append(ManagedProc(pid=pid, started_at=float(started), role=role)) + return out + + +def check_residual_procs(procs: List[ManagedProc], + tol: float = IDENTITY_TOL_S) -> Dict[str, List[ManagedProc]]: + """For each recorded proc: + pid gone -> clean + pid exists, create_time matches -> RESIDUAL (real leftover) + pid exists, create_time differs -> reused (PID reuse; DO NOT kill) + No name matching, no batch kill, unrelated processes are never touched.""" + clean, residual, reused = [], [], [] + for mp in procs: + if not psutil.pid_exists(mp.pid): + clean.append(mp) + continue + try: + ct = psutil.Process(mp.pid).create_time() + except psutil.NoSuchProcess: + clean.append(mp) + continue + if abs(ct - mp.started_at) < tol: + residual.append(mp) + else: + reused.append(mp) + return {"clean": clean, "residual": residual, "reused": reused} + + +# --------------------------------------------------------------------------- # +# STRICT Group-1 gate +# --------------------------------------------------------------------------- # +def _att_reasons(att) -> List[str]: + r = [] + gid = getattr(att, "game_id", "?") + if not getattr(att, "valid", False): + r.append(f"{gid}: valid!=true (normalized={getattr(att,'normalized_result',None)})") + if getattr(att, "error_type", None): + r.append(f"{gid}: error_type={att.error_type} reason={getattr(att,'reason','')}") + if getattr(att, "wrapper_timeout", False): + r.append(f"{gid}: wrapper_timeout") + if getattr(att, "result_json_status", None) != "ok": + r.append(f"{gid}: result_json_status={att.result_json_status}") + if getattr(att, "discrepancies", None): + r.append(f"{gid}: evidence discrepancies={att.discrepancies}") + if not getattr(att, "realized_randomization", None): + r.append(f"{gid}: realized_randomization missing (replay not parseable)") + if getattr(att, "raw_winner", None) not in (0, 1): + r.append(f"{gid}: raw_winner not decisive ({att.raw_winner})") + return r + + +def group1_strict_clean(attempts, summary) -> Tuple[bool, List[str]]: + """Strict Group-1 gate. ANY anomaly blocks Group 2. An empty/missing invalid + set or missing process-identity can NEVER pass this gate.""" + reasons: List[str] = [] + if len(attempts) != 2: + reasons.append(f"attempt_count={len(attempts)} != 2") + for att in attempts: + reasons.extend(_att_reasons(att)) + # independent residual check — requires non-empty identity, never an empty free-pass + rj_path = getattr(att, "evidence_paths", {}).get("result_json") if hasattr(att, "evidence_paths") else None + procs = load_managed_procs_from_result_json(rj_path) if rj_path else [] + if not procs: + reasons.append(f"{att.game_id}: no managed-proc identity in result-json (cannot verify by empty set)") + else: + res = check_residual_procs(procs) + if res["residual"]: + reasons.append(f"{att.game_id}: RESIDUAL pids={[(p.pid, p.role) for p in res['residual']]}") + s = summary or {} + for k, want in (("attempted_games", 2), ("valid_games", 2), ("invalid_games", 0)): + if s.get(k) != want: + reasons.append(f"summary.{k}={s.get(k)} != {want}") + # events/summary recompute consistency (summary carries the runner's own check) + if s.get("evaluation_status") != "COMPLETE": + reasons.append(f"summary.evaluation_status={s.get('evaluation_status')} != COMPLETE") + return (len(reasons) == 0, reasons) + + +def should_run_group2(g1_ok: bool) -> bool: + """Group 2 is run ONLY when Group 1 is fully clean.""" + return bool(g1_ok) diff --git a/src/agentbench_frame/report/builder.py b/src/agentbench_frame/report/builder.py index 87320d3..530935a 100644 --- a/src/agentbench_frame/report/builder.py +++ b/src/agentbench_frame/report/builder.py @@ -421,7 +421,7 @@ def _render(self): html = template.render(**ctx) out_path = os.path.join(self.output_dir, output_name or template_name) - with open(out_path, "w") as f: + with open(out_path, "w", encoding="utf-8") as f: f.write(html) except jinja2.TemplateNotFound: continue diff --git a/src/agentbench_frame/tracking/providers.py b/src/agentbench_frame/tracking/providers.py index 968e051..08fb97e 100644 --- a/src/agentbench_frame/tracking/providers.py +++ b/src/agentbench_frame/tracking/providers.py @@ -10,14 +10,41 @@ import json import os +import shlex import subprocess import time from pathlib import Path -from typing import Any, Iterable, Mapping, Optional, Sequence +from typing import Any, Iterable, List, Mapping, Optional, Sequence, Union from agentbench_frame.tracking.provider import ProviderInvocation, ProviderUsage +#: Type alias accepted by ``executable`` — a single command (string, split with +#: :func:`shlex.split`) or a pre-split argv list. Accepting argv lets callers +#: use ``[sys.executable, path_to_script]`` so provider tests run on Windows +#: (where ``#!`` script execution via the bare filename is not possible). +ExecutableLike = Union[str, Sequence[str]] + + +def _split_executable(executable: ExecutableLike) -> List[str]: + """Normalize ``executable`` to a pre-split argv list. + + * ``str`` → :func:`shlex.split` (POSIX-style; equivalent on Windows when + used as ``subprocess.run([argv0, argv1, ...])`` since we never + shell out). + * iterable/list → list(obj) verbatim. + + Keeps backward compatibility with the prior single-string form + (``executable="codex"``) while enabling cross-platform argv pre-splitting + (``executable=[sys.executable, '/abs/path/fake-codex.py']``), required so + a real process can be invoked on Windows where ``WinError 193`` would + otherwise be raised by the OS loader for a non-PE ``#!`` file. + """ + if isinstance(executable, str): + return list(shlex.split(executable, posix=os.name == "posix")) + return list(executable) + + def _lines(source: str | Iterable[str]) -> tuple[list[dict], int]: if isinstance(source, str): source = source.splitlines() @@ -165,12 +192,14 @@ class _SubprocessProvider: def __init__( self, - executable: str, + executable: ExecutableLike, timeout_s: Optional[float] = None, env: Optional[Mapping[str, str]] = None, extra_args: Sequence[str] = (), ) -> None: - self.executable = executable + # Canonical form is a pre-split argv list (list[str]). Backward + # compat: a plain string (e.g. ``"codex"``) is split via shlex. + self.executable: List[str] = _split_executable(executable) self.timeout_s = timeout_s self.env = dict(env) if env is not None else None self.extra_args = list(extra_args) @@ -226,7 +255,8 @@ class CodexProvider(_SubprocessProvider): provider_name = "codex" parser = staticmethod(parse_codex_jsonl) - def __init__(self, executable: str = "codex", sandbox: str = "workspace-write", **kwargs) -> None: + def __init__(self, executable: ExecutableLike = "codex", + sandbox: str = "workspace-write", **kwargs) -> None: super().__init__(executable, **kwargs) self.sandbox = sandbox @@ -235,7 +265,7 @@ def build_command(self, context: Mapping[str, Any]) -> list[str]: if not prompt: raise ValueError("Codex provider requires context['prompt'] or context['task']") sandbox = context.get("sandbox", self.sandbox) - return [self.executable, "exec", "--json", "--sandbox", str(sandbox), + return [*self.executable, "exec", "--json", "--sandbox", str(sandbox), *self.extra_args, str(prompt)] @@ -247,7 +277,7 @@ class ClaudeCodeProvider(_SubprocessProvider): def __init__( self, - executable: str = "claude", + executable: ExecutableLike = "claude", permission_mode: str = "acceptEdits", **kwargs, ) -> None: @@ -259,7 +289,7 @@ def build_command(self, context: Mapping[str, Any]) -> list[str]: if not prompt: raise ValueError("Claude Code provider requires context['prompt'] or context['task']") permission_mode = context.get("permission_mode", self.permission_mode) - command = [self.executable, "-p", str(prompt), "--output-format", "stream-json", + command = [*self.executable, "-p", str(prompt), "--output-format", "stream-json", "--verbose", "--permission-mode", str(permission_mode)] max_turns = context.get("max_turns") if max_turns is not None: diff --git a/src/agentbench_frame/tracking/quality.py b/src/agentbench_frame/tracking/quality.py index fa81f9d..19577c8 100644 --- a/src/agentbench_frame/tracking/quality.py +++ b/src/agentbench_frame/tracking/quality.py @@ -13,6 +13,7 @@ "benchmark_game_result", "benchmark_evaluation", "evaluation", "budget", "policy_kl_trace", "occupancy", "elo", "h2h", "resource", "provider_event", "provider_invocation", + "game", }) diff --git a/src/agentbench_frame/tracking/run.py b/src/agentbench_frame/tracking/run.py index 6329d54..e8e03cc 100644 --- a/src/agentbench_frame/tracking/run.py +++ b/src/agentbench_frame/tracking/run.py @@ -60,7 +60,12 @@ class Run: def start(cls, game: str, agent: str, run_type: str = "eval", data_dir: Optional[str] = None, - config: Optional[Dict[str, Any]] = None) -> "Run": + config: Optional[Dict[str, Any]] = None, + run_id: Optional[str] = None, + append: bool = True, + created: Optional[str] = None, + git_commit: Optional[str] = None, + started_at: Optional[float] = None) -> "Run": """Create a Run. Args: @@ -69,9 +74,36 @@ def start(cls, game: str, agent: str, run_type: "rl" | "rule_iter" | "eval" data_dir: Override data root (default: $AGENTBENCH_DATA or ./agentbench_data) config: Arbitrary config dict written to run.toml [config] section + run_id: Optional caller-supplied run_id (review #1 backcompat). When + provided (e.g. ``20260722-001929_52bd14`` for matrix resume or + offline migration), the run directory reuses that id verbatim + — no second run directory is ever created. When omitted, the + framework auto-generates one via :meth:`_make_run_id`. + append: If True (default), the events.jsonl writer opens in + append mode so multiple ``Run`` objects on the same run_id + preserve history. If False, the events.jsonl is truncated at + start — used by idempotent rebuild paths (review #1 matrix + ``write_run_compatible_output``) so re-invocations do not + create duplicate event lines. + created: Optional ISO-8601 timestamp string (e.g. + ``2026-07-21T16:19:30Z``) for offline migration. When + provided, the run's ``created`` metadata (and the persisted + run.toml + summary.json ``created`` field) reflect this + authoritative evidence timestamp instead of the framework's + "now". When omitted, the current wall clock is used + (default behaviour). Must be a valid ISO-8601 string. + git_commit: Optional git commit short-SHA, overriding the + framework's auto-detected ``git rev-parse --short HEAD``. + Used in offline migration where the migrated run should carry + the original commit identity. """ data_dir = data_dir or _data_root() - run_id = cls._make_run_id() + if run_id is None or not isinstance(run_id, str) or not run_id: + run_id = cls._make_run_id() + if created is None or not isinstance(created, str) or not created: + created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + if git_commit is None or not isinstance(git_commit, str): + git_commit = _git_commit() # CI-expected structure: runs/{game}/{agent}/{run_id}/ run_path = os.path.join(data_dir, "runs", game, agent, run_id) os.makedirs(run_path, exist_ok=True) @@ -79,13 +111,13 @@ def start(cls, game: str, agent: str, meta = RunMeta( run_id=run_id, game=game, agent=agent, run_type=run_type, - created=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - git_commit=_git_commit(), - started_at=time.time(), + created=created, + git_commit=git_commit, + started_at=(started_at if started_at is not None else time.time()), config=config or {}, ) - writer = JSONLWriter(os.path.join(run_path, "events.jsonl")) + writer = JSONLWriter(os.path.join(run_path, "events.jsonl"), append=append) return cls(run_id=run_id, run_dir=run_path, meta=meta, writer=writer, config=config or {}) @staticmethod @@ -415,16 +447,94 @@ def log_h2h(self, h2h: Dict[str, Dict[str, float]]): # ---- finish ---- - def finish(self, extra_summary: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + def recompute_totals_from_events(self, *, + game_event_type: str = "game") -> Dict[str, Any]: + """Recompute ``_episodes`` / ``_total_steps`` / win-loss-draw counts + from the live ``game`` events written to disk by callers that bypass + :meth:`log_episode` (e.g. matrix re-emit, offline migration). Returns + the recalculated counters as a dict. + + Why this is needed (review #1 §5): matrix records ARE per-game events + emitted via ``run.write("game", ...)``; framework counters + (``self._episodes`` / ``self._total_steps``) are only incremented by + :meth:`log_episode`. Without this recomputation, ``_write_toml`` / + ``_build_summary`` would report zero totals even though the events + file on disk has the real per-game records. We refuse to fabricate + episode events; matrix events ARE the events. + + Semantics: + + - reads only events of ``event_type == game_event_type`` (default + ``"game"``) — game records that carry ``valid``, + ``normalized_result`` (win/loss/draw/error), ``steps``; + - increments ``_episodes`` only for valid game records (matches + :meth:`log_episode`'s ``valid=True`` semantics + matches the + Miracle aggregate's ``valid_games`` count); + - increments ``_total_steps`` by per-game steps for valid games; + - DOES NOT mutate ``_episode_rewards`` / ``_episode_winners`` / + ``_episode_agent_players`` — these bookkeeping lists are only + populated by :meth:`log_episode`, and matrix migration has no + meaningful reward / per-episode winner representation. summary + ``win_rate`` / ``wins`` / ``losses`` / ``draws`` are still set + by ``extra_summary`` passed to :meth:`finish` for matrix paths. + + Returns a dict with the recomputed counters so callers (matrix + migration) can sanity-check totals are consistent with the hard + anchors BEFORE the run is committed/promoted. + """ + # Flush buffered writes before reading. + self.writer.flush() + path = os.path.join(self.run_dir, "events.jsonl") + recomputed = dict(episodes=0, total_steps=0, wins=0, losses=0, + draws=0, valid_games=0) + if not os.path.exists(path): + return recomputed + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if not isinstance(rec, dict): + continue + if rec.get("event_type", rec.get("event")) != game_event_type: + continue + valid = bool(rec.get("valid")) + if not valid: + continue + recomputed["valid_games"] += 1 + recomputed["episodes"] += 1 + recomputed["total_steps"] += int(rec.get("steps") or 0) + nr = rec.get("normalized_result") + if nr == "win": + recomputed["wins"] += 1 + elif nr == "loss": + recomputed["losses"] += 1 + elif nr == "draw": + recomputed["draws"] += 1 + # Mutate framework counters in place so _build_summary / _write_toml + # see the recomputed totals. + self._episodes = recomputed["episodes"] + self._total_steps = recomputed["total_steps"] + return recomputed + + def finish(self, extra_summary: Optional[Dict[str, Any]] = None, + finished_at: Optional[float] = None) -> Dict[str, Any]: """Stop tracking and persist the final run summary. ``extra_summary`` lets runners add workflow-specific metrics before the JSON file is written, so the returned and persisted summaries stay consistent. + + ``finished_at`` lets offline migration preserve the original execution + wall-clock time instead of stamping the migration's own time. """ if self._sampler: self._sampler.stop() self.writer.flush() - self.meta.finished_at = time.time() + self.meta.finished_at = (finished_at if finished_at is not None else time.time()) summary = self._build_summary() summary["budget"] = self._budget.snapshot() if extra_summary: @@ -490,12 +600,12 @@ def _resource_summary(self) -> Dict: def _write_toml(self, path: str): lines = ["[run]", - f'run_id = "{self.run_id}"', - f'game = "{self.meta.game}"', - f'agent = "{self.meta.agent}"', - f'type = "{self.meta.run_type}"', - f'created = "{self.meta.created}"', - f'git_commit = "{self.meta.git_commit}"', + f'run_id = "{self._toml_escape(self.run_id)}"', + f'game = "{self._toml_escape(self.meta.game)}"', + f'agent = "{self._toml_escape(self.meta.agent)}"', + f'type = "{self._toml_escape(self.meta.run_type)}"', + f'created = "{self._toml_escape(self.meta.created)}"', + f'git_commit = "{self._toml_escape(self.meta.git_commit)}"', f"started_at = {self.meta.started_at}"] if self.meta.finished_at: lines.append(f"finished_at = {self.meta.finished_at}") @@ -504,8 +614,13 @@ def _write_toml(self, path: str): if self.config: lines.append(""); lines.append("[config]") for k, v in self.config.items(): - if isinstance(v, str): lines.append(f'{k} = "{v}"') + if isinstance(v, str): lines.append(f'{k} = "{self._toml_escape(v)}"') elif isinstance(v, bool): lines.append(f"{k} = {str(v).lower()}") elif isinstance(v, (int, float)): lines.append(f"{k} = {v}") - with open(path, "w") as f: + with open(path, "w", encoding="utf-8") as f: f.write("\n".join(lines) + "\n") + + @staticmethod + def _toml_escape(s: str) -> str: + """Escape a string for TOML double-quoted value: backslash first, then quote.""" + return str(s).replace("\\", "\\\\").replace('"', '\\"') diff --git a/tests/miracle/_fake_vendor.py b/tests/miracle/_fake_vendor.py new file mode 100644 index 0000000..3c949ae --- /dev/null +++ b/tests/miracle/_fake_vendor.py @@ -0,0 +1,117 @@ +"""Fake vendor runner for match_runner tests. Simulates vendor run_match.py +behaviours WITHOUT the real Judge, so the integration tests are fast, hermetic, +and need none of the Miracle assets. + +Accepts the same standard args as the real vendor runner (--p0-dir, --p1-dir, +--p0-name, --p1-name, --timeout, --out, --tag, --result-json) plus a --fake-mode +that selects the simulated behaviour: + normal write clean result-json + trace (3 ai_operations) + valid replay, exit 0 + no_result exit 0 but write NO result-json + corrupt write truncated/invalid result-json, exit 0 + nonzero exit 2 immediately + hang spawn a long-lived child then hang the parent (wrapper-timeout test) + bigio write several MiB to stdout AND stderr, exit 0 +""" +import argparse +import json +import pathlib +import struct +import subprocess +import sys +import time + + +def _write_replay(path: pathlib.Path, map_type=0, day_time=1): + path.write_bytes(struct.pack(">7i", 0, 0, 0, map_type, day_time, 0, 0)) + + +def _write_trace(path: pathlib.Path, events): + with path.open("w", encoding="utf-8") as f: + for e in events: + f.write(json.dumps(e) + "\n") + + +def _proc(role, cleanup=True, natural=False, term_req=False, rc=0): + return {"role": role, "cleanup_succeeded": cleanup, + "natural_exit": natural, "termination_requested": term_req, + "final_returncode": rc, "forced_kill": False, + "identity_confirmed": True, "started_at": 0.0, "pid": 0} + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--p0-dir"); p.add_argument("--p1-dir") + p.add_argument("--p0-name", default="p0"); p.add_argument("--p1-name", default="p1") + p.add_argument("--timeout", type=float, default=12.0) + p.add_argument("--out", type=pathlib.Path, default=pathlib.Path(".")) + p.add_argument("--tag", default="fake") + p.add_argument("--result-json", type=pathlib.Path, default=None) + p.add_argument("--fake-mode", default="normal") + a = p.parse_args() + + a.out.mkdir(parents=True, exist_ok=True) + trace = a.out / f"{a.tag}.jsonl" + replay = a.out / f"{a.tag}.replay" + + def write_result(d): + if a.result_json is not None: + a.result_json.parent.mkdir(parents=True, exist_ok=True) + a.result_json.write_text(json.dumps(d)) + + if a.fake_mode == "normal": + _write_trace(trace, [ + {"kind": "match_start", "players": [a.p0_name, a.p1_name]}, + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_operation", "player": 1}, + {"kind": "ai_operation", "player": 0}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + _write_replay(replay, 0, 1) + write_result({ + "schema_version": 1, "tag": a.tag, + "end_info_received": True, "end_info": {"0": 5, "1": 2}, + "scores": {"0": 5, "1": 2}, "raw_winner": 0, + "score_tie": False, "judge_tiebreak_applied": False, + "timeout": {"ai0": False, "ai1": False}, + "ai_error": {"ai0": False, "ai1": False}, + "trace_path": str(trace), "replay_path": str(replay), + "cleanup_all_succeeded": True, "exception": None, + "run_match_returncode": 0, + "judge": _proc("judge"), + "ai0": _proc("ai0", cleanup=True, natural=True, term_req=False, rc=0), + "ai1": _proc("ai1", cleanup=True, natural=True, term_req=False, rc=0), + }) + sys.stdout.write("ok\n") + return 0 + + if a.fake_mode == "no_result": + return 0 # exit clean, write nothing + + if a.fake_mode == "corrupt": + if a.result_json is not None: + a.result_json.parent.mkdir(parents=True, exist_ok=True) + a.result_json.write_text('{"schema_version": 1, "end_info": {') # truncated + return 0 + + if a.fake_mode == "nonzero": + return 2 + + if a.fake_mode == "hang": + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(3600)"]) + sys.stdout.write(f"CHILD {child.pid}\n") + sys.stdout.flush() + time.sleep(3600) + return 0 + + if a.fake_mode == "bigio": + chunk = b"x" * (1024 * 1024) + for _ in range(3): + sys.stdout.buffer.write(chunk); sys.stdout.buffer.flush() + sys.stderr.buffer.write(chunk); sys.stderr.buffer.flush() + return 0 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/miracle/_proc_helper.py b/tests/miracle/_proc_helper.py new file mode 100644 index 0000000..d77bf5e --- /dev/null +++ b/tests/miracle/_proc_helper.py @@ -0,0 +1,56 @@ +"""Harmless helper process for the process-tree tests. NOT the real Judge. + +Modes (argv[1]): + stay sleep then exit 0 (or until killed) + stay_forever sleep until killed + spawn_child + spawn `stay ` as a detached child, print + "READY spawn_child parent= child=" (flush), + then sleep and exit + +The tests use these to build real parent/child trees without touching the +Miracle Judge or any AI binary. +""" +import os +import subprocess +import sys +import time + + +def _main() -> int: + mode = sys.argv[1] + py = sys.executable + me = os.path.abspath(__file__) + + if mode == "stay": + secs = float(sys.argv[2]) if len(sys.argv) > 2 else 60.0 + print(f"READY stay pid={os.getpid()} secs={secs}", flush=True) + time.sleep(secs) + return 0 + + if mode == "stay_forever": + print(f"READY stay_forever pid={os.getpid()}", flush=True) + while True: + time.sleep(3600) + return 0 + + if mode == "spawn_child": + child_secs = float(sys.argv[2]) + parent_secs = float(sys.argv[3]) + child = subprocess.Popen( + [py, me, "stay", str(child_secs)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + print( + f"READY spawn_child parent={os.getpid()} child={child.pid}", + flush=True, + ) + time.sleep(parent_secs) + return 0 + + return 2 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/tests/miracle/test_atomic_windows_retry.py b/tests/miracle/test_atomic_windows_retry.py new file mode 100644 index 0000000..2fd4f27 --- /dev/null +++ b/tests/miracle/test_atomic_windows_retry.py @@ -0,0 +1,39 @@ +import hashlib, json +from pathlib import Path +import pytest + +def _winerror(): + e = PermissionError(5, "sharing violation") + e.winerror = 5 + return e + +def test_unique_temp_and_transient_windows_retry(tmp_path, monkeypatch): + from agentbench_frame.games.miracle import atomicio as aio + target = tmp_path / "progress.json"; calls=[]; real=aio.os.replace + monkeypatch.setattr(aio, "_is_windows", lambda: True) + def flaky(src, dst): + calls.append(Path(src).name) + if len(calls) < 3: raise _winerror() + return real(src, dst) + monkeypatch.setattr(aio.os, "replace", flaky) + aio.atomic_write_json(target, {"ok": True}) + assert len(calls) == 3 and calls[0] == calls[1] == calls[2] + assert calls[0] != "progress.json.tmp" + assert json.loads(target.read_text(encoding="utf-8")) == {"ok": True} + +def test_persistent_windows_permission_preserves_old_target(tmp_path, monkeypatch): + from agentbench_frame.games.miracle import atomicio as aio + target = tmp_path / "progress.json"; target.write_text('{"old":true}', encoding="utf-8") + before=hashlib.sha256(target.read_bytes()).hexdigest(); monkeypatch.setattr(aio, "_is_windows", lambda: True) + monkeypatch.setattr(aio.os, "replace", lambda *a: (_ for _ in ()).throw(_winerror())) + with pytest.raises(PermissionError): aio.atomic_write_json(target, {"new": True}) + assert hashlib.sha256(target.read_bytes()).hexdigest() == before + assert json.loads(target.read_text(encoding="utf-8")) == {"old": True} + assert list(tmp_path.glob(".progress.json.*.tmp")) + +def test_progress_uses_shared_unique_writer(tmp_path): + from agentbench_frame.games.miracle.matrix import write_progress_atomic + a=tmp_path/"a.json"; b=tmp_path/"b.json" + write_progress_atomic(a,{"attempts":{"a":1}}); write_progress_atomic(b,{"attempts":{"b":2}}) + assert json.loads(a.read_text()) != json.loads(b.read_text()) + assert not (tmp_path/"a.json.tmp").exists() and not (tmp_path/"b.json.tmp").exists() diff --git a/tests/miracle/test_data_check_utf8.py b/tests/miracle/test_data_check_utf8.py new file mode 100644 index 0000000..cb5ff2e --- /dev/null +++ b/tests/miracle/test_data_check_utf8.py @@ -0,0 +1,29 @@ +"""Regression: data check must handle UTF-8 Chinese content on Windows (GBK-safe read).""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +def test_data_check_handles_utf8_chinese(tmp_path): + """summary.json with Chinese config must not cause GBK decode failure.""" + rd = tmp_path / "runs" / "24_miracle" / "test" / "r1" + rd.mkdir(parents=True) + (rd / "run.toml").write_text( + '[run]\nrun_id = "r1"\ngame = "24_miracle"\nagent = "test"\n' + 'type = "eval"\ncreated = "2026-01-01T00:00:00Z"\n', encoding="utf-8") + (rd / "summary.json").write_text(json.dumps({ + "run_id": "r1", "game": "24_miracle", "agent": "test", + "wall_hours": 1.0, "total_steps": 100, "win_rate": 0.5, + "config": {"note": "中文测试字符 — 小样本声明"}, + }, ensure_ascii=False), encoding="utf-8") + env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2] / "src")} + r = subprocess.run( + [sys.executable, "-m", "agentbench_frame.cli", "data", "check", + "--data-dir", str(tmp_path)], + capture_output=True, text=True, timeout=10, env=env) + assert r.returncode == 0, f"data check failed: {r.stderr}" + assert "1 valid" in r.stdout diff --git a/tests/miracle/test_dependencies.py b/tests/miracle/test_dependencies.py new file mode 100644 index 0000000..7700c21 --- /dev/null +++ b/tests/miracle/test_dependencies.py @@ -0,0 +1,34 @@ +"""Dependency-declaration test: psutil is required by the Miracle adapter and a +missing install must produce an actionable error (not a bare ModuleNotFoundError).""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[2] +_SRC = _REPO / "src" + + +def test_missing_psutil_gives_actionable_error(): + code = ( + "import sys\n" + "sys.modules['psutil'] = None # force import psutil to fail\n" + "import agentbench_frame.games.miracle.proctree\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = str(_SRC) + os.pathsep + env.get("PYTHONPATH", "") + env.pop("PYTHONHOME", None) + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env) + assert r.returncode != 0 + combined = r.stderr + r.stdout + assert "psutil" in combined + assert "uv sync --extra miracle" in combined, combined + + +def test_pyproject_declares_miracle_extra(): + text = (_REPO / "pyproject.toml").read_text(encoding="utf-8") + assert 'miracle = ["psutil"]' in text diff --git a/tests/miracle/test_driver.py b/tests/miracle/test_driver.py new file mode 100644 index 0000000..b8478cb --- /dev/null +++ b/tests/miracle/test_driver.py @@ -0,0 +1,218 @@ +"""Run-contract tests for the 24_miracle driver (SKILL.md 测试门槛 items 11 & 12, +plus the disk-level side-swap correctness that defeats framework risks #2/#3/#5). + +These instantiate the real framework ``Run`` (no Judge, no subprocess) and +assert that the persisted ``run.toml`` / ``summary.json`` are contract-complete +and that win_rate is correct on disk after side-swapping. + + py -3.13 -m pytest tests/miracle/test_driver.py -v +""" +from __future__ import annotations + +import json +import tomllib +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.driver import feed_outcomes_to_run +from agentbench_frame.games.miracle.result import ( + GameOutcome, + compute_h2h, + compute_win_rate, + finalize, +) +from agentbench_frame.tracking.run import Run + + +# ---- helpers ------------------------------------------------------------- # +def _outcome(gid, eval_camp, raw_winner, steps=40, score0=None, score1=None, **kw): + s0 = score0 if score0 is not None else (5 if raw_winner == 0 else 2) + s1 = score1 if score1 is not None else (2 if raw_winner == 0 else 5) + return finalize( + GameOutcome( + game_id=gid, + evaluated_agent="ifelse", + opponent="rank04", + evaluated_agent_camp=eval_camp, + raw_winner=raw_winner, + score0=s0, + score1=s1, + steps=steps, + replay_path=f"/tmp/{gid}.replay", + replay_sha256="x" * 64, + **kw, + ) + ) + + +def _build_run(tmp_path: Path, outcomes, agent="ifelse", run_type="eval"): + run = Run.start( + game="24_miracle", + agent=agent, + run_type=run_type, + data_dir=str(tmp_path), + config={"opponent_set": "smoke", "n_games": str(len(outcomes))}, + ) + feed_outcomes_to_run(run, outcomes) + summary = run.finish() + run_dir = tmp_path / "runs" / "24_miracle" / agent / run.run_id + return run, summary, run_dir + + +# ---- item 11: run.toml fields -------------------------------------------- # +def test_run_toml_has_required_fields(tmp_path): + _, _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + meta = tomllib.loads((run_dir / "run.toml").read_text()) + run_section = meta["run"] + for field in ("run_id", "game", "agent", "type", "created", + "started_at", "total_steps", "total_episodes"): + assert field in run_section, f"run.toml missing [run].{field}" + assert run_section["game"] == "24_miracle" + assert run_section["agent"] == "ifelse" + assert run_section["type"] == "eval" + assert meta["config"]["opponent_set"] == "smoke" + + +def test_run_toml_finished_at_written(tmp_path): + _, _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + meta = tomllib.loads((run_dir / "run.toml").read_text()) + assert "finished_at" in meta["run"] # only after finish() + + +# ---- item 12: summary.json fields ---------------------------------------- # +REQUIRED_SUMMARY_FIELDS = ( + "run_id", "game", "agent", "run_type", "created", "git_commit", + "wall_hours", "total_episodes", "total_steps", "win_rate", + "best_elo", "final_elo", "elo_history", "h2h", "resource_summary", "config", +) + + +def test_summary_json_has_required_fields(tmp_path): + _, summary, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + disk = json.loads((run_dir / "summary.json").read_text()) + for f in REQUIRED_SUMMARY_FIELDS: + assert f in disk, f"summary.json missing {f}" + # eval run: Elo fields present but null, NOT omitted (SKILL.md) + assert disk["best_elo"] is None + assert disk["final_elo"] is None + assert disk["elo_history"] == [] + + +# ---- disk-level side-swap correctness (risks #2/#3/#5) ------------------- # +def test_persisted_win_rate_correct_under_side_swap(tmp_path): + outcomes = [ + _outcome("g1", 0, 0, steps=40), # win as camp0 + _outcome("g2", 1, 1, steps=38), # win as camp1 (swapped) + _outcome("g3", 0, 1, steps=50), # loss + ] + _, summary, run_dir = _build_run(tmp_path, outcomes) + expected = compute_win_rate(outcomes) # 2/3 + # in-memory return is correct... + assert summary["win_rate"] == pytest.approx(expected) + # ...AND the value actually persisted to disk (this is the risk-#5 check) + disk = json.loads((run_dir / "summary.json").read_text()) + assert disk["win_rate"] == pytest.approx(expected) + assert disk["total_episodes"] == 3 + assert disk["total_steps"] == 128 # 40 + 38 + 50 (valid games only) + + +def test_error_games_excluded_from_persisted_counts(tmp_path): + outcomes = [ + _outcome("g1", 0, 0, steps=40), # win + _outcome("g2", 0, 0, steps=10, ai_error_player=1), # error (opp crash) + ] + _, _, run_dir = _build_run(tmp_path, outcomes) + disk = json.loads((run_dir / "summary.json").read_text()) + # only the valid game counts toward episodes/steps/win_rate + assert disk["total_episodes"] == 1 + assert disk["total_steps"] == 40 + assert disk["win_rate"] == pytest.approx(1.0) + # but the error game IS still in events.jsonl as an audit record + events = [json.loads(l) for l in (run_dir / "events.jsonl").read_text().splitlines() if l.strip()] + game_events = [e for e in events if e.get("event") == "game"] + assert len(game_events) == 2 + assert any(e["normalized_result"] == "error" for e in game_events) + + +def test_no_runs_runs_double_dir(tmp_path): + # risk #6: Run.start must write /runs/... NOT /runs/runs/... + _, _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + assert run_dir.exists() + # the forbidden double-runs path must NOT exist + assert not (tmp_path / "runs" / "runs").exists() + + +# ---- persisted h2h (item 13 integration) --------------------------------- # +def test_summary_h2h_matches_compute(tmp_path): + outcomes = [ + _outcome("g1", 0, 0), + _outcome("g2", 1, 1), + _outcome("g3", 0, 1), + ] + _, _, run_dir = _build_run(tmp_path, outcomes) + disk = json.loads((run_dir / "summary.json").read_text()) + expected = compute_h2h(outcomes) + assert disk["h2h"] == expected + assert disk["h2h"]["ifelse"]["rank04"] == pytest.approx(2 / 3) + + +# ---- NEW framework defect: Run._write_toml must escape backslashes -------- # +def test_run_toml_valid_with_windows_path_in_config(tmp_path): + """A config value containing a Windows path (backslashes) must NOT produce + invalid TOML. This is the root cause of the smoke `data check` failure + (run.toml 'Invalid hex value' on judge_dir_resolved). Run._write_toml must + escape backslashes and quotes in string values.""" + windows_path = "C:" + "\\\\Users\\\\" + "example\\\\judge_dev_logic" + run = Run.start( + game="24_miracle", agent="x", run_type="eval", data_dir=str(tmp_path), + config={"judge_dir_resolved": windows_path}, + ) + run.log_episode(reward=1.0, steps=5, winner=0) + run.finish() + run_dir = tmp_path / "runs" / "24_miracle" / "x" / run.run_id + # run.toml must parse cleanly and round-trip the path + meta = tomllib.loads((run_dir / "run.toml").read_text(encoding="utf-8")) + assert meta["config"]["judge_dir_resolved"] == windows_path + + +# ---- _write_toml escape regression: quotes / backslashes / mixed types ---- # +def _run_with_config(tmp_path, config): + run = Run.start(game="24_miracle", agent="c", run_type="eval", + data_dir=str(tmp_path), config=config) + run.log_episode(reward=1.0, steps=1, winner=0) + run.finish() + run_dir = tmp_path / "runs" / "24_miracle" / "c" / run.run_id + return tomllib.loads((run_dir / "run.toml").read_text(encoding="utf-8")) + + +def test_run_toml_config_double_quotes_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {"note": 'he said "hi"'}) + assert meta["config"]["note"] == 'he said "hi"' + + +def test_run_toml_config_backslashes_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {"p": "a\\b\\c"}) + assert meta["config"]["p"] == "a\\b\\c" + + +def test_run_toml_config_mixed_types_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {"s": "plain", "b": True, "i": 7, "f": 1.5, + "path": r"C:\x\y"}) + c = meta["config"] + assert c["s"] == "plain" and c["b"] is True and c["i"] == 7 and c["f"] == 1.5 + assert c["path"] == r"C:\x\y" + + +def test_run_toml_normal_string_not_over_escaped(tmp_path): + # a plain ASCII value must round-trip unchanged (no double-escaping) + meta = _run_with_config(tmp_path, {"group": "GROUP1"}) + assert meta["config"]["group"] == "GROUP1" + + +def test_run_toml_run_section_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {}) + r = meta["run"] + assert r["game"] == "24_miracle" and r["agent"] == "c" and r["type"] == "eval" + assert r["total_steps"] == 1 and r["total_episodes"] == 1 + assert r["started_at"] == float(r["started_at"]) # numeric, not a quoted string diff --git a/tests/miracle/test_entry.py b/tests/miracle/test_entry.py new file mode 100644 index 0000000..f8acb91 --- /dev/null +++ b/tests/miracle/test_entry.py @@ -0,0 +1,105 @@ +"""Tests for the cross-platform AI entry resolver (阶段9A §2). + +The executable is returned as an ABSOLUTE path (Windows CreateProcess does not +search the cwd= argument for a bare name), so Popen(cmd, cwd=ai_dir) works on +every platform. + +Covers: Windows main.exe / POSIX main / main.py; explicit priority; paths with +spaces not split; main.exe+main coexistence (deterministic); missing entry; +resolver does not modify the dir; returned exe path is absolute. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.entry import resolve_ai_command + + +def _set_platform(monkeypatch, name): + monkeypatch.setattr("agentbench_frame.games.miracle.entry._IS_NT", name == "nt") + + +def _abs(tmp_path, name): + return str(Path(tmp_path).resolve() / name) + + +def test_windows_recognises_main_exe(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main.exe")] + + +def test_posix_recognises_main(tmp_path, monkeypatch): + _set_platform(monkeypatch, "posix") + (tmp_path / "main").write_bytes(b"\x7fELF") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main")] + + +def test_python_main_py(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.py").write_text("print('ai')") + assert resolve_ai_command(tmp_path) == [sys.executable, _abs(tmp_path, "main.py")] + + +def test_python_main_py_posix(tmp_path, monkeypatch): + _set_platform(monkeypatch, "posix") + (tmp_path / "main.py").write_text("print('ai')") + assert resolve_ai_command(tmp_path) == [sys.executable, _abs(tmp_path, "main.py")] + + +def test_explicit_list_takes_priority(tmp_path): + (tmp_path / "main.exe").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path, explicit=["my", "args"]) == ["my", "args"] + + +def test_explicit_str_with_spaces_not_split(tmp_path): + explicit = r"C:\Program Files\Some Dir\main.exe" + assert resolve_ai_command(tmp_path, explicit=explicit) == [explicit] + assert len(resolve_ai_command(tmp_path, explicit=explicit)) == 1 + + +def test_explicit_empty_falls_through_to_autodetect(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path, explicit="") == [_abs(tmp_path, "main.exe")] + assert resolve_ai_command(tmp_path, explicit=None) == [_abs(tmp_path, "main.exe")] + + +def test_coexistence_windows_picks_main_exe(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + (tmp_path / "main").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main.exe")] + + +def test_coexistence_posix_picks_main(tmp_path, monkeypatch): + _set_platform(monkeypatch, "posix") + (tmp_path / "main.exe").write_bytes(b"MZ") + (tmp_path / "main").write_bytes(b"\x7fELF") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main")] + + +def test_missing_entry_clear_error(tmp_path): + with pytest.raises(FileNotFoundError) as ei: + resolve_ai_command(tmp_path) + assert str(Path(tmp_path).resolve()) in str(ei.value) + + +def test_resolver_does_not_modify_strategy_dir(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + before = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + resolve_ai_command(tmp_path) + after = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + assert before == after + + +def test_returned_exe_path_is_absolute(tmp_path, monkeypatch): + # Windows CreateProcess does not search cwd= for a bare name -> must be absolute + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + cmd = resolve_ai_command(tmp_path) + assert Path(cmd[0]).is_absolute() diff --git a/tests/miracle/test_match_runner.py b/tests/miracle/test_match_runner.py new file mode 100644 index 0000000..8fb98dd --- /dev/null +++ b/tests/miracle/test_match_runner.py @@ -0,0 +1,421 @@ +"""match_runner contract tests (阶段4b-4 spec, 20 required behaviours). + +RED LIGHT FIRST: the implementation match_runner.py does not exist yet, so the +whole module fails to import; the failure is saved to +docs/games/evidence/match_runner_redlight.txt, THEN the implementation is +written to turn these green. + +Pure-logic behaviours (1, 5-16, 18) drive synthesized result-json / trace / +Replay files through match_runner's reader + cross-validation + classifier. +Integration behaviours (2, 3, 4, 17, 20) drive the harmless _fake_vendor.py +through run_match_attempt. Behaviour 19 checks cwd-independent import. + + py -3.13 -m pytest tests/miracle/test_match_runner.py -v +""" +from __future__ import annotations + +import json +import os +import struct +import subprocess +import sys +import time +from pathlib import Path + +import psutil +import pytest + +from agentbench_frame.games.miracle import match_runner +from agentbench_frame.games.miracle.match_runner import ( + Classification, + MatchAttempt, + ReplayInfo, + TraceStats, + classify, + cross_validate, + load_result_json, + read_replay_info, + run_match_attempt, + stream_trace, +) + +FAKE_VENDOR = Path(__file__).parent / "_fake_vendor.py" +REPO = Path(__file__).resolve().parents[2] +SRC = REPO / "src" + + +# ---- synthesized-data builders ------------------------------------------- # +def mk_rj(**kw): + base = { + "schema_version": 1, "tag": "g", + "end_info_received": True, "end_info": {"0": 5, "1": 2}, + "scores": {"0": 5, "1": 2}, "raw_winner": 0, + "score_tie": False, "judge_tiebreak_applied": False, + "timeout": {"ai0": False, "ai1": False}, + "ai_error": {"ai0": False, "ai1": False}, + "cleanup_all_succeeded": True, "exception": None, + "run_match_returncode": 0, + "judge": {"cleanup_succeeded": True, "natural_exit": True, "termination_requested": False, "final_returncode": 0}, + "ai0": {"cleanup_succeeded": True, "natural_exit": True, "termination_requested": False, "final_returncode": 0}, + "ai1": {"cleanup_succeeded": True, "natural_exit": True, "termination_requested": False, "final_returncode": 0}, + } + base.update(kw) + return base + + +def write_json(p: Path, d): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(d)) + + +def write_trace(p: Path, events): + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("w", encoding="utf-8") as f: + for e in events: + f.write(json.dumps(e) + "\n") + + +def write_replay(p: Path, map_type=0, day_time=1): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(struct.pack(">7i", 0, 0, 0, map_type, day_time, 0, 0)) + + +def _classify_for(rj, ts, ri, *, wrapper_timeout=False, discrepancies=None, + evaluated_agent_camp=0, vendor_returncode=0): + status, data = ("ok", rj) if rj is not None else ("missing", None) + return classify(rj_status=status, rj=data, ts=ts, ri=ri, + discrepancies=discrepancies or [], vendor_returncode=vendor_returncode, + wrapper_timeout=wrapper_timeout, evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent="ifelse", opponent="rank04") + + +# ========================================================================== # +# 1. vendor runner returns a complete result-json (integration, fake vendor) # +# ========================================================================== # +def test_01_normal_complete_result_json(tmp_path): + att = run_match_attempt( + game_id="g1", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="sampleA", p1_name="sampleB", judge_dir=tmp_path / "judge", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + evaluated_agent="sampleA", opponent="sampleB", evaluated_agent_camp=0, + extra_vendor_args=["--fake-mode", "normal"], + ) + assert att.result_json_status == "ok" + assert att.valid is True + assert att.normalized_result == "win" # raw_winner 0, evaluated camp 0 + assert att.raw_winner == 0 + assert att.steps == 3 # 3 ai_operation events + assert att.realized_randomization == {"map_type": 0, "day_time": 1} + + +# ===== 2. result-json missing ===== # +def test_02_result_json_missing(tmp_path): + att = run_match_attempt( + game_id="g2", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "no_result"], + ) + assert att.result_json_status == "missing" + assert att.valid is False + assert att.error_type == "result_json_missing" + + +# ===== 3. result-json corrupt / truncated ===== # +def test_03_result_json_corrupt(tmp_path): + att = run_match_attempt( + game_id="g3", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "corrupt"], + ) + assert att.result_json_status == "corrupt" + assert att.valid is False + assert att.error_type == "result_json_corrupt" + + +# ===== 4. vendor runner nonzero exit ===== # +def test_04_vendor_nonzero_exit(tmp_path): + att = run_match_attempt( + game_id="g4", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "nonzero"], + ) + assert att.vendor_returncode != 0 + assert att.valid is False + assert att.error_type == "result_json_missing" # no result-json produced + + +# ===== 5. result-json & trace end_info consistent ===== # +def test_05_end_info_consistent(tmp_path): + rj = mk_rj() + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_operation", "player": 0}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + ri = read_replay_info(tmp_path / "absent.replay") + assert cross_validate(rj, ts, ri) == [] + + +# ===== 6. result-json & trace scores conflict ===== # +def test_06_scores_conflict(tmp_path): + rj = mk_rj(end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, raw_winner=0) + trace = tmp_path / "t.jsonl" + write_trace(trace, [{"kind": "match_end", "end_info": json.dumps({"0": 2, "1": 5})}]) + ts = stream_trace(trace) + ri = read_replay_info(tmp_path / "absent.replay") + discs = cross_validate(rj, ts, ri) + assert any("score" in d.lower() or "end_info" in d.lower() for d in discs) + c = _classify_for(rj, ts, ri, discrepancies=discs) + assert c.error_type == "evidence_mismatch" and not c.valid + + +# ===== 7. derived raw_winner conflicts with score rule ===== # +def test_07_raw_winner_conflicts(tmp_path): + rj = mk_rj(end_info={"0": 2, "1": 5}, scores={"0": 2, "1": 5}, raw_winner=0) # 0 says p0 won but s1>s0 + ri = read_replay_info(tmp_path / "absent.replay") + ts = TraceStats() # empty + discs = cross_validate(rj, ts, ri) + assert any("raw_winner" in d.lower() or "winner" in d.lower() for d in discs) + c = _classify_for(rj, ts, ri, discrepancies=discs) + assert c.error_type == "evidence_mismatch" and not c.valid + + +# ===== 8. trace has ai_error ===== # +def test_08_trace_ai_error(tmp_path): + rj = mk_rj() + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_error", "player": 1, "state": 3}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + assert 1 in ts.ai_error_players + ri = read_replay_info(tmp_path / "absent.replay") + c = _classify_for(rj, ts, ri) + assert c.error_type == "ai_crash" and c.ai_crash_player == 1 and not c.valid + + +# ===== 9. trace has ai_timeout ===== # +def test_09_trace_ai_timeout(tmp_path): + rj = mk_rj() + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_timeout", "player": 0, "state": 2}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + assert 0 in ts.ai_timeout_players + ri = read_replay_info(tmp_path / "absent.replay") + c = _classify_for(rj, ts, ri) + assert c.error_type == "ai_timeout" and c.ai_timeout_player == 0 and not c.valid + + +# ===== 10. AI natural exit before end_info ===== # +def test_10_ai_natural_exit_before_end_info(tmp_path): + rj = mk_rj(end_info_received=False, end_info=None, scores=None, raw_winner=None, + ai0={"cleanup_succeeded": True, "natural_exit": True, + "termination_requested": False, "final_returncode": 1}) + ri = read_replay_info(tmp_path / "absent.replay") + ts = TraceStats() + c = _classify_for(rj, ts, ri) + assert c.error_type == "ai_crash" and c.ai_crash_player == 0 and not c.valid + + +# ===== 11. AI cleanup-killed after end_info, nonzero returncode -> NOT crash # +def test_11_ai_cleanup_nonzero_after_end_info_is_normal(tmp_path): + rj = mk_rj(ai0={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": True, "final_returncode": 1}, + ai1={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": True, "final_returncode": 1}) + trace = tmp_path / "t.jsonl" + write_trace(trace, [{"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}]) + ts = stream_trace(trace) + replay = tmp_path / "r.replay"; write_replay(replay) + ri = read_replay_info(replay) + c = _classify_for(rj, ts, ri) + assert c.valid is True + assert c.normal_cleanup_nonzero is True + assert c.error_type is None + assert c.normalized_result == "win" + + +# ===== 12. Judge exits before end_info ===== # +def test_12_judge_exit_before_end_info(tmp_path): + rj = mk_rj(end_info_received=False, end_info=None, scores=None, raw_winner=None, + judge={"cleanup_succeeded": True, "natural_exit": True, + "termination_requested": False, "final_returncode": 1}, + ai0={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": False, "final_returncode": None}, + ai1={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": False, "final_returncode": None}) + ri = read_replay_info(tmp_path / "absent.replay") + ts = TraceStats() + c = _classify_for(rj, ts, ri) + assert c.error_type == "judge_crash" and c.judge_crash and not c.valid + + +# ===== 13. Replay missing ===== # +def test_13_replay_missing(tmp_path): + rj = mk_rj() + ts = TraceStats() + ri = read_replay_info(tmp_path / "absent.replay") + assert ri.exists is False + c = _classify_for(rj, ts, ri) + assert c.error_type == "replay_missing" and not c.valid + + +# ===== 14. Replay corrupt / header too short ===== # +def test_14_replay_short_header(tmp_path): + rj = mk_rj() + ts = TraceStats() + replay = tmp_path / "r.replay" + replay.write_bytes(b"\x00" * 10) # too short for 7 int32 + ri = read_replay_info(replay) + assert ri.exists is True and ri.header_valid is False + c = _classify_for(rj, ts, ri) + assert c.error_type == "replay_corrupt" and not c.valid + + +# ===== 15. Replay map_type/day_time parseable ===== # +def test_15_replay_header_parseable(tmp_path): + replay = tmp_path / "r.replay" + write_replay(replay, map_type=1, day_time=0) + ri = read_replay_info(replay) + assert ri.header_valid is True + assert ri.map_type == 1 and ri.day_time == 0 + assert ri.sha256 and len(ri.sha256) == 64 + + +# ===== 16. ai_operation streaming count ===== # +def test_16_ai_operation_streaming_count(tmp_path): + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_operation", "player": 1}, + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_operation", "player": 1}, + {"kind": "ai_operation", "player": 0}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + assert ts.n_ai_operation == 5 + assert ts.end_info_seen is True + + +# ===== 17. wrapper timeout cleans vendor + child tree ===== # +def test_17_wrapper_timeout_kills_vendor_tree(tmp_path): + import os as _os + att = run_match_attempt( + game_id="g17", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + wrapper_timeout_s=2.0, + extra_vendor_args=["--fake-mode", "hang"], + ) + assert att.wrapper_timeout is True + assert att.valid is False + assert att.error_type == "wrapper_timeout" + # the fake vendor printed "CHILD " to its stdout (captured to a file) + stdout_file = Path(att.evidence_paths["stdout"]) + child_pid = None + for line in stdout_file.read_text(errors="replace").splitlines(): + if line.startswith("CHILD "): + child_pid = int(line.split()[1]) + assert child_pid is not None, "fake vendor did not report a child PID" + # grace for the OS to reap + deadline = time.time() + 5 + while time.time() < deadline and psutil.pid_exists(child_pid): + time.sleep(0.1) + assert not psutil.pid_exists(child_pid), "orphan child survived wrapper-timeout cleanup" + + +# ===== 18. duplicate game_id / output-dir collision ===== # +def test_18_duplicate_game_id_collision(tmp_path): + common = dict( + p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", p0_name="a", p1_name="b", + judge_dir=tmp_path / "j", work_dir=tmp_path / "work", + vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "normal"], + ) + first = run_match_attempt(game_id="dup", evaluated_agent_camp=0, **common) + assert first.valid is True + first_json = Path(first.evidence_paths["result_json"]) + first_content = first_json.read_text() + second = run_match_attempt(game_id="dup", evaluated_agent_camp=0, **common) + assert second.collision_detected is True + # the first attempt's result-json must NOT have been clobbered + assert first_json.read_text() == first_content + + +# ===== 19. import works from any cwd ===== # +def test_19_import_from_any_cwd(tmp_path): + code = ( + "import sys\n" + f"sys.path.insert(0, {str(SRC)!r})\n" + "from agentbench_frame.games.miracle import match_runner\n" + "print('IMPORT_OK')\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = str(SRC) + os.pathsep + env.get("PYTHONPATH", "") + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, + env=env, cwd=str(tmp_path)) + assert r.returncode == 0, r.stderr + assert "IMPORT_OK" in r.stdout + + +# ===== 20. large stdout/stderr does not deadlock ===== # +def test_20_large_stdout_stderr_no_deadlock(tmp_path): + t0 = time.time() + att = run_match_attempt( + game_id="g20", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "bigio"], + ) + elapsed = time.time() - t0 + assert elapsed < 30, f"wrapper deadlocked or hung: {elapsed:.1f}s" + stdout_size = Path(att.evidence_paths["stdout"]).stat().st_size + stderr_size = Path(att.evidence_paths["stderr"]).stat().st_size + assert stdout_size >= 3 * 1024 * 1024 - 1024 + assert stderr_size >= 3 * 1024 * 1024 - 1024 + + +def test_relative_work_paths_are_absolute_before_vendor_spawn(tmp_path, monkeypatch): + """Judge cwd may differ from framework cwd; vendor gets absolute artifacts.""" + from agentbench_frame.games.miracle import match_runner as mod + monkeypatch.chdir(tmp_path) + real_popen = mod.subprocess.Popen + seen = {} + + def checked_popen(cmd, *args, **kwargs): + for flag in ("--out", "--result-json"): + value = Path(cmd[cmd.index(flag) + 1]) + assert value.is_absolute(), f"{flag} was relative: {value}" + seen[flag] = value + assert seen["--out"].exists(), "replay/trace parent absent before process start" + return real_popen(cmd, *args, **kwargs) + + monkeypatch.setattr(mod.subprocess, "Popen", checked_popen) + att = run_match_attempt( + game_id="relative", p0_dir=Path("p0"), p1_dir=Path("p1"), + p0_name="a", p1_name="b", judge_dir=Path("judge-cwd"), + work_dir=Path("relative") / "work", vendor_script=FAKE_VENDOR, + framework_src=Path("framework-src"), extra_vendor_args=["--fake-mode", "normal"], + ) + assert att.valid is True + assert Path(att.evidence_paths["replay"]).is_absolute() + assert Path(att.evidence_paths["trace"]).is_absolute() + + +def test_judge_start_path_error_is_not_collapsed_to_vendor_eof(tmp_path): + rj = mk_rj(end_info_received=False, end_info=None, scores=None, raw_winner=None) + rj["exception"] = "FileNotFoundError: replay parent missing" + c = _classify_for(rj, TraceStats(), read_replay_info(tmp_path / "absent.replay"), + vendor_returncode=1) + assert c.error_type == "judge_start_path_error" + assert "startup path" in c.reason diff --git a/tests/miracle/test_matrix.py b/tests/miracle/test_matrix.py new file mode 100644 index 0000000..e2296b2 --- /dev/null +++ b/tests/miracle/test_matrix.py @@ -0,0 +1,223 @@ +"""Red-light tests for the 32-game matrix orchestrator (阶段正式矩阵 §3 TDD gate). + +Tests MUST pass before the first real game starts. They cover the spec's safety +requirements using synthesized MatchAttempt-like objects + tmp progress files — +no Judge, no real match. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle import matrix # red-light import +from agentbench_frame.games.miracle.matrix import ( + aggregate, + classify_game, + is_done, + load_progress, + make_attempt_plan, + mark_done, + next_incomplete, + rank_audit, + should_stop, + write_progress_atomic, +) + + +# ---- fake MatchAttempt (same shape as match_runner.MatchAttempt for classify_game) ---- # +class FakeAtt: + def __init__(self, *, game_id="g", rank=None, camp=None, valid=True, normalized_result="win", + raw_winner=None, error_type=None, wrapper_timeout=False, + result_json_status="ok", discrepancies=None, realized_randomization=None, + scores=None, steps=10, ai_crash_player=None, judge_crash=False, + cleanup_procs=None, evidence_paths=None): + self.game_id = game_id; self.rank = rank; self.camp = camp + self.valid = valid; self.normalized_result = normalized_result + self.raw_winner = raw_winner if raw_winner is not None else ( + None if camp is None else (camp if normalized_result == "win" else 1 - camp)) + self.error_type = error_type; self.wrapper_timeout = wrapper_timeout + self.result_json_status = result_json_status + self.discrepancies = discrepancies or [] + self.realized_randomization = realized_randomization or {"map_type": 0, "day_time": 1} + self.scores = scores or {"0": 5, "1": 2}; self.steps = steps + self.ai_crash_player = ai_crash_player; self.judge_crash = judge_crash + self.winner_agent = "miracle_ifelse" if normalized_result == "win" else "rank" + self.normal_cleanup_nonzero = False; self.reason = error_type or "" + self.evidence_paths = evidence_paths or {"result_json": ""} + + +# ---- 1. exactly 32 unique attempts ---- # +def test_plan_has_32_unique_attempts(): + plan = make_attempt_plan() + ids = [a["game_id"] for a in plan] + assert len(plan) == 32 + assert len(set(ids)) == 32 + + +# ---- 2. each rank camp0/camp1 once ---- # +def test_plan_each_rank_both_camps_once(): + plan = make_attempt_plan() + for rank in range(1, 17): + camps = sorted(a["camp"] for a in plan if a["rank"] == rank) + assert camps == [0, 1], f"rank{rank} camps={camps}" + + +def test_plan_order_is_rank_then_camp(): + plan = make_attempt_plan() + seq = [(a["rank"], a["camp"]) for a in plan] + assert seq[0] == (1, 0) and seq[1] == (1, 1) and seq[-1] == (16, 1) + + +# ---- 3. completed never rerun ---- # +def test_next_incomplete_skips_done(): + plan = make_attempt_plan() + prog = {"attempts": {}} + first = next_incomplete(prog, plan) + assert first["game_id"] == plan[0]["game_id"] + mark_done(prog, plan[0]["game_id"], {"valid": True, "normalized_result": "win"}) + assert is_done(prog, plan[0]["game_id"]) is True + nxt = next_incomplete(prog, plan) + assert nxt["game_id"] == plan[1]["game_id"] + + +def test_next_incomplete_none_when_all_done(): + plan = make_attempt_plan() + prog = {"attempts": {}} + for a in plan: + mark_done(prog, a["game_id"], {"valid": True}) + assert next_incomplete(prog, plan) is None + + +# ---- 4. session exists not overwrite/delete (reuses smoke_audit) ---- # +def test_fresh_session_refused_if_exists(tmp_path): + from agentbench_frame.games.miracle.smoke_audit import ensure_fresh_session + sid = matrix.make_session_id() + sd = ensure_fresh_session(tmp_path, sid) + (sd / "x").write_text("keep") + with pytest.raises(FileExistsError): + ensure_fresh_session(tmp_path, sid) + assert (sd / "x").read_text() == "keep" + + +# ---- 5. progress atomic write ---- # +def test_progress_atomic_write(tmp_path): + prog = {"session_id": "s1", "attempts": {"g1": {"state": "done"}}} + p = tmp_path / "progress.json" + write_progress_atomic(p, prog) + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded == prog + assert not (tmp_path / "progress.json.tmp").exists() + + +def test_progress_load_handles_missing(tmp_path): + assert load_progress(tmp_path / "nope.json") == {"attempts": {}} + + +# ---- 6. result append no dup (events append) ---- # +def test_append_event_no_duplicate(tmp_path): + evp = tmp_path / "events.jsonl" + matrix.append_event_atomic(evp, {"event": "game", "game_id": "g1"}) + matrix.append_event_atomic(evp, {"event": "game", "game_id": "g1"}) + matrix.append_event_atomic(evp, {"event": "game", "game_id": "g2"}) + lines = [l for l in evp.read_text(encoding="utf-8").splitlines() if l.strip()] + ids = [json.loads(l)["game_id"] for l in lines] + # no duplicate game_id entries + assert len(ids) == len(set(ids)) == 2 + + +# ---- 7 + 8. classify: valid/invalid; AI crash is NOT a valid win ---- # +def test_classify_valid_win_camp0(): + rec = classify_game(FakeAtt(game_id="m_rank01_camp0", rank=1, camp=0, normalized_result="win", raw_winner=0)) + assert rec["valid"] is True and rec["normalized_result"] == "win" + assert rec["rank"] == 1 and rec["camp"] == 0 + + +def test_classify_ai_crash_not_valid_win(): + # Judge may award ifelse the win (raw_winner=camp), but AI crash => invalid, not a capability win + att = FakeAtt(game_id="m_rank03_camp0", rank=3, camp=0, normalized_result="error", + error_type="ai_crash", ai_crash_player=1, raw_winner=0, valid=False) + rec = classify_game(att) + assert rec["valid"] is False + assert rec["normalized_result"] != "win" + + +# ---- 10. camp-swap winner normalization ---- # +def test_classify_camp_swap_normalization(): + # camp0 raw_winner=0 -> ifelse win; camp1 raw_winner=0 -> ifelse loss + r0 = classify_game(FakeAtt(game_id="a", rank=1, camp=0, normalized_result="win", raw_winner=0)) + r1 = classify_game(FakeAtt(game_id="b", rank=1, camp=1, normalized_result="loss", raw_winner=0)) + assert r0["normalized_result"] == "win" and r1["normalized_result"] == "loss" + assert r0["ifelse_camp"] == 0 and r1["ifelse_camp"] == 1 + + +# ---- 11. recovery audit distinguishes not_started/running/done ---- # +def test_progress_states_distinguished(): + plan = make_attempt_plan() + prog = {"attempts": {}} + assert matrix.state_of(prog, plan[0]["game_id"]) == "not_started" + matrix.mark_running(prog, plan[0]["game_id"]) + assert matrix.state_of(prog, plan[0]["game_id"]) == "running" + matrix.mark_done(prog, plan[0]["game_id"], {"valid": True}) + assert matrix.state_of(prog, plan[0]["game_id"]) == "done" + + +# ---- 9 + 13. infra error blocks next batch; success/invalid/infra distinct ---- # +def test_should_stop_on_infra_anomalies(): + assert should_stop(FakeAtt(wrapper_timeout=True)) == ("stop", "wrapper_timeout") + assert should_stop(FakeAtt(error_type="evidence_mismatch"))[0] == "stop" + assert should_stop(FakeAtt(result_json_status="missing"))[0] == "stop" + # a clean valid game does NOT stop + assert should_stop(FakeAtt(normalized_result="win"))[0] != "stop" + # an AI-crash invalid does NOT stop the matrix (recorded, continue) + assert should_stop(FakeAtt(normalized_result="error", error_type="ai_crash"))[0] != "stop" + + +# ---- rank audit ---- # +def test_rank_audit_pass_for_two_clean_games(): + games = [FakeAtt(game_id="m_rank01_camp0", rank=1, camp=0, normalized_result="win"), + FakeAtt(game_id="m_rank01_camp1", rank=1, camp=1, normalized_result="loss")] + ok, reasons = rank_audit(1, games) + assert ok is True, reasons + + +def test_rank_audit_fail_on_missing_camp(): + games = [FakeAtt(game_id="m_rank01_camp0", rank=1, camp=0, normalized_result="win"), + FakeAtt(game_id="m_rank01_camp0b", rank=1, camp=0, normalized_result="win")] # both camp0 + ok, reasons = rank_audit(1, games) + assert ok is False + + +# ---- 12. PID residual check wired into stop ---- # +def test_residual_proc_triggers_stop(tmp_path): + # write a result-json with a fake live-managed-proc identity pointing at this process + import os, psutil + me = psutil.Process(os.getpid()) + rj = tmp_path / "r.json" + rj.write_text(json.dumps({"judge": {"pid": os.getpid(), "started_at": me.create_time(), "role": "judge"}})) + att = FakeAtt(evidence_paths={"result_json": str(rj)}) + stop, reason = matrix.should_stop_with_residual(att) + assert stop == "stop" and "residual" in reason + + +# ---- aggregate ---- # +def test_aggregate_counts_and_winrate(): + recs = [] + for rank in range(1, 4): + recs.append({"rank": rank, "camp": 0, "valid": True, "normalized_result": "win", "raw_winner": 0, "steps": 40, "realized_randomization": {"map_type": 0, "day_time": 1}, "scores": {"0": 5, "1": 2}, "error_type": None}) + recs.append({"rank": rank, "camp": 1, "valid": True, "normalized_result": "loss", "raw_winner": 0, "steps": 38, "realized_randomization": {"map_type": 1, "day_time": 0}, "scores": {"0": 5, "1": 2}, "error_type": None}) + recs.append({"rank": 3, "camp": 0, "valid": False, "normalized_result": "error", "raw_winner": None, "steps": 5, "realized_randomization": None, "scores": None, "error_type": "ai_crash"}) + agg = aggregate(recs) + assert agg["total_attempts"] == 7 + assert agg["valid_games"] == 6 and agg["invalid_games"] == 1 + assert agg["wins"] == 3 and agg["losses"] == 3 + assert agg["win_rate"] == pytest.approx(0.5) + assert agg["per_rank"][1]["win_rate"] == pytest.approx(0.5) # rank1: camp0 win + camp1 loss + assert agg["per_rank"][3]["invalid"] == 1 + + +def test_aggregate_no_valid_winrate_null(): + recs = [{"rank": 1, "camp": 0, "valid": False, "normalized_result": "error", "raw_winner": None, "steps": 1, "realized_randomization": None, "scores": None, "error_type": "ai_crash"}] + agg = aggregate(recs) + assert agg["win_rate"] is None diff --git a/tests/miracle/test_matrix_cli_flow.py b/tests/miracle/test_matrix_cli_flow.py new file mode 100644 index 0000000..1a242c5 --- /dev/null +++ b/tests/miracle/test_matrix_cli_flow.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path + +import pytest + + +def _import_cli_module(monkeypatch): + repo = Path(__file__).resolve().parents[2] + tools_dir = str(repo / "tools") + if tools_dir not in sys.path: + sys.path.insert(0, tools_dir) + monkeypatch.setenv("AGENTBENCH_ROOT", str(repo)) + monkeypatch.setenv("MIRACLE_IFELSE_DIR", str(repo)) + sys.modules.pop("miracle_matrix", None) + return importlib.import_module("miracle_matrix") + + +def test_main_missing_protocol_returns_preflight_error_without_session( + tmp_path, monkeypatch, capsys +): + mm = _import_cli_module(monkeypatch) + rc = mm.main([ + "--dry-run", + "--protocol", str(tmp_path / "missing.json"), + "--roster", str(tmp_path / "roster.json"), + "--session-root", str(tmp_path / "sessions"), + ]) + + assert rc == 2 + assert "protocol file missing" in capsys.readouterr().err + assert not (tmp_path / "sessions").exists() + + +def test_main_missing_explicit_asset_stops_before_runner_or_session(tmp_path, monkeypatch, capsys): + mm = _import_cli_module(monkeypatch) + invoked = [] + monkeypatch.setattr(mm, "MatrixRunner", lambda **_kwargs: invoked.append(True)) + rc = mm.main(["--dry-run", "--judge-dir", str(tmp_path / "missing-judge"), + "--session-root", str(tmp_path / "sessions")]) + assert rc == 2 + assert "judge directory missing" in capsys.readouterr().err + assert invoked == [] + assert not (tmp_path / "sessions").exists() + + +def test_cli_resume_verifier_receives_control_hashes(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + sid = "sid" + session_dir = tmp_path / sid + session_dir.mkdir() + monkeypatch.setattr(mm, "SESSION_ROOT", tmp_path) + observed = {} + + def fake_verify(_session_dir, **kwargs): + observed.update(kwargs) + return False, ["stop"] + + monkeypatch.setattr(mm, "verify_session_for_resume", fake_verify) + + class FakeRunner: + def __init__(self): + self.session_dir = session_dir + + control_inputs = mm.ControlInputs( + protocol=json.loads(mm.PROTOCOL.read_text(encoding="utf-8")), + roster=json.loads(mm.ROSTER.read_text(encoding="utf-8")), + hashes={"protocol": "p", "roster": "r"}, + ) + roots = {name: tmp_path / name for name in ( + "judge_dir", "ifelse_dir", "extracted_root", "archives_root", + "precheck_root", "rank16_build_root", + )} + rc = mm._run_resume(FakeRunner(), sid, control_inputs=control_inputs, **roots) + + assert rc == 2 + assert observed["expected_control_inputs"] == {"protocol": "p", "roster": "r"} + + +def test_main_manifest_hashes_explicit_judge_and_ifelse_dirs(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + explicit_ifelse = tmp_path / "ifelse"; explicit_ifelse.mkdir() + explicit_judge = tmp_path / "judge"; explicit_judge.mkdir() + (explicit_ifelse / "main.py").write_text("ifelse", encoding="utf-8") + (explicit_judge / "main.py").write_text("judge", encoding="utf-8") + missing = tmp_path / "missing-global" + monkeypatch.setattr(mm, "IFELSE", missing) + monkeypatch.setattr(mm, "JUDGE", missing) + monkeypatch.setattr(mm, "validate_runtime_paths", lambda **_: None) + def verified_hashes(*_args, asset_digests, **_kwargs): + asset_digests.update({ + "ifelse": mm.sha(explicit_ifelse / "main.py"), + "judge": mm.sha(explicit_judge / "main.py"), + }) + return [] + monkeypatch.setattr(mm, "verify_hashes", verified_hashes) + seen = {} + + class FakeRunner: + def __init__(self, **_kwargs): + self.session_id = "fake"; self.run_id = "run"; self.session_dir = tmp_path / "session" + self.timeout = 8.0; self.wrapper_timeout_s = 180.0 + def prepare_session(self): self.session_dir.mkdir() + def record_manifest(self, **kwargs): seen.update(kwargs) + def dry_run(self): return {"plan_count": 1, "plan": [{"game_id": "first"}]} + + monkeypatch.setattr(mm, "MatrixRunner", FakeRunner) + assert mm.main(["--dry-run", "--ifelse-dir", str(explicit_ifelse), + "--judge-dir", str(explicit_judge)]) == 0 + assert seen["ifelse_sha"] == mm.sha(explicit_ifelse / "main.py") + assert seen["judge_sha"] == mm.sha(explicit_judge / "main.py") + + +def _make_main_assets(tmp_path, mm): + roots = {name: tmp_path / name for name in ( + "judge", "ifelse", "extracted", "archives", "precheck", "rank16", + )} + for root in roots.values(): + root.mkdir() + (roots["judge"] / "main.py").write_text("judge\n", encoding="utf-8") + (roots["ifelse"] / "main.py").write_text("ifelse\n", encoding="utf-8") + strategies = [] + builds = {} + archives = {} + runnables = {} + for rank in range(1, 17): + archive = roots["archives"] / f"rank{rank:02d}__fixture.zip" + archive.write_bytes(f"archive-{rank}".encode("ascii")) + archives[rank] = archive + strategy = {"rank": rank, "archive_sha256": mm.sha(archive)} + if rank in mm.PYTHON_RANKS: + directory = roots["extracted"] / f"rank{rank:02d}__fixture" + directory.mkdir() + runnable = directory / "main.py" + runnable.write_text(f"python-{rank}\n", encoding="utf-8") + runnables[rank] = runnable + strategy.update({"entry": "main.py", "runnable_sha256": mm.sha(runnable)}) + else: + directory = (roots["rank16"] / "rank16_copy" if rank == 16 + else roots["precheck"] / "strategies" / f"rank{rank:02d}") + directory.mkdir(parents=True) + executable = directory / "main.exe" + executable.write_bytes(f"exe-{rank}".encode("ascii")) + runnables[rank] = executable + builds[f"rank{rank:02d}"] = mm.sha(executable) + strategies.append(strategy) + protocol = { + "frozen_identities": { + "evaluated_agent": {"sha256": mm.sha(roots["ifelse"] / "main.py")}, + "judge": {"main_py_sha256": mm.sha(roots["judge"] / "main.py")}, + "build_artifacts_win64_mingw": builds, + } + } + protocol_path = tmp_path / "protocol.json" + roster_path = tmp_path / "roster.json" + protocol_path.write_text(json.dumps(protocol), encoding="utf-8") + roster_path.write_text(json.dumps({"strategies": strategies}), encoding="utf-8") + return roots, archives, runnables, protocol_path, roster_path + + +@pytest.mark.parametrize("asset", ["judge", "ifelse", "archive", "python", "cpp"]) +def test_main_unreadable_asset_stops_before_manifest_or_execution(tmp_path, monkeypatch, capsys, asset): + mm = _import_cli_module(monkeypatch) + roots, archives, runnables, protocol_path, roster_path = _make_main_assets(tmp_path, mm) + unreadable = { + "judge": roots["judge"] / "main.py", + "ifelse": roots["ifelse"] / "main.py", + "archive": archives[1], + "python": runnables[4], + "cpp": runnables[1], + }[asset] + original_sha = mm.sha + + def injected_sha(path): + if Path(path) == unreadable: + raise OSError("injected unreadable asset") + return original_sha(path) + + monkeypatch.setattr(mm, "sha", injected_sha) + recorded = [] + dry_runs = [] + executes = [] + session_root = tmp_path / "sessions" + + class FakeRunner: + def __init__(self, **_kwargs): + self.session_dir = session_root / "fake" + self.session_id = "fake" + self.run_id = "run" + + def prepare_session(self): + self.session_dir.mkdir(parents=True) + + def record_manifest(self, **_kwargs): + recorded.append(True) + + def dry_run(self): + dry_runs.append(True) + + def execute(self): + executes.append(True) + + monkeypatch.setattr(mm, "MatrixRunner", FakeRunner) + rc = mm.main([ + "--dry-run", "--protocol", str(protocol_path), "--roster", str(roster_path), + "--session-root", str(session_root), "--judge-dir", str(roots["judge"]), + "--ifelse-dir", str(roots["ifelse"]), "--extracted-root", str(roots["extracted"]), + "--archives-root", str(roots["archives"]), "--precheck-root", str(roots["precheck"]), + "--rank16-build-root", str(roots["rank16"]), + ]) + + assert rc == 2 + assert "FATAL: hash mismatches" in capsys.readouterr().out + assert recorded == [] + assert dry_runs == [] + assert executes == [] + + +def test_resume_current_asset_mismatch_stops_before_log_or_resume(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + sid = "sid"; session = tmp_path / sid; session.mkdir() + monkeypatch.setattr(mm, "verify_session_for_resume", lambda *_a, **_k: (True, [])) + roots = {name: tmp_path / name for name in ("judge", "ifelse", "extracted", "archives", "precheck", "rank16")} + seen = {} + def mismatch(*_args, **kwargs): + seen.update(kwargs); return ["judge sha mismatch"] + monkeypatch.setattr(mm, "verify_hashes", mismatch) + class FakeRunner: + session_dir = session + def resume(self, _sid): raise AssertionError("resume must not run") + control = mm.ControlInputs( + json.loads(mm.PROTOCOL.read_text(encoding="utf-8")), + json.loads(mm.ROSTER.read_text(encoding="utf-8")), + {"protocol": "p", "roster": "r"}, + ) + before = {p.name: p.read_bytes() for p in session.iterdir()} + rc = mm._run_resume(FakeRunner(), sid, control_inputs=control, session_root=tmp_path, + judge_dir=roots["judge"], ifelse_dir=roots["ifelse"], + extracted_root=roots["extracted"], archives_root=roots["archives"], + precheck_root=roots["precheck"], rank16_build_root=roots["rank16"]) + assert rc == 2 + assert seen["judge_root"] == roots["judge"] + assert not (session / "matrix.full.log").exists() + assert before == {p.name: p.read_bytes() for p in session.iterdir()} diff --git a/tests/miracle/test_matrix_cli_inputs.py b/tests/miracle/test_matrix_cli_inputs.py new file mode 100644 index 0000000..1df8e01 --- /dev/null +++ b/tests/miracle/test_matrix_cli_inputs.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path + +import pytest + + +def _valid_protocol(mm): + return { + "frozen_identities": { + "evaluated_agent": {"sha256": "a" * 64}, + "judge": {"main_py_sha256": "b" * 64}, + "build_artifacts_win64_mingw": { + f"rank{rank:02d}": "c" * 64 for rank in mm.CPP_RANKS + }, + } + } + + +def _valid_roster(mm): + strategies = [] + for rank in range(1, 17): + strategy = {"rank": rank, "archive_sha256": "d" * 64} + if rank in mm.PYTHON_RANKS: + strategy.update({"entry": "main.py", "runnable_sha256": "e" * 64}) + strategies.append(strategy) + return {"strategies": strategies} + + +def _write_controls(protocol_path, roster_path, protocol, roster): + protocol_path.write_text(json.dumps(protocol), encoding="utf-8") + roster_path.write_text(json.dumps(roster), encoding="utf-8") + + +def _import_cli_module(monkeypatch): + repo = Path(__file__).resolve().parents[2] + tools_dir = str(repo / "tools") + if tools_dir not in sys.path: + sys.path.insert(0, tools_dir) + monkeypatch.setenv("AGENTBENCH_ROOT", str(repo)) + monkeypatch.setenv("MIRACLE_IFELSE_DIR", str(repo)) + sys.modules.pop("miracle_matrix", None) + return importlib.import_module("miracle_matrix") + + +def test_load_control_inputs_reports_missing_file(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + + with pytest.raises(mm.PreflightError, match="protocol file missing"): + mm.load_control_inputs(tmp_path / "missing.json", tmp_path / "roster.json") + + +def test_parse_args_accepts_explicit_control_paths(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + args = mm.parse_args([ + "--dry-run", + "--protocol", str(tmp_path / "p.json"), + "--roster", str(tmp_path / "r.json"), + ]) + + assert args.dry_run is True + assert args.protocol == tmp_path / "p.json" + assert args.roster == tmp_path / "r.json" + + +def test_load_control_inputs_validates_protocol_and_roster(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + protocol = _valid_protocol(mm) + roster = _valid_roster(mm) + protocol_path = tmp_path / "protocol.json" + roster_path = tmp_path / "roster.json" + protocol_path.write_text(json.dumps(protocol), encoding="utf-8") + roster_path.write_text(json.dumps(roster), encoding="utf-8") + + inputs = mm.load_control_inputs(protocol_path, roster_path) + + assert inputs.protocol == protocol + assert inputs.roster == roster + assert set(inputs.hashes) == {"protocol", "roster"} + assert len(inputs.hashes["protocol"]) == 64 + assert len(inputs.hashes["roster"]) == 64 + + +@pytest.mark.parametrize( + "missing_field", + [ + "frozen_identities", + "evaluated_agent_sha", + "judge_sha", + *[f"cpp_sha:{rank}" for rank in (1, 2, 3, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16)], + *[f"archive_sha:{rank}" for rank in range(1, 17)], + *[f"python_entry:{rank}" for rank in (4, 5, 7)], + *[f"python_runnable_sha:{rank}" for rank in (4, 5, 7)], + ], +) +def test_main_rejects_incomplete_control_schema_before_session_write( + tmp_path, monkeypatch, capsys, missing_field +): + mm = _import_cli_module(monkeypatch) + protocol = _valid_protocol(mm) + roster = _valid_roster(mm) + identities = protocol["frozen_identities"] + if missing_field == "frozen_identities": + protocol.pop("frozen_identities") + elif missing_field == "evaluated_agent_sha": + identities["evaluated_agent"].pop("sha256") + elif missing_field == "judge_sha": + identities["judge"].pop("main_py_sha256") + elif missing_field.startswith("cpp_sha:"): + rank = int(missing_field.split(":", 1)[1]) + identities["build_artifacts_win64_mingw"].pop(f"rank{rank:02d}") + elif missing_field.startswith("archive_sha:"): + rank = int(missing_field.split(":", 1)[1]) + roster["strategies"][rank - 1].pop("archive_sha256") + elif missing_field.startswith("python_entry:"): + rank = int(missing_field.split(":", 1)[1]) + roster["strategies"][rank - 1].pop("entry") + else: + rank = int(missing_field.split(":", 1)[1]) + roster["strategies"][rank - 1].pop("runnable_sha256") + + protocol_path = tmp_path / "protocol.json" + roster_path = tmp_path / "roster.json" + session_root = tmp_path / "sessions" + _write_controls(protocol_path, roster_path, protocol, roster) + constructed = [] + prepared = [] + + class FakeRunner: + def __init__(self, **_kwargs): + constructed.append(True) + self.session_dir = session_root / "unexpected" + self.session_id = "unexpected" + self.run_id = "unexpected" + + def prepare_session(self): + prepared.append(True) + self.session_dir.mkdir(parents=True) + + monkeypatch.setattr(mm, "validate_runtime_paths", lambda **_kwargs: None) + monkeypatch.setattr(mm, "MatrixRunner", FakeRunner) + + rc = mm.main([ + "--dry-run", "--protocol", str(protocol_path), "--roster", str(roster_path), + "--session-root", str(session_root), + ]) + + assert rc == 2 + assert "FATAL:" in capsys.readouterr().err + assert constructed == [] + assert prepared == [] + assert not session_root.exists() + assert not (session_root / "unexpected" / "matrix.full.log").exists() + + +@pytest.mark.parametrize("field", [ + "evaluated_agent", "judge", "cpp_build", "archive", "python_runnable", +]) +@pytest.mark.parametrize("malformed_sha", [ + "a" * 63, "a" * 65, "g" * 64, "A" * 64, " " + "a" * 64 + " ", None, +]) +def test_main_rejects_malformed_control_sha_before_session_write( + tmp_path, monkeypatch, capsys, field, malformed_sha +): + mm = _import_cli_module(monkeypatch) + protocol = _valid_protocol(mm) + roster = _valid_roster(mm) + if field == "evaluated_agent": + protocol["frozen_identities"]["evaluated_agent"]["sha256"] = malformed_sha + elif field == "judge": + protocol["frozen_identities"]["judge"]["main_py_sha256"] = malformed_sha + elif field == "cpp_build": + protocol["frozen_identities"]["build_artifacts_win64_mingw"]["rank01"] = malformed_sha + elif field == "archive": + roster["strategies"][0]["archive_sha256"] = malformed_sha + else: + roster["strategies"][3]["runnable_sha256"] = malformed_sha + + protocol_path = tmp_path / "protocol.json" + roster_path = tmp_path / "roster.json" + session_root = tmp_path / "sessions" + _write_controls(protocol_path, roster_path, protocol, roster) + constructed = [] + prepared = [] + + class FakeRunner: + def __init__(self, **_kwargs): + constructed.append(True) + + def prepare_session(self): + prepared.append(True) + + monkeypatch.setattr(mm, "validate_runtime_paths", lambda **_kwargs: None) + monkeypatch.setattr(mm, "MatrixRunner", FakeRunner) + + rc = mm.main([ + "--dry-run", "--protocol", str(protocol_path), "--roster", str(roster_path), + "--session-root", str(session_root), + ]) + + assert rc == 2 + assert "FATAL: control schema" in capsys.readouterr().err + assert constructed == [] + assert prepared == [] + assert not session_root.exists() + + +def test_control_text_sha_is_identical_for_lf_and_crlf(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + lf = tmp_path / "lf.json" + crlf = tmp_path / "crlf.json" + lf.write_bytes(b'{\n "protocol_version": "test"\n}\n') + crlf.write_bytes(b'{\r\n "protocol_version": "test"\r\n}\r\n') + assert mm.control_text_sha(lf) == mm.control_text_sha(crlf) + + +def test_windows_crlf_protocol_checkout_passes_expected_hash(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + protocol = tmp_path / "protocol.json" + roster = tmp_path / "roster.json" + protocol.write_bytes(json.dumps(_valid_protocol(mm), indent=2).replace("\n", "\r\n").encode("utf-8")) + roster.write_text(json.dumps(_valid_roster(mm)), encoding="utf-8") + mm.load_control_inputs(protocol, roster, mm.control_text_sha(protocol)) + + +def test_tracked_control_files_have_no_machine_paths(monkeypatch): + mm = _import_cli_module(monkeypatch) + framework_marker = "AgentBench" + "Framework.framework" + markers = ("C:" + "/Users/", "C:" + "\\\\Users\\\\", "/home/", "/Users/", framework_marker) + for path in (mm.PROTOCOL, mm.ROSTER, mm.REPO / "vendor" / "miracle_local" / "run_match.py"): + text = path.read_text(encoding="utf-8") + for marker in markers: + assert marker not in text + + +def test_load_control_inputs_rejects_non_contiguous_roster(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + protocol_path = tmp_path / "protocol.json" + roster_path = tmp_path / "roster.json" + protocol_path.write_text("{}", encoding="utf-8") + roster_path.write_text( + json.dumps({"strategies": [{"rank": 1}, {"rank": 3}]}), + encoding="utf-8", + ) + + with pytest.raises(mm.PreflightError, match="roster ranks"): + mm.load_control_inputs(protocol_path, roster_path) + + +def test_load_control_inputs_rejects_non_object_strategy(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + protocol_path = tmp_path / "protocol.json" + roster_path = tmp_path / "roster.json" + protocol_path.write_text("{}", encoding="utf-8") + roster_path.write_text( + json.dumps([{"rank": rank} for rank in range(1, 17)] + ["not-an-object"]), + encoding="utf-8", + ) + + with pytest.raises(mm.PreflightError, match="roster JSON must be an object"): + mm.load_control_inputs(protocol_path, roster_path) + + +def test_cli_module_import_does_not_require_external_env(monkeypatch): + repo = Path(__file__).resolve().parents[2] + monkeypatch.delenv("AGENTBENCH_ROOT", raising=False) + monkeypatch.delenv("MIRACLE_IFELSE_DIR", raising=False) + sys.modules.pop("miracle_matrix", None) + tools_dir = str(repo / "tools") + if tools_dir not in sys.path: + sys.path.insert(0, tools_dir) + + mm = importlib.import_module("miracle_matrix") + + assert mm.PROTOCOL.name == "24_miracle_evaluation_protocol.v0.3.json" diff --git a/tests/miracle/test_matrix_identity.py b/tests/miracle/test_matrix_identity.py new file mode 100644 index 0000000..2005e4d --- /dev/null +++ b/tests/miracle/test_matrix_identity.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import hashlib +import importlib +import sys +from pathlib import Path + +import pytest + + +def _import_cli_module(monkeypatch): + repo = Path(__file__).resolve().parents[2] + tools_dir = str(repo / "tools") + if tools_dir not in sys.path: + sys.path.insert(0, tools_dir) + monkeypatch.setenv("AGENTBENCH_ROOT", str(repo)) + monkeypatch.setenv("MIRACLE_IFELSE_DIR", str(repo)) + sys.modules.pop("miracle_matrix", None) + return importlib.import_module("miracle_matrix") + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_resolve_unique_dir_rejects_ambiguous_matches(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + (tmp_path / "rank04__a").mkdir() + (tmp_path / "rank04__b").mkdir() + + with pytest.raises(RuntimeError, match="multiple opponent directories"): + mm.resolve_unique_dir(tmp_path, "rank04__*") + + +def test_verify_python_hashes_detects_modified_entry(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + python_dir = tmp_path / "rank04__fixture" + python_dir.mkdir() + (python_dir / "main.py").write_text("modified\n", encoding="utf-8") + strategy = { + "rank": 4, + "type": "python_script", + "entry": "main.py", + "runnable_sha256": "expected", + } + + errors = mm.verify_python_strategy_hashes(strategy, tmp_path) + + assert any("rank04" in error and "runnable sha" in error for error in errors) + + +def test_verify_python_hashes_checks_entry(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + extracted = tmp_path / "extracted" + archives = tmp_path / "archives" + extracted.mkdir() + archives.mkdir() + python_dir = extracted / "rank04__fixture" + python_dir.mkdir() + entry = python_dir / "main.py" + entry.write_text("print('ok')\n", encoding="utf-8") + archive = archives / "rank04__fixture.zip" + archive.write_bytes(b"archive-bytes") + strategy = { + "rank": 4, + "type": "python_script", + "entry": "main.py", + "runnable_sha256": _sha(entry), + "archive_sha256": _sha(archive), + } + + errors = mm.verify_python_strategy_hashes(strategy, extracted) + + assert errors == [] + + +def test_verify_archive_hash_rejects_duplicate_archives(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + extracted = tmp_path / "extracted" + archives = tmp_path / "archives" + extracted.mkdir() + archives.mkdir() + python_dir = extracted / "rank04__fixture" + python_dir.mkdir() + (python_dir / "main.py").write_text("print('ok')\n", encoding="utf-8") + (archives / "rank04__a.zip").write_bytes(b"a") + (archives / "rank04__b.zip").write_bytes(b"b") + strategy = { + "rank": 4, + "type": "python_script", + "entry": "main.py", + "runnable_sha256": _sha(python_dir / "main.py"), + "archive_sha256": _sha(archives / "rank04__a.zip"), + } + + errors = mm.verify_archive_hash(strategy, archives) + + assert any("multiple archive files" in error for error in errors) + + +def test_verify_archive_hash_requires_archive_identity(tmp_path, monkeypatch): + mm = _import_cli_module(monkeypatch) + extracted = tmp_path / "extracted" + archives = tmp_path / "archives" + extracted.mkdir() + archives.mkdir() + python_dir = extracted / "rank04__fixture" + python_dir.mkdir() + entry = python_dir / "main.py" + entry.write_text("print('ok')\n", encoding="utf-8") + (archives / "rank04__fixture.zip").write_bytes(b"archive-bytes") + strategy = { + "rank": 4, + "type": "python_script", + "entry": "main.py", + "runnable_sha256": _sha(entry), + } + + errors = mm.verify_archive_hash(strategy, archives) + + assert any("archive sha missing" in error for error in errors) diff --git a/tests/miracle/test_matrix_resume_assets.py b/tests/miracle/test_matrix_resume_assets.py new file mode 100644 index 0000000..4bfa382 --- /dev/null +++ b/tests/miracle/test_matrix_resume_assets.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path + +import pytest + + +PYTHON_RANKS = {4, 5, 7} + + +def _import_cli_module(monkeypatch): + repo = Path(__file__).resolve().parents[2] + tools_dir = str(repo / "tools") + if tools_dir not in sys.path: + sys.path.insert(0, tools_dir) + monkeypatch.setenv("AGENTBENCH_ROOT", str(repo)) + monkeypatch.setenv("MIRACLE_IFELSE_DIR", str(repo)) + sys.modules.pop("miracle_matrix", None) + return importlib.import_module("miracle_matrix") + + +def _tree_bytes(root: Path) -> dict[str, bytes]: + return { + str(path.relative_to(root)): path.read_bytes() + for path in sorted(root.rglob("*")) if path.is_file() + } + + +def _make_assets(tmp_path: Path, mm): + roots = {name: tmp_path / name for name in ( + "judge", "ifelse", "extracted", "archives", "precheck", "rank16", + )} + for root in roots.values(): + root.mkdir() + (roots["judge"] / "main.py").write_text("judge\n", encoding="utf-8") + (roots["ifelse"] / "main.py").write_text("ifelse\n", encoding="utf-8") + + strategies = [] + build_hashes = {} + archive_paths = {} + executable_paths = {} + for rank in range(1, 17): + archive = roots["archives"] / f"rank{rank:02d}__fixture.zip" + archive.write_bytes(f"archive-{rank}".encode("ascii")) + archive_paths[rank] = archive + strategy = {"rank": rank, "archive_sha256": mm.sha(archive)} + if rank in PYTHON_RANKS: + directory = roots["extracted"] / f"rank{rank:02d}__fixture" + directory.mkdir() + entry = directory / "main.py" + entry.write_text(f"python-{rank}\n", encoding="utf-8") + strategy.update({"entry": "main.py", "runnable_sha256": mm.sha(entry)}) + executable_paths[rank] = entry + else: + if rank == 16: + directory = roots["rank16"] / "rank16_copy" + else: + directory = roots["precheck"] / "strategies" / f"rank{rank:02d}" + directory.mkdir(parents=True) + executable = directory / "main.exe" + executable.write_bytes(f"exe-{rank}".encode("ascii")) + executable_paths[rank] = executable + build_hashes[f"rank{rank:02d}"] = mm.sha(executable) + strategies.append(strategy) + + protocol = { + "frozen_identities": { + "build_artifacts_win64_mingw": build_hashes, + "evaluated_agent": {"sha256": mm.sha(roots["ifelse"] / "main.py")}, + "judge": {"main_py_sha256": mm.sha(roots["judge"] / "main.py")}, + } + } + control = mm.ControlInputs(protocol, {"strategies": strategies}, {"protocol": "p", "roster": "r"}) + return roots, control, archive_paths, executable_paths + + +class _FakeRunner: + def __init__(self, session_dir: Path): + self.session_dir = session_dir + self.resume_count = 0 + self.execute_count = 0 + + def resume(self, _sid): + self.resume_count += 1 + + def execute(self): + self.execute_count += 1 + return {"completed": True, "halted": False} + + def write_run_compatible_output(self): + return self.session_dir + + def aggregate_from_events(self): + return {"total_attempts": 0, "valid_games": 0, "invalid_games": 0, "win_rate": None} + + +def _run_with_assets(tmp_path, monkeypatch, mutate=None, missing_root=None): + mm = _import_cli_module(monkeypatch) + roots, control, archives, executables = _make_assets(tmp_path, mm) + session = tmp_path / "session"; session.mkdir() + (session / "manifest.json").write_text(json.dumps({"fixture": True}), encoding="utf-8") + (session / "progress.json").write_text('{"attempts": {}}', encoding="utf-8") + monkeypatch.setattr(mm, "verify_session_for_resume", lambda *_a, **_k: (True, [])) + if mutate is not None: + mutate(roots, archives, executables) + runner = _FakeRunner(session) + before = _tree_bytes(session) + kwargs = { + "judge_dir": roots["judge"], "ifelse_dir": roots["ifelse"], + "extracted_root": roots["extracted"], "archives_root": roots["archives"], + "precheck_root": roots["precheck"], "rank16_build_root": roots["rank16"], + } + if missing_root is not None: + kwargs[missing_root] = None + rc = mm._run_resume(runner, "session", control_inputs=control, session_root=tmp_path, **kwargs) + return rc, runner, session, before + + +@pytest.mark.parametrize("name,mutate", [ + ("judge", lambda roots, _archives, _executables: (roots["judge"] / "main.py").write_text("changed\n", encoding="utf-8")), + ("ifelse", lambda roots, _archives, _executables: (roots["ifelse"] / "main.py").write_text("changed\n", encoding="utf-8")), + ("python runnable", lambda _roots, _archives, executables: executables[4].write_text("changed\n", encoding="utf-8")), + ("python archive", lambda _roots, archives, _executables: archives[4].write_bytes(b"changed")), + ("c++ archive", lambda _roots, archives, _executables: archives[1].write_bytes(b"changed")), + ("c++ executable", lambda _roots, _archives, executables: executables[1].write_bytes(b"changed")), + ("c++ executable is directory", lambda _roots, _archives, executables: (executables[1].unlink(), executables[1].mkdir())), + ("archive missing", lambda _roots, archives, _executables: archives[1].unlink()), + ("archive duplicate", lambda roots, _archives, _executables: (roots["archives"] / "rank01__duplicate.zip").write_bytes(b"duplicate")), +]) +def test_resume_rejects_changed_real_runtime_assets_before_session_write(tmp_path, monkeypatch, name, mutate): + rc, runner, session, before = _run_with_assets(tmp_path, monkeypatch, mutate=mutate) + + assert rc == 2, name + assert runner.resume_count == 0 + assert runner.execute_count == 0 + assert _tree_bytes(session) == before + assert not (session / "matrix.full.log").exists() + + +@pytest.mark.parametrize("rank", range(1, 17)) +def test_resume_verifies_the_unique_archive_for_every_rank(tmp_path, monkeypatch, rank): + rc, runner, session, before = _run_with_assets( + tmp_path, monkeypatch, + mutate=lambda _roots, archives, _executables: archives[rank].write_bytes(b"changed"), + ) + + assert rc == 2 + assert runner.resume_count == 0 + assert runner.execute_count == 0 + assert _tree_bytes(session) == before + assert not (session / "matrix.full.log").exists() + + +@pytest.mark.parametrize("missing_root", [ + "judge_dir", "ifelse_dir", "extracted_root", "archives_root", "precheck_root", "rank16_build_root", +]) +def test_resume_rejects_any_missing_runtime_root_before_session_write(tmp_path, monkeypatch, capsys, missing_root): + rc, runner, session, before = _run_with_assets(tmp_path, monkeypatch, missing_root=missing_root) + + assert rc == 2 + assert f"runtime root missing: {missing_root}" in capsys.readouterr().err + assert runner.resume_count == 0 + assert runner.execute_count == 0 + assert _tree_bytes(session) == before + assert not (session / "matrix.full.log").exists() + + +def test_resume_with_all_real_runtime_assets_calls_resume_once(tmp_path, monkeypatch): + rc, runner, _session, _before = _run_with_assets(tmp_path, monkeypatch) + + assert rc == 0 + assert runner.resume_count == 1 + assert runner.execute_count == 1 diff --git a/tests/miracle/test_matrix_runner.py b/tests/miracle/test_matrix_runner.py new file mode 100644 index 0000000..df512ea --- /dev/null +++ b/tests/miracle/test_matrix_runner.py @@ -0,0 +1,218 @@ +"""Tests for the matrix runner (tools/miracle_matrix.py logic). + +Uses an injected FAKE attempt_fn (signature matches the runner's keyword call: +``attempt_fn(game_id=..., p0_dir=..., ...)``) — no Judge, no AI subprocess. +Covers the 15 runner safety requirements. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle import matrix_runner +from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + + +def _att(game_id, **over): + rc = matrix_runner.parse_game_id(game_id) + d = dict(valid=True, normalized_result="win", raw_winner=None, error_type=None, + wrapper_timeout=False, result_json_status="ok", discrepancies=None, + realized_randomization=None, scores=None, steps=10, ai_crash_player=None, + judge_crash=False) + d.update(over) + rank, camp = rc + rw = d["raw_winner"] if d["raw_winner"] is not None else ( + camp if d["normalized_result"] == "win" else 1 - camp) + class A: pass + a = A() + a.game_id = game_id; a.rank = rank; a.camp = camp + a.valid = d["valid"]; a.normalized_result = d["normalized_result"]; a.raw_winner = rw + a.error_type = d["error_type"]; a.wrapper_timeout = d["wrapper_timeout"] + a.result_json_status = d["result_json_status"]; a.discrepancies = d["discrepancies"] or [] + a.realized_randomization = d["realized_randomization"] or {"map_type": 0, "day_time": 1} + a.scores = d["scores"] or {"0": 5, "1": 2}; a.steps = d["steps"] + a.ai_crash_player = d["ai_crash_player"]; a.judge_crash = d["judge_crash"] + a.winner_agent = "miracle_ifelse" if d["normalized_result"] == "win" else "opp" + a.normal_cleanup_nonzero = False; a.reason = d["error_type"] or "" + a.evidence_paths = {"stdout": "", "stderr": "", "trace": "", "replay": "", + "result_json": str(Path(game_id + ".rj"))} + return a + + +def _runner(tmp_path, fn, **kw): + return MatrixRunner(session_root=tmp_path, judge_dir=tmp_path / "j", + ifelse_dir=tmp_path / "ifelse", opponent_dir_of=lambda r: tmp_path / f"opp{r}", + vendor_script=tmp_path / "v.py", framework_src=tmp_path / "src", + timeout=8.0, wrapper_timeout_s=60.0, attempt_fn=fn, + protocol_sha="p" * 64, run_id="RID", **kw) + + +# 1. dry-run never starts Judge/AI +def test_dry_run_starts_nothing(tmp_path): + called = [] + r = _runner(tmp_path, lambda **k: called.append(1)) + r.prepare_session() + out = r.dry_run() + assert called == [] + assert out["plan_count"] == 32 + + +# 2. plan rank01-16 camp0/camp1, 32 unique +def test_dry_run_plan_correct(tmp_path): + r = _runner(tmp_path, lambda **k: None) + r.prepare_session() + out = r.dry_run() + ids = [a["game_id"] for a in out["plan"]] + assert len(ids) == 32 and len(set(ids)) == 32 + assert out["plan"][0]["game_id"] == "m_rank01_camp0" + assert out["plan"][-1]["game_id"] == "m_rank16_camp1" + + +# 3. existing session refused +def test_existing_session_refused(tmp_path): + r = _runner(tmp_path, lambda **k: None) + r.prepare_session() + r2 = _runner(tmp_path, lambda **k: None) + with pytest.raises(FileExistsError): + r2.prepare_session_for_existing(r.session_id) + + +# 4+5. manifest atomic + hashes +def test_manifest_atomic_with_hashes(tmp_path): + r = _runner(tmp_path, lambda **k: None) + r.prepare_session() + r.record_manifest(opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in [1, 2, 3, 6] + list(range(8, 17))}, + ifelse_sha="IF", judge_sha="JD", code_hashes={"matrix_runner": "MR"}) + m = json.loads((r.session_dir / "manifest.json").read_text(encoding="utf-8")) + assert m["protocol_sha256"] == "p" * 64 + assert m["plan_count"] == 32 and m["run_id"] == "RID" + assert m["ifelse_sha256"] == "IF" and m["judge_sha256"] == "JD" + assert not (r.session_dir / "manifest.json.tmp").exists() + + +def test_manifest_records_control_input_hashes(tmp_path): + r = _runner(tmp_path, lambda **k: None) + r.prepare_session() + r.record_manifest( + opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}, + control_inputs={"protocol": {"path": "/p.json", "sha256": "p"}, + "roster": {"path": "/r.json", "sha256": "r"}}, + ) + manifest = json.loads((r.session_dir / "manifest.json").read_text(encoding="utf-8")) + assert manifest["control_inputs"]["protocol"]["sha256"] == "p" + + +# 6. running -> UNCERTAIN_IN_FLIGHT, no auto-rerun +def test_running_state_uncertain_no_rerun(tmp_path): + r = _runner(tmp_path, lambda **k: None) + r.prepare_session() + plan = matrix_runner.make_attempt_plan() + matrix_runner.mark_running(r.progress, plan[0]["game_id"]) + matrix_runner.write_progress_atomic(r.progress_path, r.progress) + res = r.execute() + assert res["halted"] is True + assert "UNCERTAIN_IN_FLIGHT" in res["reason"] + + +# 7. done only after attempt returns +def test_done_only_after_attempt_returns(tmp_path): + seen = [] + def fn(**k): + seen.append(k["game_id"]); return _att(k["game_id"], normalized_result="win") + r = _runner(tmp_path, fn) + r.prepare_session() + r.execute_rank(rank=1) + prog = json.loads(r.progress_path.read_text(encoding="utf-8")) + assert prog["attempts"]["m_rank01_camp0"]["state"] == "done" + assert prog["attempts"]["m_rank01_camp1"]["state"] == "done" + + +# 8+9. valid/AI-invalid/infra distinct; AI-invalid continues, not in win-rate denom +def test_ai_invalid_continues_not_in_winrate(tmp_path): + calls = [0] + def fn(**k): + i = calls[0]; calls[0] += 1 + if i == 0: + return _att(k["game_id"], normalized_result="error", error_type="ai_crash", valid=False) + return _att(k["game_id"], normalized_result="win") + r = _runner(tmp_path, fn) + r.prepare_session() + r.execute_rank(rank=1) + agg = r.aggregate_from_events() + assert agg["total_attempts"] == 2 and agg["valid_games"] == 1 and agg["invalid_games"] == 1 + assert agg["wins"] == 1 and agg["win_rate"] == pytest.approx(1.0) + + +# 10. infra failure blocks next +def test_infra_failure_halts(tmp_path): + calls = [0] + def fn(**k): + calls[0] += 1 + if calls[0] == 1: + return _att(k["game_id"], error_type="evidence_mismatch", valid=False) + return _att(k["game_id"]) + r = _runner(tmp_path, fn) + r.prepare_session() + res = r.execute_rank(rank=1) + assert res["halted"] is True and res["state"] == "HALTED_INFRA_FAILURE" + assert calls[0] == 1 + + +# 11. rank audit after two games +def test_rank_audit_runs_after_two_games(tmp_path): + r = _runner(tmp_path, lambda **k: _att(k["game_id"])) + r.prepare_session() + r.execute_rank(rank=1) + assert (r.session_dir / "audit" / "rank01.json").exists() + + +# 12. progress/events no duplicate +def test_progress_and_events_no_duplicate(tmp_path): + r = _runner(tmp_path, lambda **k: _att(k["game_id"])) + r.prepare_session() + r.execute_rank(rank=1) + ev = [json.loads(l) for l in r.events_path.read_text(encoding="utf-8").splitlines() if l.strip()] + ids = [e["game_id"] for e in ev] + assert len(ids) == len(set(ids)) + + +# 13. normal cleanup nonzero not crash +def test_normal_cleanup_nonzero_not_crash(tmp_path): + def fn(**k): + a = _att(k["game_id"]); a.normal_cleanup_nonzero = True; return a + r = _runner(tmp_path, fn) + r.prepare_session() + res = r.execute_rank(rank=1) + assert res["halted"] is False + + +# 14. PID residual blocks continue +def test_residual_blocks_continue(tmp_path): + import os as _os, psutil + me = psutil.Process(_os.getpid()) + def fn(**k): + a = _att(k["game_id"]) + rjp = r.session_dir / (k["game_id"] + ".rj") + rjp.write_text(json.dumps({"judge": {"pid": _os.getpid(), "started_at": me.create_time(), "role": "judge"}})) + a.evidence_paths = {"result_json": str(rjp)} + return a + r = _runner(tmp_path, fn) + r.prepare_session() + res = r.execute_rank(rank=1) + assert res["halted"] is True and "residual" in res["reason"] + + +# 15. partial/complete summary independently re-computable +def test_summary_recomputable_from_events(tmp_path): + r = _runner(tmp_path, lambda **k: _att(k["game_id"])) + r.prepare_session() + r.execute_rank(rank=1) + agg = r.aggregate_from_events() + raw = [json.loads(l) for l in r.events_path.read_text(encoding="utf-8").splitlines() if l.strip()] + assert agg["total_attempts"] == 2 and len(raw) == 2 + assert agg["wins"] == sum(1 for e in raw if e["normalized_result"] == "win") diff --git a/tests/miracle/test_migration_review1.py b/tests/miracle/test_migration_review1.py new file mode 100644 index 0000000..dbacb34 --- /dev/null +++ b/tests/miracle/test_migration_review1.py @@ -0,0 +1,86 @@ +"""Regression coverage for the portable 24_miracle review-1 migrator.""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[2] +MIGRATOR = REPO / "tools" / "migrate_24_miracle_to_review1.py" + + +def _run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(MIGRATOR), *args], capture_output=True, text=True + ) + + +def test_migrator_requires_all_explicit_paths(): + result = _run() + assert result.returncode == 2 + assert "--session" in result.stderr + assert "--original-run-dir" in result.stderr + assert "--output-root" in result.stderr + + +def test_migrator_refuses_legacy_preview_name(tmp_path): + session = tmp_path / "session" + original_run = tmp_path / "results" / "runs" / "game" / "agent" / "rid" + session.mkdir() + original_run.mkdir(parents=True) + legacy = tmp_path / "24_miracle_results_migration_preview_v2" + result = _run( + "--session", str(session), + "--original-run-dir", str(original_run), + "--output-root", str(legacy), + ) + assert result.returncode == 2 + assert "legacy preview" in result.stderr + assert not legacy.exists() + + +def test_migrator_refuses_output_under_authority_input(tmp_path): + session = tmp_path / "session" + original_run = tmp_path / "results" / "runs" / "game" / "agent" / "rid" + session.mkdir() + original_run.mkdir(parents=True) + result = _run( + "--session", str(session), + "--original-run-dir", str(original_run), + "--output-root", str(session / "new_preview"), + ) + assert result.returncode == 2 + assert "must not be inside input paths" in result.stderr + + +def test_migrator_source_has_no_machine_specific_user_path(): + source = MIGRATOR.read_text(encoding="utf-8") + windows_path = "C:" + "\\Users" + "\\gongh" + posix_path = "C:/" + "Users/gongh" + assert windows_path not in source + assert posix_path not in source + + +def test_run_lifecycle_preserves_historical_execution_time(tmp_path): + from agentbench_frame.tracking.run import Run + + run = Run.start( + game="24_miracle", agent="migrated", data_dir=str(tmp_path), + run_id="historical", append=False, + created="2026-07-21T16:19:30Z", git_commit="", + started_at=1784650770.0399437, + config={"migration_started_at": 1785000000.0}, + ) + run.write("game", game_id="g", valid=True, normalized_result="win", steps=3) + run.recompute_totals_from_events(game_event_type="game") + summary = run.finish(finished_at=1784651015.0399437) + assert summary["created"] == "2026-07-21T16:19:30Z" + assert summary["started_at"] == 1784650770.0399437 + assert summary["finished_at"] == 1784651015.0399437 + assert summary["wall_hours"] == round((245.0 / 3600), 2) + run_toml = (tmp_path / "runs" / "24_miracle" / "migrated" / "historical" / "run.toml").read_text() + assert "started_at = 1784650770.0399437" in run_toml + assert "finished_at = 1784651015.0399437" in run_toml + assert "migration_started_at = 1785000000.0" in run_toml diff --git a/tests/miracle/test_pr1_expanded.py b/tests/miracle/test_pr1_expanded.py new file mode 100644 index 0000000..772cce8 --- /dev/null +++ b/tests/miracle/test_pr1_expanded.py @@ -0,0 +1,563 @@ +"""PR#1 expanded tests: end-to-end fields, classification, resume, atomic interrupt.""" +from __future__ import annotations + +import json +import os +import struct +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.match_runner import MatchAttempt, classify, ReplayInfo, TraceStats +from agentbench_frame.games.miracle.runner import attempt_to_outcome +from agentbench_frame.games.miracle.result import to_event_record + + +# ---- helper: full MatchAttempt ---- # +def _full_att(**over): + base = dict( + game_id="m_rank01_camp0", evaluated_agent="ifelse", opponent="rank01", + evaluated_agent_camp=0, valid=False, normalized_result="error", + error_type="ai_crash", reason="opponent AI crashed", + raw_winner=None, winner_agent=None, + scores={"0": 5, "1": 2}, steps=42, + realized_randomization={"map_type": 1, "day_time": 0}, + result_json_status="ok", discrepancies=[], + ai_crash_player=1, ai_timeout_player=None, + judge_crash=False, wrapper_timeout=False, normal_cleanup_nonzero=False, + evidence_paths={"stdout": "rel/stdout", "stderr": "rel/stderr", + "trace": "rel/trace", "replay": "rel/replay", + "result_json": "rel/result.json"}, + collision_detected=False, + process_cleanup=[{"role": "judge", "cleanup_succeeded": True, "final_returncode": 0}, + {"role": "ai0", "cleanup_succeeded": True, "final_returncode": 0}, + {"role": "ai1", "cleanup_succeeded": True, "final_returncode": 1}], + vendor_returncode=0, started_at=1000.0, finished_at=1005.0, duration_s=5.0, + exception="RuntimeError('AI read EOF')", + judge_exit=0, ai0_exit=0, ai1_exit=1, + replay_sha256="abc123", + ) + base.update(over) + return MatchAttempt(**base) + + +# ===== SECTION 1: end-to-end field propagation ===== # +def test_end_to_end_field_propagation(tmp_path): + att = _full_att() + o = attempt_to_outcome(att) + rec = to_event_record(o) + # write to disk and re-read (simulates events.jsonl) + p = tmp_path / "event.json" + p.write_text(json.dumps(rec, default=str), encoding="utf-8") + loaded = json.loads(p.read_text(encoding="utf-8")) + # assert every key field + assert loaded["judge_exit"] == 0 + assert loaded["ai0_exit"] == 0 + assert loaded["ai1_exit"] == 1 + assert loaded["exception"] == "RuntimeError('AI read EOF')" + assert loaded["reason"] == "opponent AI crashed" + assert loaded["replay_sha256"] == "abc123" + assert loaded["process_cleanup"] == att.process_cleanup + assert loaded["result_json_status"] == "ok" + assert loaded["vendor_returncode"] == 0 + assert loaded["evidence_paths"]["replay"] == "rel/replay" + assert loaded["steps"] == 42 + assert loaded["seed"]["realized_randomization"] == {"map_type": 1, "day_time": 0} + assert loaded["error_type"] == att.error_type # NOT overwritten by normalized_result + + +def test_exception_and_reason_separately_preserved(): + att = _full_att(exception="FileNotFoundError('no file')", reason="vendor file missing") + o = attempt_to_outcome(att) + rec = to_event_record(o) + assert rec["exception"] == "FileNotFoundError('no file')" + assert rec["reason"] == "vendor file missing" + + +# ===== SECTION 2: expanded classification (10 cases) ===== # +def _ri_ok(): + return ReplayInfo(exists=True, header_valid=True, map_type=0, day_time=1, sha256="x") + + +def _ts_ok(end_info): + return TraceStats(end_info_seen=True, end_info=end_info, n_ai_operation=10) + + +def test_class_vendor_rc_nonzero_no_exception(): + ri = _ri_ok(); ts = _ts_ok({"0": 5, "1": 2}) + c = classify(rj_status="ok", rj={"end_info_received": True, "cleanup_all_succeeded": True, + "run_match_returncode": 0, "raw_winner": 0}, ts=ts, ri=ri, + discrepancies=[], vendor_returncode=1, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "vendor_exception" + + +def test_class_result_json_internal_rc_nonzero(): + ri = _ri_ok(); ts = _ts_ok({"0": 5, "1": 2}) + c = classify(rj_status="ok", rj={"end_info_received": True, "cleanup_all_succeeded": True, + "run_match_returncode": 1, "exception": None, "raw_winner": 0}, ts=ts, ri=ri, + discrepancies=[], vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "vendor_exception" + + +def test_class_vendor_exception_with_end_info(): + ri = _ri_ok(); ts = _ts_ok({"0": 5, "1": 2}) + c = classify(rj_status="ok", rj={"end_info_received": True, "cleanup_all_succeeded": True, + "exception": "ValueError('bad')", "raw_winner": 0}, ts=ts, ri=ri, + discrepancies=[], vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "vendor_exception" + + +def test_class_result_json_missing(): + ri = ReplayInfo(exists=False) + c = classify(rj_status="missing", rj=None, ts=TraceStats(), ri=ri, discrepancies=[], + vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "result_json_missing" + + +def test_class_result_json_corrupt(): + c = classify(rj_status="corrupt", rj=None, ts=TraceStats(), ri=ReplayInfo(exists=False), + discrepancies=[], vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "result_json_corrupt" + + +def test_class_cleanup_failure(): + ri = _ri_ok(); ts = _ts_ok({"0": 5, "1": 2}) + c = classify(rj_status="ok", rj={"end_info_received": True, "cleanup_all_succeeded": False, + "raw_winner": 0}, ts=ts, ri=ri, discrepancies=[], + vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "cleanup_failure" + + +def test_class_normal_cleanup_nonzero_still_valid(): + ri = _ri_ok(); ts = _ts_ok({"0": 5, "1": 2}) + c = classify(rj_status="ok", rj={"end_info_received": True, "cleanup_all_succeeded": True, + "raw_winner": 0, + "ai0": {"termination_requested": True, "final_returncode": 1}, + "ai1": {"termination_requested": True, "final_returncode": 1}}, + ts=ts, ri=ri, discrepancies=[], vendor_returncode=0, wrapper_timeout=False, + evaluated_agent_camp=0) + assert c.valid is True + assert c.normal_cleanup_nonzero is True + + +def test_class_ai_crash_allows_continue(): + ri = ReplayInfo(exists=True, header_valid=True) + ts = TraceStats(ai_error_players=[1], end_info_seen=True, end_info={"0": 5, "1": 2}) + c = classify(rj_status="ok", rj={"end_info_received": True, "cleanup_all_succeeded": True, + "raw_winner": 0}, ts=ts, ri=ri, discrepancies=[], + vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert not c.valid and c.error_type == "ai_crash" + # matrix should_stop: ai_crash is NOT in INFRA_STOP → allows continue + from agentbench_frame.games.miracle.matrix import should_stop + assert should_stop(_full_att(error_type="ai_crash", valid=False))[0] != "stop" + + +def test_class_infra_triggers_matrix_stop(): + from agentbench_frame.games.miracle.matrix import should_stop + assert should_stop(_full_att(error_type="cleanup_failure", valid=False))[0] == "stop" + assert should_stop(_full_att(error_type="vendor_exception", valid=False))[0] == "stop" + + +# ===== SECTION 3: resume expanded ===== # +def test_resume_does_not_create_new_session(tmp_path): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + r = MatrixRunner(session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: _full_att()) + r.prepare_session() + sid = r.session_id + sessions_before = list(tmp_path.glob("*")) + r.resume(sid) + sessions_after = list(tmp_path.glob("*")) + assert sessions_before == sessions_after # no new session created + + +def test_resume_corrupted_progress_rejected(tmp_path): + """Corrupt progress.json must be rejected by verify_session_for_resume (read-only).""" + _wm(tmp_path) + (tmp_path / "progress.json").write_text("NOT JSON {{{", encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("progress" in e.lower() and "corrupt" in e.lower() for e in errs) + + +def test_resume_running_state_uncertain(tmp_path): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + from agentbench_frame.games.miracle import matrix + r = MatrixRunner(session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None) + r.prepare_session() + plan = matrix.make_attempt_plan() + matrix.mark_running(r.progress, plan[0]["game_id"]) + matrix.write_progress_atomic(r.progress_path, r.progress) + r.resume(r.session_id) + res = r.execute() + assert res["halted"] is True and "UNCERTAIN" in res["reason"] + + +# ===== SECTION 4: atomic write subprocess interrupt ===== # +def test_atomic_write_survives_subprocess_kill(tmp_path): + from agentbench_frame.games.miracle.atomicio import atomic_write_json + target = tmp_path / "target.json" + # write old complete content + atomic_write_json(target, {"old": True}) + old_content = target.read_text(encoding="utf-8") + # simulate interrupted write: monkeypatch os.replace to raise (process killed before replace) + import agentbench_frame.games.miracle.atomicio as aio + original_replace = aio.os.replace + def kill_before_replace(*a, **k): + raise ProcessLookupError("process killed") + aio.os.replace = kill_before_replace + try: + with pytest.raises(ProcessLookupError): + atomic_write_json(target, {"new": False}) + finally: + aio.os.replace = original_replace + # target must still have old complete content (not partial) + assert target.read_text(encoding="utf-8") == old_content + # temp file exists and is auditable + temps = list(tmp_path.glob(".target.json.*.tmp")) + assert len(temps) >= 1, "temp file should be detectable after interrupt" + + +def test_atomic_write_first_write_killed_no_partial(tmp_path): + from agentbench_frame.games.miracle.atomicio import atomic_write_json + target = tmp_path / "first.json" + assert not target.exists() + import agentbench_frame.games.miracle.atomicio as aio + original_replace = aio.os.replace + aio.os.replace = lambda *a, **k: (_ for _ in ()).throw(ProcessLookupError("killed")) + try: + with pytest.raises(ProcessLookupError): + atomic_write_json(target, {"data": 1}) + finally: + aio.os.replace = original_replace + # target must NOT exist (not a partial file) + assert not target.exists() + + +# ===== SECTION 2: manifest verification (7 cases) ===== # +from agentbench_frame.games.miracle.matrix_runner import verify_session_for_resume + +def _write_manifest(tmp_path, **over): + plan = [{"rank": r, "camp": c, "game_id": f"m_rank{r:02d}_camp{c}"} + for r in range(1, 17) for c in (0, 1)] + m = {"protocol_sha256": "a"*64, "code_hashes": {"matrix": "abc"}, + "plan_count": 32, "plan": plan, "run_id": "RID", + "timeout": 8.0, "wrapper_timeout_s": 180.0, + "session_id": tmp_path.name} + m.update(over) + (tmp_path / "manifest.json").write_text(json.dumps(m), encoding="utf-8") + if not (tmp_path / "progress.json").exists(): + (tmp_path / "progress.json").write_text('{"attempts": {}}', encoding="utf-8") + +def test_manifest_missing(tmp_path): + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and "manifest missing" in errs[0] + +def test_manifest_corrupt(tmp_path): + (tmp_path / "manifest.json").write_text("NOT JSON {{{", encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and "corrupt" in errs[0] + +def test_manifest_protocol_hash_change(tmp_path): + _write_manifest(tmp_path, protocol_sha256="wrong") + ok, errs = verify_session_for_resume(tmp_path, protocol_sha="a"*64) + assert not ok and any("protocol" in e for e in errs) + +def test_manifest_code_hash_change(tmp_path): + codefile = tmp_path / "matrix.py"; codefile.write_text("x=1") + _write_manifest(tmp_path, code_hashes={"matrix": "wrong"}) + ok, errs = verify_session_for_resume(tmp_path, code_files={"matrix": str(codefile)}) + assert not ok and any("code hash" in e for e in errs) + +def test_manifest_plan_change(tmp_path): + _write_manifest(tmp_path, plan=[{"rank": 1, "camp": 0, "game_id": "x"}], plan_count=1) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("plan" in e for e in errs) + +def test_manifest_run_id_missing(tmp_path): + _write_manifest(tmp_path, run_id="") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("run_id" in e for e in errs) + +def test_manifest_timeout_change(tmp_path): + _write_manifest(tmp_path, timeout=99.0) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("timeout" in e for e in errs) + +def test_manifest_valid_passes(tmp_path): + _write_manifest(tmp_path) + ok, errs = verify_session_for_resume(tmp_path, protocol_sha="a"*64) + assert ok and errs == [] + + +# ===== SECTION: asset/session/python identity + write-safety + CLI mutex ===== # +import hashlib as _hl + +def _wm(tmp_path, **over): + """Full manifest with all identity fields.""" + plan = [{"rank": r, "camp": c, "game_id": f"m_rank{r:02d}_camp{c}"} + for r in range(1, 17) for c in (0, 1)] + m = { + "protocol_sha256": "a"*64, "code_hashes": {"matrix": "abc"}, + "plan_count": 32, "plan": plan, "run_id": "RID", + "timeout": 8.0, "wrapper_timeout_s": 180.0, + "ifelse_sha256": "if_sha", "judge_sha256": "j_sha", + "opponent_archive_sha256": {f"rank{r:02d}": f"opp_{r}" for r in range(1,17)}, + "cpp_build_sha256": {f"rank{r:02d}": f"bld_{r}" for r in [1,2,3,6]+list(range(8,17))}, + "session_id": tmp_path.name, "python_version": "3.11.15", "platform": "test_plat", + } + m.update(over) + (tmp_path / "manifest.json").write_text(json.dumps(m), encoding="utf-8") + if not (tmp_path / "progress.json").exists(): + (tmp_path / "progress.json").write_text('{"attempts": {}}', encoding="utf-8") + + +def test_verify_ifelse_sha_mismatch(tmp_path): + _wm(tmp_path, ifelse_sha256="wrong") + ok, errs = verify_session_for_resume(tmp_path, expected_ifelse_sha="if_sha") + assert not ok and any("if-else" in e for e in errs) + + +def test_verify_judge_sha_mismatch(tmp_path): + _wm(tmp_path, judge_sha256="wrong") + ok, errs = verify_session_for_resume(tmp_path, expected_judge_sha="j_sha") + assert not ok and any("judge" in e for e in errs) + + +def test_verify_opponent_sha_mismatch(tmp_path): + _wm(tmp_path, opponent_archive_sha256={f"rank{r:02d}": f"opp_{r}" for r in range(1,17)}, + **{"opponent_archive_sha256.rank01": "wrong"}) # this won't work as **kw + # simpler: just write a bad opponent hash directly + import json as _j + m = _j.loads((tmp_path / "manifest.json").read_text()) + m["opponent_archive_sha256"]["rank01"] = "WRONG" + (tmp_path / "manifest.json").write_text(_j.dumps(m)) + ok, errs = verify_session_for_resume(tmp_path, expected_opponent_shas={1: "opp_1"}) + assert not ok and any("opponent" in e for e in errs) + + +def test_verify_build_sha_mismatch(tmp_path): + _wm(tmp_path) + import json as _j + m = _j.loads((tmp_path / "manifest.json").read_text()) + m["cpp_build_sha256"]["rank01"] = "WRONG" + (tmp_path / "manifest.json").write_text(_j.dumps(m)) + ok, errs = verify_session_for_resume(tmp_path, expected_build_shas={1: "bld_1"}) + assert not ok and any("build" in e for e in errs) + + +def test_verify_session_id_mismatch(tmp_path): + _wm(tmp_path, session_id="WRONG_SID") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("session_id" in e for e in errs) + + +def test_verify_python_version_mismatch(tmp_path): + _wm(tmp_path, python_version="3.99.0") + ok, errs = verify_session_for_resume(tmp_path, expected_python="3.11.15") + assert not ok and any("python" in e.lower() for e in errs) + + +def test_verify_failure_leaves_session_unchanged(tmp_path): + _wm(tmp_path, protocol_sha256="wrong") + before = {p.name: _hl.sha256(p.read_bytes()).hexdigest() + for p in tmp_path.rglob("*") if p.is_file()} + ok, errs = verify_session_for_resume(tmp_path, protocol_sha="a"*64) + assert not ok + after = {p.name: _hl.sha256(p.read_bytes()).hexdigest() + for p in tmp_path.rglob("*") if p.is_file()} + assert before == after, "session files changed during verify!" + + +def test_cli_resume_dry_run_mutex(): + """--resume and --dry-run together must fail immediately (exit 2).""" + repo = Path(__file__).resolve().parents[2] + env = {**os.environ, "AGENTBENCH_ROOT": str(repo), "MIRACLE_IFELSE_DIR": "dummy"} + r = subprocess.run( + [sys.executable, str(repo / "tools" / "miracle_matrix.py"), + "--dry-run", "--resume", "xxx"], + capture_output=True, text=True, timeout=15, cwd=str(repo), env=env) + assert r.returncode == 2 + assert "mutually exclusive" in r.stderr.lower() + + +# ===== FINAL GAP: plan rank/camp + wrapper_timeout_s + platform ===== # +def test_plan_rank_tamper_rejected(tmp_path): + _wm(tmp_path) + import json as _j + m = _j.loads((tmp_path / "manifest.json").read_text()) + m["plan"][0]["rank"] = 99 # game_id unchanged, rank wrong + (tmp_path / "manifest.json").write_text(_j.dumps(m)) + from agentbench_frame.games.miracle.matrix import make_attempt_plan + ok, errs = verify_session_for_resume(tmp_path, expected_plan=make_attempt_plan()) + assert not ok and any("rank" in e for e in errs) + + +def test_plan_camp_tamper_rejected(tmp_path): + _wm(tmp_path) + import json as _j + m = _j.loads((tmp_path / "manifest.json").read_text()) + m["plan"][1]["camp"] = 99 # game_id unchanged, camp wrong + (tmp_path / "manifest.json").write_text(_j.dumps(m)) + from agentbench_frame.games.miracle.matrix import make_attempt_plan + ok, errs = verify_session_for_resume(tmp_path, expected_plan=make_attempt_plan()) + assert not ok and any("camp" in e for e in errs) + + +def test_wrapper_timeout_missing_rejected(tmp_path): + _wm(tmp_path) + import json as _j + m = _j.loads((tmp_path / "manifest.json").read_text()) + del m["wrapper_timeout_s"] + (tmp_path / "manifest.json").write_text(_j.dumps(m)) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("wrapper_timeout_s" in e for e in errs) + + +def test_wrapper_timeout_null_rejected(tmp_path): + _wm(tmp_path, wrapper_timeout_s=None) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("wrapper_timeout_s" in e for e in errs) + + +def test_wrapper_timeout_value_change_rejected(tmp_path): + _wm(tmp_path, wrapper_timeout_s=99.0) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok and any("wrapper_timeout_s" in e for e in errs) + + +def test_platform_mismatch_rejected(tmp_path): + _wm(tmp_path, platform="wrong_platform") + ok, errs = verify_session_for_resume(tmp_path, expected_platform="test_plat") + assert not ok and any("platform" in e for e in errs) + + +# ===== SECTION 2: partial-rank resume + run_id restore end-to-end ===== # +def test_resume_partial_rank_restores_run_id_and_skips_done(tmp_path): + """rank01 camp0 done + 1 event, camp1 not_started, no audit → resume + restores original run_id, executes only camp1, camp0 not re-run.""" + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + from agentbench_frame.games.miracle import matrix + + calls = [] + def fake_fn(**kw): + calls.append(kw["game_id"]) + return _full_att(game_id=kw["game_id"], valid=True, normalized_result="win") + + r = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=fake_fn, run_id="ORIGINAL_RID") + r.prepare_session() + original_run_id = r.run_id + # write manifest with the run_id + r.record_manifest(opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in [1, 2, 3, 6] + list(range(8, 17))}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}) + # simulate camp0 done + matrix.mark_done(r.progress, "m_rank01_camp0", + {"valid": True, "normalized_result": "win", "rank": 1, "camp": 0}) + matrix.write_progress_atomic(r.progress_path, r.progress) + matrix.append_event_atomic(r.events_path, + {"event": "game", "game_id": "m_rank01_camp0", + "valid": True, "normalized_result": "win"}) + + # new runner with DIFFERENT run_id → resume should restore original + r2 = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=fake_fn, run_id="WRONG") + r2.resume(r.session_id) + assert r2.run_id == original_run_id, f"run_id not restored: {r2.run_id}" + assert "WRONG" not in str(r2.run_dir) + + res = r2.execute_rank(rank=1) + assert res.get("halted") is False + assert "m_rank01_camp0" not in calls # done NOT re-run + assert "m_rank01_camp1" in calls # camp1 executed + ev = [json.loads(l)["game_id"] for l in r2.events_path.read_text(encoding="utf-8").splitlines() if l.strip()] + assert len(ev) == 2 and len(set(ev)) == 2 + + +# ===== SECTION 3: real subprocess kill atomic write test ===== # +def test_atomic_write_real_subprocess_kill(tmp_path): + """Kill a real subprocess mid-write: target keeps old complete JSON.""" + from agentbench_frame.games.miracle.atomicio import atomic_write_json + target = tmp_path / "target.json" + atomic_write_json(target, {"old": True}) + old_content = target.read_text(encoding="utf-8") + + child = tmp_path / "child.py" + src_path = str(Path(__file__).resolve().parents[2] / "src") + marker = str(tmp_path / "ready.marker") + child.write_text(f""" +import sys, json, os, time +from pathlib import Path +sys.path.insert(0, {src_path!r}) +from agentbench_frame.games.miracle.atomicio import atomic_write_json +import agentbench_frame.games.miracle.atomicio as aio +_marker = Path({marker!r}) +_orig = aio.os.replace +def slow_replace(src, dst): + _marker.write_text("ready") + time.sleep(30) + _orig(src, dst) +aio.os.replace = slow_replace +atomic_write_json({str(target)!r}, {{"new": False}}) +""", encoding="utf-8") + + proc = subprocess.Popen([sys.executable, str(child)], cwd=str(tmp_path)) + mpath = tmp_path / "ready.marker" + deadline = time.time() + 10 + while not mpath.exists() and time.time() < deadline: + time.sleep(0.05) + assert mpath.exists(), "child never reached replace" + proc.terminate() + proc.wait(timeout=5) + + content = target.read_text(encoding="utf-8") + assert content == old_content, "target changed during killed write!" + json.loads(content) # parses cleanly + for t in tmp_path.glob(".target.json.*.tmp"): + json.loads(t.read_text(encoding="utf-8")) # temp is complete JSON + + +def test_atomic_write_first_write_killed_no_target(tmp_path): + """First write killed mid-replace: target must not exist (no partial).""" + from agentbench_frame.games.miracle.atomicio import atomic_write_json + target = tmp_path / "first.json" + assert not target.exists() + + child = tmp_path / "child2.py" + src_path = str(Path(__file__).resolve().parents[2] / "src") + marker = str(tmp_path / "ready2.marker") + child.write_text(f""" +import sys, time +from pathlib import Path +sys.path.insert(0, {src_path!r}) +from agentbench_frame.games.miracle.atomicio import atomic_write_json +import agentbench_frame.games.miracle.atomicio as aio +_orig = aio.os.replace +def slow_replace(src, dst): + Path({marker!r}).write_text("ready") + time.sleep(30) + _orig(src, dst) +aio.os.replace = slow_replace +atomic_write_json({str(target)!r}, {{"data": 1}}) +""", encoding="utf-8") + + proc = subprocess.Popen([sys.executable, str(child)], cwd=str(tmp_path)) + mpath = tmp_path / "ready2.marker" + deadline = time.time() + 10 + while not mpath.exists() and time.time() < deadline: + time.sleep(0.05) + assert mpath.exists() + proc.terminate() + proc.wait(timeout=5) + assert not target.exists(), "partial target created!" diff --git a/tests/miracle/test_pr1_fixes.py b/tests/miracle/test_pr1_fixes.py new file mode 100644 index 0000000..98182e5 --- /dev/null +++ b/tests/miracle/test_pr1_fixes.py @@ -0,0 +1,142 @@ +"""PR#1 review#2-6 red-light tests (方案C: 基线 main，review#1 统一封装待定). + +Each group must FAIL against current code (the gap), then turn green after fix. +review#1 (unified event envelope) is DEFERRED (framework-dependent). +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +# ---------- review#4: atomic result-json write (temp + fsync + os.replace) ---------- # +def test_atomic_write_json_exists_and_atomic(tmp_path): + from agentbench_frame.games.miracle.atomicio import atomic_write_json + p = tmp_path / "r.json" + atomic_write_json(p, {"a": 1, "b": "x"}) + assert json.loads(p.read_text(encoding="utf-8")) == {"a": 1, "b": "x"} + # no leftover temp file + assert not list(tmp_path.glob("*.tmp")) and not list(tmp_path.glob("r.json.*")) + + +def test_atomic_write_json_preserves_old_on_failure(tmp_path, monkeypatch): + from agentbench_frame.games.miracle.atomicio import atomic_write_json + p = tmp_path / "r.json" + p.write_text('{"old": true}', encoding="utf-8") + # inject failure during the replace step + import agentbench_frame.games.miracle.atomicio as aio + def boom(*a, **k): + raise OSError("injected") + monkeypatch.setattr(aio.os, "replace", boom) + with pytest.raises(OSError): + atomic_write_json(p, {"new": 1}) + # old complete file intact (or file unchanged) + assert json.loads(p.read_text(encoding="utf-8")) == {"old": True} + + +# ---------- review#5: first-hand process fields on GameOutcome / event record ---------- # +def test_gameoutcome_has_first_hand_fields(): + from agentbench_frame.games.miracle.result import GameOutcome, finalize + o = finalize(GameOutcome(game_id="g", evaluated_agent="a", opponent="b", evaluated_agent_camp=0)) + for f in ("judge_exit", "ai0_exit", "ai1_exit", "replay_sha256", "process_cleanup"): + assert hasattr(o, f), f"GameOutcome missing {f}" + + +def test_event_record_includes_first_hand_fields(): + from agentbench_frame.games.miracle.result import GameOutcome, finalize, to_event_record + o = finalize(GameOutcome(game_id="g", evaluated_agent="a", opponent="b", evaluated_agent_camp=0, + judge_exit=0, ai0_exit=0, ai1_exit=1, replay_sha256="abc", + process_cleanup=[{"role": "judge", "cleanup_succeeded": True}])) + rec = to_event_record(o) + for f in ("judge_exit", "ai0_exit", "ai1_exit", "replay_sha256", "process_cleanup"): + assert f in rec, f"event record missing {f}" + + +# ---------- review#6: classification of vendor/cleanup/result-json infra failures ---------- # +class _Att: + def __init__(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + +def _base_att(**over): + base = dict(wrapper_timeout=False, error_type=None, result_json_status="ok", + discrepancies=[], realized_randomization={"map_type": 0, "day_time": 1}, + raw_winner=0, ai_crash_player=None, ai_timeout_player=None, + judge_crash=False, normal_cleanup_nonzero=False, evaluated_agent_camp=0, + evaluated_agent="ifelse", opponent="r", game_id="g") + base.update(over) + return _Att(**base) + +def test_classify_cleanup_failure_is_infra_invalid(): + from agentbench_frame.games.miracle.match_runner import classify, ReplayInfo, TraceStats + # valid end_info + replay ok, but cleanup failed -> infra invalid (not valid) + ri = ReplayInfo(exists=True, header_valid=True) + c = classify(rj_status="ok", rj={"end_info_received": True, "ai0": {"termination_requested": True, "final_returncode": 0}, + "ai1": {"termination_requested": True, "final_returncode": 0}, + "raw_winner": 0, "cleanup_all_succeeded": False, "run_match_returncode": 0}, + ts=TraceStats(end_info_seen=True, end_info={"0": 5, "1": 2}), ri=ri, discrepancies=[], + vendor_returncode=0, wrapper_timeout=False, evaluated_agent_camp=0) + assert c.valid is False + assert c.error_type in ("cleanup_failure", "infra_failure") + + +def test_classify_vendor_nonzero_with_end_info_is_infra(): + from agentbench_frame.games.miracle.match_runner import classify, ReplayInfo, TraceStats + ri = ReplayInfo(exists=True, header_valid=True) + c = classify(rj_status="ok", rj={"end_info_received": True, + "ai0": {"termination_requested": False, "final_returncode": 0}, + "ai1": {"termination_requested": False, "final_returncode": 0}, + "raw_winner": 0, "cleanup_all_succeeded": True, "run_match_returncode": 1, + "exception": "RuntimeError('x')"}, + ts=TraceStats(end_info_seen=True, end_info={"0": 5, "1": 2}), ri=ri, discrepancies=[], + vendor_returncode=1, wrapper_timeout=False, evaluated_agent_camp=0) + assert c.valid is False + assert c.error_type in ("vendor_exception", "infra_failure") + + +# ---------- review#2: --resume ---------- # +def test_matrix_runner_resume_method_exists(): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + assert hasattr(MatrixRunner, "resume"), "MatrixRunner missing resume()" + + +def test_resume_skips_done_no_rerun(tmp_path, monkeypatch): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + from agentbench_frame.games.miracle import matrix + calls = [] + r = MatrixRunner(session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: calls.append(k["game_id"]) or _base_att()) + r.prepare_session() + plan = matrix.make_attempt_plan() + # mark rank01 both camps done already + matrix.mark_done(r.progress, "m_rank01_camp0", {"valid": True, "normalized_result": "win"}) + matrix.mark_done(r.progress, "m_rank01_camp1", {"valid": True, "normalized_result": "loss"}) + matrix.write_progress_atomic(r.progress_path, r.progress) + monkeypatch.setattr(r, "session_id", r.session_id) # no-op; resume reuses existing session + r.resume(r.session_id) + r.execute_rank(rank=1) + # done games not re-run + assert "m_rank01_camp0" not in calls + assert "m_rank01_camp1" not in calls + + +# ---------- review#3: POSIX reap (wait after terminate/kill) ---------- # +def test_proctree_cleanup_reaps_popen(): + from agentbench_frame.games.miracle.proctree import ProcessTreeManager + # spawn a long-lived child via Popen + proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + mgr = ProcessTreeManager() + mp = mgr.register_popen(proc, "child") + mgr.cleanup_all("test") + # reap: Popen.returncode must be set (wait was called), not None (zombie/unreaped) + assert proc.returncode is not None, "Popen not reaped (returncode None)" + finally: + if proc.returncode is None: + proc.kill(); proc.wait(timeout=5) diff --git a/tests/miracle/test_proctree.py b/tests/miracle/test_proctree.py new file mode 100644 index 0000000..8280257 --- /dev/null +++ b/tests/miracle/test_proctree.py @@ -0,0 +1,201 @@ +"""Process-tree manager contract tests (SKILL.md 测试门槛 item 17 + the 8 +process-safety requirements from the 阶段4b spec). + +These run FIRST as a RED LIGHT (the implementation proctree.py does not exist +yet), the failure is saved as evidence, THEN the implementation is written to +turn them green. + +Validated behaviours: + 1. normal child exits naturally (natural_exit, no kill) + 2. parent exits but child still survives -> child is reachable + cleanable + 3. timeout -> the whole recorded tree is cleaned + 4. cleanup is idempotent + 5. an already-exited PID is skipped, never a false kill of others + 6. only this run's recorded exact PIDs are cleaned + 7. after cleanup neither parent nor child PID remains + 8. NO name-based / fuzzy kill (structural scan of the implementation) + + PID identity (create_time) must match before any kill; a stale/reused PID + is never killed. + + py -3.13 -m pytest tests/miracle/test_proctree.py -v +""" +from __future__ import annotations + +import pathlib +import re +import subprocess +import sys +import time + +import psutil +import pytest + +from agentbench_frame.games.miracle import proctree # noqa: F401 (red-light import) +from agentbench_frame.games.miracle.proctree import ManagedProcess, ProcessTreeManager + +HELPER = pathlib.Path(__file__).parent / "_proc_helper.py" +PY = sys.executable + + +# ---- spawn helpers ------------------------------------------------------- # +def _spawn(args, **kw): + return subprocess.Popen( + [PY, str(HELPER), *args], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **kw, + ) + + +def _ready(proc, timeout=5.0): + """Read the single READY line the helper prints on startup.""" + line = proc.stdout.readline() + assert line.startswith("READY"), f"unexpected helper output: {line!r}" + return line + + +def _child_pid_from(line: str) -> int: + return int(line.split("child=")[1].strip()) + + +def _alive(pid: int, started_at: float, tol: float = 1.0) -> bool: + """Identity-checked liveness: PID exists AND its create_time matches.""" + if not psutil.pid_exists(pid): + return False + try: + return abs(psutil.Process(pid).create_time() - started_at) < tol + except psutil.NoSuchProcess: + return False + + +# ---- 1: natural exit ----------------------------------------------------- # +def test_normal_child_exits_naturally(): + mgr = ProcessTreeManager() + p = _spawn(["stay", "0.3"]) + try: + _ready(p) + mp = mgr.register_popen(p, "child") + p.wait(timeout=5) + mgr.poll() + assert mp.natural_exit is True + assert mp.natural_returncode == 0 + assert mp.termination_requested is False + mgr.cleanup_all("test-done") + assert mp.cleanup_succeeded is True + finally: + if p.poll() is None: + p.kill() + + +# ---- 2: parent exits, child survives, then cleaned ----------------------- # +def test_parent_exits_child_survives_then_cleaned(): + mgr = ProcessTreeManager() + p = _spawn(["spawn_child", "30", "0.2"]) # child lives 30s, parent 0.2s + line = _ready(p) + child_pid = _child_pid_from(line) + p.wait(timeout=5) # parent exits naturally + cmp = mgr.register_pid(child_pid, "grandchild") + assert _alive(cmp.pid, cmp.started_at) is True # child still alive + mgr.cleanup_all("parent-gone") + assert _alive(cmp.pid, cmp.started_at) is False # child now cleaned + assert cmp.cleanup_succeeded is True + + +# ---- 3: timeout cleans the whole recorded tree --------------------------- # +def test_timeout_kills_whole_tree(): + mgr = ProcessTreeManager() + p = _spawn(["spawn_child", "30", "30"]) # both long-lived + line = _ready(p) + child_pid = _child_pid_from(line) + mp = mgr.register_popen(p, "parent") + cmp = mgr.register_pid(child_pid, "child") + assert _alive(mp.pid, mp.started_at) and _alive(cmp.pid, cmp.started_at) + mgr.cleanup_all("timeout") + assert not _alive(mp.pid, mp.started_at) + assert not _alive(cmp.pid, cmp.started_at) + assert mp.cleanup_succeeded and cmp.cleanup_succeeded + + +# ---- 4: idempotent ------------------------------------------------------- # +def test_cleanup_is_idempotent(): + mgr = ProcessTreeManager() + p = _spawn(["stay", "30"]) + _ready(p) + mp = mgr.register_popen(p, "x") + mgr.cleanup_all("first") + assert not _alive(mp.pid, mp.started_at) + # second cleanup must be a no-op without raising + mgr.cleanup_all("second") + mgr.cleanup_one(mp, "third") + assert not _alive(mp.pid, mp.started_at) + + +# ---- 5: already-exited PID is skipped, no false kill --------------------- # +def test_exited_pid_does_not_cause_false_kill(): + mgr = ProcessTreeManager() + dead = _spawn(["stay", "0.2"]) + _ready(dead) + dead.wait(timeout=5) + dmp = mgr.register_popen(dead, "dead") # already exited + other = _spawn(["stay", "30"]) + _ready(other) + other_mp = mgr.register_popen(other, "other") + mgr.cleanup_one(dmp, "noop") # must skip the dead one + assert _alive(other_mp.pid, other_mp.started_at) is True # other untouched + mgr.cleanup_all("final") + assert not _alive(other_mp.pid, other_mp.started_at) + + +# ---- 6: only recorded exact PIDs are cleaned ----------------------------- # +def test_only_recorded_pids_are_cleaned(): + mgr = ProcessTreeManager() + a = _spawn(["stay", "30"]); _ready(a) + b = _spawn(["stay", "30"]); _ready(b) + b_ct = psutil.Process(b.pid).create_time() + ma = mgr.register_popen(a, "a") # register ONLY a + mgr.cleanup_all("partial") # should clean a only + assert not _alive(ma.pid, ma.started_at) + # b was never registered -> must still be alive + assert _alive(b.pid, b_ct) is True + b.terminate(); b.wait(timeout=5) + + +# ---- 7: no residual parent nor child PID --------------------------------- # +def test_cleanup_leaves_neither_parent_nor_child_pid(): + mgr = ProcessTreeManager() + p = _spawn(["spawn_child", "30", "30"]) + line = _ready(p) + child_pid = _child_pid_from(line) + mp = mgr.register_popen(p, "parent") + cmp = mgr.register_pid(child_pid, "child") + mgr.cleanup_all("done") + assert not _alive(mp.pid, mp.started_at) + assert not _alive(cmp.pid, cmp.started_at) + + +# ---- 8: no name-based / fuzzy kill (structural) -------------------------- # +def test_no_name_based_or_fuzzy_kill(): + src = pathlib.Path(proctree.__file__).read_text(encoding="utf-8") + forbidden = ["/im ", "/im\"", "/IM ", "/IM\"", "tskill ", "imagename", + "Get-Process", "wmic process where name"] + hits = [f for f in forbidden if f in src] + assert not hits, f"fuzzy/name-based kill patterns found in proctree.py: {hits}" + # every taskkill must be /PID-based (exact PID), never by image name + for m in re.finditer(r"taskkill", src, re.IGNORECASE): + ctx = src[m.start(): m.start() + 160] + assert "/PID" in ctx or "/pid" in ctx, f"non-PID taskkill: {ctx!r}" + + +# ---- +: PID identity (create_time) guards against PID reuse -------------- # +def test_identity_mismatch_does_not_kill(): + mgr = ProcessTreeManager() + p = _spawn(["stay", "30"]) + _ready(p) + mp = mgr.register_popen(p, "x") + mp.started_at = 1.0 # forge a stale start time -> simulates PID reuse + mgr.cleanup_one(mp, "stale") + assert mp.identity_confirmed is False + assert _alive(p.pid, psutil.Process(p.pid).create_time()) is True # NOT killed + # restore real identity and clean up properly + mp.started_at = psutil.Process(p.pid).create_time() + mgr.cleanup_one(mp, "real") + assert mp.identity_confirmed is True + assert _alive(mp.pid, mp.started_at) is False diff --git a/tests/miracle/test_result.py b/tests/miracle/test_result.py new file mode 100644 index 0000000..243e746 --- /dev/null +++ b/tests/miracle/test_result.py @@ -0,0 +1,363 @@ +"""Contract tests for the 24_miracle pure result module. + +Covers SKILL.md 测试门槛 items that are pure logic (no subprocess, no Judge): + 1. Agent wins at camp0 + 2. Agent wins at camp1 + 3. side-swap tallying (raw_winner==0 is NOT always the evaluated agent's win) + 4. draw recorded separately, not a win + 5. opponent crash -> error (not a capability win) + 6. agent crash -> error + 7. judge crash -> error + 8. timeout -> error + 9. replay missing -> error + 10. duplicate game_id resumability + 13. h2h direction +plus end_info parsing, replay-header parsing, file hashing, and event-record +contract completeness. + +Run with the 3.11+ interpreter: + py -3.13 -m pytest tests/miracle/test_result.py -v +""" +from __future__ import annotations + +import json +import struct + +import pytest + +from agentbench_frame.games.miracle.result import ( + DRAW, + ERROR, + GameOutcome, + LOSS, + VALID_RESULTS, + WIN, + build_seed_provenance, + compute_h2h, + compute_run_stats, + compute_win_rate, + derive_raw_winner, + finalize, + normalize, + outcome_counts, + read_replay_header, + scores_from_end_info, + select_games_to_run, + sha256_file, + to_event_record, + would_rerun_successful, +) + + +# ---- helpers ------------------------------------------------------------- # +def outcome(eval_camp=0, raw_winner=None, **kw): + """Build a finalized GameOutcome with sane defaults for tests.""" + o = GameOutcome( + game_id=kw.pop("game_id", "g1"), + evaluated_agent=kw.pop("evaluated_agent", "ifelse"), + opponent=kw.pop("opponent", "rank04"), + evaluated_agent_camp=eval_camp, + raw_winner=raw_winner, + **kw, + ) + return finalize(o) + + +# ---- 1 & 2: win at camp0 / camp1 ----------------------------------------- # +def test_win_at_camp0(): + o = outcome(eval_camp=0, raw_winner=0) + assert o.normalized_result == WIN + assert o.winner_agent == "ifelse" + + +def test_win_at_camp1(): + # evaluated agent on camp 1, Judge says camp 1 won -> evaluated agent wins + o = outcome(eval_camp=1, raw_winner=1) + assert o.normalized_result == WIN + assert o.winner_agent == "ifelse" + + +def test_loss_at_camp1_when_raw_winner_zero(): + # side-swapped: evaluated on camp1, raw_winner==0 (opponent camp) -> LOSS, + # NOT a win. This is the exact bug the framework's Match has. + o = outcome(eval_camp=1, raw_winner=0) + assert o.normalized_result == LOSS + assert o.winner_agent == "rank04" + + +# ---- 3: side-swap tallying ----------------------------------------------- # +def test_side_swap_win_rate_counts_both_sides(): + # two wins, one as camp0, one as camp1 (swapped) -> 100% + games = [ + outcome(game_id="a", eval_camp=0, raw_winner=0), + outcome(game_id="b", eval_camp=1, raw_winner=1), + ] + assert compute_win_rate(games) == 1.0 + + +def test_side_swap_raw_winner_zero_is_not_always_eval_win(): + # raw_winner==0 once means eval win (camp0) and once means eval loss (camp1) + games = [ + outcome(game_id="a", eval_camp=0, raw_winner=0), # win + outcome(game_id="b", eval_camp=1, raw_winner=0), # loss (swapped) + ] + assert compute_win_rate(games) == pytest.approx(0.5) + # naive "raw_winner==0 => win" would wrongly give 1.0 + + +# ---- 4: draw ------------------------------------------------------------- # +def test_draw_recorded_not_a_win(): + # Miracle Judge never draws; simulate a hypothetical draw (raw_winner=-1). + games = [ + outcome(game_id="a", eval_camp=0, raw_winner=0), # win + finalize(GameOutcome(game_id="b", evaluated_agent="ifelse", + opponent="rank04", evaluated_agent_camp=0, + raw_winner=-1)), # draw + ] + counts = outcome_counts(games) + assert counts["win"] == 1 and counts["draw"] == 1 + # draw is a valid game but not a win -> win_rate = 1/2 + assert compute_win_rate(games) == pytest.approx(0.5) + + +# ---- 5-9: anomaly -> error, never a capability win ----------------------- # +def test_opponent_crash_is_error_not_win(): + # Judge gave camp0 the win (raw_winner=0, evaluated on camp0), but the + # OPPONENT crashed -> invalid evidence, must NOT count as a capability win. + o = outcome(eval_camp=0, raw_winner=0, ai_error_player=1) + assert o.normalized_result == ERROR + assert o.valid is False + + +def test_agent_crash_is_error(): + o = outcome(eval_camp=0, raw_winner=1, ai_error_player=0) + assert o.normalized_result == ERROR + + +def test_judge_crash_is_error(): + o = outcome(eval_camp=0, judge_ok=False) + assert o.normalized_result == ERROR + assert o.raw_winner is None or o.normalized_result == ERROR + + +def test_timeout_is_error(): + o = outcome(eval_camp=0, raw_winner=0, ai_timeout_player=0) + assert o.normalized_result == ERROR + o2 = outcome(eval_camp=0, raw_winner=1, ai_timeout_player=1) + assert o2.normalized_result == ERROR + + +def test_replay_missing_is_error(): + o = outcome(eval_camp=0, raw_winner=0, replay_ok=False) + assert o.normalized_result == ERROR + + +def test_error_games_excluded_from_win_rate(): + games = [ + outcome(game_id="ok", eval_camp=0, raw_winner=0), # win + outcome(game_id="crash", eval_camp=0, raw_winner=0, + ai_error_player=1), # error + ] + # only 1 valid game (the win) -> win_rate 1.0, not 0.5 + assert compute_win_rate(games) == 1.0 + assert outcome_counts(games)["error"] == 1 + + +# ---- 10: duplicate game_id resumability ---------------------------------- # +def test_select_games_skips_completed_successful(): + planned = ["g1", "g2", "g3", "g4"] + done_valid = ["g2"] # g2 already has a valid result + assert select_games_to_run(planned, done_valid) == ["g1", "g3", "g4"] + + +def test_would_rerun_successful_detected(): + done = ["g2"] + assert would_rerun_successful("g2", done) is True + assert would_rerun_successful("g1", done) is False + + +def test_no_valid_games_means_rerun_all(): + planned = ["g1", "g2"] + assert select_games_to_run(planned, []) == ["g1", "g2"] + + +# ---- 13: h2h direction --------------------------------------------------- # +def test_h2h_direction_no_draws_sums_to_one(): + # ifelse vs rank04: ifelse won 3, rank04 won 1, across side-swaps + games = [ + outcome(game_id="1", eval_camp=0, raw_winner=0), # ifelse win + outcome(game_id="2", eval_camp=1, raw_winner=1), # ifelse win (swapped) + outcome(game_id="3", eval_camp=0, raw_winner=0), # ifelse win + outcome(game_id="4", eval_camp=0, raw_winner=1), # rank04 win + ] + h2h = compute_h2h(games) + assert h2h["ifelse"]["rank04"] == pytest.approx(0.75) + assert h2h["rank04"]["ifelse"] == pytest.approx(0.25) + # no draws -> symmetric pair sums to 1 + assert h2h["ifelse"]["rank04"] + h2h["rank04"]["ifelse"] == pytest.approx(1.0) + + +def test_h2h_with_draw_does_not_sum_to_one(): + games = [ + outcome(game_id="1", eval_camp=0, raw_winner=0), # win + finalize(GameOutcome(game_id="2", evaluated_agent="ifelse", + opponent="rank04", evaluated_agent_camp=0, + raw_winner=-1)), # draw + ] + h2h = compute_h2h(games) + assert h2h["ifelse"]["rank04"] == pytest.approx(0.5) # 1 win / 2 valid + assert h2h["rank04"]["ifelse"] == pytest.approx(0.0) + assert h2h["ifelse"]["rank04"] + h2h["rank04"]["ifelse"] == pytest.approx(0.5) + + +def test_h2h_excludes_error_games(): + games = [ + outcome(game_id="1", eval_camp=0, raw_winner=0), # win + outcome(game_id="2", eval_camp=0, raw_winner=0, ai_error_player=1), # error + ] + h2h = compute_h2h(games) + assert h2h["ifelse"]["rank04"] == pytest.approx(1.0) # only the valid game counts + + +# ---- end_info parsing (Judge main.py semantics) -------------------------- # +def test_derive_raw_winner_player0_wins(): + assert derive_raw_winner({"0": 5, "1": 2}) == 0 + + +def test_derive_raw_winner_player1_wins_on_tie(): + # Judge breaks ties toward player1 (main.py:448-449) + assert derive_raw_winner({"0": 3, "1": 3}) == 1 + assert derive_raw_winner({"0": 2, "1": 5}) == 1 + + +def test_derive_raw_winner_none_when_missing(): + assert derive_raw_winner(None) is None + assert derive_raw_winner({}) is None + assert derive_raw_winner({"0": 1}) is None + assert derive_raw_winner({"0": "x", "1": "y"}) is None + + +def test_scores_from_end_info(): + assert scores_from_end_info({"0": 7, "1": 4}) == (7, 4) + assert scores_from_end_info(None) == (None, None) + + +# ---- replay header + file hashing ---------------------------------------- # +def test_read_replay_header(tmp_path): + # [0,0,0,map_type,day_time,0,0] as big-endian signed int32 + blob = struct.pack(">7i", 0, 0, 0, 1, 0, 0, 0) + p = tmp_path / "g.replay" + p.write_bytes(blob) + assert read_replay_header(p) == {"map_type": 1, "day_time": 0} + + +def test_read_replay_header_missing(tmp_path): + assert read_replay_header(tmp_path / "nope.replay") is None + + +def test_read_replay_header_too_short(tmp_path): + p = tmp_path / "short.replay" + p.write_bytes(b"\x00" * 10) + assert read_replay_header(p) is None + + +def test_sha256_file(tmp_path): + p = tmp_path / "x.bin" + p.write_bytes(b"hello") + # known sha256 of "hello" + assert sha256_file(p) == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + + +def test_sha256_file_missing(tmp_path): + assert sha256_file(tmp_path / "nope") is None + + +# ---- event-record contract completeness ---------------------------------- # +def test_event_record_has_all_required_fields(): + o = outcome(game_id="g1", eval_camp=0, raw_winner=0, + replay_path="/tmp/g1.replay", replay_sha256="abc", + evaluated_source_sha256="d1", opponent_source_sha256="e1", + judge_exit=0, ai0_exit=0, ai1_exit=0, steps=42, + started_at=1.0, finished_at=2.0, duration_s=1.0) + rec = to_event_record(o) + missing = [f for f in ( + "game_id", "seed", "policy_ids", "policy_source_sha256", "camps", + "raw_winner", "winner_agent", "normalized_result", "scores", "draw", + "started_at", "finished_at", "duration", "judge_exit", "ai0_exit", + "ai1_exit", "timeout_s", "exception", "replay_path", "replay_sha256", + "valid", "is_resume", "is_rerun") if f not in rec] + assert missing == [] + # direction-critical fields must be present and correct + assert rec["raw_winner"] == 0 + assert rec["winner_agent"] == "ifelse" + assert rec["normalized_result"] == WIN + assert rec["evaluated_agent_camp"] == 0 + assert rec["valid"] is True + + +def test_event_record_seed_provenance_and_serializable(): + o = outcome(game_id="g1", eval_camp=0, raw_winner=0, + realized_randomization={"map_type": 1, "day_time": 0}) + rec = to_event_record(o) + # seed field is the structured provenance, NOT map_type/day_time called a seed + assert rec["seed"] == { + "requested_seed": None, "effective_seed": None, + "deterministic_seed_supported": False, "reproducible_from_seed": False, + "realized_randomization": {"map_type": 1, "day_time": 0}, + } + # must round-trip through JSON (events.jsonl is newline-delimited JSON) + s = json.dumps(rec, default=str) + assert json.loads(s)["game_id"] == "g1" + + +# ---- run-level statistics ------------------------------------------------ # +def test_compute_run_stats_counts_and_steps(): + outcomes = [ + outcome(game_id="1", eval_camp=0, raw_winner=0, steps=40), # win + outcome(game_id="2", eval_camp=1, raw_winner=1, steps=38), # win (swapped) + outcome(game_id="3", eval_camp=0, raw_winner=1, steps=50), # loss + outcome(game_id="4", eval_camp=0, raw_winner=0, steps=10, ai_error_player=1), # error + ] + stats = compute_run_stats(outcomes) + assert stats["attempted_games"] == 4 + assert stats["valid_games"] == 3 + assert stats["invalid_games"] == 1 + assert (stats["wins"], stats["losses"], stats["draws"]) == (2, 1, 0) + assert stats["win_rate_denominator"] == 3 + assert stats["win_rate"] == pytest.approx(2 / 3) + assert stats["attempted_steps"] == 138 # 40+38+50+10 + assert stats["total_steps"] == 128 # 40+38+50 (valid only) + assert stats["evaluation_status"] == "COMPLETE" + + +def test_compute_run_stats_no_valid_games_win_rate_null(): + outcomes = [ + outcome(game_id="1", eval_camp=0, raw_winner=0, steps=10, ai_error_player=1), + outcome(game_id="2", eval_camp=0, judge_ok=False), + ] + stats = compute_run_stats(outcomes) + assert stats["valid_games"] == 0 + assert stats["win_rate"] is None + assert stats["evaluation_status"] == "NO_VALID_GAMES" + assert stats["win_rate_denominator"] == 0 + + +# ---- tie semantics: Judge resolves ties to player1, NOT a draw ----------- # +def test_tie_resolved_to_player1_not_draw(): + o_win = outcome(eval_camp=1, raw_winner=1, score0=4, score1=4) # evaluated camp1 wins the tiebreak + assert o_win.score_tie is True and o_win.judge_tiebreak_applied is True + assert o_win.normalized_result == "win" + assert o_win.draw is False + o_loss = outcome(eval_camp=0, raw_winner=1, score0=4, score1=4) # evaluated camp0 loses the tiebreak + assert o_loss.normalized_result == "loss" + assert o_loss.score_tie is True + assert o_loss.draw is False + + +def test_build_seed_provenance_structure(): + p = build_seed_provenance({"map_type": 0, "day_time": 1}) + assert p == { + "requested_seed": None, "effective_seed": None, + "deterministic_seed_supported": False, "reproducible_from_seed": False, + "realized_randomization": {"map_type": 0, "day_time": 1}, + } diff --git a/tests/miracle/test_results_pipeline.py b/tests/miracle/test_results_pipeline.py new file mode 100644 index 0000000..8226280 --- /dev/null +++ b/tests/miracle/test_results_pipeline.py @@ -0,0 +1,151 @@ +"""End-to-end Results-pipeline tests (SKILL.md 测试门槛 items 14, 15, 16). + +Drive the real framework ``Run`` to produce a contract-complete run, then run +the AgentBenchResults scripts that SKILL.md 本地验收 mandates, and assert: + + 14. aggregate.py picks up the 24_miracle run into registry.toml + 15. report_builder.py renders a static site with data.json + 16. web win_rate == summary.json win_rate == win_rate recomputed from events + (three-way consistency; "CI成功不等于数据正确") + +UPSTREAM BUG (documented, not hidden): + The upstream ``aggregate.py`` writes ``path = str(run_dir.relative_to(...))``, + which on Windows emits backslashes and makes registry.toml invalid TOML, so + report_builder aborts. We vendor a one-line portability patch + (vendor/results_local/aggregate.py, as_posix) for local verification and pin + the upstream bug with a dedicated regression test below. report_builder.py is + used unmodified (it works once registry.toml is valid). + + py -3.13 -m pytest tests/miracle/test_results_pipeline.py -v +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.driver import feed_outcomes_to_run +from agentbench_frame.games.miracle.result import GameOutcome, finalize +from agentbench_frame.tracking.run import Run + +_REPO_ROOT = Path(__file__).resolve().parents[2] +from agentbench_frame.games.miracle.paths import results_repo as _results_repo +RESULTS_REPO = _results_repo() or Path("") +UPSTREAM_AGGREGATE = RESULTS_REPO / "scripts" / "aggregate.py" +UPSTREAM_REPORT_BUILDER = RESULTS_REPO / "scripts" / "report_builder.py" +# local portability-patched copy (as_posix); see header in that file. +LOCAL_AGGREGATE = _REPO_ROOT / "vendor" / "results_local" / "aggregate.py" + +pytestmark = pytest.mark.skipif( + not (LOCAL_AGGREGATE.exists() and UPSTREAM_REPORT_BUILDER.exists()), + reason="local aggregate + upstream report_builder required", +) + + +# ---- helpers (self-contained) -------------------------------------------- # +def _outcome(gid, eval_camp, raw_winner, steps=40, **kw): + s0 = 5 if raw_winner == 0 else 2 + s1 = 2 if raw_winner == 0 else 5 + return finalize( + GameOutcome( + game_id=gid, evaluated_agent="ifelse", opponent="rank04", + evaluated_agent_camp=eval_camp, raw_winner=raw_winner, + score0=s0, score1=s1, steps=steps, + replay_path=f"/tmp/{gid}.replay", replay_sha256="x" * 64, **kw, + ) + ) + + +def _build_run(tmp_path: Path, outcomes): + run = Run.start( + game="24_miracle", agent="ifelse", run_type="eval", + data_dir=str(tmp_path), + config={"opponent_set": "smoke", "n_games": str(len(outcomes))}, + ) + feed_outcomes_to_run(run, outcomes) + summary = run.finish() + return summary, tmp_path / "runs" / "24_miracle" / "ifelse" / run.run_id + + +def _recompute_win_rate_from_events(run_dir: Path) -> float: + games = [ + json.loads(line) + for line in (run_dir / "events.jsonl").read_text().splitlines() + if line.strip() and json.loads(line).get("event") == "game" + ] + valid = [g for g in games if g["normalized_result"] in ("win", "loss", "draw")] + if not valid: + return 0.0 + wins = sum(1 for g in valid if g["normalized_result"] == "win") + return wins / len(valid) + + +def _run_py(script: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(script), *args], capture_output=True, text=True) + + +# ---- 14: aggregate (local patched copy) ---------------------------------- # +def test_aggregate_picks_up_miracle_run(tmp_path): + _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + out = _run_py(LOCAL_AGGREGATE, "--data-dir", str(tmp_path), + "--output", str(tmp_path / "registry.toml")) + assert out.returncode == 0, out.stderr + reg = tomllib.loads((tmp_path / "registry.toml").read_text()) # parses cleanly + runs = reg.get("runs", []) + assert len(runs) == 1 + assert runs[0]["game"] == "24_miracle" + assert runs[0]["agent"] == "ifelse" + assert runs[0]["path"].startswith("runs/24_miracle/") # forward slashes + + +# ---- 15 + 16: render + three-way win_rate consistency -------------------- # +def test_results_pipeline_renders_and_winrate_is_consistent(tmp_path): + outcomes = [ + _outcome("g1", 0, 0, steps=40), # win + _outcome("g2", 1, 1, steps=38), # win (swapped) + _outcome("g3", 0, 1, steps=50), # loss + _outcome("g4", 0, 0, steps=10, ai_error_player=1), # error (excluded) + ] + summary, run_dir = _build_run(tmp_path, outcomes) + + agg = _run_py(LOCAL_AGGREGATE, "--data-dir", str(tmp_path), + "--output", str(tmp_path / "registry.toml")) + assert agg.returncode == 0, agg.stderr + + rep = _run_py(UPSTREAM_REPORT_BUILDER, "--data-dir", str(tmp_path), + "--output", str(tmp_path / "_site")) + assert rep.returncode == 0, rep.stderr + assert (tmp_path / "_site" / "data.json").exists() + + summary_wr = summary["win_rate"] + events_wr = _recompute_win_rate_from_events(run_dir) + site = json.loads((tmp_path / "_site" / "data.json").read_text()) + web_wr = site["runs"][0]["metrics"]["win_rate"] + + assert events_wr == pytest.approx(2 / 3) # 2 wins / 3 valid (error excluded) + assert summary_wr == pytest.approx(events_wr) # summary == events + assert web_wr == pytest.approx(events_wr) # web == events + + +# ---- upstream bug regression (pins the finding, does NOT hide it) -------- # +@pytest.mark.skipif(sys.platform != "win32", + reason="upstream path-separator bug only manifests on Windows") +@pytest.mark.skipif(not UPSTREAM_AGGREGATE.exists(), reason="upstream aggregate not present") +def test_upstream_aggregate_emits_invalid_toml_on_windows(tmp_path): + """Locks in the documented upstream bug: on Windows, upstream aggregate.py + writes backslash paths into registry.toml, which tomllib rejects. The local + vendored copy (LOCAL_AGGREGATE) fixes this; this test proves the fix is + necessary and that we are not masking a contract problem in our adapter.""" + _, _ = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + out = _run_py(UPSTREAM_AGGREGATE, "--data-dir", str(tmp_path), + "--output", str(tmp_path / "registry_up.toml")) + assert out.returncode == 0 # upstream runs, but its output is invalid TOML + text = (tmp_path / "registry_up.toml").read_text() + assert "\\" in text, "expected backslash path separators on Windows" + with pytest.raises(tomllib.TOMLDecodeError): + tomllib.loads(text) diff --git a/tests/miracle/test_review1_idempotent_write.py b/tests/miracle/test_review1_idempotent_write.py new file mode 100644 index 0000000..e533d05 --- /dev/null +++ b/tests/miracle/test_review1_idempotent_write.py @@ -0,0 +1,130 @@ +"""review #1 §8: ``write_run_compatible_output`` must be idempotent. + +Repeated calls (e.g. matrix resume, re-runs in CI) must not append duplicate +events to ``events.jsonl``; each rebuild yields the same per-game record count +and the same envelope (event_id disallowed to vary across calls — we don't +re-seed; each call produces a fresh set of UUID event_ids but the gameId- +count is what determines reproducibility). For the test we verify: + + - repeated calls do not increase the number of ``game`` events; + - the run directory stays at one entry (no second run_id); + - win_rate and per-game lines count are deterministic. + +Also covers review #1 §5: framework envelope is present on every emitted +event (``event``, ``event_type``, ``schema_version``, ``event_id``, +``run_id``, ``created_at``, ``timestamp``). +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +def _make_runner(tmp_path): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + from agentbench_frame.games.miracle.matrix import mark_done, write_progress_atomic, append_event_atomic + r = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None, + run_id="FIXED_RID_for_idempotency", + ) + r.prepare_session() + r.record_manifest(opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}) + # write three sample session events + for gid, valid, norm in [("m_rank01_camp0", True, "win"), + ("m_rank01_camp1", True, "loss"), + ("m_rank02_camp0", False, "error")]: + mark_done(r.progress, gid, {"valid": valid, "normalized_result": norm, + "rank": int(gid.split("_")[1][5:]), "camp": int(gid[-1])}) + append_event_atomic(r.events_path, { + "event": "game", "game_id": gid, + "valid": valid, "normalized_result": norm, + "rank": int(gid.split("_")[1][5:]), "camp": int(gid[-1]), + "judge_exit": 0, "ai0_exit": 0, "ai1_exit": 1, + "replay_sha256": "x" * 64, + "process_cleanup": [{"role": "vendor"}, {"role": "judge"}], + }) + write_progress_atomic(r.progress_path, r.progress) + return r + + +def test_write_run_compatible_output_is_idempotent(tmp_path: Path): + r = _make_runner(tmp_path) + run_dir = r.write_run_compatible_output() + ev = Path(run_dir) / "events.jsonl" + assert ev.exists() + lines1 = [l for l in ev.read_text(encoding="utf-8").splitlines() if l.strip()] + assert len([l for l in lines1 if json.loads(l).get("event_type") == "game"]) == 3, \ + "first call should produce exactly 3 game events" + + # second call on the SAME runner+session — must NOT duplicate + run_dir2 = r.write_run_compatible_output() + assert Path(run_dir) == Path(run_dir2), "run_dir path must be reused, not duplicated" + + # only ONE run directory under data_dir/runs/24_miracle/... + data_runs = Path(run_dir).parent + sib = list(data_runs.iterdir()) + assert len([p for p in sib if p.is_dir()]) == 1, \ + f"second run_id created: {[p.name for p in sib]}" + + # events count must remain exactly 3 (no duplicate append) + lines2 = [l for l in ev.read_text(encoding="utf-8").splitlines() if l.strip()] + assert len(lines2) == 3, \ + f"events doubled after re-call: {len(lines2)} (expected 3)" + + # all events carry framework envelope after re-call + for l in lines2: + e = json.loads(l) + assert e.get("event") == "game" and e.get("event_type") == "game" + assert e.get("schema_version") == "1.0" + assert e.get("event_id", "").startswith("evt_") + assert e.get("run_id") == "FIXED_RID_for_idempotency" + assert "created_at" in e and "timestamp" in e + + # a THIRD re-call must still be exactly 3 (idempotent at N invocations) + r.write_run_compatible_output() + lines3 = [l for l in ev.read_text(encoding="utf-8").splitlines() if l.strip()] + assert len(lines3) == 3, f"3rd call doubled: {len(lines3)}" + + +def test_envelope_present_on_every_game_event(tmp_path: Path): + r = _make_runner(tmp_path) + run_dir = r.write_run_compatible_output() + events = [json.loads(l) for l in (run_dir / "events.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()] + assert events + for e in events: + for k in ("event", "event_type", "schema_version", "event_id", "run_id", "created_at", "timestamp"): + assert k in e, f"event missing envelope key {k}: {sorted(e)}" + # Miracle first-hand fields preserved verbatim + for k in ("game_id", "judge_exit", "ai0_exit", "ai1_exit", "replay_sha256", "process_cleanup"): + assert k in e, f"first-hand Miracle field {k} missing after envelope: {sorted(e)}" + + +def test_summary_json_win_rate_matches_aggregate(tmp_path: Path): + r = _make_runner(tmp_path) + run_dir = r.write_run_compatible_output() + summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + # 2 valid games / 1 win / 1 loss → win_rate 0.5 (not 0.0 from framework default) + assert summary["win_rate"] == 0.5 + assert summary["total_episodes"] == 2 + assert summary["wins"] == 1 + assert summary["losses"] == 1 + # matrix aggregate preserved + assert summary["matrix_aggregate"]["valid_games"] == 2 + assert summary["matrix_aggregate"]["wins"] == 1 + + +def test_run_toml_round_trips_after_review1_lifecycle(tmp_path: Path): + import tomllib + r = _make_runner(tmp_path) + run_dir = r.write_run_compatible_output() + meta = tomllib.loads((run_dir / "run.toml").read_text(encoding="utf-8")) + assert meta["run"]["run_id"] == "FIXED_RID_for_idempotency" + assert meta["run"]["game"] == "24_miracle" + assert meta["run"]["type"] == "eval" + assert meta["config"]["matrix"] == "plan_a_32" \ No newline at end of file diff --git a/tests/miracle/test_review1_promotion_safety.py b/tests/miracle/test_review1_promotion_safety.py new file mode 100644 index 0000000..f18152d --- /dev/null +++ b/tests/miracle/test_review1_promotion_safety.py @@ -0,0 +1,303 @@ +"""review #1 收口第 2 项：不可破坏重建测试。 + +确保 ``MatrixRunner.write_run_compatible_output`` 不会因中途异常丢弃既有 +完整 run。如果重建在第 N 条事件或 summary 阶段失败，原 run 三文件（events ++ summary + run.toml）必须逐字节不变，且没有任何「半截 events」或 +events/summary 不一致状态被暴露给后续读者。 + +策略： +- 在同文件系统临时目录生成完整候选 run； +- 对候选 run 执行事件质量、32 unique game_id、summary 重算、run_id 一致性检查； +- 全部通过后原子或带回滚地 promotion；promotion 失败必须能恢复旧版本； +- 正常连续调用仍保持一个 run_id、无重复事件。 + +注入点（覆盖 3 种失败窗口）： +- 写入候选 events.jsonl 第 7 条后异常； +- 写候选 summary.json 阶段异常； +- promotion 阶段（os.replace）失败 → 回滚。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +def _runner_with_three_games(tmp_path: Path): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + from agentbench_frame.games.miracle.matrix import ( + append_event_atomic, mark_done, write_progress_atomic, + ) + r = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None, + run_id="PROMO_SAFE_RID", + ) + r.prepare_session() + r.record_manifest(opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}) + for gid, valid, norm, rank, camp in [("m_rank01_camp0", True, "win", 1, 0), + ("m_rank01_camp1", True, "loss", 1, 1), + ("m_rank02_camp0", False, "error", 2, 0)]: + mark_done(r.progress, gid, {"valid": valid, "normalized_result": norm, + "rank": rank, "camp": camp, + "steps": 100 if valid else 0}) + append_event_atomic(r.events_path, { + "event": "game", "game_id": gid, + "valid": valid, "normalized_result": norm, + "rank": rank, "camp": camp, + "steps": 100 if valid else 0, + "judge_exit": 0, "ai0_exit": 0, "ai1_exit": 1, + "replay_sha256": "x" * 64, + "process_cleanup": [{"role": "vendor"}, {"role": "judge"}], + }) + write_progress_atomic(r.progress_path, r.progress) + return r + + +def _sha(path: Path) -> str: + if not path.exists(): + return "" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _snapshot(run_dir: Path) -> dict: + return {f: _sha(run_dir / f) for f in ("events.jsonl", "summary.json", "run.toml")} + + +# --------------------------------------------------------------------------- # +# Tests +# --------------------------------------------------------------------------- # +def test_normal_consecutive_calls_keep_single_run_id_and_no_duplicates(tmp_path): + """正常连续调用：保持一个 run_id、events 不重复。""" + r = _runner_with_three_games(tmp_path) + rd1 = r.write_run_compatible_output() + rd2 = r.write_run_compatible_output() + assert Path(rd1) == Path(rd2) + # only ONE run directory + sib = list(Path(rd1).parent.iterdir()) + assert len([p for p in sib if p.is_dir()]) == 1 + # events not duplicated + lines = [l for l in (Path(rd1) / "events.jsonl").read_text(encoding="utf-8").splitlines() + if l.strip()] + assert len(lines) == 3 + + +def test_failure_during_event_emit_keeps_existing_run_byte_for_byte(tmp_path): + """已存在完整 run 第 1 次成功；第 2 次重建写入 candidate 时在第 2 条事件 + 抛异常——原 run 三文件必须逐字节不变。 + + 注：第 2 次调用会写入 3 个事件（plus internal write 用于 events、budget 等）。 + 第 1 次调用成功后 ``state['n']`` 已经历 N 次写入；第 2 次调用进入 build + 阶段时，我们从那里写入事件开始计时。 + """ + from agentbench_frame.games.miracle import matrix_runner as mr_mod + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + before = _snapshot(Path(rd)) + from agentbench_frame.tracking import run as run_mod + orig_write = run_mod.Run.write + + # we want to fire on the SECOND call's 2nd game event; the bomb state + # counts only writes AFTER we install it. + state = {"n": 0, "see_re": re.compile(r"^game$")} + + def bomb(self, event_type=None, **kw): + state["n"] += 1 + # only raise on the 2nd *game* event during this rebuild to target + # a mid-stream failure window that doesn't deadlock the writer. + if event_type == "game": + state.setdefault("game_n", 0) + state["game_n"] += 1 + if state["game_n"] == 2: + raise RuntimeError("injected during event emit") + return orig_write(self, event_type, **kw) + run_mod.Run.write = bomb + try: + with pytest.raises(RuntimeError): + r.write_run_compatible_output() + finally: + run_mod.Run.write = orig_write + after = _snapshot(Path(rd)) + assert before == after, f"existing run mutated by aborted rebuild: {before} vs {after}" + + +def test_failure_during_summary_keeps_existing_run_byte_for_byte(tmp_path): + """已存在完整 run；第 2 次重建在 run.finish 阶段失败——原 run 不变。""" + from agentbench_frame.tracking import run as run_mod + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + before = _snapshot(Path(rd)) + orig_finish = run_mod.Run.finish + + def bomb(self, extra_summary=None): + raise RuntimeError("injected during finish() summary build") + run_mod.Run.finish = bomb + try: + with pytest.raises(RuntimeError): + r.write_run_compatible_output() + finally: + run_mod.Run.finish = orig_finish + after = _snapshot(Path(rd)) + assert before == after + + +def test_failure_during_promotion_keeps_existing_run_byte_for_byte(tmp_path): + """Directory-level promotion failure (candidate→live rename) — must rollback, + old live run byte-for-byte restored.""" + import os + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + before = _snapshot(Path(rd)) + # New directory-level _atomic_promote uses os.rename (not os.replace). + # It does: rename(live→backup), rename(candidate→live). + # Bomb only when the SOURCE is a staging/candidate dir (so the rollback + # rename(backup→live) is NOT affected). + from agentbench_frame.games.miracle import matrix_runner as mr_mod + orig_rename = os.rename + + def bomb(src, dst): + if ".staging" in str(src) or "staging" in str(src): + raise OSError("injected promotion failure") + return orig_rename(src, dst) + + orig = mr_mod.os.rename + mr_mod.os.rename = bomb + try: + with pytest.raises(OSError, match="injected promotion"): + r.write_run_compatible_output() + finally: + mr_mod.os.rename = orig + after = _snapshot(Path(rd)) + assert before == after + + +def test_initial_build_when_no_existing_run(tmp_path): + """当目标 run 目录不存在时，正常写成功；无回滚需求。""" + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + assert Path(rd).exists() + assert (Path(rd) / "events.jsonl").exists() + assert (Path(rd) / "summary.json").exists() + assert (Path(rd) / "run.toml").exists() + + +def test_candidate_run_dir_is_temporary_not_under_live(tmp_path): + """候选 run 不应暴露在最终 `runs///` 下。一次成功调用 + 结束后，没有 leftover candidate/staging sibling.""" + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + sib = [p.name for p in Path(rd).parent.iterdir() if p.is_dir()] + # exactly one: the live run_id directory + assert "PROMO_SAFE_RID" in sib + # nothing ending in .tmp / candidate / staging + assert all(not (n.endswith(".tmp") or ".candidate" in n or ".staging" in n) + for n in sib), f"leftover staging dirs: {sib}" + + +def test_candidate_validation_runs_event_quality_32_unique(tmp_path, monkeypatch): + """强制最终 live events 包含 exactly3 events（此处用 3-cases 测试）；并且 + event_id 全局唯一（framework 的 inspect_event_file 自动断言）。""" + from agentbench_frame.tracking.quality import inspect_event_file + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + rep = inspect_event_file(Path(rd) / "events.jsonl") + assert rep.total_lines == 3 + assert rep.duplicate_event_ids == 0 + assert rep.missing_event_ids == 0 + assert rep.unknown_event_types == 0 + + +def test_summary_recompute_matches_events_independent(tmp_path): + """events↔summary 独立重算一致（保持 3 valid / 1 win / 1 loss / win_rate 0.5）。""" + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + evs = [json.loads(l) for l in (Path(rd) / "events.jsonl").read_text(encoding="utf-8").splitlines() + if l.strip()] + valid = [e for e in evs if e.get("valid")] + wins = [e for e in valid if e.get("normalized_result") == "win"] + losses = [e for e in valid if e.get("normalized_result") == "loss"] + expected_wr = len(wins) / len(valid) + s = json.loads((Path(rd) / "summary.json").read_text(encoding="utf-8")) + assert abs(s["win_rate"] - expected_wr) < 1e-9 + assert s["total_episodes"] == len(valid) + assert s["wins"] == len(wins) + assert s["losses"] == len(losses) + + +def test_run_toml_totals_match_events(tmp_path): + """run.toml 的 total_steps/total_episodes 应等于 events 有效游戏重算。""" + import tomllib + r = _runner_with_three_games(tmp_path) + rd = r.write_run_compatible_output() + evs = [json.loads(l) for l in (Path(rd) / "events.jsonl").read_text(encoding="utf-8").splitlines() + if l.strip()] + valid = [e for e in evs if e.get("valid")] + expected_steps = sum(int(e.get("steps", 0)) for e in valid) + meta = tomllib.loads((Path(rd) / "run.toml").read_text(encoding="utf-8")) + assert meta["run"]["total_episodes"] == len(valid) + assert meta["run"]["total_steps"] == expected_steps + + +def test_real_child_kill_after_live_rename_recovers_original_bytes(tmp_path): + """A terminated writer must restore all three old live files byte-for-byte. + + This is intentionally a real subprocess and ``os._exit``, not an + ``os.rename`` mock. The child exits in the window after live->backup and + before candidate->live; the recovery entry must prefer the old backup. + """ + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + + live = tmp_path / "live" + candidate = tmp_path / "candidate" + live.mkdir() + candidate.mkdir() + original = { + "events.jsonl": b'{"old": "events"}\n', + "summary.json": b'{"old": "summary"}\n', + "run.toml": b'[run]\nold = "toml"\n', + } + replacement = { + "events.jsonl": b'{"new": "events"}\n', + "summary.json": b'{"new": "summary"}\n', + "run.toml": b'[run]\nnew = "toml"\n', + } + for name, data in original.items(): + (live / name).write_bytes(data) + for name, data in replacement.items(): + (candidate / name).write_bytes(data) + + child = ( + "from pathlib import Path; import sys; " + "from agentbench_frame.games.miracle.matrix_runner import MatrixRunner; " + "MatrixRunner._atomic_promote(Path(sys.argv[1]), Path(sys.argv[2]))" + ) + env = os.environ.copy() + repo_src = str(Path(__file__).resolve().parents[2] / "src") + env["PYTHONPATH"] = repo_src + os.pathsep + env.get("PYTHONPATH", "") + env["MIRACLE_TEST_KILL_AFTER_LIVE_RENAME"] = "1" + result = subprocess.run( + [sys.executable, "-c", child, str(candidate), str(live)], + env=env, capture_output=True, text=True, + ) + assert result.returncode == 86, result.stderr + backup = live.with_name(live.name + ".backup_promote") + marker = live.with_name(live.name + ".promote_marker.json") + assert not live.exists() and backup.exists() and marker.exists() + + MatrixRunner._recover_promotion_transaction(live, backup, marker) + assert all((live / name).read_bytes() == data for name, data in original.items()) + assert not backup.exists() and not marker.exists() diff --git a/tests/miracle/test_runner.py b/tests/miracle/test_runner.py new file mode 100644 index 0000000..ec1c59f --- /dev/null +++ b/tests/miracle/test_runner.py @@ -0,0 +1,129 @@ +"""MiracleEvalRunner tests (阶段4b-6). Uses an injected fake attempt_fn so the +runner logic is exercised without running real matches (real-match behaviour is +covered by match_runner's own tests + smoke). + +Verifies: + * run-level statistics land in the persisted summary (attempted/valid/invalid/ + wins/losses/draws/win_rate_denominator/attempted_steps/total_steps/evaluation_status); + * win_rate == wins/valid_games and matches an independent recompute from events; + * valid_games == 0 -> win_rate is null, evaluation_status NO_VALID_GAMES; + * side-swapped wins both count (raw_winner differs across games); + * no runs/runs double-dir (risk #6); disk summary (not just in-memory) is correct; + * Run is driven directly (no BaseRunner). +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from agentbench_frame.games.miracle.match_runner import MatchAttempt +from agentbench_frame.games.miracle.runner import MiracleEvalRunner + +SRC = Path(__file__).resolve().parents[2] / "src" + + +def _mk_att(*, game_id, evaluated_agent_camp, evaluated_agent="ifelse", + opponent="rank04", normalized="error", error_type="ai_crash", + raw_winner=None, scores=None, steps=30, + realized=None, reason="synthetic") -> MatchAttempt: + valid = normalized in ("win", "loss", "draw") + if raw_winner is None and valid: + raw_winner = evaluated_agent_camp if normalized == "win" else (1 - evaluated_agent_camp) + winner = None + if valid and raw_winner in (0, 1): + winner = evaluated_agent if raw_winner == evaluated_agent_camp else opponent + return MatchAttempt( + game_id=game_id, evaluated_agent=evaluated_agent, opponent=opponent, + evaluated_agent_camp=evaluated_agent_camp, + valid=valid, normalized_result=normalized, + error_type=(None if valid else error_type), reason=reason, + raw_winner=raw_winner, winner_agent=winner, scores=scores, steps=steps, + realized_randomization=realized, + result_json_status=("ok" if error_type not in ("result_json_missing", "result_json_corrupt") else "missing"), + discrepancies=[], ai_crash_player=(0 if error_type == "ai_crash" else None), + ai_timeout_player=(0 if error_type == "ai_timeout" else None), + judge_crash=(error_type == "judge_crash"), + wrapper_timeout=(error_type == "wrapper_timeout"), + normal_cleanup_nonzero=False, + evidence_paths={"stdout": "", "stderr": "", "trace": "", "replay": "", "result_json": ""}, + collision_detected=False, process_cleanup=[], vendor_returncode=0, + started_at=0.0, finished_at=1.0, duration_s=1.0, exception=None, + ) + + +def _runner_with_plan(tmp_path: Path, plan: List[Dict[str, Any]], *, n_games=None): + counter = {"i": 0} + n = n_games if n_games is not None else len(plan) + + def fake(*, game_id, evaluated_agent_camp, evaluated_agent, opponent, **_): + i = counter["i"] + counter["i"] += 1 + p = plan[i % len(plan)] + return _mk_att( + game_id=game_id, evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent=evaluated_agent, opponent=opponent, **p, + ) + + return MiracleEvalRunner( + agent="ifelse", data_dir=str(tmp_path), judge_dir=tmp_path / "judge", + vendor_script=tmp_path / "vendor.py", framework_src=str(SRC), + evaluated_dir=tmp_path / "eval", opponent_dir=tmp_path / "opp", n_games=n, + opponent="rank04", work_dir=tmp_path / "work", attempt_fn=fake, + config={"source": "test"}, + ) + + +def test_mixed_run_statistics_and_consistency(tmp_path): + plan = [ + {"normalized": "win", "scores": {"0": 5, "1": 2}, "steps": 40, "realized": {"map_type": 0, "day_time": 1}}, + {"normalized": "win", "scores": {"0": 2, "1": 5}, "steps": 38, "realized": {"map_type": 1, "day_time": 0}}, + {"normalized": "loss", "scores": {"0": 2, "1": 5}, "steps": 50, "realized": {"map_type": 0, "day_time": 0}}, + {"normalized": "error", "error_type": "ai_crash", "reason": "ai crash", "steps": 10}, + ] + runner = _runner_with_plan(tmp_path, plan) + summary = runner.run() + # persisted on disk + run_dir = tmp_path / "runs" / "24_miracle" / "ifelse" / runner_run_id(runner, summary) + disk = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + for k in ("attempted_games", "valid_games", "invalid_games", "wins", "losses", + "draws", "win_rate_denominator", "attempted_steps", "total_steps", + "evaluation_status", "win_rate_available", "h2h"): + assert k in disk, f"summary missing {k}" + assert disk["attempted_games"] == 4 + assert disk["valid_games"] == 3 and disk["invalid_games"] == 1 + assert (disk["wins"], disk["losses"], disk["draws"]) == (2, 1, 0) + assert disk["win_rate_denominator"] == 3 + assert disk["win_rate"] == pytest.approx(2 / 3) + assert disk["attempted_steps"] == 138 and disk["total_steps"] == 128 + assert disk["evaluation_status"] == "COMPLETE" and disk["win_rate_available"] is True + assert disk["total_episodes"] == 3 # == valid_games + assert disk["win_rate"] == pytest.approx(summary["_recompute_check"]["win_rate"]) + # no double-runs + assert not (tmp_path / "runs" / "runs").exists() + + +def test_no_valid_games_win_rate_null(tmp_path): + plan = [{"normalized": "error", "error_type": "ai_crash", "reason": "x", "steps": 5}] + runner = _runner_with_plan(tmp_path, plan, n_games=2) + summary = runner.run() + assert summary["valid_games"] == 0 + assert summary["win_rate"] is None + assert summary["evaluation_status"] == "NO_VALID_GAMES" + assert summary["win_rate_available"] is False + assert summary["_recompute_check"]["win_rate"] is None + + +def test_side_swap_both_wins_count(tmp_path): + plan = [{"normalized": "win", "scores": {"0": 5, "1": 2}, "steps": 40, + "realized": {"map_type": 0, "day_time": 0}}] + runner = _runner_with_plan(tmp_path, plan, n_games=2) # camp0 then camp1 + summary = runner.run() + assert summary["wins"] == 2 + assert summary["win_rate"] == pytest.approx(1.0) + + +def runner_run_id(runner, summary) -> str: + return summary["run_id"] diff --git a/tests/miracle/test_section2_strictness.py b/tests/miracle/test_section2_strictness.py new file mode 100644 index 0000000..f0ec3b3 --- /dev/null +++ b/tests/miracle/test_section2_strictness.py @@ -0,0 +1,620 @@ +"""Section 2 余下严格性缺口针对性红灯测试（24_miracle resume 验证链）。 + +覆盖上一位 Agent 完成的 verify_session_for_resume 与 resume() 之外的剩余缺口： + + 1. progress 中未知 game_id 必须明确拒绝 + 2. progress 中非法 state（非 not_started/running/done）必须明确拒绝 + 3. opponent / build / code_hashes 字典必须精确键集合匹配（拒绝额外键） + 4. manifest 缺 session_id 必须拒绝 + 5. plan_count 必须与真实 plan 长度 AND 32 严格一致 + 6. complete-rank audit 必须解析、rank 一致、ok=True、两 game_id/camp 与 progress 一致 + 7. partial rank（仅一 camp done）提前出现声称完整的 audit 文件必须拒绝 + 8. CLI 级 fake 恢复必须经过 tools/miracle_matrix._run_resume 的 verify 链 + 9. resume() 后不应再走宽松 load_progress（必须严格解析同一文件） + 10. events.jsonl 非 JSON 对象行（int/str/list 等顶层非 dict）必须保守拒绝 + +只读：所有测试均使用 tmp_path 临时夹具，不触碰权威 session；不启动 Judge/AI； +无 git 操作。 +""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.matrix_runner import verify_session_for_resume + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +_PLAN32 = [{"rank": r, "camp": c, "game_id": f"m_rank{r:02d}_camp{c}"} + for r in range(1, 17) for c in (0, 1)] +_KNOWN_GIDS = {a["game_id"] for a in _PLAN32} + + +def _wm(tmp_path, *, progress=None, **over): + """Write a full manifest with all identity fields. session_id 默认等于 dirname.""" + m = { + "protocol_sha256": "a" * 64, + "code_hashes": {"matrix": "abc", "matrix_runner": "def", + "match_runner": "ghi", "vendor_run_match": "jkl"}, + "plan_count": 32, "plan": _PLAN32, "run_id": "RID", + "timeout": 8.0, "wrapper_timeout_s": 180.0, + "ifelse_sha256": "if_sha", "judge_sha256": "j_sha", + "opponent_archive_sha256": {f"rank{r:02d}": f"opp_{r}" for r in range(1, 17)}, + "cpp_build_sha256": {f"rank{r:02d}": f"bld_{r}" for r in range(1, 17)}, + "session_id": tmp_path.name, "python_version": "3.13.5", "platform": "test_plat", + } + m.update(over) + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + pr = progress if progress is not None else {"attempts": {}} + (tmp_path / "progress.json").write_text(json.dumps(pr, ensure_ascii=False), encoding="utf-8") + return m + + +def _done_entry(*, valid=True, norm="win", rank=1, camp=0): + return {"state": "done", "valid": valid, "normalized_result": norm, + "rank": rank, "camp": camp} + + +# =========================================================================== # +# 1. progress 未知 game_id 拒绝 +# =========================================================================== # +def test_progress_unknown_game_id_rejected(tmp_path): + proc = {"attempts": {"m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank99_camp5": _done_entry(rank=99, camp=5)}} + _wm(tmp_path, progress=proc) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("unknown" in e.lower() or "m_rank99" in e for e in errs), errs + + +# =========================================================================== # +# 2. progress 非法 state 拒绝 +# =========================================================================== # +def test_progress_illegal_state_rejected(tmp_path): + proc = {"attempts": {"m_rank01_camp0": {"state": "tilted", "rank": 1, "camp": 0}}} + _wm(tmp_path, progress=proc) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("state" in e.lower() and ("illegal" in e.lower() or "invalid" in e.lower()) + for e in errs), errs + + +def test_progress_not_started_explicit_state_accepted(tmp_path): + """state=not_started 显式出现也应被接受（合法，无事件对应）。""" + proc = {"attempts": {"m_rank01_camp0": {"state": "not_started", "rank": 1, "camp": 0}}} + _wm(tmp_path, progress=proc) + # without events / running it should be all clean — but we don't pass expected_* IDs + # so the unknown check must NOT flag a known id with legal not_started state. + ok, errs = verify_session_for_resume(tmp_path) + assert not any("m_rank01_camp0" in e and "unknown" in e.lower() for e in errs), errs + + +# =========================================================================== # +# 3. opponent/build/code_hashes 精确键集合匹配（拒绝额外键） +# =========================================================================== # +def test_manifest_opponent_extra_key_rejected(tmp_path): + m = _wm(tmp_path) + m["opponent_archive_sha256"]["rank99"] = "should_not_be_here" + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errs = verify_session_for_resume( + tmp_path, + expected_opponent_shas={r: f"opp_{r}" for r in range(1, 17)}, + ) + assert not ok + assert any("opponent" in e.lower() and ("extra" in e.lower() or "unexpected" in e.lower()) + for e in errs), errs + + +def test_manifest_build_extra_key_rejected(tmp_path): + m = _wm(tmp_path) + m["cpp_build_sha256"]["rank99"] = "should_not_be_here" + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errs = verify_session_for_resume( + tmp_path, + expected_build_shas={r: f"bld_{r}" for r in range(1, 17)}, + ) + assert not ok + assert any("build" in e.lower() and ("extra" in e.lower() or "unexpected" in e.lower()) + for e in errs), errs + + +def test_manifest_code_hash_extra_key_rejected(tmp_path): + m = _wm(tmp_path) + m["code_hashes"]["rogue_module"] = "should_not_be_here" + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errs = verify_session_for_resume( + tmp_path, + code_files={"matrix": str(tmp_path / "manifest.json"), # any existing file + "matrix_runner": str(tmp_path / "manifest.json"), + "match_runner": str(tmp_path / "manifest.json")}, + ) + assert not ok + assert any("code" in e.lower() and ("extra" in e.lower() or "unexpected" in e.lower()) + for e in errs), errs + + +def test_resume_rejects_control_input_hash_change(tmp_path): + m = _wm(tmp_path) + m["control_inputs"] = { + "protocol": {"path": "/p.json", "sha256": "old"}, + "roster": {"path": "/r.json", "sha256": "same"}, + } + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errors = verify_session_for_resume( + tmp_path, + expected_control_inputs={"protocol": "new", "roster": "same"}, + ) + assert not ok + assert any("control input hash mismatch" in error for error in errors), errors + + +def test_manifest_opponent_missing_key_rejected(tmp_path): + m = _wm(tmp_path) + del m["opponent_archive_sha256"]["rank16"] + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errs = verify_session_for_resume( + tmp_path, + expected_opponent_shas={r: f"opp_{r}" for r in range(1, 17)}, + ) + assert not ok + assert any("opponent" in e.lower() for e in errs), errs + + +# =========================================================================== # +# 4. manifest 缺 session_id 必须拒绝 +# =========================================================================== # +def test_manifest_missing_session_id_rejected(tmp_path): + m = _wm(tmp_path) + del m["session_id"] + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("session_id" in e.lower() and ("missing" in e.lower() or "absent" in e.lower()) + for e in errs), errs + + +# =========================================================================== # +# 5. plan_count 与 plan 长度 AND 32 严格一致 +# =========================================================================== # +def test_manifest_plan_count_mismatch_len_rejected(tmp_path): + """plan_count 写 5 但 plan 还是 32 项 -> 拒绝。""" + _wm(tmp_path, plan_count=5) + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("plan_count" in e.lower() for e in errs), errs + + +def test_manifest_plan_count_field_missing_rejected(tmp_path): + m = _wm(tmp_path) + del m["plan_count"] + (tmp_path / "manifest.json").write_text(json.dumps(m, ensure_ascii=False), encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("plan_count" in e.lower() for e in errs), errs + + +def test_manifest_plan_count_correct_passes(tmp_path): + """plan_count=32 且 plan 长 32 时不报错（正反向安抚）。""" + _wm(tmp_path, plan_count=32) + ok, errs = verify_session_for_resume(tmp_path) + assert not any("plan_count" in e.lower() for e in errs), errs + + +# =========================================================================== # +# 6. complete-rank audit 必须解析、rank 一致、ok=True、game_id/camp 与 progress 一致 +# =========================================================================== # +def _mk_audit_dir(tmp_path, rank): + d = tmp_path / "audit" + d.mkdir(exist_ok=True) + return d / f"rank{rank:02d}.json" + + +def test_complete_rank_audit_corrupt_rejected(tmp_path): + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank01_camp1": _done_entry(rank=1, camp=1, norm="loss"), + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text("NOT JSON {{{", encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("rank01" in e and ("audit" in e.lower() and "corrupt" in e.lower()) + for e in errs), errs + + +def test_complete_rank_audit_rank_wrong_rejected(tmp_path): + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank01_camp1": _done_entry(rank=1, camp=1, norm="loss"), + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text(json.dumps({"rank": 99, "ok": True, "games": []}), encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("rank01" in e and "audit" in e.lower() for e in errs), errs + + +def test_complete_rank_audit_not_ok_rejected(tmp_path): + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank01_camp1": _done_entry(rank=1, camp=1, norm="loss"), + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text(json.dumps({"rank": 1, "ok": False, "reasons": ["x"], + "games": [{"game_id": "m_rank01_camp0", "camp": 0}, + {"game_id": "m_rank01_camp1", "camp": 1}]}), + encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("rank01" in e and "audit" in e.lower() and "ok" in e.lower() for e in errs), errs + + +def test_complete_rank_audit_game_ids_mismatch_rejected(tmp_path): + """audit JSON 声称的 games 与 progress 的 done 状态 game_id 不一致 -> 拒绝。""" + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank01_camp1": _done_entry(rank=1, camp=1, norm="loss"), + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text(json.dumps({"rank": 1, "ok": True, + "games": [{"game_id": "m_rank01_camp0", "camp": 0}, + {"game_id": "m_rank01_camp0", "camp": 0}]}), + encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("rank01" in e and "audit" in e.lower() + and ("game_id" in e.lower() or "mismatch" in e.lower()) for e in errs), errs + + +def test_complete_rank_audit_camps_mismatch_rejected(tmp_path): + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank01_camp1": _done_entry(rank=1, camp=1, norm="loss"), + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text(json.dumps({"rank": 1, "ok": True, + "games": [{"game_id": "m_rank01_camp0", "camp": 1}, + {"game_id": "m_rank01_camp1", "camp": 0}]}), + encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("rank01" in e and "audit" in e.lower() for e in errs), errs + + +def test_complete_rank_audit_valid_passes(tmp_path): + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + "m_rank01_camp1": _done_entry(rank=1, camp=1, norm="loss"), + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text(json.dumps({"rank": 1, "ok": True, "reasons": [], + "games": [{"game_id": "m_rank01_camp0", "camp": 0}, + {"game_id": "m_rank01_camp1", "camp": 1}]}), + encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + # the only errs possibly left must NOT be about audit + assert not any("rank01" in e and "audit" in e.lower() for e in errs), errs + + +# =========================================================================== # +# 7. partial rank 提前出现声称完整的 audit 文件必须拒绝 +# =========================================================================== # +def test_partial_rank_premature_complete_audit_rejected(tmp_path): + """只有 camp0 done（part），却已存在一份标准 audit —— 必须拒绝。""" + proc = {"attempts": { + "m_rank01_camp0": _done_entry(rank=1, camp=0), + # camp1 not_started (absent from attempts) + }} + _wm(tmp_path, progress=proc) + af = _mk_audit_dir(tmp_path, 1) + af.write_text(json.dumps({"rank": 1, "ok": True, "reasons": [], + "games": [{"game_id": "m_rank01_camp0", "camp": 0}, + {"game_id": "m_rank01_camp1", "camp": 1}]}), + encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("rank01" in e.lower() and + ("premature" in e.lower() or "partial" in e.lower() or "pre" in e.lower()) + for e in errs), errs + + +# =========================================================================== # +# 8. CLI 级 fake 恢复必须经过 tools/miracle_matrix._run_resume 的 verify 链 +# =========================================================================== # +def _import_cli_module(monkeypatch): + """Import tools/miracle_matrix as a module. The module top-level calls + paths.judge_dir() / ifelse_dir() which require AGENTBENCH_ROOT and + MIRACLE_IFELSE_DIR env vars to be set (they only BUILD paths, never read + them, so dummy values are safe — no Judge/AI is touched).""" + repo = Path(__file__).resolve().parents[2] + tools_dir = str(repo / "tools") + if tools_dir not in sys.path: + sys.path.insert(0, tools_dir) + # tools/miracle_matrix.py only ever uses these to construct Path objects; + # it never reads the file system at import time. Dummy values are safe and + # do NOT touch the authoritative Judge/AI matrix (see SKILL.md boundary). + monkeypatch.setenv("AGENTBENCH_ROOT", str(repo)) + monkeypatch.setenv("MIRACLE_IFELSE_DIR", str(repo)) + # fresh import: drop any cached broken partial module so env stubs take effect + sys.modules.pop("miracle_matrix", None) + import importlib + return importlib.import_module("miracle_matrix") + + +def test_cli_run_resume_invokes_verify_and_proceeds(tmp_path, monkeypatch): + """verify 通过 -> resume() 被调用 + matrix.full.log 被写；execute() 不跑真实比赛。""" + mm = _import_cli_module(monkeypatch) + sid = "fake_sid_via_cli" + sd = tmp_path / sid + sd.mkdir(parents=True) + (sd / "manifest.json").write_text( + json.dumps({"run_id": "RID", "plan_count": 32, "plan": _PLAN32, + "timeout": 8.0, "wrapper_timeout_s": 180.0, + "session_id": sid}, ensure_ascii=False), encoding="utf-8") + (sd / "progress.json").write_text('{"attempts":{}}', encoding="utf-8") + monkeypatch.setattr(mm, "SESSION_ROOT", tmp_path) + + verified = [] + def fake_verify(sdp, **kw): + verified.append(str(sdp)) + return (True, []) + monkeypatch.setattr(mm, "verify_session_for_resume", fake_verify) + + resumed = {"count": 0} + executed = {"count": 0} + + class FakeRunner: + def __init__(self): + self.session_dir = sd + self.run_id = "RID" + self.session_id = sid + + def resume(self, s): + resumed["count"] += 1 + self.session_id = s + + def execute(self): + executed["count"] += 1 + return {"completed": True, "halted": False} + + def write_run_compatible_output(self): + return sd + + def aggregate_from_events(self): + return {"total_attempts": 0, "valid_games": 0, + "invalid_games": 0, "win_rate": None} + + roots = {name: tmp_path / name for name in ( + "judge_dir", "ifelse_dir", "extracted_root", "archives_root", + "precheck_root", "rank16_build_root", + )} + monkeypatch.setattr(mm, "verify_hashes", lambda *_a, **_k: []) + rc = mm._run_resume(FakeRunner(), sid, **roots) + assert rc == 0 + assert len(verified) == 1, "verify_session_for_resume must be called exactly once" + assert resumed["count"] == 1, "r.resume() must be called after verify OK" + assert executed["count"] == 1 + # matrix.full.log appended (resume path proceeds past verify) + log_path = sd / "matrix.full.log" + assert log_path.exists(), "matrix.full.log not written after resume" + + +def test_cli_run_resume_no_resume_when_verify_fails(tmp_path, monkeypatch): + """verify 失败 -> 不调用 r.resume()/execute()/write，返回 2。""" + mm = _import_cli_module(monkeypatch) + sid = "fake_sid_fail" + sd = tmp_path / sid + sd.mkdir(parents=True) + (sd / "manifest.json").write_text( + json.dumps({"run_id": "RID", "session_id": sid}, ensure_ascii=False), encoding="utf-8") + (sd / "progress.json").write_text('{"attempts":{}}', encoding="utf-8") + monkeypatch.setattr(mm, "SESSION_ROOT", tmp_path) + monkeypatch.setattr(mm, "verify_session_for_resume", + lambda *a, **k: (False, ["synthetic mismatch"])) + + resumed = {"count": 0} + executed = {"count": 0} + + class FakeRunner: + def __init__(self): + self.session_dir = sd + self.run_id = "RID" + self.session_id = sid + + def resume(self, s): + resumed["count"] += 1 + + def execute(self): + executed["count"] += 1 + return {"completed": True} + + def write_run_compatible_output(self): + return sd + + def aggregate_from_events(self): + return {} + + roots = {name: tmp_path / name for name in ( + "judge_dir", "ifelse_dir", "extracted_root", "archives_root", + "precheck_root", "rank16_build_root", + )} + rc = mm._run_resume(FakeRunner(), sid, **roots) + assert rc == 2, "verify-fail must return 2" + assert resumed["count"] == 0, "must NOT resume when verify fails" + assert executed["count"] == 0 + assert not (sd / "matrix.full.log").exists(), "log must NOT be opened when verify fails" + + +# =========================================================================== # +# 9. resume() 后不再走宽松 load_progress（同一文件严格解析一致） +# =========================================================================== # +def test_resume_strict_rejects_corrupt_progress(tmp_path): + """验证通过后，manifest 没变、progress 立即被人为破坏成 NOT JSON —— 此时 + resume() 必须严格解析并拒绝，而不是宽松 load_progress 把它当成空 progress + 默默通过。""" + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + + r = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None, + run_id="RID", + ) + r.prepare_session() + sid = r.session_id + # write a valid manifest (so a fresh verify_session_for_resume passes by itself) + r.record_manifest(opponent_hashes={i: "h" for i in range(1, 17)}, + build_hashes={i: "b" for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}) + # corrupt progress (NOT JSON) + (r.progress_path).write_text("NOT JSON {{{", encoding="utf-8") + # resume() MUST raise on the strict parse — never silently return empty + with pytest.raises((ValueError, json.JSONDecodeError, RuntimeError)): + r2 = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None, + run_id="RID2", + ) + r2.resume(sid) + + +def test_resume_strict_rejects_non_dict_progress(tmp_path): + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + r = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None, run_id="RID", + ) + r.prepare_session() + sid = r.session_id + r.record_manifest(opponent_hashes={i: "h" for i in range(1, 17)}, + build_hashes={i: "b" for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}) + # progress is a JSON array, not an object + (r.progress_path).write_text("[1, 2, 3]", encoding="utf-8") + with pytest.raises((ValueError, TypeError, RuntimeError)): + r2 = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=lambda **k: None, run_id="RID2", + ) + r2.resume(sid) + + +# =========================================================================== # +# 10. events.jsonl 非 JSON 对象行（int / str / 数组顶层）必须保守拒绝 +# =========================================================================== # +def test_events_non_json_object_line_rejected(tmp_path): + """events.jsonl 含一行 `42`（合法 JSON 但顶层不是对象）—— 必须报错，不能崩。""" + proc = {"attempts": {}} + _wm(tmp_path, progress=proc) + (tmp_path / "events.jsonl").write_text("42\n", encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("events" in e.lower() and + ("corrupt" in e.lower() or "object" in e.lower() or "invalid" in e.lower()) + for e in errs), errs + + +def test_events_string_top_level_rejected(tmp_path): + proc = {"attempts": {}} + _wm(tmp_path, progress=proc) + (tmp_path / "events.jsonl").write_text('"hello"\n', encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("events" in e.lower() for e in errs), errs + + +def test_events_array_top_level_rejected(tmp_path): + proc = {"attempts": {}} + _wm(tmp_path, progress=proc) + (tmp_path / "events.jsonl").write_text('[1, 2, 3]\n', encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + assert not ok + assert any("events" in e.lower() for e in errs), errs + + +def test_events_no_keys_object_with_no_game_id_is_acceptable(tmp_path): + """一个合法 JSON 对象但没有 game_id —— 不是 corrupt，不应崩，但因为无 game_id + 所以不影响 done/not_started 比对。只是确保 isinstance(e, dict) 保护生效，不报 corrupt。""" + proc = {"attempts": {}} + _wm(tmp_path, progress=proc) + (tmp_path / "events.jsonl").write_text('{"event":"meta","note":"x"}\n', encoding="utf-8") + ok, errs = verify_session_for_resume(tmp_path) + # 不应出现 events corrupt 错误 + assert not any("events" in e.lower() and "corrupt" in e.lower() for e in errs), errs + + +# =========================================================================== # +# 额外: done 局 attempt_fn 调用次数严格为 0；不生成第二个 run 目录 +# =========================================================================== # +def test_resume_done_games_call_attempt_fn_zero_times(tmp_path): + """session 中 rank01 两 camp done；resume 后 execute() 必须不调用 attempt_fn。""" + from agentbench_frame.games.miracle.matrix_runner import MatrixRunner + from agentbench_frame.games.miracle import matrix as mx + + calls = [] + def fake_fn(**kw): + calls.append(kw["game_id"]) + return None # should not be reached + + r = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=fake_fn, run_id="ORIGINAL_RID", + ) + r.prepare_session() + r.record_manifest(opponent_hashes={i: "h" + str(i) for i in range(1, 17)}, + build_hashes={i: "b" + str(i) for i in range(1, 17)}, + ifelse_sha="IF", judge_sha="JD", code_hashes={}) + mx.mark_done(r.progress, "m_rank01_camp0", _done_entry(rank=1, camp=0)) + mx.mark_done(r.progress, "m_rank01_camp1", _done_entry(rank=1, camp=1, norm="loss")) + mx.write_progress_atomic(r.progress_path, r.progress) + # events for the done games (audit requires 1 event each? no - audit checks + # events equivalence; let's write minimal events) + mx.append_event_atomic(r.events_path, + {"event": "game", "game_id": "m_rank01_camp0", + "valid": True, "normalized_result": "win"}) + mx.append_event_atomic(r.events_path, + {"event": "game", "game_id": "m_rank01_camp1", + "valid": True, "normalized_result": "loss"}) + # rank audit should exist for the done pair (so we don't trigger + # "complete rank missing audit" — we write a VALID audit) + audit_dir = r.session_dir / "audit" + audit_dir.mkdir(parents=True, exist_ok=True) + (audit_dir / "rank01.json").write_text(json.dumps({ + "rank": 1, "ok": True, "reasons": [], + "games": [{"game_id": "m_rank01_camp0", "camp": 0}, + {"game_id": "m_rank01_camp1", "camp": 1}], + }, ensure_ascii=False), encoding="utf-8") + + # new runner with DIFFERENT run_id -- resume must keep the original run_id + r2 = MatrixRunner( + session_root=tmp_path, judge_dir=tmp_path / "j", ifelse_dir=tmp_path / "i", + opponent_dir_of=lambda rk: tmp_path / f"o{rk}", vendor_script=tmp_path / "v.py", + framework_src=tmp_path / "src", attempt_fn=fake_fn, run_id="WRONG", + ) + r2.resume(r.session_id) + assert r2.run_id == "ORIGINAL_RID" + # only execute rank 1 (both camps done) — should not call attempt_fn + res = r2.execute_rank(rank=1) + assert res.get("halted") is False + assert calls == [], f"attempt_fn called {len(calls)} times for done games: {calls}" + # run_dir baked from ORIGINAL_RID — no second run dir created + assert "WRONG" not in str(r2.run_dir) + # session inventory unchanged (only ONE session dir under tmp_path) + sessions = [p for p in tmp_path.iterdir() if p.is_dir()] + assert len(sessions) == 1, f"resume created extra session dirs: {sessions}" diff --git a/tests/miracle/test_section4_full_chain.py b/tests/miracle/test_section4_full_chain.py new file mode 100644 index 0000000..4a126bd --- /dev/null +++ b/tests/miracle/test_section4_full_chain.py @@ -0,0 +1,683 @@ +"""Section 4 端到端贯通测试（MatchAttempt → GameOutcome → to_event_record → JSON）。 + +约束（与事前提上的契约一致）： + +1. 不只构造最终 GameOutcome，必须经过包含完整 fake result-json 的 + ``_build_attempt_from_files`` 入口逐值验证。 +2. 验证以下字段在「result-json→MatchAttempt→GameOutcome→to_event_record→JSON 落盘重读」 + 全链逐字一致： + * vendor exception 原文（``vendor_exception``，逐字 result-json 的 ``exception``） + * wrapper exception 原文（``wrapper_exception``，外层 wrapper Popen/包装异常） + * 有 reason 但无真实 exception 时 ``exception`` MUST 为 null（不得用 reason 填） + * ``timeout``（result-json 原始 ``timeout`` 形态）与 ``timeout_s``（实际传入的逐步超时） + * Judge/ai0/ai1 不同 final_returncode + * ``process_cleanup`` 包含四个 role，且来源明确 + （vendor = ProcessTreeManager 外层；judge/ai0/ai1 = result-json 内层） + * cleanup failure 走分类 ``cleanup_failure`` + * 内部 ``run_match_returncode`` 与外部 ``vendor_returncode`` 各自落事件且不混淆 + * Replay SHA-256 + * ``evidence_paths`` 全套保留 +""" +from __future__ import annotations + +import json +import os +import struct +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +SRC = REPO / "src" + + +# --------------------------------------------------------------------------- # +# fake result-json shape (mirrors vendor/miracle_local/run_match.py output) +# --------------------------------------------------------------------------- # +def _inner_status(*, role, pid, final_returncode, **over): + """Per-role status block the vendor writes into result-json.""" + s = { + "role": role, "pid": pid, "started_at": 1000.0, + "natural_exit": False, "natural_returncode": None, + "termination_requested": False, "termination_reason": None, + "final_returncode": final_returncode, + "forced_kill": False, "cleanup_succeeded": True, "identity_confirmed": True, + } + s.update(over) + return s + + +def _fake_result_json(tmp_path, tag, *, vendor_exception=None, + cleanup_all_succeeded=True, run_match_returncode=0, + end_info_received=True, end_info=None, + raw_winner=0, timeout_flag=None, ai_error=None, + ai0_final_returncode=0, ai1_final_returncode=0, + judge_final_returncode=0, scores=None, + score_tie=False): + if timeout_flag is None: + timeout_flag = {"ai0": False, "ai1": False} + if ai_error is None: + ai_error = {"ai0": False, "ai1": False} + if end_info is None: + end_info = {"0": 5, "1": 2} + if scores is None: + scores = {"0": 5, "1": 2} + return { + "schema_version": 1, "tag": str(tag), + "started_at": 1000.0, "finished_at": 1005.0, "duration_s": 5.0, + "judge_dir_resolved": str(tmp_path / "fake_judge"), + "p0": {"name": "ifelse", "dir": str(tmp_path / "p0")}, + "p1": {"name": "rank01", "dir": str(tmp_path / "p1")}, + "judge": _inner_status(role="judge", pid=111, final_returncode=judge_final_returncode, + natural_exit=True, natural_returncode=judge_final_returncode), + "ai0": _inner_status(role="ai0", pid=112, final_returncode=ai0_final_returncode), + "ai1": _inner_status(role="ai1", pid=113, final_returncode=ai1_final_returncode), + "end_info_received": end_info_received, + "end_info": end_info, + "scores": scores, + "raw_winner": raw_winner, + "score_tie": score_tie, + "judge_tiebreak_applied": score_tie, + "timeout": timeout_flag, + "ai_error": ai_error, + "trace_path": str(tmp_path / "work" / f"{tag}.jsonl"), + "replay_path": str(tmp_path / "work" / f"{tag}.replay"), + "cleanup_all_succeeded": cleanup_all_succeeded, + "exception": vendor_exception, + "run_match_returncode": run_match_returncode, + } + + +def _vendor_manager_status(*, vendor_final_rc=-1, cleanup_succeeded=True, + forced_kill=False, termination_requested=True, + termination_reason="wrapper-timeout"): + return [{ + "role": "vendor", "pid": 999, "started_at": 1000.0, + "natural_exit": False, "natural_returncode": None, + "termination_requested": termination_requested, + "termination_reason": termination_reason, + "final_returncode": vendor_final_rc, + "forced_kill": forced_kill, + "cleanup_succeeded": cleanup_succeeded, + "identity_confirmed": True, + }] + + + + + +# =========================================================================== # +# 1. Main full-chain propagation test (vendor exception + wrapper exception + +# timeout/timeout_s + per-role exits + 4-role cleanup + internal/external rc +# + replay SHA + evidence_paths + reason without exception) +# =========================================================================== # +def test_full_chain_field_propagation_via_fake_result_json(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + from agentbench_frame.games.miracle.atomicio import atomic_write_json + from agentbench_frame.games.miracle.result import sha256_file + + tag = "m_rank01_camp0" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + + vendor_exception_text = "RuntimeError('vendor AI read EOF before end')" + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=vendor_exception_text, + cleanup_all_succeeded=False, + run_match_returncode=7, + ai0_final_returncode=1, ai1_final_returncode=0, judge_final_returncode=0, + timeout_flag={"ai0": False, "ai1": True}, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + pop = payload.pop # local alias + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + replay_bytes = struct.pack(">7i", 0, 0, 0, 1, 0, 0, 0) + b"\x00" * 16 + replay_path = work_dir / f"{tag}.replay" + replay_path.write_bytes(replay_bytes) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + wrapper_exception_text = "OSError('wrapper Popen failed: phantom')" + mgr_status = _vendor_manager_status(vendor_final_rc=-1, cleanup_succeeded=True, + forced_kill=True, + termination_reason="wrapper-timeout") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank01", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=-1, wrapper_timeout=True, + wrapper_exception=wrapper_exception_text, + vendor_manager_status=mgr_status, + timeout_s=8.0, started=999.0, finished=1006.0, + collision_detected=False, + ) + + # === MatchAttempt 级断言 === + assert att.vendor_exception == vendor_exception_text, "vendor_exception must be verbatim from result-json" + assert att.wrapper_exception == wrapper_exception_text, "wrapper_exception must be verbatim outer Popen exc" + assert att.timeout == {"ai0": False, "ai1": True}, "timeout must be the raw result-json timeout dict" + assert att.timeout_s == 8.0, "timeout_s must be the run_match_attempt timeout kwarg" + assert att.wrapper_timeout is True + assert att.judge_exit == 0 + assert att.ai0_exit == 1 + assert att.ai1_exit == 0 + assert att.run_match_returncode == 7 + assert att.vendor_returncode == -1 + assert att.replay_sha256 == sha256_file(replay_path) + assert att.reason and isinstance(att.reason, str) + # exception compat rule: wrapper_exception 拥有最高优先级，无 reason 填充 + assert att.exception == wrapper_exception_text, \ + "exception compat field must equal wrapper_exception (NOT reason)" + # evidence_paths 5 项保留 + assert set(att.evidence_paths.keys()) == {"stdout", "stderr", "trace", "replay", "result_json"} + + # process_cleanup 必须含四个 role + source 标记 + roles = sorted(r["role"] for r in att.process_cleanup) + assert roles == ["ai0", "ai1", "judge", "vendor"], \ + f"process_cleanup must contain 4 roles, got {roles}" + sources = {(r["role"], r["source"]) for r in att.process_cleanup} + assert ("vendor", "process_tree_manager") in sources, \ + "vendor must be tagged source=process_tree_manager" + assert ("judge", "result_json") in sources + assert ("ai0", "result_json") in sources + assert ("ai1", "result_json") in sources + + # === GameOutcome 级断言 === + o = attempt_to_outcome(att) + assert o.vendor_exception == vendor_exception_text + assert o.wrapper_exception == wrapper_exception_text + assert o.timeout == {"ai0": False, "ai1": True} + assert o.timeout_s == 8.0 + assert o.run_match_returncode == 7 + assert o.judge_exit == 0 and o.ai0_exit == 1 and o.ai1_exit == 0 + assert o.vendor_returncode == -1 + assert o.replay_sha256 == att.replay_sha256 + assert o.exception == wrapper_exception_text + assert o.reason and o.reason == att.reason + + # === to_event_record + JSON 落盘重读 === + rec = to_event_record(o) + rec_path = tmp_path / "sample_event.jsonl" + rec_path.write_text(json.dumps(rec, default=str, ensure_ascii=False) + "\n", + encoding="utf-8") + loaded = json.loads(rec_path.read_text(encoding="utf-8").strip()) + + assert loaded["timeout"] == {"ai0": False, "ai1": True} + assert loaded["timeout_s"] == 8.0 + assert loaded["wrapper_timeout"] is True + assert loaded["wrapper_exception"] == wrapper_exception_text + assert loaded["vendor_exception"] == vendor_exception_text + assert loaded["run_match_returncode"] == 7 + assert loaded["vendor_returncode"] == -1 + assert loaded["judge_exit"] == 0 + assert loaded["ai0_exit"] == 1 + assert loaded["ai1_exit"] == 0 + assert loaded["result_json_status"] == "ok" + assert loaded["reason"] == att.reason + # exception is null when no real exception — but here we have a real exception, + # so it must equal wrapper_exception (the highest priority real exception source) + assert loaded["exception"] == wrapper_exception_text + # internal rc NOT silently feeding reason / exception: + assert "RuntimeError" not in loaded.get("reason", "") # reason doesn't echo exception text + # replay sha same as the MatchAttempt's hash: + assert loaded["replay_sha256"] == att.replay_sha256 + # evidence_paths preserved on the event layer too: + assert loaded["evidence_paths"]["replay"].endswith(".replay") + assert loaded["evidence_paths"]["result_json"].endswith(".result.json") + + +# =========================================================================== # +# 2. reason populated but no real exception → exception MUST be null +# =========================================================================== # +def test_reason_without_real_exception_yields_null_exception(tmp_path): + """result-json MISSING case: classify produces a populated reason, but + there is no real exception (no wrapper exception, no vendor exception), + so the exception (compat field) MUST be null — never filled by reason.""" + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + + tag = "m_rank02_camp0" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + # do NOT write a result-json → load_result_json returns ("missing", None) + (work_dir / f"{tag}.jsonl").write_text("", encoding="utf-8") # empty trace + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + # no replay file + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank02", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=0, wrapper_timeout=False, + wrapper_exception=None, # no wrapper exception + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=True, + termination_requested=False, termination_reason=None, + ), + timeout_s=12.0, started=1000.0, finished=1001.0, collision_detected=False, + ) + + # classify flags result_json_missing with a populated reason + assert att.error_type == "result_json_missing" + assert att.reason, "classify must produce a reason for the missing case" + assert att.reason and att.reason is not None + # but there is NO real exception anywhere → compat exception MUST be null + assert att.exception is None, "no real exception → compat exception MUST be None" + assert att.wrapper_exception is None + assert att.vendor_exception is None + assert att.valid is False + + o = attempt_to_outcome(att) + assert o.exception is None + assert o.reason and o.reason == att.reason + + rec = to_event_record(o) + rec_path = tmp_path / "ev2.jsonl" + rec_path.write_text(json.dumps(rec, default=str, ensure_ascii=False), + encoding="utf-8") + loaded = json.loads(rec_path.read_text(encoding="utf-8")) + assert loaded["exception"] is None, "exception sentinel must be null in JSON" + # reason is still preserved verbatim — proves exception wasn't filled by reason + assert loaded["reason"] == att.reason + assert loaded["error_type"] == "result_json_missing" + + +# =========================================================================== # +# 3. vendor_exception only (no wrapper exception) — compat exception = vendor_exception +# =========================================================================== # +def test_vendor_exception_only_path(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + + tag = "m_rank03_camp1" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + from agentbench_frame.games.miracle.atomicio import atomic_write_json + vendor_exc = "ValueError('vendor side bad JSON')" + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=vendor_exc, + cleanup_all_succeeded=True, run_match_returncode=1, + ai0_final_returncode=0, ai1_final_returncode=0, judge_final_returncode=0, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 1, 1, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank03", + evaluated_agent_camp=1, + work_dir=work_dir, tag=tag, + vendor_returncode=0, wrapper_timeout=False, + wrapper_exception=None, + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=True, + termination_requested=False, termination_reason=None, + ), + timeout_s=8.0, started=1000.0, finished=1002.0, collision_detected=False, + ) + assert att.vendor_exception == vendor_exc + assert att.wrapper_exception is None + # compat exception rule: wrapper_exception (None) OR vendor_exception → vendor_exception + assert att.exception == vendor_exc, "compat exception must promote vendor_exception when no wrapper exception" + o = attempt_to_outcome(att) + rec = to_event_record(o) + p = tmp_path / "ev3.jsonl" + p.write_text(json.dumps(rec, default=str, ensure_ascii=False), encoding="utf-8") + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded["vendor_exception"] == vendor_exc + assert loaded["wrapper_exception"] is None + assert loaded["exception"] == vendor_exc + + +# =========================================================================== # +# 4. cleanup failure classification path (no wrapper_timeout, both rcs 0, no exc) +# =========================================================================== # +def test_cleanup_failure_classification_path(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + + tag = "m_rank04_camp0" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + from agentbench_frame.games.miracle.atomicio import atomic_write_json + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=None, + cleanup_all_succeeded=False, + run_match_returncode=0, + ai0_final_returncode=0, ai1_final_returncode=0, judge_final_returncode=0, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 0, 0, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank04", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=0, wrapper_timeout=False, + wrapper_exception=None, + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=False, + termination_requested=True, termination_reason="post_end_info", + ), + timeout_s=8.0, started=1000.0, finished=1005.0, collision_detected=False, + ) + assert att.error_type == "cleanup_failure", f"cleanup_failure not classified, got {att.error_type}: {att.reason}" + assert att.valid is False + o = attempt_to_outcome(att) + rec = to_event_record(o) + p = tmp_path / "ev4.jsonl" + p.write_text(json.dumps(rec, default=str, ensure_ascii=False), encoding="utf-8") + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded["error_type"] == "cleanup_failure" + assert loaded["valid"] is False + # cleanup failed role vendor still in process_cleanup, sourced from mgr + assert any(r.get("role") == "vendor" and r.get("cleanup_succeeded") is False + for r in loaded["process_cleanup"]) + + +# =========================================================================== # +# 5. internal run_match_returncode lands separately from external vendor_returncode +# =========================================================================== # +def test_internal_run_match_returncode_lands_on_event_separately(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + from agentbench_frame.games.miracle.atomicio import atomic_write_json + + tag = "m_rank05_camp0" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=None, + cleanup_all_succeeded=True, run_match_returncode=3, + ai0_final_returncode=0, ai1_final_returncode=0, judge_final_returncode=0, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 1, 1, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank05", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=0, # outer wrapper returned 0 cleanly + wrapper_timeout=False, wrapper_exception=None, + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=True, + termination_requested=False, termination_reason=None, + ), + timeout_s=8.0, started=1000.0, finished=1005.0, collision_detected=False, + ) + # classify must flag vendor_exception (internal rc != 0) and stop the matrix + assert att.error_type == "vendor_exception", att.reason + # the external rc stays 0, internal stays 3 (must not be conflated) + o = attempt_to_outcome(att) + rec = to_event_record(o) + p = tmp_path / "ev5.jsonl" + p.write_text(json.dumps(rec, default=str, ensure_ascii=False), encoding="utf-8") + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded["run_match_returncode"] == 3 + assert loaded["vendor_returncode"] == 0 # both keys present and distinct: + assert "run_match_returncode" in loaded + assert "vendor_returncode" in loaded + + +# =========================================================================== # +# 6. evidence_paths 落事件且 result_json 路径以 .result.json 结尾 +# =========================================================================== # +def test_evidence_paths_lands_on_event(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + from agentbench_frame.games.miracle.atomicio import atomic_write_json + + tag = "m_rank06_camp1" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=None, cleanup_all_succeeded=True, run_match_returncode=0, + ai0_final_returncode=0, ai1_final_returncode=0, judge_final_returncode=0, + raw_winner=1, end_info={"0": 2, "1": 5}, scores={"0": 2, "1": 5}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 2, "1": 5}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 1, 0, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank06", + evaluated_agent_camp=1, + work_dir=work_dir, tag=tag, + vendor_returncode=0, wrapper_timeout=False, wrapper_exception=None, + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=True, + termination_requested=False, termination_reason=None, + ), + timeout_s=8.0, started=1000.0, finished=1001.0, collision_detected=False, + ) + o = attempt_to_outcome(att) + rec = to_event_record(o) + assert all(k in rec["evidence_paths"] for k in + ("stdout", "stderr", "trace", "replay", "result_json")) + assert rec["evidence_paths"]["result_json"].endswith(f"{tag}.result.json") + assert rec["evidence_paths"]["replay"].endswith(f"{tag}.replay") + + +# =========================================================================== # +# 7. timeout field shape — even when both ai flags are False, raw result-json +# timeout dict must be preserved verbatim on the event +# =========================================================================== # +def test_timeout_field_raw_dict_preserved(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.runner import attempt_to_outcome + from agentbench_frame.games.miracle.result import to_event_record + from agentbench_frame.games.miracle.atomicio import atomic_write_json + + tag = "m_rank07_camp0" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + custom_timeout = {"ai0": True, "ai1": False} + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=None, cleanup_all_succeeded=True, run_match_returncode=0, + ai0_final_returncode=2, ai1_final_returncode=0, judge_final_returncode=0, + timeout_flag=custom_timeout, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "ai_timeout", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 0, 1, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank07", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=0, wrapper_timeout=False, wrapper_exception=None, + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=True, + termination_requested=False, termination_reason=None, + ), + timeout_s=11.0, started=1000.0, finished=1002.0, collision_detected=False, + ) + assert att.timeout == custom_timeout # raw verbatim from result-json + assert att.timeout_s == 11.0 + o = attempt_to_outcome(att) + rec = to_event_record(o) + p = tmp_path / "ev7.jsonl" + p.write_text(json.dumps(rec, default=str, ensure_ascii=False), encoding="utf-8") + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded["timeout"] == custom_timeout + assert loaded["timeout_s"] == 11.0 + + +# =========================================================================== # +# 8. boundary: _build_attempt_from_files must work even with vendor_manager_status=[] +# (no vendor process; only inner judge/ai0/ai1 from result-json land in process_cleanup) +# =========================================================================== # +def test_process_cleanup_handles_empty_vendor_manager_status(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.atomicio import atomic_write_json + + tag = "m_rank08_camp0" + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=None, cleanup_all_succeeded=True, run_match_returncode=0, + ai0_final_returncode=0, ai1_final_returncode=0, judge_final_returncode=0, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 0, 0, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("out", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("err", encoding="utf-8") + + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="rank08", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=0, wrapper_timeout=False, wrapper_exception=None, + vendor_manager_status=[], # empty — only result-json inner procs come in + timeout_s=8.0, started=1000.0, finished=1001.0, collision_detected=False, + ) + roles = sorted(r["role"] for r in att.process_cleanup) + assert roles == ["ai0", "ai1", "judge"] # no vendor, only inner 3 + # each comes from result-json + assert all(r["source"] == "result_json" for r in att.process_cleanup) + + +# =========================================================================== # +# 9. compat exception rule is deterministic & test-covered: +# wrapper_exception + vendor_exception both present → wrapper wins (no overwrite) +# wrapper=None, vendor=None → None +# wrapper set, vendor None → wrapper_exception +# wrapper None, vendor set → vendor_exception +# =========================================================================== # +def test_compat_exception_rule_deterministic(tmp_path): + from agentbench_frame.games.miracle.match_runner import _build_attempt_from_files + from agentbench_frame.games.miracle.atomicio import atomic_write_json + + def _build_for(*, wrapper_exception, vendor_exception): + tag = f"test_{abs(hash((wrapper_exception, vendor_exception))) % 100000}" + # use unique tag per call within the same tmp_path dir + work_dir = tmp_path / f"wd_{tag}" + work_dir.mkdir(parents=True, exist_ok=True) + payload = _fake_result_json( + tmp_path, tag, + vendor_exception=vendor_exception, + cleanup_all_succeeded=(vendor_exception is None), + run_match_returncode=0 if vendor_exception is None else 1, + ai0_final_returncode=0, ai1_final_returncode=0, judge_final_returncode=0, + timeout_flag={"ai0": False, "ai1": False}, + raw_winner=0, end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, + ) + atomic_write_json(work_dir / f"{tag}.result.json", payload) + ei = json.dumps({"0": 5, "1": 2}) + (work_dir / f"{tag}.jsonl").write_text( + json.dumps({"kind": "ai_operation", "player": 0}) + "\n" + + json.dumps({"kind": "match_end", "end_info": ei}) + "\n", + encoding="utf-8", + ) + (work_dir / f"{tag}.replay").write_bytes( + struct.pack(">7i", 0, 0, 0, 0, 0, 0, 0) + b"\x00" * 16) + (work_dir / f"{tag}.stdout").write_text("o", encoding="utf-8") + (work_dir / f"{tag}.stderr").write_text("e", encoding="utf-8") + att = _build_attempt_from_files( + game_id=tag, evaluated_agent="ifelse", opponent="opp", + evaluated_agent_camp=0, + work_dir=work_dir, tag=tag, + vendor_returncode=(0 if wrapper_exception is None else -1), + wrapper_timeout=(wrapper_exception is not None), + wrapper_exception=wrapper_exception, + vendor_manager_status=_vendor_manager_status( + vendor_final_rc=0, cleanup_succeeded=True, + termination_requested=(wrapper_exception is not None), + termination_reason="wrapper-timeout" if wrapper_exception else None, + ), + timeout_s=8.0, started=1000.0, finished=1002.0, collision_detected=False, + ) + return att + + # case 1: wrapper + vendor → wrapper wins + att = _build_for(wrapper_exception="WrapperErr('w')", vendor_exception="VendorErr('v')") + assert att.wrapper_exception == "WrapperErr('w')" and att.vendor_exception == "VendorErr('v')" + assert att.exception == "WrapperErr('w')" + # case 2: both None → None + att = _build_for(wrapper_exception=None, vendor_exception=None) + assert att.exception is None + # case 3: wrapper alone → wrapper exception + att = _build_for(wrapper_exception="WrapperErr('w2')", vendor_exception=None) + assert att.exception == "WrapperErr('w2')" + # case 4: vendor alone → vendor_exception + att = _build_for(wrapper_exception=None, vendor_exception="VendorErr('v2')") + assert att.exception == "VendorErr('v2')" \ No newline at end of file diff --git a/tests/miracle/test_smoke_audit.py b/tests/miracle/test_smoke_audit.py new file mode 100644 index 0000000..3b89a79 --- /dev/null +++ b/tests/miracle/test_smoke_audit.py @@ -0,0 +1,182 @@ +"""Tests for the smoke evidence-safety helpers (阶段4b smoke driver fixes).""" +from __future__ import annotations + +import json +import subprocess +import sys +import time +from pathlib import Path + +import psutil +import pytest + +from agentbench_frame.games.miracle.smoke_audit import ( + ManagedProc, + build_manifest, + check_residual_procs, + ensure_fresh_session, + group1_strict_clean, + load_managed_procs_from_result_json, + make_session_id, + session_exists, + should_run_group2, + write_manifest_atomic, +) + + +# ---- 1. existing session is rejected, never deleted ---- # +def test_existing_session_rejected_and_not_deleted(tmp_path): + sid = make_session_id() + sd = ensure_fresh_session(tmp_path, sid) + (sd / "marker").write_text("prior evidence") + # a second ensure with the SAME id must refuse + with pytest.raises(FileExistsError): + ensure_fresh_session(tmp_path, sid) + # and the prior evidence must still be there + assert (sd / "marker").read_text() == "prior evidence" + + +def test_session_ids_are_unique(): + ids = {make_session_id() for _ in range(50)} + assert len(ids) == 50 + + +# ---- a fake MatchAttempt-shaped object for gate tests ---- # +class FakeAtt: + def __init__(self, *, game_id="g1_00_camp0", valid=True, normalized_result="win", + error_type=None, wrapper_timeout=False, result_json_status="ok", + discrepancies=None, realized_randomization=None, raw_winner=0, + reason="", evidence_paths=None): + self.game_id = game_id + self.valid = valid + self.normalized_result = normalized_result + self.error_type = error_type + self.wrapper_timeout = wrapper_timeout + self.result_json_status = result_json_status + self.discrepancies = discrepancies or [] + self.realized_randomization = realized_randomization or {"map_type": 0, "day_time": 1} + self.raw_winner = raw_winner + self.reason = reason + self.evidence_paths = evidence_paths or {"result_json": ""} + + +def _good_summary(): + return {"attempted_games": 2, "valid_games": 2, "invalid_games": 0, + "evaluation_status": "COMPLETE"} + + +def _two_good_attempts_with_procs(tmp_path): + # give each attempt a result-json with judge/ai0/ai1 identity for the residual check + rj = tmp_path / "rj.json" + rj.write_text(json.dumps({ + "judge": {"pid": 999999, "started_at": 1.0, "role": "judge"}, + "ai0": {"pid": 999998, "started_at": 1.0, "role": "ai0"}, + "ai1": {"pid": 999997, "started_at": 1.0, "role": "ai1"}, + })) + a0 = FakeAtt(game_id="g1_00_camp0", raw_winner=0, + evidence_paths={"result_json": str(rj)}) + a1 = FakeAtt(game_id="g1_01_camp1", raw_winner=1, + evidence_paths={"result_json": str(rj)}) + return [a0, a1] + + +# ---- 2. invalid attempt blocks Group 2 ---- # +def test_invalid_attempt_blocks_group2(tmp_path): + attempts = _two_good_attempts_with_procs(tmp_path) + attempts[0].valid = False + attempts[0].error_type = "ai_crash" + ok, reasons = group1_strict_clean(attempts, _good_summary()) + assert ok is False + assert should_run_group2(ok) is False + + +# ---- 3. attempt count < or > 2 blocks ---- # +def test_attempt_count_not_two_blocks(tmp_path): + one = _two_good_attempts_with_procs(tmp_path)[:1] + ok, reasons = group1_strict_clean(one, _good_summary()) + assert ok is False and any("attempt_count" in r for r in reasons) + three = _two_good_attempts_with_procs(tmp_path) + [FakeAtt()] + ok3, _ = group1_strict_clean(three, _good_summary()) + assert ok3 is False + + +# ---- 4. summary count mismatch blocks ---- # +def test_summary_count_mismatch_blocks(tmp_path): + attempts = _two_good_attempts_with_procs(tmp_path) + bad = {"attempted_games": 2, "valid_games": 1, "invalid_games": 1, + "evaluation_status": "COMPLETE"} + ok, reasons = group1_strict_clean(attempts, bad) + assert ok is False + assert any("valid_games" in r for r in reasons) + + +# ---- 5. PID alive + same create_time -> residual ---- # +def test_residual_detected_for_live_pid_same_createtime(): + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + ct = psutil.Process(p.pid).create_time() + res = check_residual_procs([ManagedProc(pid=p.pid, started_at=ct, role="judge")]) + assert res["residual"] and res["residual"][0].pid == p.pid + assert res["clean"] == [] and res["reused"] == [] + finally: + p.terminate(); p.wait(timeout=5) + + +# ---- 6. PID reuse (wrong create_time) -> NOT residual, NOT killed ---- # +def test_pid_reuse_not_flagged_as_residual(): + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + res = check_residual_procs([ManagedProc(pid=p.pid, started_at=1.0, role="ai0")]) + assert res["residual"] == [] # not flagged + assert res["reused"] and res["reused"][0].pid == p.pid + assert psutil.pid_exists(p.pid) # still alive — we did NOT kill it + finally: + p.terminate(); p.wait(timeout=5) + + +# ---- 7. missing process identity -> cannot pass via empty set ---- # +def test_missing_proc_identity_blocks_gate(tmp_path): + a0 = FakeAtt(game_id="g1_00_camp0", evidence_paths={"result_json": ""}) # no identity + a1 = FakeAtt(game_id="g1_01_camp1", evidence_paths={"result_json": ""}) + ok, reasons = group1_strict_clean([a0, a1], _good_summary()) + assert ok is False + assert any("no managed-proc identity" in r for r in reasons) + + +def test_load_managed_procs_handles_missing_and_identity(): + assert load_managed_procs_from_result_json("/no/such/file.json") == [] + import tempfile + f = Path(tempfile.mkdtemp()) / "r.json" + f.write_text(json.dumps({"judge": {"pid": 123, "started_at": 9.0}})) + procs = load_managed_procs_from_result_json(f) + assert len(procs) == 1 and procs[0].pid == 123 and procs[0].role == "judge" + + +# ---- 8. full manifest + log file generated ---- # +def test_manifest_written_atomically(tmp_path): + code = tmp_path / "code.py"; code.write_text("print(1)") + asset = tmp_path / "asset.bin"; asset.write_bytes(b"xyz") + m = build_manifest(session_id="SID", auth_cap=4, python_executable="py", + python_version="3.13.5", code_files=[code], asset_files=[asset], + groups_planned=[{"group": "GROUP1", "n_games": 2}], + notes=["prior 429 was a tool rejection, not a game result"]) + p = write_manifest_atomic(tmp_path, m) + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded["session_id"] == "SID" + assert loaded["auth_cap_games"] == 4 + assert loaded["code_hashes"][str(code)] + assert loaded["notes"][0].startswith("prior 429") + # atomic temp must be gone + assert not (tmp_path / "manifest.json.tmp").exists() + + +# ---- 9. Group 2 never called when Group 1 not fully passing ---- # +def test_group2_only_when_group1_clean(): + assert should_run_group2(True) is True + assert should_run_group2(False) is False + + +def test_strict_clean_passes_for_two_clean_attempts(tmp_path): + attempts = _two_good_attempts_with_procs(tmp_path) + ok, reasons = group1_strict_clean(attempts, _good_summary()) + assert ok is True, reasons diff --git a/tests/miracle/test_vendor_boundary.py b/tests/miracle/test_vendor_boundary.py new file mode 100644 index 0000000..079f385 --- /dev/null +++ b/tests/miracle/test_vendor_boundary.py @@ -0,0 +1,63 @@ +"""vendor aggregate boundary check (阶段4b-8 / spec #11). + +Proves the vendored ``vendor/results_local/aggregate.py`` differs from the +upstream AgentBenchResults ``aggregate.py`` ONLY by the Windows path-separator +patch (and the explanatory header). Specifically: after reverting the single +``as_posix()`` change, the AST of every function (find_runs / _toml_val / main) +is byte-identical to upstream — so metrics, h2h, and run-discovery logic are +unchanged. Also pins the upstream SHA256 so drift is detected. +""" +from __future__ import annotations + +import ast +import hashlib +import os +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +from agentbench_frame.games.miracle.paths import results_repo as _results_repo +UPSTREAM = (_results_repo() or Path("")) / "scripts" / "aggregate.py" +VENDOR = REPO / "vendor" / "results_local" / "aggregate.py" +UPSTREAM_SHA256 = "126796cd3626deb814fff3efef82f64844954d11a7817ae43c17e58dd3667411" + +pytestmark = pytest.mark.skipif(not UPSTREAM.exists(), reason="upstream aggregate not present") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _funcs_from_text(text: str): + tree = ast.parse(text) + return {n.name: ast.dump(n) for n in tree.body if isinstance(n, ast.FunctionDef)} + + +def test_upstream_aggregate_sha_pinned(): + assert _sha256(UPSTREAM) == UPSTREAM_SHA256 + + +def test_vendor_only_differs_by_path_separator(): + assert VENDOR.exists() + up_text = UPSTREAM.read_text(encoding="utf-8") + vn_text = VENDOR.read_text(encoding="utf-8") + # revert the one patched line at the SOURCE level, then every function's AST + # must match upstream byte-for-byte (metrics / h2h / discovery unchanged) + vn_text_norm = vn_text.replace( + '"path": run_dir.relative_to(data_dir).as_posix(),', + '"path": str(run_dir.relative_to(data_dir)),', + ) + assert vn_text_norm != vn_text, "vendor path line not found / already native" + up = _funcs_from_text(up_text) + vn = _funcs_from_text(vn_text_norm) + assert set(up) == set(vn), "function set differs" + for name, dump in up.items(): + assert dump == vn[name], f"function {name} differs beyond the path patch" + + +def test_vendor_path_uses_posix_and_upstream_uses_native(): + up_text = UPSTREAM.read_text(encoding="utf-8") + vn_text = VENDOR.read_text(encoding="utf-8") + assert '"path": str(run_dir.relative_to(data_dir))' in up_text + assert '"path": run_dir.relative_to(data_dir).as_posix()' in vn_text diff --git a/tests/test_local_report_research.py b/tests/test_local_report_research.py index 9160498..98c46be 100644 --- a/tests/test_local_report_research.py +++ b/tests/test_local_report_research.py @@ -113,7 +113,7 @@ def test_local_report_reads_first_hand_events_and_renders_research_fields(self): ig_history = builder.runs[0]["research"]["ig_history"] ig_chart = builder.runs[0]["research"]["ig_chart"] quality = builder.runs[0]["research"]["quality"] - html = (output / "index.html").read_text() + html = (output / "index.html").read_text(encoding="utf-8") self.assertAlmostEqual(ig_history[0]["trajectory_kl_episode"], 0.4) self.assertAlmostEqual(ig_history[0]["mean_local_policy_kl"], 0.2) diff --git a/tests/test_provider_integration_contracts.py b/tests/test_provider_integration_contracts.py index a1d9892..8e47b8d 100644 --- a/tests/test_provider_integration_contracts.py +++ b/tests/test_provider_integration_contracts.py @@ -102,26 +102,66 @@ def invoke(self, context): self.assertEqual(events[0]["provider_metadata"]["provider_status"], "success") def test_codex_provider_captures_process_stream_and_return_status(self): + """Codex provider invokes a real subprocess and parses its JSONL stream. + + Cross-platform real subprocess: instead of a ``#!/bin/sh`` script (which + the Windows loader rejects with ``WinError 193``), this test runs an + argv form ``[sys.executable, path_to_fake_codex.py]``. The fake codex + is a real Python script that emits the same two JSONL events. The + provider's ``executable`` parameter accepts either a single string + (legacy) or a pre-split argv list — this uses argv to exercise the + cross-platform path. + """ from agentbench_frame.tracking.providers import CodexProvider with tempfile.TemporaryDirectory() as tmp: - executable = Path(tmp) / "fake-codex" - executable.write_text( - "#!/bin/sh\n" - "printf '%s\\n' '{\"type\":\"thread.started\",\"thread_id\":\"fake\"}'\n" - "printf '%s\\n' '{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":2,\"output_tokens\":3}}'\n" + fake = Path(tmp) / "fake_codex.py" + fake.write_text( + # Emit the two codex-JSONL events every real Codex CLI prints: + # 1) thread.started + # 2) turn.completed with usage (token count) + "import json, sys; " + "print(json.dumps({'type': 'thread.started', 'thread_id': 'fake'})); " + "print(json.dumps({'type': 'turn.completed', " + "'usage': {'input_tokens': 2, 'output_tokens': 3}}))", + encoding="utf-8", ) - executable.chmod(executable.stat().st_mode | stat.S_IXUSR) raw = Path(tmp) / "raw.jsonl" - result = CodexProvider(executable=str(executable)).invoke({ + # argv pre-split guaranteed cross-platform;[sys.executable, + # script.py] runs a real process on Windows AND POSIX. + argv = [sys.executable, str(fake)] + result = CodexProvider(executable=argv, sandbox="workspace-write").invoke({ "prompt": "do it", "workspace_root": tmp, "raw_output_path": str(raw), }) self.assertEqual(result.status, "completed") self.assertEqual(result.usage.total_tokens, 5) + self.assertEqual(result.usage.prompt_tokens, 2) + self.assertEqual(result.usage.completion_tokens, 3) self.assertEqual(result.raw_output_ref, str(raw)) - self.assertIn("turn.completed", raw.read_text()) + self.assertIn("turn.completed", raw.read_text(encoding="utf-8")) + # provider should record return_code 0 (the fake script exits cleanly) + self.assertEqual(result.metadata.get("return_code"), 0) + + def test_codex_provider_accepts_legacy_string_executable(self): + """Backward compatibility: a plain string ``executable`` continues to + be split by shlex and used as the argv[0:] prefix. We exercise this + without invoking a shell: route through ``sys.executable`` so the path + round-trips even when it contains spaces (Windows Python install path). + """ + from agentbench_frame.tracking.providers import CodexProvider + + p = CodexProvider(executable="codex") + # ``executable`` may be stored as either a string or a list; both + # backward-compatible forms are accepted. The contract: build_command + # prepends whatever we passed. + cmd = p.build_command({"prompt": "hi", "workspace_root": "/nowhere"}) + # first element is the literal "codex" string, then the subcommand + # tokens the Codex CLI accepts; the prompt is appended last. + self.assertEqual(cmd[0], "codex") + self.assertEqual(cmd[1:5], ["exec", "--json", "--sandbox", "workspace-write"]) + self.assertEqual(cmd[-1], "hi") if __name__ == "__main__": diff --git a/tests/test_run_lifecycle_review1.py b/tests/test_run_lifecycle_review1.py new file mode 100644 index 0000000..f7e1d49 --- /dev/null +++ b/tests/test_run_lifecycle_review1.py @@ -0,0 +1,76 @@ +"""review #1 on framework: optional ``run_id`` in ``Run.start`` (backcompat TDD). + +This covers the matrix-resume / migration scenario where the caller already +has a fixed ``run_id`` (e.g. ``20260722-001929_52bd14``) and must NOT let +``Run.start`` fabricate a different one — otherwise a new run directory is +created and resume integrity breaks (review #1 §1 constraint). + +Sequence: + 1. Write a red-light test that asserts ``Run.start(run_id=...)`` accepts + a caller-supplied id and reuses it (rewrite events.jsonl + run.toml). + 2. Apply the minimal back-compat patch on ``Run.start``. + 3. Re-run: must be GREEN. + +Also covers idempotent re-write: ``Run.start(run_id=X)`` called twice in the +same directory must not produce a second run directory (run_dir reuses the +same path), and events must not be double-appended on a ``Run`` object reach +(although the normal lifecycle recreates the writer fresh each time). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + + +def test_run_start_accepts_caller_run_id(tmp_path: Path): + from agentbench_frame.tracking.run import Run + rid = "20260722-001929_52bd14" + run = Run.start(game="24_miracle", agent="miracle_ifelse", run_type="eval", + data_dir=str(tmp_path), run_id=rid, + config={"matrix": "plan_a_32"}) + assert run.run_id == rid + run_dir = tmp_path / "runs" / "24_miracle" / "miracle_ifelse" / rid + assert run.run_dir == str(run_dir) + assert run_dir.exists() + # write an event, finish, and confirm run_id round-trips to disk + run.log_episode(reward=1.0, steps=5, winner=0) + run.finish() + summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + assert summary["run_id"] == rid + toml = (run_dir / "run.toml").read_text(encoding="utf-8") + assert f'run_id = "{rid}"' in toml + events = [json.loads(l) for l in (run_dir / "events.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()] + assert events and all(e.get("run_id") == rid for e in events), \ + "every event must carry the same caller-supplied run_id" + + +def test_run_start_default_run_id_unchanged(tmp_path: Path): + """Backward compatibility: no run_id kwarg -> framework auto-generates.""" + from agentbench_frame.tracking.run import Run + run = Run.start(game="28_generals", agent="x", run_type="eval", data_dir=str(tmp_path)) + assert run.run_id # non-empty, framework-generated form "YYYYMMDD_HHMM_<8hex>" + # must look like an auto-generated id (timestamp prefix + 8 hex suffix) + parts = run.run_id.split("_") + assert len(parts) == 3 + assert parts[0].isdigit() and parts[1].isdigit() + assert len(parts[2]) == 8 + run.finish() + + +def test_run_start_run_id_reuses_same_dir_no_second(tmp_path: Path): + """Two Run.start calls with the SAME caller run_id must reuse the same + directory path; no second run directory is created.""" + from agentbench_frame.tracking.run import Run + rid = "20260722-001929_52bd14" + r1 = Run.start(game="24_miracle", agent="a", run_type="eval", + data_dir=str(tmp_path), run_id=rid) + r2 = Run.start(game="24_miracle", agent="a", run_type="eval", + data_dir=str(tmp_path), run_id=rid) + assert r1.run_dir == r2.run_dir + siblings = list((tmp_path / "runs" / "24_miracle" / "a").iterdir()) + assert len([p for p in siblings if p.is_dir()]) == 1 + r1.finish() + r2.finish() \ No newline at end of file diff --git a/tools/migrate_24_miracle_to_review1.py b/tools/migrate_24_miracle_to_review1.py new file mode 100644 index 0000000..3518184 --- /dev/null +++ b/tools/migrate_24_miracle_to_review1.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""24_miracle offline migration tool — re-generates CI-compatible run output +from the original 32-game session's first-hand evidence. + +No hardcoded paths: all I/O via CLI args. No subprocess/Judge/AI calls. + +Usage: + python tools/migrate_24_miracle_to_review1.py \ + --session \ + --original-run-dir \ + --output-root + +Outputs: /data/runs/24_miracle///{events.jsonl,summary.json,run.toml} + /migration_audit.json +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import struct +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.atomicio import atomic_write_json +from agentbench_frame.games.miracle.match_runner import ( + REPLAY_HEADER_BYTES, _VALID_MAP_VALUES, load_result_json, + read_replay_info, stream_trace, sha256_file, +) +from agentbench_frame.games.miracle.result import build_seed_provenance +from agentbench_frame.tracking.run import Run + +EXPECTED = dict(attempts=32, valid=20, invalid=12, wins=2, losses=18, + win_rate=0.1, total_steps=18144, total_episodes=20) + +# Original execution provenance (NOT migration provenance) +ORIGINAL_FRAMEWORK_HEAD = "4bd67fa" +MIGRATION_SCHEMA_COMMIT = "1a61320" +ORIGINAL_STARTED_AT = 1784650770.0399437 +ORIGINAL_FINISHED_AT = 1784651015.0399437 + + +def _same_or_within(child: Path, parent: Path) -> bool: + """Return whether ``child`` is ``parent`` or lives below it.""" + try: + child.resolve().relative_to(parent.resolve()) + return True + except ValueError: + return False + + +def _deterministic_event_id(game_id: str, run_id: str) -> str: + h = hashlib.sha256(f"{game_id}|{run_id}".encode("utf-8")).hexdigest() + return f"evt_{h[:16]}" + + +def _build_game_record(work_dir: Path, tag: str, run_id: str, + orig_created_iso: str) -> Dict[str, Any]: + rj_path = work_dir / f"{tag}.result.json" + trace_path = work_dir / f"{tag}.jsonl" + replay_path = work_dir / f"{tag}.replay" + stdout_path = work_dir / f"{tag}.stdout" + stderr_path = work_dir / f"{tag}.stderr" + + rj_status, rj = load_result_json(rj_path) + rj = rj or {} + ts = stream_trace(trace_path) + ri = read_replay_info(replay_path) + + rr = {"map_type": ri.map_type, "day_time": ri.day_time} if ri.header_valid else None + replay_sha = sha256_file(replay_path) if replay_path.exists() else None + judge_block = rj.get("judge") or {} + ai0_block = rj.get("ai0") or {} + ai1_block = rj.get("ai1") or {} + + vendor = {"role": "vendor", "source": "result_json_derived", + "final_returncode": rj.get("run_match_returncode"), + "cleanup_succeeded": bool(rj.get("cleanup_all_succeeded"))} + process_cleanup = [vendor] + for role_name, block in (("judge", judge_block), ("ai0", ai0_block), ("ai1", ai1_block)): + if isinstance(block, dict): + row = dict(block) + row.setdefault("role", role_name) + row["source"] = "result_json" + process_cleanup.append(row) + + rank, camp = int(tag.replace("m_rank", "").split("_camp")[0]), \ + int(tag.replace("m_rank", "").split("_camp")[1]) + + end_info = rj.get("end_info") + raw_winner = rj.get("raw_winner") + if raw_winner is None and isinstance(end_info, dict): + try: + s0, s1 = int(end_info["0"]), int(end_info["1"]) + raw_winner = 0 if s0 > s1 else 1 + except (KeyError, ValueError, TypeError): + pass + end_received = bool(rj.get("end_info_received")) + + error_type = None + reason = "" + normalized_result = "error" + valid = False + ai_err = rj.get("ai_error", {}) + if isinstance(ai_err, dict): + if ai_err.get("ai0"): error_type, reason = "ai_crash", "trace ai_error player 0" + elif ai_err.get("ai1"): error_type, reason = "ai_crash", "trace ai_error player 1" + elif rj.get("timeout", {}).get("ai0"): error_type, reason = "ai_timeout", "trace ai_timeout player 0" + elif rj.get("timeout", {}).get("ai1"): error_type, reason = "ai_timeout", "trace ai_timeout player 1" + if error_type is None and not end_received: + if raw_winner is None: error_type, reason = "judge_crash", "no legal end_info produced" + if error_type is None and raw_winner is None: + error_type, reason = "no_decisive_winner", "raw_winner is None" + if error_type is None: + valid = True + normalized_result = "win" if raw_winner == camp else "loss" + + scores = rj.get("scores") + if scores is None and isinstance(end_info, dict): + scores = {"0": end_info.get("0"), "1": end_info.get("1")} + s0 = scores.get("0") if isinstance(scores, dict) else None + s1 = scores.get("1") if isinstance(scores, dict) else None + + return { + "event": "game", "schema_version": "1.0", "run_id": run_id, + "event_id": _deterministic_event_id(tag, run_id), + "created_at": orig_created_iso, + "timestamp": rj.get("finished_at"), + "game_id": tag, "rank": rank, "camp": camp, "ifelse_camp": camp, + "valid": valid, "normalized_result": normalized_result, + "raw_winner": raw_winner, + "winner_agent": ("miracle_ifelse" if raw_winner == camp else f"rank{rank:02d}") + if raw_winner in (0, 1) else None, + "scores": {"0": s0, "1": s1} if scores is not None else None, + "steps": ts.n_ai_operation, + "realized_randomization": rr, + "seed": build_seed_provenance(rr), + "error_type": error_type, "reason": reason, + "wrapper_timeout": False, + "ai_crash_player": int("ai0" in reason) if error_type == "ai_crash" else None, + "judge_crash": bool(error_type == "judge_crash"), + "normal_cleanup_nonzero": False, + "result_json_status": rj_status, + "judge_exit": judge_block.get("final_returncode"), + "ai0_exit": ai0_block.get("final_returncode"), + "ai1_exit": ai1_block.get("final_returncode"), + "replay_sha256": replay_sha, + "process_cleanup": process_cleanup, + "timeout": rj.get("timeout"), "timeout_s": 8.0, + "wrapper_exception": None, + "vendor_exception": rj.get("exception"), + "run_match_returncode": rj.get("run_match_returncode"), + "evidence_paths": { + "stdout": f"work/{tag}.stdout" if stdout_path.exists() else "", + "stderr": f"work/{tag}.stderr" if stderr_path.exists() else "", + "trace": f"work/{tag}.jsonl", + "replay": f"work/{tag}.replay", + "result_json": f"work/{tag}.result.json", + }, + "is_resume": False, "is_rerun": False, + "started_at": rj.get("started_at"), + "finished_at": rj.get("finished_at"), + "duration": rj.get("duration_s"), + } + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--session", required=True, type=Path, + help="Path to the original matrix session directory") + ap.add_argument("--original-run-dir", required=True, type=Path, + help="Path to the original Results run directory") + ap.add_argument("--output-root", required=True, type=Path, + help="Path to the output preview root (must not exist or be empty)") + args = ap.parse_args() + + session: Path = args.session + orig_run_dir: Path = args.original_run_dir + out_root: Path = args.output_root + work = session / "work" + + # --- input validation --- + if not session.exists(): + print(f"FATAL: session not found: {session}", file=sys.stderr); return 2 + if not orig_run_dir.exists(): + print(f"FATAL: original run dir not found: {orig_run_dir}", file=sys.stderr); return 2 + if out_root.name in { + "24_miracle_results_migration_preview", + "24_miracle_results_migration_preview_v2", + }: + print("FATAL: output root must be a new preview, not a legacy preview", file=sys.stderr); return 2 + if out_root.exists() and any(out_root.iterdir()): + print(f"FATAL: output root not empty: {out_root}", file=sys.stderr); return 2 + # A child of an authority input would be just as destructive as an equal + # path. Do not guess local defaults and never write under either input. + try: + if (_same_or_within(out_root, session) + or _same_or_within(out_root, orig_run_dir) + or _same_or_within(out_root, orig_run_dir.parent)): + print("FATAL: output root must not be inside input paths", file=sys.stderr); return 2 + except OSError: + pass + + # --- load original run metadata --- + import re + def _seg(text, key): + m = re.search(rf'^\s*{key}\s*=\s*"([^"]*)"', text, re.MULTILINE) + return m.group(1) if m else None + run_toml_text = (orig_run_dir / "run.toml").read_text(encoding="utf-8") + orig_summary = json.loads((orig_run_dir / "summary.json").read_text(encoding="utf-8")) + run_id = _seg(run_toml_text, "run_id") or orig_summary.get("run_id") + agent = _seg(run_toml_text, "agent") or orig_summary.get("agent") + game = _seg(run_toml_text, "game") or orig_summary.get("game") or "24_miracle" + orig_created = _seg(run_toml_text, "created") or orig_summary.get("created") or "" + if not run_id: + print("FATAL: run_id missing", file=sys.stderr); return 2 + + out_data_root = out_root / "data" + out_run_dir = out_data_root / "runs" / game / agent / run_id + print(f"[migrate] run_id={run_id} created={orig_created!r}") + + # --- load original manifest --- + orig_manifest = json.loads((session / "manifest.json").read_text(encoding="utf-8")) + + # --- compose 32 per-game records --- + plan_ids = [f"m_rank{r:02d}_camp{c}" for r in range(1, 17) for c in (0, 1)] + records = [_build_game_record(work, gid, run_id, orig_created) for gid in plan_ids] + + # --- hard-assert anchors --- + n = len(records) + valid_recs = [r for r in records if r["valid"]] + invalid_recs = [r for r in records if not r["valid"]] + wins = [r for r in valid_recs if r["normalized_result"] == "win"] + losses = [r for r in valid_recs if r["normalized_result"] == "loss"] + total_steps = sum(int(r["steps"]) for r in valid_recs) + metrics = dict(attempts=n, valid=len(valid_recs), invalid=len(invalid_recs), + wins=len(wins), losses=len(losses), + win_rate=round(len(wins) / len(valid_recs), 4) if valid_recs else None, + total_steps=total_steps, total_episodes=len(valid_recs)) + print(f"[migrate] metrics={metrics}") + failed = [f"{k}: expected {v!r}, got {metrics.get(k)!r}" + for k, v in EXPECTED.items() if metrics.get(k) != v] + if failed: + print("FATAL: hard-assert violation", file=sys.stderr) + for f in failed: print(f" - {f}", file=sys.stderr) + return 1 + + # --- drive framework Run lifecycle with ORIGINAL execution timestamps --- + migration_started_at = time.time() + run = Run.start( + game=game, agent=agent, run_type="eval", + data_dir=str(out_data_root), run_id=run_id, append=False, + created=orig_created, git_commit="", + started_at=ORIGINAL_STARTED_AT, + config={ + "matrix": "plan_a_32", "total_attempts": 32, + "original_framework_head": ORIGINAL_FRAMEWORK_HEAD, + "migration_schema_commit": MIGRATION_SCHEMA_COMMIT, + "original_code_hashes": orig_manifest.get("code_hashes", {}), + "protocol_sha256": orig_manifest.get("protocol_sha256"), + "migration_started_at": migration_started_at, + }, + ) + for rec in records: + run.write("game", **rec) + + recomputed = run.recompute_totals_from_events(game_event_type="game") + assert recomputed["total_steps"] == EXPECTED["total_steps"] + assert recomputed["episodes"] == EXPECTED["total_episodes"] + + agg = { + "total_attempts": n, "valid_games": len(valid_recs), + "invalid_games": len(invalid_recs), "wins": len(wins), "losses": len(losses), + "win_rate": EXPECTED["win_rate"], + "steps_stats": {"n": len(valid_recs), "total": total_steps}, + "small_sample_note": "32 局为小样本；含 invalid；不把对手崩溃包装为 Agent 胜利；win_rate 分母仅 valid games。", + } + run.log_h2h({}) + + run.finish( + extra_summary={ + "win_rate": EXPECTED["win_rate"], + "win_rate_available": (len(valid_recs) > 0), + "total_episodes": recomputed["episodes"], + "total_steps": recomputed["total_steps"], + "wins": recomputed["wins"], "losses": recomputed["losses"], + "matrix_aggregate": agg, "evaluation_status": "COMPLETE", + }, + finished_at=ORIGINAL_FINISHED_AT, + ) + + # --- audit JSON (portable: NO absolute paths) --- + audit = { + "run_id": run_id, + "orig_created": orig_created, + "metrics": metrics, + "expected_metrics": EXPECTED, + "hard_asserts_passed": not failed, + "events_sha256": sha256_file(out_run_dir / "events.jsonl"), + "summary_sha256": sha256_file(out_run_dir / "summary.json"), + "run_toml_sha256": sha256_file(out_run_dir / "run.toml"), + "event_id_service": "deterministic sha256(game_id|run_id)[:16]", + "timestamp_source": "result-json finished_at (real per-game epoch)", + "created_at_source": "original run.toml 'created' field", + "git_commit": "", + "original_framework_head": ORIGINAL_FRAMEWORK_HEAD, + "migration_schema_commit": MIGRATION_SCHEMA_COMMIT, + "original_code_hashes": orig_manifest.get("code_hashes", {}), + "protocol_sha256": orig_manifest.get("protocol_sha256"), + "original_started_at": ORIGINAL_STARTED_AT, + "original_finished_at": ORIGINAL_FINISHED_AT, + "migration_started_at": migration_started_at, + "totals_source": "Run.recompute_totals_from_events", + "path_sanitization": "evidence_paths=session-relative; audit=logical refs only", + } + atomic_write_json(out_root / "migration_audit.json", audit, indent=2) + print(f"[migrate] output: {out_run_dir}") + print("[migrate] DONE - all hard-asserts passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/miracle_iteration.py b/tools/miracle_iteration.py new file mode 100644 index 0000000..f8a324a --- /dev/null +++ b/tools/miracle_iteration.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""One authorized, bounded 24_miracle v0 -> v1 iteration smoke.""" +from __future__ import annotations + +import argparse, difflib, hashlib, json, os, shutil, subprocess, sys, time +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) +from agentbench_frame.games.miracle.entry import resolve_ai_command +from agentbench_frame.games.miracle.runner import MiracleEvalRunner +from agentbench_frame.tracking import CodexProvider, Run +from agentbench_frame.tracking.controller import CodingAgentController + + +def sha(p: Path) -> str: + h = hashlib.sha256() + for f in sorted(x for x in p.rglob("*") if x.is_file() and "__pycache__" not in x.parts): + h.update(f.relative_to(p).as_posix().encode()); h.update(f.read_bytes()) + return h.hexdigest() + +def copy(src: Path, dst: Path) -> None: + shutil.copytree(src, dst) + +_NON_INFRA_INVALID = {"ai_crash", "ai_timeout"} + + +def run_pair(version: str, opponent: str, vdir: Path, odir: Path, judge: Path, vendor: Path, session: Path) -> dict: + # Keep the framework Run agent path flat; version identity belongs in the + # immutable version metadata, not in a path component. + r = MiracleEvalRunner(agent=f"miracle_ifelse_{version}", opponent=opponent, + evaluated_dir=vdir, opponent_dir=odir, n_games=2, data_dir=str(session / "data"), + judge_dir=judge, vendor_script=vendor, framework_src=REPO / "src", + work_dir=session / "work", timeout=10.0, wrapper_timeout_s=180.0, + prefix=f"{version}_{opponent}", config={"version":version,"opponent":opponent}) + default_attempt = r._default_attempt_fn + def guarded_attempt(**kwargs): + attempt = default_attempt(**kwargs) + # A game-level AI invalid is evidence and may proceed to the side swap. + # Every other non-valid classification is infrastructure and must stop + # before another game (and, in particular, before validation). + if not attempt.valid and attempt.error_type not in _NON_INFRA_INVALID: + raise RuntimeError( + f"infrastructure anomaly in {attempt.game_id}: {attempt.error_type}" + ) + return attempt + r.attempt_fn = guarded_attempt + summary = r.run() + attempts=[] + for a in r.attempts: + attempts.append({"game_id":a.game_id,"valid":a.valid,"result":a.normalized_result, + "steps":a.steps,"camp":a.evaluated_agent_camp,"error_type":a.error_type, + "evidence_paths":a.evidence_paths,"process_cleanup":a.process_cleanup}) + return {"summary":summary,"attempts":attempts} + +def totals(groups: dict) -> dict: + atts=[a for g in groups.values() for a in g["attempts"]] + valid=[a for a in atts if a["valid"]]; wins=[a for a in valid if a["result"]=="win"] + return {"attempts":len(atts),"valid":len(valid),"invalid":len(atts)-len(valid), + "wins":len(wins),"losses":sum(a["result"]=="loss" for a in valid), + "draws":sum(a["result"]=="draw" for a in valid), + "win_rate":(len(wins)/len(valid) if valid else None),"total_steps":sum(a["steps"] or 0 for a in valid)} + +def main() -> int: + ap=argparse.ArgumentParser() + ap.add_argument("--session-root",type=Path,required=True); ap.add_argument("--agentbench-root",type=Path,required=True) + ap.add_argument("--ifelse-source",type=Path,required=True) + ap.add_argument("--codex", nargs="+", required=True, + help="explicit provider argv; never fall back to PATH") + a=ap.parse_args(); stamp=datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S_%f") + session=a.session_root / stamp + if session.exists(): raise RuntimeError("refusing existing iteration session") + judge=a.agentbench_root/"backend_sources/corpus/24_miracle/logic/judge_dev_logic" + extracted=a.agentbench_root/"top_algorithms/corpus/24_miracle_final/extracted" + r04=next(extracted.glob("rank04__*"),None); r09=next(extracted.glob("rank09__*"),None) + if not ((judge/"main.py").exists() and r04 and r09 and (a.ifelse_source/"main.py").exists()): + raise RuntimeError("required authority assets missing") + session.mkdir(parents=True); (session/"work").mkdir(); (session/"data").mkdir() + versions=session/"versions"; opponents=session/"opponents"; versions.mkdir(); opponents.mkdir() + v0=versions/"v0"; copy(a.ifelse_source,v0); copy(r04,opponents/"rank04"); copy(r09,opponents/"rank09") + # rank09 must be genuinely buildable/launchable before the first game. + build=subprocess.run(["make"],cwd=opponents/"rank09",capture_output=True,text=True,timeout=180) + if build.returncode or not (opponents/"rank09"/"main.exe").exists(): + (session/"precheck_rank09.json").write_text(json.dumps({"returncode":build.returncode,"stdout":build.stdout,"stderr":build.stderr}),encoding="utf-8") + raise RuntimeError("rank09 compile precheck failed") + for name in ("rank04","rank09"): + cmd=resolve_ai_command(opponents/name); p=subprocess.Popen(cmd,cwd=opponents/name,stdin=subprocess.DEVNULL,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + try: p.communicate(timeout=2) + except subprocess.TimeoutExpired: p.kill(); p.communicate() + history=a.agentbench_root/"24_miracle_results_migration_preview_v3/data/runs/24_miracle/miracle_ifelse/20260722-001929_52bd14/events.jsonl" + hist=[json.loads(x) for x in history.read_text(encoding="utf-8").splitlines() if x.strip()] + historic={f"rank{n:02d}":[x for x in hist if x.get("rank")==n and x.get("valid")] for n in (4,9)} + if not all(len(x)>=2 for x in historic.values()): raise RuntimeError("rank04/rank09 historical-validity check failed") + population={"rank04":{"role":"train","sha256":sha(opponents/"rank04")},"rank09":{"role":"validation","sha256":sha(opponents/"rank09")}} + meta={"version_id":"miracle_ifelse/v0","parent":None,"source_sha256":sha(a.ifelse_source),"workspace_sha256":sha(v0),"entry_command":resolve_ai_command(v0),"created_at":datetime.now(timezone.utc).isoformat(),"immutable":True} + (session/"v0.json").write_text(json.dumps(meta,indent=2),encoding="utf-8"); (session/"population.json").write_text(json.dumps(population,indent=2),encoding="utf-8") + vendor=REPO/"vendor/miracle_local/run_match.py" + g0={"train":run_pair("v0","rank04",v0,opponents/"rank04",judge,vendor,session),"validation":run_pair("v0","rank09",v0,opponents/"rank09",judge,vendor,session)} + (session/"v0_evaluation.json").write_text(json.dumps({"groups":g0,"totals":totals(g0)},indent=2,default=str),encoding="utf-8") + # Only training evidence is exposed to the provider. + replay={"rules":"Score more than the opposing player; no deterministic seed.","allowed_for_strategy":True, + "episodes":[{"game_id":x["game_id"],"camp":x["camp"],"valid":x["valid"],"result":x["result"],"steps":x["steps"],"error_type":x["error_type"]} for x in g0["train"]["attempts"]], + "forbidden":"rank09 validation replay, result, summary, and all opponent source"} + candidate=session/"workspace_for_agent"; candidate.mkdir(); shutil.copy2(v0/"main.py",candidate/"main.py"); (candidate/"TRAINING_REPLAY.json").write_text(json.dumps(replay,indent=2),encoding="utf-8") + tracking=Run.start("24_miracle","miracle_ifelse",run_type="rule_iter",data_dir=str(session/"tracking"),config={"information_gain":None,"information_gain_status":"pending_authoritative_definition"}) + prompt="Read only TRAINING_REPLAY.json and main.py in this workspace. Make one minimal rule improvement in main.py only. Do not read validation data, network, or external files; do not change protocols, timeouts, framework, or opponents." + controller=CodingAgentController(CodexProvider(executable=a.codex,timeout_s=1800),recorder=tracking._act_recorder,budget=tracking._budget,budget_phase="learning",raw_output_dir=str(session/"provider_output")) + act=controller.run_act({"prompt":prompt,"workspace_root":str(candidate),"sandbox":"workspace-write"},workspace_root=str(candidate),version_before="v0") + tracking.finish(); (session/"coding_agent_act.json").write_text(json.dumps(act.__dict__,indent=2,default=str),encoding="utf-8") + if act.status != "completed": raise RuntimeError(f"provider status uncertain or failed: {act.status}") + before=(v0/"main.py").read_text(encoding="utf-8").splitlines(keepends=True); after=(candidate/"main.py").read_text(encoding="utf-8").splitlines(keepends=True) + diff="".join(difflib.unified_diff(before,after,fromfile="v0/main.py",tofile="v1/main.py")); (session/"workspace.diff").write_text(diff,encoding="utf-8") + v1=versions/"v1"; copy(v0,v1); shutil.copy2(candidate/"main.py",v1/"main.py"); diff_hash=hashlib.sha256(diff.encode()).hexdigest(); m1={"version_id":"miracle_ifelse/v1","parent":"miracle_ifelse/v0","source_sha256":sha(v1),"workspace_sha256":sha(v1),"diff_sha256":diff_hash,"provider_invocation_id":act.act_id,"created_at":datetime.now(timezone.utc).isoformat(),"immutable":True} + (session/"v1.json").write_text(json.dumps(m1,indent=2),encoding="utf-8") + g1={"train":run_pair("v1","rank04",v1,opponents/"rank04",judge,vendor,session),"validation":run_pair("v1","rank09",v1,opponents/"rank09",judge,vendor,session)} + t0,t1=totals(g0),totals(g1); report={"v0":t0,"v1":t1,"raw":t0["win_rate"],"evo":t1["win_rate"],"gain":(t1["win_rate"]-t0["win_rate"] if t0["win_rate"] is not None and t1["win_rate"] is not None else None),"small_sample_note":"Non-deterministic smoke only; not statistically significant; v1 degradation does not invalidate the loop.","agent_episode_read":2,"agent_step_read":sum(x["steps"] or 0 for x in g0["train"]["attempts"]),"information_gain":None,"information_gain_status":"pending_authoritative_definition"} + (session/"final_report.json").write_text(json.dumps(report,indent=2),encoding="utf-8"); print(session); return 0 +if __name__=="__main__": raise SystemExit(main()) diff --git a/tools/miracle_matrix.py b/tools/miracle_matrix.py new file mode 100644 index 0000000..b8576a3 --- /dev/null +++ b/tools/miracle_matrix.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python3 +"""24_miracle A-plan 32-game formal matrix runner (single process). + +Binds match_runner + matrix.py + matrix_runner + Framework Run/events/summary. +Run under ONE Python 3.11.15 ``uv run`` process (constant env). Modes: + --dry-run : create session + manifest + verify hashes + print plan; NO game/subprocess. + (default) : verify, then execute() the 32 attempts (per-rank audit, infra-stop), + then write Run-compatible output + aggregate. + +Opponent runnable dirs (frozen builds, NOT recompiled): + rank04/05/07 (Python): protected extracted dirs + rank01/02/03/06/08-15 (C++): .smoke/precheck/20260721-184652_da899d/strategies/rankNN + rank16 (C++): .smoke/rank16build/20260721-195738_baff71/rank16_copy +""" +from __future__ import annotations + +import hashlib +import argparse +import json +import platform +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.matrix_runner import ( # noqa: E402 + MatrixRunner, + verify_session_for_resume, +) +from agentbench_frame.games.miracle.matrix import make_attempt_plan # noqa: E402 + +from agentbench_frame.games.miracle.paths import judge_dir, ifelse_dir, extracted_dir, archives_dir + + +def _safe_default(factory, fallback: Path) -> Path: + try: + return factory() + except RuntimeError: + return fallback + + +JUDGE = _safe_default(judge_dir, REPO / ".external" / "judge") +IFELSE = _safe_default(ifelse_dir, REPO / ".external" / "ifelse") +EXTRACTED = _safe_default(extracted_dir, REPO / ".external" / "extracted") +ARCHIVES = _safe_default(archives_dir, REPO / ".external" / "archives") +VENDOR = REPO / "vendor" / "miracle_local" / "run_match.py" +FW_SRC = REPO / "src" +PROTOCOL = REPO / "docs" / "games" / "24_miracle_evaluation_protocol.v0.3.json" +ROSTER = REPO / "docs" / "games" / "24_miracle_roster_manifest.json" +PRECHECK_9A = REPO / ".smoke" / "precheck" / "20260721-184652_da899d" +RANK16_BUILD = REPO / ".smoke" / "rank16build" / "20260721-195738_baff71" +SESSION_ROOT = REPO / ".smoke" / "matrix" +PROTOCOL_SHA = "f64b948c3dfef1e59d0d2db9ea747ed1d742d054bafd9abe6e6b824d65723b99" +AUTH_TEXT = "用户授权 A 方案 32 局正式矩阵（v0.3）；逐对手分批审计；成功局不重跑；infra 即停；完成后聚合+本地 Results 验收；不 push/不上传。" + +PYTHON_RANKS = (4, 5, 7) +CPP_RANKS = (1, 2, 3, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16) +LOG = None + + +class PreflightError(RuntimeError): + """A stable, user-facing input validation failure.""" + + +@dataclass(frozen=True) +class ControlInputs: + protocol: dict + roster: dict + hashes: dict[str, str] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run the 24_miracle evaluation matrix") + parser.add_argument("--dry-run", action="store_true", help="plan only; do not start a game") + parser.add_argument("--resume", metavar="SESSION_ID", help="resume an existing session") + parser.add_argument("--protocol", type=Path, default=PROTOCOL) + parser.add_argument("--roster", type=Path, default=ROSTER) + parser.add_argument("--protocol-sha", default=None) + parser.add_argument("--session-root", type=Path, default=SESSION_ROOT) + parser.add_argument("--judge-dir", type=Path, default=JUDGE) + parser.add_argument("--ifelse-dir", type=Path, default=IFELSE) + parser.add_argument("--extracted-root", type=Path, default=EXTRACTED) + parser.add_argument("--archives-root", type=Path, default=ARCHIVES) + parser.add_argument("--precheck-root", type=Path, default=PRECHECK_9A) + parser.add_argument("--rank16-build-root", type=Path, default=RANK16_BUILD) + return parser.parse_args(argv) + + +def log(msg=""): + print(msg, flush=True) + if LOG: + LOG.write(str(msg) + "\n"); LOG.flush() + + +def sha(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + + +def control_text_sha(path: Path) -> str: + """Hash UTF-8 control text after normalizing physical newlines to LF.""" + text = Path(path).read_text(encoding="utf-8") + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def _load_json_object(path: Path, label: str) -> dict: + if not path.exists(): + raise PreflightError(f"{label} file missing: {path}") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + raise PreflightError(f"{label} JSON invalid: {path}: {exc}") from exc + if not isinstance(value, dict): + raise PreflightError(f"{label} JSON must be an object: {path}") + return value + + +def _require_control_mapping(value, label: str) -> dict: + if not isinstance(value, dict): + raise PreflightError(f"control schema: {label} missing or not an object") + return value + + +def _require_control_sha(value, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(char not in "0123456789abcdef" for char in value) + ): + raise PreflightError(f"control schema: {label} missing or not a 64-character lowercase hexadecimal SHA256") + return value + + +def _validate_control_schema(protocol: dict, strategies: list[dict]) -> None: + """Reject incomplete control inputs before a runner or session exists.""" + identities = _require_control_mapping(protocol.get("frozen_identities"), "frozen_identities") + evaluated_agent = _require_control_mapping( + identities.get("evaluated_agent"), "frozen_identities.evaluated_agent" + ) + _require_control_sha( + evaluated_agent.get("sha256"), "frozen_identities.evaluated_agent.sha256" + ) + judge = _require_control_mapping(identities.get("judge"), "frozen_identities.judge") + _require_control_sha(judge.get("main_py_sha256"), "frozen_identities.judge.main_py_sha256") + builds = _require_control_mapping( + identities.get("build_artifacts_win64_mingw"), + "frozen_identities.build_artifacts_win64_mingw", + ) + for rank in CPP_RANKS: + _require_control_sha( + builds.get(f"rank{rank:02d}"), + f"frozen_identities.build_artifacts_win64_mingw.rank{rank:02d}", + ) + for strategy in strategies: + rank = strategy["rank"] + label = f"roster.strategies.rank{rank:02d}" + _require_control_sha(strategy.get("archive_sha256"), f"{label}.archive_sha256") + if rank in PYTHON_RANKS: + entry = strategy.get("entry") + if not isinstance(entry, str) or not entry: + raise PreflightError(f"control schema: {label}.entry missing or not a string") + _require_control_sha(strategy.get("runnable_sha256"), f"{label}.runnable_sha256") + + +def load_control_inputs( + protocol_path: Path, + roster_path: Path, + expected_protocol_sha: str | None = None, +) -> ControlInputs: + protocol_path = Path(protocol_path) + roster_path = Path(roster_path) + protocol = _load_json_object(protocol_path, "protocol") + roster = _load_json_object(roster_path, "roster") + strategies = roster.get("strategies") + if not isinstance(strategies, list): + raise PreflightError("roster strategies must be a list") + if len(strategies) != 16 or any(not isinstance(item, dict) for item in strategies): + raise PreflightError("roster ranks must be exactly 1..16: invalid strategy entry") + ranks = [item.get("rank") for item in strategies if isinstance(item, dict)] + if ranks != list(range(1, 17)): + raise PreflightError(f"roster ranks must be exactly 1..16: {ranks}") + _validate_control_schema(protocol, strategies) + protocol_hash = control_text_sha(protocol_path) + if expected_protocol_sha and protocol_hash != expected_protocol_sha: + raise PreflightError( + f"protocol sha mismatch: got {protocol_hash}, expected {expected_protocol_sha}" + ) + return ControlInputs( + protocol=protocol, + roster=roster, + hashes={"protocol": protocol_hash, "roster": control_text_sha(roster_path)}, + ) + + +def resolve_unique_dir(root: Path, pattern: str) -> Path: + root = Path(root) + matches = sorted(path for path in root.glob(pattern) if path.is_dir()) + if not matches: + raise FileNotFoundError(f"no opponent directory matching {pattern}: {root}") + if len(matches) > 1: + names = ", ".join(str(path) for path in matches) + raise RuntimeError(f"multiple opponent directories matching {pattern}: {names}") + return matches[0] + + +def resolve_unique_file(root: Path, pattern: str) -> Path: + root = Path(root) + matches = sorted(path for path in root.glob(pattern) if path.is_file()) + if not matches: + raise FileNotFoundError(f"no archive file matching {pattern}: {root}") + if len(matches) > 1: + names = ", ".join(str(path) for path in matches) + raise RuntimeError(f"multiple archive files matching {pattern}: {names}") + return matches[0] + + +def resolve_opponent_dir( + rank: int, + roster: dict | None = None, + *, + extracted_root: Path = EXTRACTED, + precheck_root: Path = PRECHECK_9A, + rank16_build_root: Path = RANK16_BUILD, +) -> Path: + if rank in PYTHON_RANKS: + return resolve_unique_dir(extracted_root, f"rank{rank:02d}__*") + if rank == 16: + return Path(rank16_build_root) / "rank16_copy" + return Path(precheck_root) / "strategies" / f"rank{rank:02d}" + + +def opponent_dir_of(rank: int) -> Path: + return resolve_opponent_dir(rank) + + +def _strategy_label(strategy: dict) -> str: + rank = strategy.get("rank", "unknown") + return f"rank{int(rank):02d}" if isinstance(rank, int) else f"rank{rank}" + + +def verify_archive_hash(strategy: dict, archives_root: Path) -> list[str]: + """Verify the unique archive for one roster entry using raw-byte hashing.""" + rank = strategy.get("rank") + label = _strategy_label(strategy) + if not isinstance(rank, int): + return [f"{label}: invalid roster rank"] + try: + archive = resolve_unique_file(Path(archives_root), f"rank{rank:02d}__*.zip") + except (FileNotFoundError, RuntimeError) as exc: + return [f"{label}: {exc}"] + expected_archive_sha = strategy.get("archive_sha256") + if not expected_archive_sha: + return [f"{label}: archive sha missing from roster"] + try: + actual_archive_sha = sha(archive) + except OSError as exc: + return [f"{label}: archive sha unreadable: {exc}"] + if actual_archive_sha != expected_archive_sha: + return [f"{label}: archive sha mismatch"] + return [] + + +def verify_python_strategy_hashes(strategy: dict, extracted_root: Path) -> list[str]: + """Verify a Python opponent's extracted runnable entry only. + + Archives are verified once for every roster entry by ``verify_archive_hash``. + """ + rank = strategy.get("rank", "unknown") + label = _strategy_label(strategy) + errors: list[str] = [] + try: + directory = resolve_unique_dir(extracted_root, f"rank{int(rank):02d}__*") + except (FileNotFoundError, RuntimeError) as exc: + return [f"{label}: {exc}"] + entry = strategy.get("entry") + if not isinstance(entry, str) or not entry or Path(entry).is_absolute() or ".." in Path(entry).parts: + return [f"{label}: invalid runnable entry"] + entry_path = directory / entry + if not entry_path.is_file(): + errors.append(f"{label}: runnable entry missing: {entry_path}") + else: + expected_entry_sha = strategy.get("runnable_sha256") + if not expected_entry_sha: + errors.append(f"{label}: runnable sha missing from roster") + else: + try: + actual_entry_sha = sha(entry_path) + except OSError as exc: + errors.append(f"{label}: runnable sha unreadable: {exc}") + else: + if actual_entry_sha != expected_entry_sha: + errors.append(f"{label}: runnable sha mismatch") + return errors + + +def verify_hashes( + v3: dict, + roster: dict, + *, + extracted_root: Path = EXTRACTED, + archives_root: Path = ARCHIVES, + precheck_root: Path = PRECHECK_9A, + rank16_build_root: Path = RANK16_BUILD, + ifelse_root: Path = IFELSE, + judge_root: Path = JUDGE, + asset_digests: dict[str, str] | None = None, +): + """Verify every opponent's runnable identity matches the frozen hashes.""" + mismatches = [] + ba = v3["frozen_identities"]["build_artifacts_win64_mingw"] + strategies = {item["rank"]: item for item in roster.get("strategies", []) if isinstance(item, dict)} + for rank in range(1, 17): + strategy = strategies.get(rank) + if strategy is None: + mismatches.append(f"rank{rank:02d}: roster strategy missing") + else: + mismatches.extend(verify_archive_hash(strategy, archives_root)) + try: + d = resolve_opponent_dir( + rank, + roster, + extracted_root=extracted_root, + precheck_root=precheck_root, + rank16_build_root=rank16_build_root, + ) + except (FileNotFoundError, RuntimeError) as exc: + mismatches.append(f"rank{rank:02d}: {exc}") + continue + if not d.exists(): + mismatches.append(f"rank{rank:02d}: opponent dir missing {d}") + continue + if rank in PYTHON_RANKS: + if strategy is None: + continue + else: + mismatches.extend(verify_python_strategy_hashes(strategy, extracted_root)) + else: + me = d / "main.exe" + if not me.is_file(): + mismatches.append(f"rank{rank:02d}: main.exe missing") + else: + want = ba[f"rank{rank:02d}"] + try: + got = sha(me) + except OSError as exc: + mismatches.append(f"rank{rank:02d}: main.exe sha unreadable: {exc}") + else: + if got != want: + mismatches.append(f"rank{rank:02d}: main.exe sha {got} != frozen {want}") + ifelse_main = Path(ifelse_root) / "main.py" + if not ifelse_main.is_file(): + mismatches.append(f"ifelse main.py missing: {ifelse_main}") + else: + try: + ifelse_sha = sha(ifelse_main) + except OSError as exc: + mismatches.append(f"ifelse sha unreadable: {exc}") + else: + if ifelse_sha != v3["frozen_identities"]["evaluated_agent"]["sha256"]: + mismatches.append("ifelse sha mismatch") + elif asset_digests is not None: + asset_digests["ifelse"] = ifelse_sha + judge_main = Path(judge_root) / "main.py" + if not judge_main.is_file(): + mismatches.append(f"judge main.py missing: {judge_main}") + else: + try: + judge_sha = sha(judge_main) + except OSError as exc: + mismatches.append(f"judge sha unreadable: {exc}") + else: + if judge_sha != v3["frozen_identities"]["judge"]["main_py_sha256"]: + mismatches.append("judge sha mismatch") + elif asset_digests is not None: + asset_digests["judge"] = judge_sha + return mismatches + + +def _run_resume( + r, + resume_sid, + *, + control_inputs: ControlInputs | None = None, + session_root: Path | None = None, + judge_dir: Path | None = None, + ifelse_dir: Path | None = None, + extracted_root: Path | None = None, + archives_root: Path | None = None, + precheck_root: Path | None = None, + rank16_build_root: Path | None = None, +): + """Resume an existing session. VERIFY FIRST (read-only); only if ALL checks + pass, call resume() + write. Verification failure leaves session untouched.""" + global LOG + session_root = Path(session_root or SESSION_ROOT) + sd = session_root / resume_sid + if not sd.exists(): + print(f"FATAL: session not found: {sd}", file=sys.stderr) + return 2 + # READ-ONLY verification — load protocol + roster for full identity + if control_inputs is None: + control_inputs = load_control_inputs(PROTOCOL, ROSTER, PROTOCOL_SHA) + v3 = control_inputs.protocol + roster = control_inputs.roster + ba = v3["frozen_identities"]["build_artifacts_win64_mingw"] + ok, errs = verify_session_for_resume( + sd, + protocol_sha=control_inputs.hashes["protocol"], + code_files={ + "matrix": str(REPO / "src/agentbench_frame/games/miracle/matrix.py"), + "matrix_runner": str(REPO / "src/agentbench_frame/games/miracle/matrix_runner.py"), + "match_runner": str(REPO / "src/agentbench_frame/games/miracle/match_runner.py"), + "vendor_run_match": str(VENDOR), + }, + expected_plan=make_attempt_plan(), + expected_ifelse_sha=v3["frozen_identities"]["evaluated_agent"]["sha256"], + expected_judge_sha=v3["frozen_identities"]["judge"]["main_py_sha256"], + expected_opponent_shas={s["rank"]: s["archive_sha256"] for s in roster["strategies"]}, + expected_build_shas={int(k.replace("rank", "")): v for k, v in ba.items() + if k.startswith("rank") and isinstance(v, str) and "_" not in k}, + expected_platform=platform.platform(), + expected_control_inputs=control_inputs.hashes, + ) + if not ok: + print("FATAL: resume verification failed:", file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + return 2 + runtime_roots = ( + ("judge_dir", judge_dir), ("ifelse_dir", ifelse_dir), + ("extracted_root", extracted_root), ("archives_root", archives_root), + ("precheck_root", precheck_root), ("rank16_build_root", rank16_build_root), + ) + missing_roots = [name for name, value in runtime_roots if value is None] + if missing_roots: + print(f"FATAL: resume runtime root missing: {', '.join(missing_roots)}", file=sys.stderr) + return 2 + mismatches = verify_hashes( + v3, roster, + extracted_root=Path(extracted_root), archives_root=Path(archives_root), + precheck_root=Path(precheck_root), rank16_build_root=Path(rank16_build_root), + ifelse_root=Path(ifelse_dir), judge_root=Path(judge_dir), + ) + if mismatches: + print("FATAL: resume current asset verification failed:", file=sys.stderr) + for mismatch in mismatches: + print(f" - {mismatch}", file=sys.stderr) + return 2 + # only now: open session + write + r.resume(resume_sid) + LOG = open(r.session_dir / "matrix.full.log", "a", encoding="utf-8") + log(f"RESUME session: {resume_sid} (verified)") + result = r.execute() + log(f"resume result: {json.dumps({k: v for k, v in result.items() if k != 'record'}, ensure_ascii=False)}") + r.write_run_compatible_output() + agg = r.aggregate_from_events() + log(f"aggregate: total={agg['total_attempts']} valid={agg['valid_games']} invalid={agg['invalid_games']} win_rate={agg['win_rate']}") + return 0 if result.get("completed") else 1 + + +def validate_runtime_paths(*, judge_root: Path, ifelse_root: Path, extracted_root: Path, + archives_root: Path, precheck_root: Path, + rank16_build_root: Path) -> None: + for label, path in ( + ("judge directory", judge_root), + ("ifelse directory", ifelse_root), + ("extracted opponent directory", extracted_root), + ("opponent archives directory", archives_root), + ("precheck directory", precheck_root), + ("rank16 build directory", rank16_build_root), + ): + if not Path(path).is_dir(): + raise PreflightError(f"{label} missing: {path}") + for label, path in (("judge main.py", Path(judge_root) / "main.py"), + ("ifelse main.py", Path(ifelse_root) / "main.py")): + if not path.is_file(): + raise PreflightError(f"{label} missing: {path}") + + +def main(argv: Sequence[str] | None = None) -> int: + global LOG + args = parse_args(argv) + if args.dry_run and args.resume: + print("FATAL: --resume and --dry-run are mutually exclusive", file=sys.stderr) + return 2 + expected_protocol_sha = args.protocol_sha + if expected_protocol_sha is None and args.protocol == PROTOCOL: + expected_protocol_sha = PROTOCOL_SHA + try: + control_inputs = load_control_inputs(args.protocol, args.roster, expected_protocol_sha) + validate_runtime_paths( + judge_root=args.judge_dir, + ifelse_root=args.ifelse_dir, + extracted_root=args.extracted_root, + archives_root=args.archives_root, + precheck_root=args.precheck_root, + rank16_build_root=args.rank16_build_root, + ) + except PreflightError as exc: + print(f"FATAL: {exc}", file=sys.stderr) + return 2 + v3 = control_inputs.protocol + roster = control_inputs.roster + + def opponent_resolver(rank: int) -> Path: + return resolve_opponent_dir( + rank, + roster, + extracted_root=args.extracted_root, + precheck_root=args.precheck_root, + rank16_build_root=args.rank16_build_root, + ) + + r = MatrixRunner( + session_root=args.session_root, judge_dir=args.judge_dir, ifelse_dir=args.ifelse_dir, + opponent_dir_of=opponent_resolver, vendor_script=VENDOR, framework_src=FW_SRC, + timeout=8.0, wrapper_timeout_s=180.0, + protocol_sha=control_inputs.hashes["protocol"], + python=sys.executable, evaluated_agent="miracle_ifelse", auth_text=AUTH_TEXT, + ) + if args.resume: + return _run_resume( + r, + args.resume, + control_inputs=control_inputs, + session_root=args.session_root, + judge_dir=args.judge_dir, + ifelse_dir=args.ifelse_dir, + extracted_root=args.extracted_root, + archives_root=args.archives_root, + precheck_root=args.precheck_root, + rank16_build_root=args.rank16_build_root, + ) + r.prepare_session() + LOG = open(r.session_dir / "matrix.full.log", "w", encoding="utf-8") + log(f"session_id: {r.session_id}") + log(f"run_id: {r.run_id}") + log(f"python: {sys.executable} ({platform.python_version()})") + log(f"protocol v0.3 sha: {control_inputs.hashes['protocol']} (verified)") + log(f"mode: {'DRY_RUN' if args.dry_run else 'EXECUTE'}") + + asset_digests: dict[str, str] = {} + mismatches = verify_hashes( + v3, + roster, + extracted_root=args.extracted_root, + archives_root=args.archives_root, + precheck_root=args.precheck_root, + rank16_build_root=args.rank16_build_root, + ifelse_root=args.ifelse_dir, + judge_root=args.judge_dir, + asset_digests=asset_digests, + ) + if mismatches: + log("FATAL: hash mismatches -> STOP before any game:") + for m in mismatches: + log(" - " + m) + return 2 + ifelse_sha = asset_digests.get("ifelse") + judge_sha = asset_digests.get("judge") + if ifelse_sha is None or judge_sha is None: + log("FATAL: verified ifelse/judge digest missing -> STOP before any game") + return 2 + opp_arch = {s["rank"]: s["archive_sha256"] for s in roster["strategies"]} + cpp_build = {int(k.replace("rank", "")): v for k, v in v3["frozen_identities"]["build_artifacts_win64_mingw"].items() + if k.startswith("rank") and isinstance(v, str) and "_" not in k} + r.record_manifest( + opponent_hashes=opp_arch, build_hashes=cpp_build, + ifelse_sha=ifelse_sha, + judge_sha=judge_sha, + code_hashes={"matrix": sha(REPO / "src/agentbench_frame/games/miracle/matrix.py"), + "matrix_runner": sha(REPO / "src/agentbench_frame/games/miracle/matrix_runner.py"), + "match_runner": sha(REPO / "src/agentbench_frame/games/miracle/match_runner.py"), + "vendor_run_match": sha(VENDOR)}, + control_inputs={ + "protocol": {"path": str(args.protocol), "sha256": control_inputs.hashes["protocol"]}, + "roster": {"path": str(args.roster), "sha256": control_inputs.hashes["roster"]}, + }, + ) + log(f"manifest: {r.session_dir / 'manifest.json'}") + log("hash verification: ALL_MATCH") + + if args.dry_run: + out = r.dry_run() + log(f"plan: {out['plan_count']} attempts; first={out['plan'][0]['game_id']} last={out['plan'][-1]['game_id']}") + log(f"timeout={r.timeout} wrapper_timeout_s={r.wrapper_timeout_s}") + log("DRY_RUN complete — no game/subprocess started.") + return 0 + + log("\n===== EXECUTE 32 attempts (single process) =====") + t0 = time.time() + result = r.execute() + log(f"execute result: {json.dumps({k: v for k, v in result.items() if k != 'record'}, ensure_ascii=False)}") + log(f"elapsed: {round(time.time() - t0, 1)}s") + run_dir = r.write_run_compatible_output() + log(f"Run-compatible output: {run_dir}") + agg = r.aggregate_from_events() + log(f"aggregate: total={agg['total_attempts']} valid={agg['valid_games']} invalid={agg['invalid_games']} " + f"wins={agg['wins']} losses={agg['losses']} win_rate={agg['win_rate']}") + (r.session_dir / "aggregate.json").write_text(json.dumps(agg, ensure_ascii=False, indent=2), encoding="utf-8") + log(f"aggregate.json: {r.session_dir / 'aggregate.json'}") + return 0 if result.get("completed") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/miracle_precheck.py b/tools/miracle_precheck.py new file mode 100644 index 0000000..8383704 --- /dev/null +++ b/tools/miracle_precheck.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""24_miracle 阶段9A 预检：隔离编译 13 个 C++ 策略 + 无对局启动预检。 + +绝不启动 Judge / server / 真实对局。对每个 C++ 策略： + 1. 从受保护 extracted 源目录复制到唯一全新 session 目录（不动原策略）。 + 2. 用策略自带 makefile 原样编译（不改逻辑/优化）。 + 3. 记录源哈希、编译命令、完整 stdout/stderr/exit、产物类型/大小/SHA256、PE/DLL 静态检查。 + 4. 用 entry.resolve_ai_command 构造命令（与正式 runner 一致），短时 Popen（stdin=DEVNULL， + 不接 Judge），按 PID+create_time 精确清理。 + 5. 分类：COMPILE_PASS / OS_SPAWN_PASS / COMMAND_RESOLUTION_PASS / PROTOCOL_NOT_VALIDATED。 + +无 Judge 启动 → 协议/比赛可用性未验证（PROTOCOL_NOT_VALIDATED），不得表述为"策略已成功完成比赛"。 +rank03 同样编译记录，但保留"运行时崩溃风险"（编译/启动成功 ≠ 比赛可用）。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.entry import resolve_ai_command # noqa: E402 +from agentbench_frame.games.miracle.proctree import ProcessTreeManager # noqa: E402 +from agentbench_frame.games.miracle.smoke_audit import ( # noqa: E402 + ensure_fresh_session, make_session_id, write_manifest_atomic, +) + +from agentbench_frame.games.miracle.paths import extracted_dir +EXTRACTED = extracted_dir() +ROSTER = REPO / "docs" / "games" / "24_miracle_roster_manifest.json" +PRECHECK_ROOT = REPO / ".smoke" / "precheck" +LOG = None + + +def log(msg=""): + print(msg, flush=True) + if LOG: + LOG.write(str(msg) + "\n"); LOG.flush() + + +def sha256_file(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + + +def is_pe(p: Path) -> bool: + try: + return p.read_bytes()[:2] == b"MZ" + except OSError: + return False + + +def dll_deps(p: Path): + """Static DLL-dependency check via objdump (no execution of the AI).""" + try: + r = subprocess.run(["objdump", "-p", str(p)], capture_output=True, text=True, timeout=15) + deps = [ln.split(":", 1)[1].strip() for ln in r.stdout.splitlines() + if ln.strip().startswith("DLL Name")] + return deps + except Exception as e: + return [f""] + + +def find_binary(d: Path): + for name in ("main.exe", "main"): + if (d / name).exists(): + return d / name + return None + + +def cpp_strategies(): + roster = json.loads(ROSTER.read_text(encoding="utf-8")) + out = [] + for s in roster["strategies"]: + if s.get("type") == "cpp_source": + rank = s["rank"] + cand = list(EXTRACTED.glob(f"rank{rank:02d}__*")) + out.append({"rank": rank, "name": s["entity"], "extracted": cand[0] if cand else None, + "archive_sha256": s["archive_sha256"], "invalid_now": s.get("invalid_now", False)}) + return out + + +def compile_one(copy_dir: Path): + """Run the strategy's own makefile. Record everything.""" + t0 = time.time() + cmd = ["make"] + try: + r = subprocess.run(cmd, cwd=str(copy_dir), capture_output=True, text=True, timeout=180) + return {"command": cmd, "stdout": r.stdout, "stderr": r.stderr, + "returncode": r.returncode, "duration_s": round(time.time() - t0, 2), + "exception": None} + except subprocess.TimeoutExpired as e: + return {"command": cmd, "stdout": e.stdout or "", "stderr": e.stderr or "", + "returncode": None, "duration_s": round(time.time() - t0, 2), + "exception": "timeout"} + except Exception as e: + return {"command": cmd, "stdout": "", "stderr": str(e), + "returncode": None, "duration_s": round(time.time() - t0, 2), + "exception": repr(e)} + + +def spawn_check(copy_dir: Path, timeout_s=2.0): + """Short isolated process-spawn check. NO Judge. stdin=DEVNULL so the AI + sees EOF and exits fast; we only verify the OS can create the process.""" + result = {"command_resolution": None, "command_resolution_pass": False, + "os_spawn_pass": None, "returncode": None, "stderr_head": None, + "cleanup_all_succeeded": None, "exception": None, "protocol_validated": False} + # 1. command resolution (same path the real runner uses) + try: + cmd = resolve_ai_command(copy_dir) + result["command_resolution"] = cmd + result["command_resolution_pass"] = True + except FileNotFoundError as e: + result["command_resolution"] = f"FileNotFoundError: {e}" + result["command_resolution_pass"] = False + return result + # 2. OS spawn (NO judge; stdin closed) + mgr = ProcessTreeManager() + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0 + try: + proc = subprocess.Popen(cmd, cwd=str(copy_dir), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + creationflags=creationflags) + mgr.register_popen(proc, f"ai_spawn") + result["os_spawn_pass"] = True + try: + proc.communicate(timeout=timeout_s) + result["returncode"] = proc.returncode + except subprocess.TimeoutExpired: + result["returncode"] = "timeout" + except (OSError, FileNotFoundError) as e: + result["os_spawn_pass"] = False + result["exception"] = repr(e) + return result + finally: + # precise PID + create_time cleanup + procs = mgr.cleanup_all("spawn-check") + result["cleanup_all_succeeded"] = all(p.cleanup_succeeded for p in procs) + result["cleanup_detail"] = mgr.status() # list[dict], JSON-serializable + # capture a short stderr head if any + return result + + +def main() -> int: + global LOG + session_id = make_session_id() + session_dir = ensure_fresh_session(PRECHECK_ROOT, session_id) + LOG = open(session_dir / "precheck.full.log", "w", encoding="utf-8") + log(f"precheck session: {session_dir}") + log(f"compiler: g++ {subprocess.run(['g++','--version'],capture_output=True,text=True).stdout.splitlines()[0]}") + log(f"make: {subprocess.run(['make','--version'],capture_output=True,text=True).stdout.splitlines()[0]}") + log("NOTE: 无 Judge / 无对局启动；仅编译 + 短时进程创建预检。PROTOCOL_NOT_VALIDATED。") + + targets = cpp_strategies() + reports = [] + for t in targets: + rank = t["rank"]; name = t["name"] + log(f"\n===== rank{rank:02d} ({name}) =====") + if t["extracted"] is None: + log(f" EXTRACTED_DIR_MISSING"); reports.append({"rank": rank, "status": "MISSING_SOURCE"}); continue + copy_dir = session_dir / "strategies" / f"rank{rank:02d}" + shutil.copytree(t["extracted"], copy_dir) + # source hashes (record, do not modify) + src_files = sorted(p for p in copy_dir.iterdir() if p.suffix in (".cpp", ".c", ".h", ".hpp", ".json") or p.name == "makefile") + source_hashes = {p.name: sha256_file(p) for p in src_files} + log(f" copied {len(src_files)} source files from {t['extracted'].name}") + # compile + comp = compile_one(copy_dir) + binary = find_binary(copy_dir) + compile_pass = (comp["returncode"] == 0 and binary is not None) + log(f" compile: returncode={comp['returncode']} binary={'main.exe' if binary and binary.name=='main.exe' else binary.name if binary else None} -> {'COMPILE_PASS' if compile_pass else 'COMPILE_FAIL'}") + if comp["stderr"]: + log(f" [stderr tail] {comp['stderr'][-300:]!r}") + bin_info = None + if binary: + bin_info = {"name": binary.name, "size": binary.stat().st_size, + "sha256": sha256_file(binary), "is_pe": is_pe(binary), + "dll_deps": dll_deps(binary)} + log(f" binary: size={bin_info['size']} pe={bin_info['is_pe']} sha256={bin_info['sha256'][:16]}… dlls={bin_info['dll_deps']}") + # spawn pre-check (only if compiled) + spawn = None + if compile_pass: + spawn = spawn_check(copy_dir) + log(f" spawn: cmd_res={spawn['command_resolution_pass']} os_spawn={spawn['os_spawn_pass']} rc={spawn['returncode']} cleanup={spawn['cleanup_all_succeeded']}") + report = { + "rank": rank, "name": name, "invalid_now": t["invalid_now"], + "source_dir_original": str(t["extracted"]), + "source_dir_copy": str(copy_dir), + "archive_sha256": t["archive_sha256"], + "source_hashes": source_hashes, + "compile": comp, "binary": bin_info, + "compile_pass": compile_pass, + "spawn_precheck": spawn, + "classification": { + "COMPILE_PASS": compile_pass, + "OS_SPAWN_PASS": bool(spawn and spawn["os_spawn_pass"]), + "COMMAND_RESOLUTION_PASS": bool(spawn and spawn["command_resolution_pass"]), + "PROTOCOL_NOT_VALIDATED": True, + }, + "verification_note": "仅静态编译 + 短时进程创建预检；未启动 Judge/对局，不证明协议或比赛可用。" + + (" rank03 历史运行时崩溃风险保留，编译/启动成功不等于比赛可用。" if rank == 3 else ""), + } + reports.append(report) + (session_dir / "reports" ).mkdir(exist_ok=True) + (session_dir / "reports" / f"rank{rank:02d}.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + + manifest = { + "session_id": session_id, "kind": "stage9A_precheck", + "created_unix": time.time(), "python": sys.executable, + "python_version": platform.python_version(), + "compiler": "g++ 15.2.0 (MinGW)", "make": "GNU Make 4.4.1", + "no_judge_no_match": True, + "strategy_count": len(reports), + "summary": { + "compile_pass": sum(1 for r in reports if r.get("compile_pass")), + "compile_fail": sum(1 for r in reports if not r.get("compile_pass") and r.get("status") != "MISSING_SOURCE"), + "missing_source": sum(1 for r in reports if r.get("status") == "MISSING_SOURCE"), + }, + } + write_manifest_atomic(session_dir, manifest) + (session_dir / "summary.json").write_text(json.dumps({"reports": reports}, ensure_ascii=False, indent=2), encoding="utf-8") + log(f"\n===== SUMMARY =====") + log(f"compile_pass={manifest['summary']['compile_pass']}/{len(reports)} compile_fail={manifest['summary']['compile_fail']} missing={manifest['summary']['missing_source']}") + for r in reports: + c = r.get("classification", {}) + log(f" rank{r['rank']:02d}: compile={c.get('COMPILE_PASS')} os_spawn={c.get('OS_SPAWN_PASS')} cmd_res={c.get('COMMAND_RESOLUTION_PASS')} protocol=NOT_VALIDATED{' [invalid_now]' if r.get('invalid_now') else ''}") + log(f"\nsession dir: {session_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/miracle_rank16_build.py b/tools/miracle_rank16_build.py new file mode 100644 index 0000000..504bef1 --- /dev/null +++ b/tools/miracle_rank16_build.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""阶段9B：rank16 隔离构建修复 + 无对局启动预检。 + +用户决定：允许只在 rank16 的隔离构建副本中预先创建空 ``build/`` 目录（构建环境 +准备，非策略修复）。不改源码 / Makefile / 编译参数 / 优化。 + +步骤： + 1. 全新唯一 session；从受保护 extracted 源复制 rank16。 + 2. 记录编译前全部源文件 + Makefile SHA256。 + 3. 仅在副本中创建 Makefile 预期的空 build/。 + 4. 用原 Makefile 编译一次。 + 5. 编译后再次核对源文件 + Makefile 哈希 == 编译前（必须一致）。 + 6. 记录新增文件清单（仅构建产物 + 空目录准备）。 + 7. 若成功：PE/大小/SHA256/依赖 + 绝对路径命令解析 + 短时无 Judge 启动 + PID+create_time 清理。 +不删除此前 rank16 失败证据。不启动 Judge / 真实对局。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.entry import resolve_ai_command # noqa: E402 +from agentbench_frame.games.miracle.proctree import ProcessTreeManager # noqa: E402 +from agentbench_frame.games.miracle.smoke_audit import ( # noqa: E402 + ensure_fresh_session, make_session_id, write_manifest_atomic, +) + +from agentbench_frame.games.miracle.paths import extracted_dir +EXTRACTED = extracted_dir() +RANK16_SRC = next(EXTRACTED.glob("rank16__*")) +SESSION_ROOT = REPO / ".smoke" / "rank16build" +LOG = None + +SOURCE_SUFFIXES = (".cpp", ".c", ".h", ".hpp", ".json") + + +def log(msg=""): + print(msg, flush=True) + if LOG: + LOG.write(str(msg) + "\n"); LOG.flush() + + +def sha256_file(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + + +def is_pe(p: Path) -> bool: + try: + return p.read_bytes()[:2] == b"MZ" + except OSError: + return False + + +def dll_deps(p: Path): + try: + r = subprocess.run(["objdump", "-p", str(p)], capture_output=True, text=True, timeout=15) + return [ln.split(":", 1)[1].strip() for ln in r.stdout.splitlines() + if ln.strip().startswith("DLL Name")] + except Exception as e: + return [f""] + + +def source_hashes(d: Path): + """Hash strategy SOURCE files only (not build/ artifacts, not main exe).""" + out = {} + for p in sorted(d.iterdir()): + if p.is_dir(): + continue + if p.suffix in SOURCE_SUFFIXES or p.name == "makefile": + out[p.name] = sha256_file(p) + return out + + +def spawn_check(copy_dir: Path, timeout_s=2.0): + res = {"command_resolution": None, "command_resolution_pass": False, + "os_spawn_pass": None, "returncode": None, "cleanup_all_succeeded": None, + "exception": None} + try: + cmd = resolve_ai_command(copy_dir) + res["command_resolution"] = cmd + res["command_resolution_pass"] = True + except FileNotFoundError as e: + res["command_resolution"] = f"FileNotFoundError: {e}" + return res + mgr = ProcessTreeManager() + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0 + try: + proc = subprocess.Popen(cmd, cwd=str(copy_dir), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + creationflags=creationflags) + mgr.register_popen(proc, "rank16_spawn") + res["os_spawn_pass"] = True + try: + proc.communicate(timeout=timeout_s) + res["returncode"] = proc.returncode + except subprocess.TimeoutExpired: + res["returncode"] = "timeout" + except (OSError, FileNotFoundError) as e: + res["os_spawn_pass"] = False + res["exception"] = repr(e) + return res + finally: + procs = mgr.cleanup_all("spawn-check") + res["cleanup_all_succeeded"] = all(p.cleanup_succeeded for p in procs) + res["cleanup_detail"] = mgr.status() + return res + + +def main() -> int: + global LOG + session_id = make_session_id() + session_dir = ensure_fresh_session(SESSION_ROOT, session_id) + LOG = open(session_dir / "rank16_build.full.log", "w", encoding="utf-8") + log(f"rank16 build session: {session_dir}") + log(f"source (protected, read-only): {RANK16_SRC}") + log("用户决定：仅在隔离副本创建空 build/（构建环境准备，非策略修复）；不改源码/Makefile。") + + copy_dir = session_dir / "rank16_copy" + shutil.copytree(RANK16_SRC, copy_dir) + log(f"copied to: {copy_dir}") + + hashes_before = source_hashes(copy_dir) + log(f"source files before: {len(hashes_before)} files hashed") + + # build-env prep ONLY in the copy: empty build/ dir the Makefile assumes + (copy_dir / "build").mkdir(exist_ok=False) + log("created empty build/ in copy (Makefile assumes it exists)") + + files_before_make = {p.name for p in copy_dir.iterdir() if p.is_file()} | {"build"} + + # compile with the ORIGINAL makefile (no parameter/optimization change) + cmd = ["make"] + t0 = time.time() + comp = subprocess.run(cmd, cwd=str(copy_dir), capture_output=True, text=True, timeout=180) + comp_rec = {"command": cmd, "stdout": comp.stdout, "stderr": comp.stderr, + "returncode": comp.returncode, "duration_s": round(time.time() - t0, 2)} + + hashes_after = source_hashes(copy_dir) + source_integrity_ok = (hashes_before == hashes_after) + + # new files after make (build artifacts + exe only) + files_after = set() + for p in copy_dir.rglob("*"): + if p.is_file(): + files_after.add(str(p.relative_to(copy_dir)).replace("\\", "/")) + files_before_set = set() + for p in copy_dir.rglob("*"): + pass + # compute new files = files present now that are NOT original source files + original_source_names = set(hashes_before.keys()) | {"Data.json"} if "Data.json" in hashes_before else set(hashes_before.keys()) + new_files = sorted(f for f in files_after if Path(f).name not in hashes_before) + + binary = None + for name in ("main.exe", "main"): + if (copy_dir / name).exists(): + binary = copy_dir / name + break + compile_pass = (comp_rec["returncode"] == 0 and binary is not None) + + log(f"compile: returncode={comp_rec['returncode']} binary={binary.name if binary else None} -> {'COMPILE_PASS' if compile_pass else 'COMPILE_FAIL'}") + log(f"source_integrity (before==after): {source_integrity_ok}") + log(f"new files after make: {new_files}") + if comp_rec["stderr"]: + log(f"[stderr tail] {comp_rec['stderr'][-400:]!r}") + + binary_info = None + spawn = None + if binary: + binary_info = {"name": binary.name, "size": binary.stat().st_size, + "sha256": sha256_file(binary), "is_pe": is_pe(binary), + "dll_deps": dll_deps(binary)} + log(f"binary: size={binary_info['size']} pe={binary_info['is_pe']} sha256={binary_info['sha256']} dlls={binary_info['dll_deps']}") + if compile_pass: + spawn = spawn_check(copy_dir) + log(f"spawn: cmd_res={spawn['command_resolution_pass']} os_spawn={spawn['os_spawn_pass']} rc={spawn['returncode']} cleanup={spawn['cleanup_all_succeeded']}") + + report = { + "session_id": session_id, + "rank": 16, + "source_dir_original": str(RANK16_SRC), + "source_dir_copy": str(copy_dir), + "build_env_prep": "created empty build/ in copy ONLY (Makefile assumes it); no source/Makefile/param/optimization change", + "source_hashes_before": hashes_before, + "source_hashes_after": hashes_after, + "source_integrity_ok": source_integrity_ok, + "new_files_after_make": new_files, + "make": comp_rec, + "binary": binary_info, + "compile_pass": compile_pass, + "spawn_precheck": spawn, + "classification": { + "COMPILE_PASS": compile_pass, + "OS_SPAWN_PASS": bool(spawn and spawn["os_spawn_pass"]), + "COMMAND_RESOLUTION_PASS": bool(spawn and spawn["command_resolution_pass"]), + "PROTOCOL_NOT_VALIDATED": True, + }, + "verification_note": "仅隔离编译（副本内建空 build/）+ 短时无 Judge 进程创建预检；未启动 Judge/对局，不证明协议或比赛可用。", + "prior_failure_evidence_preserved": ".smoke/precheck/20260721-184652_da899d/ (rank16 COMPILE_FAIL 历史保留)", + } + (session_dir / "rank16_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + manifest = { + "session_id": session_id, "kind": "stage9B_rank16_isolated_build", + "created_unix": time.time(), "python": sys.executable, + "compiler": subprocess.run(["g++", "--version"], capture_output=True, text=True).stdout.splitlines()[0], + "make": subprocess.run(["make", "--version"], capture_output=True, text=True).stdout.splitlines()[0], + "compile_pass": compile_pass, + "source_integrity_ok": source_integrity_ok, + "no_source_or_makefile_modification": source_integrity_ok, + } + write_manifest_atomic(session_dir, manifest) + log(f"\nRESULT: compile_pass={compile_pass} source_integrity_ok={source_integrity_ok}") + log(f"session dir: {session_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/miracle_smoke.py b/tools/miracle_smoke.py new file mode 100644 index 0000000..0e58fa3 --- /dev/null +++ b/tools/miracle_smoke.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""24_miracle local smoke (阶段6, pre-authorized, AT MOST 4 games total). + +Evidence-safety design (this round's fixes): + * Every run gets a unique session dir under .smoke/sessions//. + A pre-existing session id is REFUSED — this script NEVER deletes or + overwrites prior sessions, and NEVER re-runs a successful game. + * sampleA/sampleB are ASSET dirs, kept separate from session products. + * A manifest (time, code+asset hashes, python version, auth cap=4) is written + atomically before any match starts. + * Group 1 (sampleA vs sampleB, 2 games, camps swapped) must pass a STRICT gate + before Group 2 (if-else vs sampleB, 2 games, camps swapped) is even considered. + * Residual processes are checked INDEPENDENTLY by exact PID + psutil + create_time (never trusting cleanup_succeeded, never killing by name). + * Full stdout/stderr are written to UTF-8 files in the session dir; the console + shows a summary. Tool rejections / exceptions are recorded separately and are + NEVER written as game results. + +Run from the repo root with a 3.11+ interpreter, UTF-8 mode: + PYTHONUTF8=1 py -3.13 tools/miracle_smoke.py +""" +from __future__ import annotations + +import json +import platform +import sys +import traceback +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.runner import MiracleEvalRunner # noqa: E402 +from agentbench_frame.games.miracle.smoke_audit import ( # noqa: E402 + build_manifest, + check_residual_procs, + ensure_fresh_session, + group1_strict_clean, + load_managed_procs_from_result_json, + make_session_id, + should_run_group2, + write_manifest_atomic, +) + +SESSIONS_ROOT = REPO / ".smoke" / "sessions" +from agentbench_frame.games.miracle.paths import judge_dir, ifelse_dir, sample_ai_dir +JUDGE = judge_dir() +SAMPLE_AI_SRC = sample_ai_dir() +VENDOR = REPO / "vendor" / "miracle_local" / "run_match.py" +FW_SRC = REPO / "src" +SAMPLE_A = REPO / ".smoke" / "sampleA" +SAMPLE_B = REPO / ".smoke" / "sampleB" +IFELSE = ifelse_dir() +AUTH_CAP = 4 + +_LOG = None + + +def log(msg=""): + print(msg, flush=True) + if _LOG is not None: + _LOG.write(str(msg) + "\n") + _LOG.flush() + + +def _attempt_report(att) -> dict: + rr = att.realized_randomization or {} + rj_path = att.evidence_paths.get("result_json") + procs = load_managed_procs_from_result_json(rj_path) if rj_path else [] + res = check_residual_procs(procs) if procs else {"clean": [], "residual": [], "reused": []} + rec = { + "game_id": att.game_id, + "valid": att.valid, + "normalized_result": att.normalized_result, + "raw_winner": att.raw_winner, + "winner_agent": att.winner_agent, + "scores": att.scores, + "evaluated_agent_camp": att.evaluated_agent_camp, + "steps": att.steps, + "seed": {"requested_seed": None, "effective_seed": None, + "reproducible_from_seed": False, + "realized_randomization": rr}, + "error_type": att.error_type, + "reason": att.reason, + "wrapper_timeout": att.wrapper_timeout, + "normal_cleanup_nonzero": att.normal_cleanup_nonzero, + "result_json_status": att.result_json_status, + "discrepancies": att.discrepancies, + "evidence_paths": att.evidence_paths, + "residual_check": { + "n_identity_procs": len(procs), + "clean": [(p.pid, p.role) for p in res["clean"]], + "residual": [(p.pid, p.role) for p in res["residual"]], + "reused": [(p.pid, p.role) for p in res["reused"]], + }, + } + return rec + + +def _run_group(name, *, agent, opponent, evaluated_dir, opponent_dir, data_root, work_dir, + timeout, wrapper_timeout_s, prefix): + log(f"\n===== {name}: {agent} vs {opponent} (2 games, camps swapped) =====") + runner = MiracleEvalRunner( + agent=agent, opponent=opponent, + evaluated_dir=evaluated_dir, opponent_dir=opponent_dir, + n_games=2, data_dir=str(data_root), judge_dir=JUDGE, + vendor_script=VENDOR, framework_src=FW_SRC, work_dir=work_dir, + timeout=timeout, wrapper_timeout_s=wrapper_timeout_s, prefix=prefix, + config={"group": name, "judge_dir_resolved": str(JUDGE.resolve())}, + ) + summary = runner.run() + reports = [_attempt_report(att) for att in runner.attempts] + for rec in reports: + rr = rec["seed"]["realized_randomization"] + log(f" {rec['game_id']}: valid={rec['valid']} result={rec['normalized_result']} " + f"raw_winner={rec['raw_winner']} camp={rec['evaluated_agent_camp']} " + f"scores={rec['scores']} steps={rec['steps']} " + f"map=(mt={rr.get('map_type')},dt={rr.get('day_time')}) " + f"err={rec['error_type']} residual={rec['residual_check']['residual']}") + s = {k: summary.get(k) for k in + ("attempted_games", "valid_games", "invalid_games", "wins", "losses", "draws", + "win_rate_denominator", "win_rate", "evaluation_status", "total_steps", "attempted_steps")} + log(f" summary[{agent}]: {json.dumps(s, ensure_ascii=False)}") + ok, reasons = group1_strict_clean(runner.attempts, summary) + log(f" {name} strict gate: {'PASS' if ok else 'FAIL'}" + ("" if ok else f" -> {reasons}")) + return {"ok": ok, "reasons": reasons, "summary": s, "reports": reports, + "run_dir": f"{data_root}/runs/24_miracle/{agent}//"} + + +def main() -> int: + global _LOG + session_id = make_session_id() + session_dir = ensure_fresh_session(SESSIONS_ROOT, session_id) + _LOG = open(session_dir / "smoke.full.log", "w", encoding="utf-8") + data_root = session_dir / "data" + work_dir = session_dir / "work" + data_root.mkdir(parents=True, exist_ok=True) + work_dir.mkdir(parents=True, exist_ok=True) + + log(f"session_id: {session_id}") + log(f"session_dir: {session_dir}") + log(f"Judge (authoritative): {JUDGE.resolve()}") + log(f"vendor runner: {VENDOR}") + log(f"sample A / B: {SAMPLE_A} / {SAMPLE_B}") + log(f"if-else bot: {IFELSE}") + log(f"auth_cap_games: {AUTH_CAP}") + log("NOTE: the previous smoke attempt was rejected by a 429/tool error before " + "any subprocess launched; that is NOT a game result.") + + manifest = build_manifest( + session_id=session_id, auth_cap=AUTH_CAP, + python_executable=sys.executable, python_version=platform.python_version(), + code_files=[REPO / "tools" / "miracle_smoke.py", REPO / "src" / "agentbench_frame" / "games" / "miracle" / "runner.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "match_runner.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "proctree.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "result.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "driver.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "smoke_audit.py", + VENDOR], + asset_files=[JUDGE / "main.py", JUDGE / "Data.json", + SAMPLE_AI_SRC / "main.py", + IFELSE / "main.py", SAMPLE_A / "main.py", SAMPLE_B / "main.py"], + groups_planned=[{"group": "GROUP1", "agent": "sampleA", "opponent": "sampleB", "n_games": 2}, + {"group": "GROUP2", "agent": "miracle_ifelse", "opponent": "sampleB", "n_games": 2, + "conditional_on": "GROUP1 strict pass"}], + notes=["429/tool rejection on prior attempt was not a game result", + "no session is ever deleted or overwritten; successful games are never re-run"], + ) + mp = write_manifest_atomic(session_dir, manifest) + log(f"manifest: {mp}") + + try: + g1 = _run_group("GROUP1", agent="sampleA", opponent="sampleB", + evaluated_dir=SAMPLE_A, opponent_dir=SAMPLE_B, + data_root=data_root, work_dir=work_dir, + timeout=8.0, wrapper_timeout_s=120.0, prefix="g1") + (session_dir / "group1.json").write_text( + json.dumps(g1, ensure_ascii=False, indent=2), encoding="utf-8") + + g2 = None + if should_run_group2(g1["ok"]): + log("\n>> GROUP 1 strict pass — proceeding to GROUP 2.") + g2 = _run_group("GROUP2", agent="miracle_ifelse", opponent="sampleB", + evaluated_dir=IFELSE, opponent_dir=SAMPLE_B, + data_root=data_root, work_dir=work_dir, + timeout=10.0, wrapper_timeout_s=180.0, prefix="g2") + (session_dir / "group2.json").write_text( + json.dumps(g2, ensure_ascii=False, indent=2), encoding="utf-8") + else: + log("\n>> GROUP 1 strict gate FAILED — GROUP 2 NOT started (per spec).") + + log("\n===== residual process check (independent, exact PID + create_time) =====") + for grp_name, grp in (("GROUP1", g1), ("GROUP2", g2)): + if grp is None: + continue + for rec in grp["reports"]: + rc = rec["residual_check"] + log(f" {rec['game_id']}: identity_procs={rc['n_identity_procs']} " + f"clean={len(rc['clean'])} residual={rc['residual']} reused={len(rc['reused'])}") + + overall = g1["ok"] and (g2 is None or g2["ok"]) and (g2 is not None) + # overall green only if BOTH groups ran and passed; G2-not-started is not green + log(f"\nSMOKE RESULT: {'BOTH_GROUPS_PASS' if (g1['ok'] and g2 and g2['ok']) else 'NOT_FULLY_PASSING'}") + return 0 if (g1["ok"] and g2 is not None and g2["ok"]) else 1 + except BaseException as exc: # noqa: BLE001 + # record the exception SEPARATELY from game results; preserve the session + tb = traceback.format_exc() + (session_dir / "exception.log").write_text(tb, encoding="utf-8") + log(f"\nEXCEPTION (not a game result): {exc!r}") + log(tb) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/miracle_local/run_match.py b/vendor/miracle_local/run_match.py new file mode 100644 index 0000000..0743724 --- /dev/null +++ b/vendor/miracle_local/run_match.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""LOCAL VERIFICATION COPY of 高翔's ``tools/miracle/run_match.py``. + +VENDOR PROVENANCE + Original source : external_asset:AgentBench-gaoxiang/tools/miracle/run_match.py + Original SHA256 : 91d5693651402a3579146181ee6ada5586c103f38ba5f321b92c23a17a7a8ae2 + Authoritative Judge : external_asset:24_miracle/judge_dev_logic + (高翔's in-tree Judge copy is byte-identical to it, diff verified.) + Role : local Windows verification only; the original stays frozen, this is a + minimally-patched vendored copy used by match_runner.py. + +MODIFICATIONS (minimal, cross-platform portability; NO protocol/contract change) + 1. JUDGE_DIR overridable via env MIRACLE_JUDGE_DIR (default unchanged). Formal + smoke sets it to the authoritative Judge path. + 2. ``terminate()`` (Unix-only ``os.killpg``) replaced by + ``agentbench_frame.games.miracle.proctree.ProcessTreeManager`` — graceful + terminate -> grace window -> force-kill tree, by EXACT pid with psutil + create_time identity check. Works on Windows and POSIX, no name-based kill. + 3. ``spawn`` flags: POSIX ``start_new_session=True``; Windows + ``CREATE_NEW_PROCESS_GROUP``. + 4. ``read_ai_operation`` no longer uses ``selectors`` (broken on Windows pipes); + it uses a daemon reader thread + queue with timeout — cross-platform. + 5. NEW ``--result-json ``: writes a machine-readable structured result + (process metadata, end_info, derived raw_winner, scores, tie flags, + per-AI timeout/error, trace/replay paths, cleanup status, timings, any + exception). match_runner.py reads THIS file, not the human-readable stdout. + 6. Per-process metadata (pid, started_at, natural_exit, natural_returncode, + termination_requested, termination_reason, final_returncode, forced_kill, + cleanup_succeeded, identity_confirmed) is captured so the adapter can tell a + post-end_info cleanup-kill of an idling AI apart from a strategy crash. + +Everything else is byte-faithful to the original protocol bridge. +""" + +from __future__ import annotations + +import argparse +import json +import os +import queue +import struct +import subprocess +import sys +import threading +import time +from pathlib import Path + +# --- make the framework's proctree importable when launched as a subprocess --- # +_FW_SRC = os.environ.get("MIRACLE_FRAMEWORK_SRC") +if _FW_SRC and _FW_SRC not in sys.path: + sys.path.insert(0, _FW_SRC) +from agentbench_frame.games.miracle.proctree import ProcessTreeManager # noqa: E402 +from agentbench_frame.games.miracle.entry import resolve_ai_command # noqa: E402 +from agentbench_frame.games.miracle.atomicio import atomic_write_json # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_JUDGE_DIR = ROOT / "backend_sources/corpus/24_miracle/logic/judge_dev_logic" +JUDGE_DIR = Path(os.environ.get("MIRACLE_JUDGE_DIR") or DEFAULT_JUDGE_DIR) + + +class ProtocolError(RuntimeError): + pass + + +# ---- low-level framed I/O (unchanged protocol) --------------------------- # +def read_exact(stream, n: int) -> bytes: + chunks = [] + remaining = n + while remaining: + chunk = stream.read(remaining) + if not chunk: + raise EOFError("unexpected EOF") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def write_judge_input(proc: subprocess.Popen, obj: dict) -> None: + payload = json.dumps(obj, separators=(",", ":")).encode("utf-8") + proc.stdin.write(struct.pack(">i", len(payload)) + payload) + proc.stdin.flush() + + +def read_judge_frame(proc: subprocess.Popen): + header = read_exact(proc.stdout, 8) + length, goal = struct.unpack(">ii", header) + if length < 0: + raise ProtocolError(f"negative judge frame length: {length}") + payload = read_exact(proc.stdout, length) + return goal, json.loads(payload.decode("utf-8")) + + +def write_ai_state(proc: subprocess.Popen, payload: str) -> None: + proc.stdin.write(payload.encode("utf-8")) + proc.stdin.flush() + + +def read_ai_operation(proc: subprocess.Popen, timeout: float): + """Read one AI operation with a timeout. Cross-platform: a daemon reader + thread + queue (the original used ``selectors`` which does not work on + Windows pipes). Raises ``TimeoutError`` on timeout.""" + q: "queue.Queue[tuple]" = queue.Queue() + + def _read(): + try: + header = read_exact(proc.stdout, 4) + length = struct.unpack(">i", header)[0] + if length < 0: + q.put(("error", ProtocolError(f"negative AI frame length: {length}"))) + return + payload = read_exact(proc.stdout, length) + q.put(("ok", json.loads(payload.decode("utf-8")))) + except BaseException as exc: # noqa: BLE001 - report any read failure upstream + q.put(("error", exc)) + + t = threading.Thread(target=_read, daemon=True) + t.start() + try: + kind, val = q.get(timeout=timeout) + except queue.Empty: + raise TimeoutError("AI operation timed out") + if kind == "error": + raise val + return val + + +# resolve_ai_command is imported from agentbench_frame.games.miracle.entry +# (cross-platform: Windows main.exe / POSIX ./main / main.py; explicit priority; +# paths with spaces kept whole). See tests/miracle/test_entry.py. + + +def _creation_flags(): + """Platform-appropriate process-group flag so the tree can be managed.""" + if os.name == "nt": + return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + return {"start_new_session": True} + + +def spawn(cmd, cwd: Path) -> subprocess.Popen: + return subprocess.Popen( + cmd, + cwd=str(cwd), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **_creation_flags(), + ) + + +def drain_stderr(proc: subprocess.Popen) -> str: + if proc.stderr is None: + return "" + try: + return proc.stderr.read().decode("utf-8", errors="replace") + except Exception: + return "" + + +def player_error(player: int, state: int, error: int = 0) -> dict: + return { + "player": -1, + "content": json.dumps({"error": error, "player": player, "state": state}), + } + + +def derive_raw_winner(end_info): + """Judge end_info = {"0": score0, "1": score1}; winner = 0 if s0>s1 else 1. + Ties are broken toward player1 (Judge main.py:448-449). Returns (winner, s0, s1, tie).""" + if not isinstance(end_info, dict) or "0" not in end_info or "1" not in end_info: + return None, None, None, False + try: + s0, s1 = int(end_info["0"]), int(end_info["1"]) + except (TypeError, ValueError): + return None, None, None, False + tie = (s0 == s1) + return (0 if s0 > s1 else 1), s0, s1, tie + + +# ---- match entry point --------------------------------------------------- # +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--p0-dir", required=True, type=Path) + parser.add_argument("--p1-dir", required=True, type=Path) + parser.add_argument("--p0-cmd") + parser.add_argument("--p1-cmd") + parser.add_argument("--p0-name", default="player0") + parser.add_argument("--p1-name", default="player1") + parser.add_argument("--timeout", type=float, default=12.0) + parser.add_argument("--out", type=Path, default=ROOT / "reports/miracle_rollouts") + parser.add_argument("--tag", default=None) + parser.add_argument("--result-json", type=Path, default=None, + help="machine-readable structured result (used by match_runner)") + args = parser.parse_args() + + args.out.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") + tag = args.tag or f"{args.p0_name}_vs_{args.p1_name}_{stamp}" + replay = args.out / f"{tag}.replay" + trace = args.out / f"{tag}.jsonl" + + mgr = ProcessTreeManager() + match_started = time.time() + end_info_received = False + end_info = None + timeout_flag = {"ai0": False, "ai1": False} + ai_error_flag = {"ai0": False, "ai1": False} + last_state = 0 + run_exc = None + + def log(obj: dict) -> None: + with trace.open("a", encoding="utf-8") as f: + f.write(json.dumps(obj, ensure_ascii=False, separators=(",", ":")) + "\n") + + try: + judge_p = spawn([sys.executable, "main.py"], JUDGE_DIR) + judge_mp = mgr.register_popen(judge_p, "judge") + ai0_p = spawn(resolve_ai_command(args.p0_dir, args.p0_cmd), args.p0_dir) + ai1_p = spawn(resolve_ai_command(args.p1_dir, args.p1_cmd), args.p1_dir) + ai0_mp = mgr.register_popen(ai0_p, "ai0") + ai1_mp = mgr.register_popen(ai1_p, "ai1") + procs = [ai0_p, ai1_p] + mps = {"ai0": ai0_mp, "ai1": ai1_mp} + names = [args.p0_name, args.p1_name] + + write_judge_input(judge_p, {"replay": str(replay), "player_list": [1, 1]}) + log({"kind": "match_start", "players": names, "replay": str(replay), + "judge_dir": str(JUDGE_DIR.resolve())}) + + while True: + goal, frame = read_judge_frame(judge_p) + last_state = frame.get("state", last_state) + log({"kind": "judge_frame", "goal": goal, "frame": frame}) + + if frame.get("state") == -1: + end_info_received = True + try: + end_info = json.loads(frame.get("end_info", "{}")) + except Exception: + end_info = None + log({"kind": "match_end", "end_info": frame.get("end_info")}) + break + + players = frame.get("player") or [] + contents = frame.get("content") or [] + if not players or not contents: + continue + + for idx, player in enumerate(players): + payload = contents[idx] + write_ai_state(procs[player], payload) + try: + operation = read_ai_operation(procs[player], args.timeout) + except TimeoutError: + timeout_flag[f"ai{player}"] = True + log({"kind": "ai_timeout", "player": player, "state": last_state}) + write_judge_input(judge_p, player_error(player, last_state, error=1)) + continue + except Exception as exc: + ai_error_flag[f"ai{player}"] = True + log({"kind": "ai_error", "player": player, "state": last_state, "error": repr(exc)}) + write_judge_input(judge_p, player_error(player, last_state, error=0)) + continue + + log({"kind": "ai_operation", "player": player, "name": names[player], "operation": operation}) + write_judge_input( + judge_p, + {"player": operation["player"], "content": json.dumps(operation, separators=(",", ":"))}, + ) + + returncode = 0 + except BaseException as exc: # noqa: BLE001 - capture ANY failure (e.g. Judge crash) + run_exc = repr(exc) + returncode = 1 + finally: + # clean up judge + AIs (graceful -> force tree, exact PID, identity-checked) + mgr.cleanup_all("match-end") + match_finished = time.time() + + # drain stderr after processes are dead (safe, non-blocking once EOF) + for idx, p in enumerate([judge_p] + [mp.popen for mp in [ai0_mp, ai1_mp]]): + err = drain_stderr(p) + if err: + who = "judge" if idx == 0 else f"ai{idx-1}" + log({"kind": "stderr", "who": who, "stderr": err[-12000:]}) + + status = {s["role"]: s for s in mgr.status()} + raw_winner, s0, s1, tie = derive_raw_winner(end_info) + result = { + "schema_version": 1, + "tag": tag, + "started_at": match_started, + "finished_at": match_finished, + "duration_s": round(match_finished - match_started, 3), + "judge_dir_resolved": str(JUDGE_DIR.resolve()), + "p0": {"name": args.p0_name, "dir": str(args.p0_dir.resolve())}, + "p1": {"name": args.p1_name, "dir": str(args.p1_dir.resolve())}, + "judge": status.get("judge"), + "ai0": status.get("ai0"), + "ai1": status.get("ai1"), + "end_info_received": end_info_received, + "end_info": end_info, + "scores": ({"0": s0, "1": s1} if s0 is not None else None), + "raw_winner": raw_winner, + "score_tie": bool(tie), + "judge_tiebreak_applied": bool(tie), + "timeout": timeout_flag, + "ai_error": ai_error_flag, + "trace_path": str(trace), + "replay_path": str(replay), + "cleanup_all_succeeded": bool(mgr.status() and all(s["cleanup_succeeded"] for s in mgr.status())), + "exception": run_exc, + "run_match_returncode": returncode, + } + if args.result_json is not None: + atomic_write_json(args.result_json, result) # temp + fsync + os.replace (atomic) + # human-readable stdout kept for back-compat (match_runner uses result-json) + print(json.dumps({"trace": str(trace), "replay": str(replay), + "end_info_received": end_info_received}, ensure_ascii=False)) + + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/results_local/aggregate.py b/vendor/results_local/aggregate.py new file mode 100644 index 0000000..933437a --- /dev/null +++ b/vendor/results_local/aggregate.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""LOCAL VERIFICATION COPY of AgentBenchResults `scripts/aggregate.py`. + +This is a faithful copy of the upstream aggregator with ONE minimal cross-platform +patch. It is used only for local schema/aggregate/web verification +(SKILL.md 阶段7) on Windows; the upstream script in the AgentBenchResults repo +remains the authoritative implementation. On Linux (production CI) the upstream +script is correct and this copy is unnecessary. + +UPSTREAM BUG (confirmed 2026-07-21 on Windows): + Upstream writes `path = str(run_dir.relative_to(data_dir))`, which on Windows + yields backslash separators (`runs\\24_miracle\\...`). Backslashes are illegal + inside a TOML basic string, so the emitted `registry.toml` fails to parse and + `report_builder.load_registry` aborts. The bug is invisible on Linux CI where + paths use forward slashes. + +PATCH (single line, zero semantic change): + `str(run_dir.relative_to(data_dir))` -> `run_dir.relative_to(data_dir).as_posix()` + i.e. serialize the run's relative path with forward slashes so registry.toml + is valid TOML on every platform. Path semantics are unchanged. + +Everything else is byte-identical to upstream. See +docs/games/24_miracle_adapter_status.md ("Framework / Results 缺陷" section). + +Usage: python vendor/results_local/aggregate.py --data-dir ./ --output registry.toml +""" + +import argparse, json, sys, tomllib +from pathlib import Path + + +def _get_meta_field(meta: dict, key: str, default=""): + """Read a field from run.toml, handling both [run] section and flat formats.""" + if "run" in meta: + return meta["run"].get(key, default) + return meta.get(key, default) + + +def find_runs(data_dir: Path) -> list[dict]: + runs = [] + runs_root = data_dir / "runs" + if not runs_root.is_dir(): + return runs + + for game_dir in sorted(runs_root.iterdir()): + if not game_dir.is_dir(): continue + for agent_dir in sorted(game_dir.iterdir()): + if not agent_dir.is_dir(): continue + for run_dir in sorted(agent_dir.iterdir()): + if not run_dir.is_dir(): continue + run_toml = run_dir / "run.toml" + if not run_toml.exists(): + continue + try: + raw = tomllib.loads(run_toml.read_text()) + except Exception: + raw = {} + + # summary.json is authoritative for metrics + s = {} + sj = run_dir / "summary.json" + if sj.exists(): + try: + s = json.loads(sj.read_text()) + except Exception: + pass + + # Merge: summary.json fields override, fallback to run.toml. + # PATCH (as_posix): forward-slash path so registry.toml is valid TOML + # on Windows too; upstream uses str(...) which emits backslashes here. + runs.append({ + "run_id": run_dir.name, + "game": game_dir.name, + "agent": agent_dir.name, + "type": s.get("run_type") or _get_meta_field(raw, "type", "eval"), + "path": run_dir.relative_to(data_dir).as_posix(), + "created": s.get("created") or _get_meta_field(raw, "created", ""), + "git_commit": s.get("git_commit") or _get_meta_field(raw, "git_commit", ""), + "summary": { + "best_elo": s.get("best_elo"), + "final_elo": s.get("final_elo"), + "wall_hours": s.get("wall_hours"), + "total_steps": s.get("total_steps"), + "win_rate": s.get("win_rate"), + }, + }) + return runs + + +def _toml_val(v): + if v is None: return "" + if isinstance(v, bool): return str(v).lower() + if isinstance(v, str): return f'"{v}"' + if isinstance(v, float): return f"{v}" + if isinstance(v, int): return f"{v}" + return f'"{v}"' + + +def main(): + p = argparse.ArgumentParser(description="Generate registry.toml from runs/") + p.add_argument("--data-dir", default=".") + p.add_argument("--output", default="registry.toml") + args = p.parse_args() + data_dir = Path(args.data_dir) + runs = find_runs(data_dir) + + lines = ["# Auto-generated by aggregate.py — do not edit manually", ""] + for r in runs: + lines.append("[[runs]]") + lines.append(f'run_id = "{r["run_id"]}"') + lines.append(f'game = "{r["game"]}"') + lines.append(f'agent = "{r["agent"]}"') + lines.append(f'type = "{r["type"]}"') + lines.append(f'path = "{r["path"]}"') + if r["created"]: lines.append(f'created = "{r["created"]}"') + if r["git_commit"]: lines.append(f'git_commit = "{r["git_commit"]}"') + # Summary as inline dict + s = r["summary"] + summary_items = [] + for k, v in s.items(): + if v is not None: + summary_items.append(f'{k} = {_toml_val(v)}') + if summary_items: + lines.append("[runs.summary]") + lines.extend(summary_items) + lines.append("") + + out = Path(args.output) + out.write_text("\n".join(lines)) + print(f"Wrote {len(runs)} runs to {out}", file=sys.stderr) + + +if __name__ == "__main__": + main()