diff --git a/.gitignore b/.gitignore index c18dd8d..a964611 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ __pycache__/ +.smoke/ +.pytest_cache/ +_site/ +docs/games/results_payload/ diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..cf7e2a2 --- /dev/null +++ b/conftest.py @@ -0,0 +1,13 @@ +"""Pytest bootstrap: put the src-layout package on sys.path without an editable +install, so `from agentbench_frame... import ...` resolves in tests and so the +`agentbench_frame.*` modules can be imported by helper scripts. + +Run tests with the Python 3.11+ interpreter (the framework imports `tomllib`): + py -3.13 -m pytest tests/ +""" +import sys +from pathlib import Path + +_SRC = Path(__file__).resolve().parent / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) diff --git a/pyproject.toml b/pyproject.toml index d18a089..7962bcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [] [project.optional-dependencies] tracking = ["psutil"] +miracle = ["psutil"] rl = ["torch", "numpy"] report = ["jinja2"] all = ["psutil", "torch", "numpy", "jinja2"] diff --git a/src/agentbench_frame/games/__init__.py b/src/agentbench_frame/games/__init__.py new file mode 100644 index 0000000..78d05ff --- /dev/null +++ b/src/agentbench_frame/games/__init__.py @@ -0,0 +1,4 @@ +"""Game-specific adapters under the AgentBench framework. + +Each subpackage adapts one Saiblo game. `miracle` adapts the 24th Miracle game +via the original external Judge (no in-process reimplementation).""" diff --git a/src/agentbench_frame/games/miracle/__init__.py b/src/agentbench_frame/games/miracle/__init__.py new file mode 100644 index 0000000..6528104 --- /dev/null +++ b/src/agentbench_frame/games/miracle/__init__.py @@ -0,0 +1,45 @@ +"""Isolated adapter for the external 24_miracle Judge runtime. + +The package wraps subprocess/runtime behavior and normalizes results without +reimplementing Judge rules in process. +""" + +from agentbench_frame.games.miracle.result import ( + DRAW, + ERROR, + LOSS, + VALID_RESULTS, + WIN, + GameOutcome, + compute_h2h, + compute_win_rate, + derive_raw_winner, + finalize, + normalize, + outcome_counts, + read_replay_header, + select_games_to_run, + sha256_file, + to_event_record, + would_rerun_successful, +) + +__all__ = [ + "GameOutcome", + "WIN", + "LOSS", + "DRAW", + "ERROR", + "VALID_RESULTS", + "derive_raw_winner", + "finalize", + "normalize", + "compute_win_rate", + "compute_h2h", + "outcome_counts", + "read_replay_header", + "sha256_file", + "select_games_to_run", + "would_rerun_successful", + "to_event_record", +] diff --git a/src/agentbench_frame/games/miracle/atomicio.py b/src/agentbench_frame/games/miracle/atomicio.py new file mode 100644 index 0000000..07f05ee --- /dev/null +++ b/src/agentbench_frame/games/miracle/atomicio.py @@ -0,0 +1,57 @@ +"""Atomic JSON file writer: temp file + fsync + os.replace. + +Guarantees the final path is never partially written. The new content is +written to a same-directory temp file, fsync'd, then atomically renamed over +the target with ``os.replace``. A crash anywhere before the rename leaves the +target with its prior complete content (or absent); a leftover temp file +(``..*.tmp``) is the only audit trace. Used for vendor result-json so a +force-killed vendor can never leave a half-written result. +""" +from __future__ import annotations + +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Any + +_SENTINEL = object() +_WINDOWS_SHARING_WINERRORS = {5, 32} + + +def _is_windows() -> bool: + """Return the host platform without making tests mutate global ``os``.""" + return os.name == "nt" + + +def _replace_with_windows_retry(tmp: str, target: str) -> None: + """Retry only transient Windows sharing/access failures within 150ms.""" + for i, delay in enumerate((0.01, 0.02, 0.04, 0.08), start=1): + try: + os.replace(tmp, target) + return + except PermissionError as exc: + if not _is_windows() or getattr(exc, "winerror", None) not in _WINDOWS_SHARING_WINERRORS or i == 4: + raise + time.sleep(delay) + + +def atomic_write_json(path, obj: Any, *, encoding: str = "utf-8", + indent: int = 2, default=_SENTINEL) -> Path: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=f".{p.name}.", suffix=".tmp", dir=str(p.parent)) + try: + with os.fdopen(fd, "w", encoding=encoding) as f: + if default is _SENTINEL: + json.dump(obj, f, ensure_ascii=False, indent=indent) + else: + json.dump(obj, f, ensure_ascii=False, indent=indent, default=default) + f.flush() + os.fsync(f.fileno()) + _replace_with_windows_retry(tmp, str(p)) + except BaseException: + # never leave the target half-written; leave the temp for audit (do NOT unlink) + raise + return p diff --git a/src/agentbench_frame/games/miracle/driver.py b/src/agentbench_frame/games/miracle/driver.py new file mode 100644 index 0000000..0e6dfde --- /dev/null +++ b/src/agentbench_frame/games/miracle/driver.py @@ -0,0 +1,65 @@ +"""Glue between finalized Miracle :class:`GameOutcome` instances and the +framework's :class:`Run`. + +This module is the single place that defeats framework risks #2 / #3 / #5 +(see docs/games/24_miracle_adapter_status.md): + +* #2 / #3 — ``Match``/``Run`` count ``raw_winner == 0`` as a win. We feed + ``Run.log_episode`` the **normalized** winner (0 = evaluated agent won), + so ``Run._build_summary`` persists a win_rate that is correct even after + side-swapping. +* #5 — ``BaseRunner.run()`` writes ``summary.json`` before merging the + subclass result. We never use ``BaseRunner``; the caller drives ``Run`` + directly and everything is injected *before* ``finish()``. + +Error games are recorded as ``game`` audit events but are NOT logged as +episodes, so they fall out of ``total_episodes`` and ``win_rate`` (有效对局). +""" +from __future__ import annotations + +from typing import Dict, Sequence + +from agentbench_frame.games.miracle.result import ( + DRAW, + LOSS, + WIN, + GameOutcome, + compute_h2h, + compute_win_rate, + to_event_record, +) +from agentbench_frame.tracking.run import Run + +#: map normalized result -> the integer winner Run.log_episode expects. +#: Run counts ``winner == 0`` as a win, so 0 = evaluated-agent win. +_EPISODE_WINNER = {WIN: 0, LOSS: 1, DRAW: -1} + + +def feed_outcomes_to_run(run: Run, outcomes: Sequence[GameOutcome]) -> Dict: + """Write every outcome as a ``game`` audit event and log the valid ones as + episodes with normalized winners + auditable step counts. + + Does **not** call ``run.finish()`` — the caller owns the run lifecycle so it + can set ``run_type`` / ``data_dir`` correctly (risk #4 / #6) before any + summary is written. + """ + for o in outcomes: + run.write(**to_event_record(o)) + if o.valid: + evaluated_score = o.score0 if o.evaluated_agent_camp == 0 else o.score1 + run.log_episode( + reward=float(evaluated_score or 0.0), + steps=int(o.steps), + winner=_EPISODE_WINNER[o.normalized_result], + info={ + "game_id": o.game_id, + "opponent": o.opponent, + "evaluated_agent_camp": o.evaluated_agent_camp, + "raw_winner": o.raw_winner, + }, + ) + run.log_h2h(compute_h2h(outcomes)) + return { + "win_rate": compute_win_rate(outcomes), + "valid_games": sum(1 for o in outcomes if o.valid), + } diff --git a/src/agentbench_frame/games/miracle/entry.py b/src/agentbench_frame/games/miracle/entry.py new file mode 100644 index 0000000..f2164c0 --- /dev/null +++ b/src/agentbench_frame/games/miracle/entry.py @@ -0,0 +1,72 @@ +"""Cross-platform AI entry resolution for the 24_miracle runner. + +Resolves the argv list used to launch an AI subprocess via ``subprocess.Popen`` +(``shell=False``). The executable is returned as an **ABSOLUTE** path because +Windows ``CreateProcess`` does NOT search the ``cwd=`` argument's directory for a +bare executable name (``Popen(["main.exe"], cwd=dir)`` raises FileNotFoundError); +an absolute path is found regardless of the child's working directory. + +Precedence: + 1. ``explicit`` (a list is kept as-is; a string is kept WHOLE as a single + executable path — never whitespace-split, so paths with spaces survive) + takes priority over auto-detection. + 2. Auto-detection (deterministic and documented): + Windows (``os.name == 'nt'``): ``main.exe`` > ``main.py`` > ``main`` + POSIX: ``main`` > ``main.py`` > ``main.exe`` + When ``main.exe`` and ``main`` coexist, Windows picks ``main.exe`` and + POSIX picks ``main`` (recorded behaviour). + ``main.py`` is launched via ``sys.executable`` + absolute script path. + +This never modifies the strategy directory and never reads strategy source. +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import List, Optional, Union + +#: platform flag read at call time (not the global ``os.name``), so tests can +#: patch it via ``monkeypatch.setattr(entry, "_IS_NT", ...)`` without disturbing +#: pathlib or other stdlib code that reads the real ``os.name``. +_IS_NT = os.name == "nt" + + +def resolve_ai_command( + ai_dir: Union[str, Path], + explicit: Optional[Union[str, List[str]]] = None, +) -> List[str]: + """Return the argv list for the AI subprocess (executable as an ABSOLUTE path). + + Raises ``FileNotFoundError`` (mentioning ``ai_dir``) when no entry exists. + """ + # 1. explicit takes priority; a string is kept whole (paths with spaces!) + if explicit is not None and explicit != "" and not ( + isinstance(explicit, (list, tuple)) and len(explicit) == 0 + ): + if isinstance(explicit, (list, tuple)): + return [str(x) for x in explicit] + return [str(explicit)] + + ai_dir = Path(ai_dir).resolve() # absolute, so Popen(cwd=...) finds the exe on Windows + is_nt = _IS_NT + + # 2. auto-detection — absolute path to the entry + if is_nt: + if (ai_dir / "main.exe").exists(): + return [str(ai_dir / "main.exe")] + if (ai_dir / "main.py").exists(): + return [sys.executable, str(ai_dir / "main.py")] + if (ai_dir / "main").exists(): # unusual on Windows (no extension) + return [str(ai_dir / "main")] + else: + if (ai_dir / "main").exists(): + return [str(ai_dir / "main")] + if (ai_dir / "main.py").exists(): + return [sys.executable, str(ai_dir / "main.py")] + if (ai_dir / "main.exe").exists(): # cross-mounted POSIX + return [str(ai_dir / "main.exe")] + + raise FileNotFoundError( + f"no AI entry found (main.exe on Windows / main on POSIX / main.py) in {ai_dir}" + ) diff --git a/src/agentbench_frame/games/miracle/match_runner.py b/src/agentbench_frame/games/miracle/match_runner.py new file mode 100644 index 0000000..b2437b4 --- /dev/null +++ b/src/agentbench_frame/games/miracle/match_runner.py @@ -0,0 +1,595 @@ +"""Miracle match wrapper: run one game through the (vendored) run_match runner, +then independently cross-verify result-json + trace + Replay and classify the +outcome on the event timeline (not just the final returncode). + +Design points (阶段4b-4 spec): + * Invokes the vendor runner with ``sys.executable -u``, ``shell=False``, an + explicit arg array, and explicit ``MIRACLE_JUDGE_DIR`` / paths / identities. + * stdout/stderr are redirected to FILES (never pipes) so huge output cannot + deadlock or grow without bound in memory. + * A wrapper-level overall timeout; on expiry the vendor subtree (and its AI + descendants) is cleaned by exact-PID process-tree kill (proctree). + * All evidence (stdout/stderr/trace/result-json) is preserved on disk. + * The vendor runner writes result-json atomically (temp + replace). If it is + missing (e.g. force-killed mid-write) the attempt is classified + ``result_json_missing`` — never guessed. + * Three-way cross-validation: result-json vs trace (streamed) vs Replay. Any + authoritative conflict -> ``evidence_mismatch``; nothing is silently picked. + * Returns a ``MatchAttempt``. It does NOT mutate any aggregate; the runner + (MiracleEvalRunner) decides what to log. + +This module is psutil-backed (via proctree); see the ``miracle`` extra. +""" +from __future__ import annotations + +import json +import os +import struct +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from agentbench_frame.games.miracle.proctree import ProcessTreeManager +from agentbench_frame.games.miracle.result import sha256_file + +#: replay header = 7 big-endian signed int32: [0,0,0,map_type,day_time,0,0] +REPLAY_HEADER_BYTES = 28 +#: the Judge draws map_type/day_time via random.randint(0,1); anything outside +#: {0,1} means the header is not a valid Miracle replay. +_VALID_MAP_VALUES = {0, 1} + + +# --------------------------------------------------------------------------- # +# streaming trace stats +# --------------------------------------------------------------------------- # +@dataclass +class TraceStats: + n_ai_operation: int = 0 + ai_error_players: List[int] = field(default_factory=list) + ai_timeout_players: List[int] = field(default_factory=list) + end_info_seen: bool = False + end_info: Optional[Dict[str, int]] = None + error_before_end: bool = False + + +def stream_trace(path) -> TraceStats: + """Stream a trace JSONL line by line (never load it whole).""" + ts = TraceStats() + p = Path(path) + if not p.exists(): + return ts + with p.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + kind = e.get("kind") + if kind == "ai_operation": + ts.n_ai_operation += 1 + elif kind == "ai_error": + pl = e.get("player") + if pl is not None: + ts.ai_error_players.append(pl) + if not ts.end_info_seen: + ts.error_before_end = True + elif kind == "ai_timeout": + pl = e.get("player") + if pl is not None: + ts.ai_timeout_players.append(pl) + if not ts.end_info_seen: + ts.error_before_end = True + elif kind == "match_end": + ts.end_info_seen = True + ei = e.get("end_info") + if isinstance(ei, str): + try: + ts.end_info = json.loads(ei) + except json.JSONDecodeError: + ts.end_info = None + elif isinstance(ei, dict): + ts.end_info = ei + return ts + + +# --------------------------------------------------------------------------- # +# result-json loader +# --------------------------------------------------------------------------- # +def load_result_json(path) -> Tuple[str, Optional[dict]]: + """Return (status, data) where status is 'ok' | 'missing' | 'corrupt'.""" + p = Path(path) + if not p.exists(): + return ("missing", None) + try: + return ("ok", json.loads(p.read_text(encoding="utf-8"))) + except (json.JSONDecodeError, ValueError, UnicodeDecodeError): + return ("corrupt", None) + + +# --------------------------------------------------------------------------- # +# replay info +# --------------------------------------------------------------------------- # +@dataclass +class ReplayInfo: + exists: bool = False + length_ok: bool = False + header_valid: bool = False + map_type: Optional[int] = None + day_time: Optional[int] = None + sha256: Optional[str] = None + + +def read_replay_info(path) -> ReplayInfo: + p = Path(path) + info = ReplayInfo(exists=p.exists()) + if not info.exists: + return info + info.sha256 = sha256_file(p) + data = p.read_bytes() + info.length_ok = len(data) >= REPLAY_HEADER_BYTES + if info.length_ok: + try: + vals = struct.unpack(">7i", data[:REPLAY_HEADER_BYTES]) + info.map_type, info.day_time = int(vals[3]), int(vals[4]) + info.header_valid = ( + info.map_type in _VALID_MAP_VALUES and info.day_time in _VALID_MAP_VALUES + ) + except struct.error: + info.header_valid = False + return info + + +# --------------------------------------------------------------------------- # +# cross-validation (result-json vs trace vs scores) +# --------------------------------------------------------------------------- # +def cross_validate(rj: Optional[dict], ts: TraceStats, ri: ReplayInfo) -> List[str]: + """Return a list of authoritative-conflict descriptions. Only CONFLICTS are + reported here (both sides present and disagreeing); missing end_info in the + trace is handled by the classifier, not treated as a field conflict.""" + discs: List[str] = [] + if rj is None: + return discs + if rj.get("schema_version") != 1: + discs.append("schema_version_unexpected") + rj_end = rj.get("end_info") if rj.get("end_info_received") else None + if rj_end is None: + return discs + # end_info conflict (only when trace also has one) + if ts.end_info is not None and ts.end_info != rj_end: + discs.append("end_info_scores_mismatch_between_result_json_and_trace") + # scores field vs end_info + rj_scores = rj.get("scores") + if rj_scores is not None: + try: + if (int(rj_scores.get("0")) != int(rj_end.get("0")) + or int(rj_scores.get("1")) != int(rj_end.get("1"))): + discs.append("scores_field_conflicts_end_info") + except (TypeError, ValueError, AttributeError): + discs.append("scores_or_end_info_unparseable") + # raw_winner vs score rule (0 if s0>s1 else 1; ties -> 1) + try: + s0, s1 = int(rj_end.get("0")), int(rj_end.get("1")) + expected = 0 if s0 > s1 else 1 + if rj.get("raw_winner") != expected: + discs.append(f"raw_winner_mismatch_expected_{expected}_got_{rj.get('raw_winner')}") + except (TypeError, ValueError, AttributeError): + discs.append("end_info_unparseable") + return discs + + +# --------------------------------------------------------------------------- # +# timeline classification +# --------------------------------------------------------------------------- # +@dataclass +class Classification: + normalized_result: str = "error" + error_type: Optional[str] = None + valid: bool = False + reason: str = "" + ai_crash_player: Optional[int] = None + ai_timeout_player: Optional[int] = None + judge_crash: bool = False + wrapper_timeout: bool = False + normal_cleanup_nonzero: bool = False + raw_winner: Optional[int] = None + winner_agent: Optional[str] = None + + +def classify(*, rj_status: str, rj: Optional[dict], ts: TraceStats, ri: ReplayInfo, + discrepancies: List[str], vendor_returncode: int, wrapper_timeout: bool, + evaluated_agent_camp: int, evaluated_agent: str = "eval", + opponent: str = "opp") -> Classification: + """Classify a game by event timeline + end state. Precedence (most severe + first): wrapper_timeout > result_json missing/corrupt > evidence_mismatch > + ai_crash > ai_timeout > judge_crash > replay_missing > replay_corrupt > valid.""" + c = Classification() + agent_at = lambda camp: evaluated_agent if camp == evaluated_agent_camp else opponent + + if wrapper_timeout: + c.wrapper_timeout = True + c.error_type = "wrapper_timeout" + c.reason = f"vendor tree exceeded wrapper timeout (vendor rc={vendor_returncode})" + return c + if rj_status == "missing": + c.error_type = "result_json_missing" + c.reason = f"no result-json produced (vendor rc={vendor_returncode})" + return c + if rj_status == "corrupt": + c.error_type = "result_json_corrupt" + c.reason = "result-json unparseable" + return c + if discrepancies: + c.error_type = "evidence_mismatch" + c.reason = "; ".join(discrepancies) + return c + + # Infrastructure failures (review#6): cleanup failure and vendor exception. + # These are more severe than game-level AI failures (which allow continue). + rj_dict = rj or {} + if rj_dict.get("cleanup_all_succeeded") is False: + c.error_type = "cleanup_failure" + c.reason = "process cleanup did not succeed (possible residual/orphan)" + return c + _rm_rc = rj_dict.get("run_match_returncode") + _vendor_exc = str(rj_dict.get("exception") or "") + if ("FileNotFoundError" in _vendor_exc and not rj_dict.get("end_info_received")): + c.error_type = "judge_start_path_error" + c.reason = f"judge startup path failure: {_vendor_exc}" + return c + if vendor_returncode != 0 or rj_dict.get("exception") or (_rm_rc is not None and _rm_rc != 0): + c.error_type = "vendor_exception" + c.reason = f"vendor runner exception (rc={vendor_returncode}, rj_rc={_rm_rc}, exc={rj_dict.get('exception')})" + return c + + end_received = bool(rj and rj.get("end_info_received")) + + # AI crash: trace ai_error, OR an AI natural-exited before end_info + if ts.ai_error_players: + c.error_type = "ai_crash" + c.ai_crash_player = ts.ai_error_players[0] + c.reason = f"trace ai_error (player {c.ai_crash_player})" + return c + if not end_received: + for role, idx in (("ai0", 0), ("ai1", 1)): + p = (rj or {}).get(role) or {} + if p.get("natural_exit") and not p.get("termination_requested"): + c.error_type = "ai_crash" + c.ai_crash_player = idx + c.reason = f"{role} exited naturally before end_info" + return c + + if ts.ai_timeout_players: + c.error_type = "ai_timeout" + c.ai_timeout_player = ts.ai_timeout_players[0] + c.reason = f"trace ai_timeout (player {c.ai_timeout_player})" + return c + + if not end_received: + c.error_type = "judge_crash" + c.judge_crash = True + c.reason = "no legal end_info produced" + return c + + if not ri.exists: + c.error_type = "replay_missing" + c.reason = "replay file absent" + return c + if not ri.header_valid: + c.error_type = "replay_corrupt" + c.reason = "replay header invalid (length/format/range)" + return c + + # valid candidate + raw = rj.get("raw_winner") + c.raw_winner = raw + c.winner_agent = agent_at(raw) if raw in (0, 1) else None + c.valid = True + c.error_type = None + if raw == evaluated_agent_camp: + c.normalized_result = "win" + elif raw in (0, 1): + c.normalized_result = "loss" + else: + c.normalized_result = "draw" + # post-end_info cleanup of idling AIs may yield nonzero returncodes; that's normal + for role in ("ai0", "ai1"): + p = (rj or {}).get(role) or {} + if p.get("termination_requested") and p.get("final_returncode") not in (0, None): + c.normal_cleanup_nonzero = True + return c + + +# --------------------------------------------------------------------------- # +# per-game attempt +# --------------------------------------------------------------------------- # +@dataclass +class MatchAttempt: + game_id: str + evaluated_agent: str + opponent: str + evaluated_agent_camp: int + valid: bool + normalized_result: str + error_type: Optional[str] + reason: str + raw_winner: Optional[int] + winner_agent: Optional[str] + scores: Optional[Dict[str, int]] + steps: int + realized_randomization: Optional[Dict[str, int]] + result_json_status: str + discrepancies: List[str] + ai_crash_player: Optional[int] + ai_timeout_player: Optional[int] + judge_crash: bool + wrapper_timeout: bool + normal_cleanup_nonzero: bool + evidence_paths: Dict[str, str] + collision_detected: bool + process_cleanup: List[dict] + vendor_returncode: int + started_at: float + finished_at: float + duration_s: float + exception: Optional[str] + judge_exit: Optional[int] = None + ai0_exit: Optional[int] = None + ai1_exit: Optional[int] = None + replay_sha256: Optional[str] = None + # ---- Section 4 first-hand process signals (one source of truth each) ---- # + #: raw ``timeout`` field from the result-json, verbatim. Vendor emits this as + #: ``{"ai0": bool, "ai1": bool}`` per-player flags, but we preserve whatever + #: the vendor wrote (never re-shape/flatten). + timeout: Optional[Any] = None + #: per-step Judge timeout actually passed to ``run_match_attempt`` (the + #: ``timeout`` kwarg). Distinct from ``timeout`` (the vendor's view). + timeout_s: Optional[float] = None + #: outer Popen / wrapper-layer exception (``repr(exc)`` or None); separate + #: from the vendor exception reported inside result-json. + wrapper_exception: Optional[str] = None + #: verbatim ``exception`` string from the result-json's own view; the vendor + #: subprocess saw an exception (or none) and recorded it. + vendor_exception: Optional[str] = None + #: internal ``run_match_returncode`` from the result-json (vendor side), + #: separate from the outer ``vendor_returncode`` (Popen.returncode of the + #: vendor script). + run_match_returncode: Optional[int] = None + + +def _merge_process_cleanup(vendor_manager_status: List[dict], + rj: Optional[dict]) -> List[dict]: + """Compose the unified ``process_cleanup`` list. Four sources, labelled: + + * vendor → ``source="process_tree_manager"`` (outer Popen we owned). + * judge → ``source="result_json"`` (vendor's inner Judge). + * ai0 → ``source="result_json"`` (vendor's inner AI #0). + * ai1 → ``source="result_json"`` (vendor's inner AI #1). + + The two exception sources must NOT overwrite each other; here they're just + appended in role-sorted order so downstream code reads both rows verbatim. + """ + rows: List[dict] = [] + for vrow in vendor_manager_status or []: + item = dict(vrow) + item.setdefault("role", "vendor") + item["source"] = "process_tree_manager" + rows.append(item) + _rj = rj or {} + for role in ("judge", "ai0", "ai1"): + v = _rj.get(role) + if isinstance(v, dict): + item = dict(v) + item.setdefault("role", role) + item["source"] = "result_json" + rows.append(item) + return rows + + +def _build_attempt_from_files(*, game_id: str, evaluated_agent: str, opponent: str, + evaluated_agent_camp: int, + work_dir, tag: str, + vendor_returncode: int, + wrapper_timeout: bool, + wrapper_exception: Optional[str], + vendor_manager_status: List[dict], + timeout_s: float, + started: float, finished: float, + collision_detected: bool = False) -> MatchAttempt: + """Build a fully classified MatchAttempt from on-disk result-json/trace/ + replay artifacts plus the wrapper-level signals (vendor_returncode, + wrapper_timeout, wrapper_exception, vendor_manager_status, ``timeout_s``). + + Pure post-subprocess construction: NO subprocess, NO Judge/AI invocations. + Used by ``run_match_attempt`` after the vendor subprocess has exited and by + end-to-end tests that drive a fake result-json through the full pipeline + (MatchAttempt → GameOutcome → to_event_record → JSON落盘重读). + """ + work_dir = Path(work_dir).expanduser().resolve() + result_json = work_dir / f"{tag}.result.json" + trace = work_dir / f"{tag}.jsonl" + replay = work_dir / f"{tag}.replay" + stdout_file = work_dir / f"{tag}.stdout" + stderr_file = work_dir / f"{tag}.stderr" + + rj_status, rj = load_result_json(result_json) + ts = stream_trace(trace) + ri = read_replay_info(replay) + discs = cross_validate(rj, ts, ri) if rj is not None else [] + c = classify(rj_status=rj_status, rj=rj, ts=ts, ri=ri, discrepancies=discs, + vendor_returncode=(vendor_returncode + if isinstance(vendor_returncode, int) else -1), + wrapper_timeout=wrapper_timeout, + evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent=evaluated_agent, opponent=opponent) + + _rj = rj or {} + judge_exit = (_rj.get("judge") or {}).get("final_returncode") + ai0_exit = (_rj.get("ai0") or {}).get("final_returncode") + ai1_exit = (_rj.get("ai1") or {}).get("final_returncode") + replay_sha = sha256_file(replay) if replay.exists() else None + # vendor_exception = verbatim result-json ``exception`` field + vendor_exception_field = _rj.get("exception") + # timeout = raw result-json ``timeout`` field, verbatim (may be dict) + timeout_field = _rj.get("timeout") + # internal run_match_returncode from the result-json + internal_rc = _rj.get("run_match_returncode") + + # ---- compat ``exception`` COMES FROM REAL EXCEPTIONS ONLY ---- + # Rule (deterministic, test-covered): + # wrapper_exception (outer Popen/包装异常) wins; + # else vendor_exception (verbatim result-json exception); + # else None. + # ``reason`` is NEVER used to fill ``exception``. + if wrapper_exception is not None: + compat_exception = wrapper_exception + elif vendor_exception_field is not None: + compat_exception = vendor_exception_field + else: + compat_exception = None + + process_cleanup = _merge_process_cleanup(vendor_manager_status, rj) + + return MatchAttempt( + game_id=game_id, evaluated_agent=evaluated_agent, opponent=opponent, + evaluated_agent_camp=evaluated_agent_camp, + valid=c.valid, normalized_result=c.normalized_result, + error_type=c.error_type, reason=c.reason, + raw_winner=c.raw_winner, winner_agent=c.winner_agent, + scores=(rj.get("scores") if rj else None), + steps=ts.n_ai_operation, + realized_randomization=( + {"map_type": ri.map_type, "day_time": ri.day_time} + if ri.header_valid else None + ), + result_json_status=rj_status, discrepancies=discs, + ai_crash_player=c.ai_crash_player, ai_timeout_player=c.ai_timeout_player, + judge_crash=c.judge_crash, wrapper_timeout=c.wrapper_timeout, + normal_cleanup_nonzero=c.normal_cleanup_nonzero, + evidence_paths={ + "stdout": str(stdout_file), "stderr": str(stderr_file), + "trace": str(trace), "replay": str(replay), + "result_json": str(result_json), + }, + collision_detected=collision_detected, + process_cleanup=process_cleanup, + vendor_returncode=(vendor_returncode if isinstance(vendor_returncode, int) else -1), + judge_exit=judge_exit, ai0_exit=ai0_exit, ai1_exit=ai1_exit, + replay_sha256=replay_sha, + started_at=started, finished_at=finished, + duration_s=round(finished - started, 3), + exception=compat_exception, + timeout=timeout_field, + timeout_s=timeout_s, + wrapper_exception=wrapper_exception, + vendor_exception=vendor_exception_field, + run_match_returncode=(internal_rc if isinstance(internal_rc, int) else None), + ) + + +def run_match_attempt(*, game_id: str, p0_dir, p1_dir, p0_name: str, p1_name: str, + judge_dir, work_dir, vendor_script, framework_src, + timeout: float = 12.0, wrapper_timeout_s: float = 60.0, + evaluated_agent_camp: int = 0, + evaluated_agent: str = "ifelse", opponent: str = "rank04", + extra_vendor_args: Optional[List[str]] = None, + extra_env: Optional[Dict[str, str]] = None, + python: Optional[str] = None) -> MatchAttempt: + """Run ONE game via the vendor runner and return a fully classified attempt. + + Does not mutate any aggregate. All evidence is preserved under work_dir. + The classification / field-packing logic lives in + :func:`_build_attempt_from_files`; this function owns the subprocess + invocation (vendor tree cleanup, wrapper timeout) and forwards the + wrapper-level signals (vendor_returncode, wrapper_timeout, + wrapper_exception, ProcessTreeManager status, per-step ``timeout``). + """ + # The vendor launches the Judge with ``cwd=judge_dir``. Every execution + # artifact therefore must be absolute before crossing this boundary: a + # relative replay path would otherwise be interpreted under Judge cwd. + work_dir = Path(work_dir).expanduser().resolve() + work_dir.mkdir(parents=True, exist_ok=True) + judge_dir = Path(judge_dir).expanduser().resolve() + p0_dir = Path(p0_dir).expanduser().resolve() + p1_dir = Path(p1_dir).expanduser().resolve() + vendor_script = Path(vendor_script).expanduser().resolve() + framework_src = Path(framework_src).expanduser().resolve() + python = python or sys.executable + + # collision handling: never clobber a prior attempt's evidence for this game_id + result_json = work_dir / f"{game_id}.result.json" + collision = result_json.exists() + tag = game_id + if collision: + i = 2 + while (work_dir / f"{game_id}__{i}.result.json").exists(): + i += 1 + tag = f"{game_id}__{i}" + result_json = work_dir / f"{tag}.result.json" + trace = work_dir / f"{tag}.jsonl" + replay = work_dir / f"{tag}.replay" + stdout_file = work_dir / f"{tag}.stdout" + stderr_file = work_dir / f"{tag}.stderr" + + cmd: List[str] = [ + python, "-u", str(vendor_script), + "--p0-dir", str(p0_dir), "--p1-dir", str(p1_dir), + "--p0-name", str(p0_name), "--p1-name", str(p1_name), + "--timeout", str(timeout), + "--out", str(work_dir), "--tag", tag, + "--result-json", str(result_json), + ] + if extra_vendor_args: + cmd.extend(extra_vendor_args) + + env = dict(os.environ) + env["MIRACLE_JUDGE_DIR"] = str(judge_dir) + env["MIRACLE_FRAMEWORK_SRC"] = str(framework_src) + if extra_env: + env.update(extra_env) + + started = time.time() + mgr = ProcessTreeManager() + out_fh = open(stdout_file, "wb") + err_fh = open(stderr_file, "wb") + wrapper_timeout = False + try: + proc = subprocess.Popen(cmd, shell=False, stdout=out_fh, stderr=err_fh, env=env) + mgr.register_popen(proc, "vendor") + try: + proc.communicate(timeout=wrapper_timeout_s) + rc = proc.returncode + except subprocess.TimeoutExpired: + wrapper_timeout = True + mgr.cleanup_all("wrapper-timeout") + try: + rc = proc.wait(timeout=5) + except Exception: + rc = proc.returncode + except Exception as exc: # noqa: BLE001 + rc = -1 + _exc = repr(exc) + else: + _exc = None + finally: + out_fh.close() + err_fh.close() + finished = time.time() + + return _build_attempt_from_files( + game_id=game_id, evaluated_agent=evaluated_agent, opponent=opponent, + evaluated_agent_camp=evaluated_agent_camp, + work_dir=work_dir, tag=tag, + vendor_returncode=(rc if isinstance(rc, int) else -1), + wrapper_timeout=wrapper_timeout, + wrapper_exception=_exc, + vendor_manager_status=mgr.status(), + timeout_s=timeout, + started=started, finished=finished, + collision_detected=collision, + ) diff --git a/src/agentbench_frame/games/miracle/paths.py b/src/agentbench_frame/games/miracle/paths.py new file mode 100644 index 0000000..707b50d --- /dev/null +++ b/src/agentbench_frame/games/miracle/paths.py @@ -0,0 +1,49 @@ +"""Environment-driven path resolution for the 24_miracle adapter tools/tests. + +No machine-specific absolute paths live in source. Callers set: + AGENTBENCH_ROOT - AgentBench corpus root (has backend_sources/ + top_algorithms/) + MIRACLE_IFELSE_DIR - 高翔 if-else bot directory (ifelse_bot/) + AGENTBENCH_RESULTS - AgentBenchResults repo root (optional; pipeline tests skip if unset) +Falls back with a clear error / None rather than embedding any local path. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Optional + + +def _required(env: str, what: str) -> Path: + v = os.environ.get(env) + if not v: + raise RuntimeError(f"env {env} not set; point it at the {what}") + return Path(v) + + +def agentbench_root() -> Path: + return _required("AGENTBENCH_ROOT", "AgentBench corpus root (backend_sources/ + top_algorithms/)") + + +def judge_dir() -> Path: + return agentbench_root() / "backend_sources" / "corpus" / "24_miracle" / "logic" / "judge_dev_logic" + + +def sample_ai_dir() -> Path: + return agentbench_root() / "backend_sources" / "corpus" / "24_miracle" / "logic" / "judge_dev_sample_ai" + + +def extracted_dir() -> Path: + return agentbench_root() / "top_algorithms" / "corpus" / "24_miracle_final" / "extracted" + + +def archives_dir() -> Path: + return agentbench_root() / "top_algorithms" / "corpus" / "24_miracle_final" / "archives" + + +def ifelse_dir() -> Path: + return _required("MIRACLE_IFELSE_DIR", "高翔 if-else bot directory (contains main.py)") + + +def results_repo() -> Optional[Path]: + v = os.environ.get("AGENTBENCH_RESULTS") + return Path(v) if v else None diff --git a/src/agentbench_frame/games/miracle/proctree.py b/src/agentbench_frame/games/miracle/proctree.py new file mode 100644 index 0000000..2510466 --- /dev/null +++ b/src/agentbench_frame/games/miracle/proctree.py @@ -0,0 +1,226 @@ +"""Cross-platform process-tree management for the Miracle subprocess wrapper. + +Safety contract (阶段4b spec): + * Only exact PIDs that this manager registered are ever signalled. + * Before any signal, the PID's psutil create_time must match the value + recorded at registration. A reused / stale PID is NEVER killed. + * Graceful terminate first, then a short grace window (polled, not slept), + then force-kill the tree (descendants included) only if still alive. + * No name-based / fuzzy kill: a process is never selected by its image + name, and signalling is never issued in bulk by interpreter or script + name. All signalling targets exact registered PIDs via psutil; descendant + discovery uses psutil's own process parentage, which is identity-confirming. + * Idempotent: cleaning an already-dead PID is a silent no-op. + +Per-process bookkeeping distinguishes a natural exit from a runner-requested +termination, so the adapter never mistakes a post-end_info cleanup-kill of an +idling AI client for a strategy crash (see match_runner.py classification). +""" +from __future__ import annotations + +import subprocess +import time +from dataclasses import dataclass +from typing import Dict, List, Optional + +try: + import psutil +except ImportError as _exc: # pragma: no cover - exercised via subprocess in tests + raise ImportError( + "Miracle process-tree cleanup requires psutil.\n" + "Install with: uv sync --extra miracle (or: pip install psutil)" + ) from _exc + +#: tolerance (seconds) for create_time comparison when confirming PID identity. +IDENTITY_TOL_S = 1.0 + + +@dataclass +class ManagedProcess: + pid: int + role: str + started_at: float # psutil create_time captured at registration + popen: Optional[object] = None # subprocess.Popen when we own the process + natural_exit: bool = False + natural_returncode: Optional[int] = None + termination_requested: bool = False + termination_reason: Optional[str] = None + final_returncode: Optional[int] = None + forced_kill: bool = False + cleanup_succeeded: bool = False + identity_confirmed: bool = True + wait_error: Optional[str] = None # create_time matched at the moment we acted + + +# --------------------------------------------------------------------------- # +# low-level helpers (all exact-PID, psutil-based) +# --------------------------------------------------------------------------- # +def _create_time(pid: int) -> Optional[float]: + try: + return psutil.Process(pid).create_time() + except (psutil.NoSuchProcess, psutil.AccessDenied): + return None + + +def _identity_ok(pid: int, started_at: float) -> bool: + """True if `pid` currently belongs to the same process recorded at `started_at`.""" + ct = _create_time(pid) + if ct is None: + return False + return abs(ct - started_at) < IDENTITY_TOL_S + + +def _exists(pid: int) -> bool: + return psutil.pid_exists(pid) + + +def _wait_for_exit(pid: int, timeout: float, step: float = 0.03) -> bool: + """Poll until `pid` is gone or `timeout` elapses. Returns True if it died.""" + deadline = time.time() + timeout + while time.time() < deadline: + if not _exists(pid): + return True + time.sleep(step) + return not _exists(pid) + + +def _graceful_terminate(pid: int) -> None: + try: + psutil.Process(pid).terminate() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +def _collect_tree(root_pid: int): + """Return [root_pid, *descendant_pids] while the root is alive, so cleanup can + reach orphaned descendants even after the root exits. Descendants come from + psutil parentage (identity-confirming); selection is never by image name. + If the root is already gone, returns [root_pid] only.""" + try: + root = psutil.Process(root_pid) + kids = root.children(recursive=True) + return [root_pid] + [k.pid for k in kids] + except psutil.NoSuchProcess: + return [root_pid] + + +def _force_kill_one(pid: int) -> None: + """Force-kill a single EXACT pid (no descendant walk, no name match).""" + try: + psutil.Process(pid).kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +# --------------------------------------------------------------------------- # +# manager +# --------------------------------------------------------------------------- # +class ProcessTreeManager: + """Tracks exact PIDs and cleans them up with graceful -> force semantics. + + Use :meth:`register_popen` for processes you spawned directly and + :meth:`register_pid` for descendant PIDs you only learned about (e.g. read + from a child's stdout). Only registered PIDs are ever touched. + """ + + def __init__(self, grace_s: float = 2.0): + self.grace_s = grace_s + self._procs: Dict[int, ManagedProcess] = {} + + # ---- registration ---- + def register_popen(self, popen, role: str) -> ManagedProcess: + mp = ManagedProcess( + pid=popen.pid, role=role, + started_at=_create_time(popen.pid) or 0.0, popen=popen, + ) + self._procs[mp.pid] = mp + return mp + + def register_pid(self, pid: int, role: str) -> ManagedProcess: + mp = ManagedProcess(pid=pid, role=role, started_at=_create_time(pid) or 0.0) + self._procs[mp.pid] = mp + return mp + + # ---- liveness / refresh ---- + def is_alive(self, mp: ManagedProcess) -> bool: + return _identity_ok(mp.pid, mp.started_at) + + def poll(self) -> None: + for mp in self._procs.values(): + self._refresh(mp) + + def _refresh(self, mp: ManagedProcess) -> None: + if mp.natural_exit: + return + if mp.popen is not None: + rc = mp.popen.poll() + if rc is not None: + mp.natural_exit = True + mp.natural_returncode = rc + if mp.final_returncode is None: + mp.final_returncode = rc + elif not _identity_ok(mp.pid, mp.started_at): + # descendant we don't own has disappeared + mp.natural_exit = True + mp.natural_returncode = None + + # ---- cleanup ---- + def cleanup_one(self, mp: ManagedProcess, reason: str) -> ManagedProcess: + self._refresh(mp) + if mp.natural_exit: + mp.cleanup_succeeded = True + return mp + if not _identity_ok(mp.pid, mp.started_at): + # PID gone OR reused (create_time mismatch): never kill an unconfirmed PID + mp.identity_confirmed = False + mp.cleanup_succeeded = True + return mp + # live + identity-confirmed: request termination of the WHOLE tree. + # Collect descendants now, while the root is alive, so we can still reach + # them after the root exits (orphaned descendants would otherwise survive). + mp.identity_confirmed = True + mp.termination_requested = True + mp.termination_reason = reason + tree_pids = _collect_tree(mp.pid) + for p in tree_pids: + _graceful_terminate(p) + _wait_for_exit(mp.pid, self.grace_s) + survivors = [p for p in tree_pids if _exists(p)] + if survivors: + for p in survivors: + _force_kill_one(p) + mp.forced_kill = True + _wait_for_exit(mp.pid, max(self.grace_s, 1.0)) + # ALWAYS reap owned Popen BEFORE identity check (POSIX zombie stays in + # the process table until parent waits; without this, _identity_ok sees + # the zombie as alive → wrongly reports cleanup_failed) + if mp.popen is not None and mp.final_returncode is None: + try: + mp.final_returncode = mp.popen.wait(timeout=max(self.grace_s, 1.0)) + except subprocess.TimeoutExpired: + mp.wait_error = "TimeoutExpired (process did not exit after wait)" + except Exception as e: + mp.wait_error = repr(e) + # THEN check identity (after reap, zombie gone → PID freed → clean) + mp.cleanup_succeeded = not _identity_ok(mp.pid, mp.started_at) + return mp + + def cleanup_all(self, reason: str) -> List[ManagedProcess]: + return [self.cleanup_one(mp, reason) for mp in list(self._procs.values())] + + def status(self) -> List[dict]: + self.poll() + return [ + { + "pid": mp.pid, "role": mp.role, "started_at": mp.started_at, + "natural_exit": mp.natural_exit, + "natural_returncode": mp.natural_returncode, + "termination_requested": mp.termination_requested, + "termination_reason": mp.termination_reason, + "final_returncode": mp.final_returncode, + "forced_kill": mp.forced_kill, + "cleanup_succeeded": mp.cleanup_succeeded, + "identity_confirmed": mp.identity_confirmed, + } + for mp in self._procs.values() + ] diff --git a/src/agentbench_frame/games/miracle/result.py b/src/agentbench_frame/games/miracle/result.py new file mode 100644 index 0000000..c96f560 --- /dev/null +++ b/src/agentbench_frame/games/miracle/result.py @@ -0,0 +1,397 @@ +"""Pure result-normalization logic for the 24_miracle adapter. + +No subprocess, no framework imports — only the standard library. Fully +unit-testable. Semantics are pinned to: + +* Judge ``main.py`` (judge_dev_logic): ``end_info`` format, winner derivation, + crash/timeout handling, replay header layout. +* ``SKILL.md`` sections: 胜负归一化, h2h方向, total_steps定义, + AgentBenchResults数据契约. + +Keeping this module dependency-free means the contract tests run without the +Judge, the framework, or any external process. +""" +from __future__ import annotations + +import hashlib +import struct +from collections import Counter +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Sequence, Tuple + +# ---- normalized-result vocabulary (SKILL.md 胜负归一化) ---- +WIN = "win" +LOSS = "loss" +DRAW = "draw" +ERROR = "error" +#: results that count as 有效对局 (valid games). Errors are excluded. +VALID_RESULTS: Tuple[str, ...] = (WIN, LOSS, DRAW) + + +# --------------------------------------------------------------------------- # +# end_info parsing (Judge main.py:440-459) +# --------------------------------------------------------------------------- # +def derive_raw_winner(end_info: Optional[dict]) -> Optional[int]: + """Map the Judge's terminal ``end_info`` to a camp index. + + ``end_info = {"0": player0_score, "1": player1_score}`` (main.py:457) and + ``winner = 0 if score0 > score1 else 1`` (main.py:450). Ties are broken + toward player1 (main.py:448-449), so a decisive 0/1 is always produced when + the Judge finishes normally. Returns ``None`` when ``end_info`` is absent or + malformed — i.e. the Judge crashed or the match ended catastrophically. + """ + if not isinstance(end_info, dict) or "0" not in end_info or "1" not in end_info: + return None + try: + s0, s1 = int(end_info["0"]), int(end_info["1"]) + except (TypeError, ValueError): + return None + return 0 if s0 > s1 else 1 + + +def scores_from_end_info(end_info: Optional[dict]) -> Tuple[Optional[int], Optional[int]]: + """Return (player0_score, player1_score) from end_info, (None, None) if absent.""" + if not isinstance(end_info, dict): + return None, None + try: + return int(end_info.get("0")), int(end_info.get("1")) + except (TypeError, ValueError): + return None, None + + +# --------------------------------------------------------------------------- # +# per-game outcome +# --------------------------------------------------------------------------- # +@dataclass +class GameOutcome: + """Everything the adapter knows about one played game. + + ``evaluated_agent_camp`` is the player index (0/1) the evaluated agent sat + at for THIS game; side-swapping is expressed by flipping it across games. + """ + + game_id: str + evaluated_agent: str + opponent: str + evaluated_agent_camp: int + + # verdict inputs + raw_winner: Optional[int] = None # camp index 0/1, or None + score0: Optional[int] = None + score1: Optional[int] = None + ai_error_player: Optional[int] = None # camp index of an AI that crashed + ai_timeout_player: Optional[int] = None # camp index of an AI that timed out + judge_ok: bool = True # end_info present AND run_match exited 0 + replay_ok: bool = True + + # process-level signals (filled by the subprocess wrapper) + judge_exit: Optional[int] = None + ai0_exit: Optional[int] = None + ai1_exit: Optional[int] = None + process_cleanup: List[Dict[str, Any]] = field(default_factory=list) + exception: Optional[str] = None + timeout_s: Optional[float] = None + #: raw ``timeout`` from the result-json (vendor per-player flags shape). + timeout: Optional[Dict[str, Any]] = None + #: outer Popen / wrapper-layer exception; separate from vendor_exception. + wrapper_exception: Optional[str] = None + #: verbatim ``exception`` reported by the vendor inside the result-json. + vendor_exception: Optional[str] = None + #: internal ``run_match_returncode`` from the result-json; distinct from + #: outer ``vendor_returncode`` (which the wrapper observed). + run_match_returncode: Optional[int] = None + #: classification-flag: wrapper-level timeout drained the vendor tree. + wrapper_timeout: Optional[bool] = None + + # provenance / budget + realized_randomization: Optional[Dict[str, int]] = None # {map_type, day_time} from replay + steps: int = 0 # ai_operation count == environment steps + score_tie: bool = False # score0 == score1 (Judge resolves to player1) + judge_tiebreak_applied: bool = False # a tie was resolved to player1 by the Judge + reason: Optional[str] = None # classification reason (separate from raw exception) + error_type: Optional[str] = None # machine-readable error type (ai_crash, cleanup_failure, ...) + result_json_status: str = "ok" + vendor_returncode: Optional[int] = None + evidence_paths: Dict[str, str] = field(default_factory=dict) + duration_s: Optional[float] = None + started_at: Optional[float] = None + finished_at: Optional[float] = None + replay_path: Optional[str] = None + replay_sha256: Optional[str] = None + evaluated_source_sha256: Optional[str] = None + opponent_source_sha256: Optional[str] = None + is_resume: bool = False + is_rerun: bool = False + + # derived — filled by finalize() + winner_agent: Optional[str] = None + normalized_result: str = ERROR + valid: bool = False + draw: bool = False + + @property + def opponent_camp(self) -> int: + return 1 - self.evaluated_agent_camp + + def agent_at_camp(self, camp: int) -> str: + return self.evaluated_agent if camp == self.evaluated_agent_camp else self.opponent + + +def normalize(outcome: GameOutcome) -> str: + """Classify a game into ``{win, loss, draw, error}`` (SKILL.md 胜负归一化). + + Any AI crash, Judge crash, timeout, or missing replay => ``error`` and is + NOT counted as a capability win (rank03's opponent crash is exactly this). + The Judge never produces a draw (ties -> player1, main.py:448-449); DRAW is + retained only for protocol safety. + """ + if (not outcome.judge_ok) or (not outcome.replay_ok) or outcome.raw_winner is None: + return ERROR + if outcome.ai_error_player is not None or outcome.ai_timeout_player is not None: + return ERROR + if outcome.raw_winner not in (0, 1): + return DRAW + return WIN if outcome.raw_winner == outcome.evaluated_agent_camp else LOSS + + +def finalize(outcome: GameOutcome) -> GameOutcome: + """Fill derived fields in place and return the outcome.""" + outcome.normalized_result = normalize(outcome) + outcome.valid = outcome.normalized_result in VALID_RESULTS + outcome.draw = outcome.normalized_result == DRAW + outcome.winner_agent = ( + outcome.agent_at_camp(outcome.raw_winner) if outcome.raw_winner in (0, 1) else None + ) + outcome.score_tie = (outcome.score0 is not None and outcome.score1 is not None + and outcome.score0 == outcome.score1) + outcome.judge_tiebreak_applied = outcome.score_tie + return outcome + + +# --------------------------------------------------------------------------- # +# aggregates +# --------------------------------------------------------------------------- # +def compute_win_rate(outcomes: Sequence[GameOutcome]) -> float: + """``win_rate = 有效胜局数 / 有效对局数`` (SKILL.md). + + Draws are valid games but not wins, so they stay in the denominator; error + games are excluded entirely. Returns 0.0 when there are no valid games + (mirrors the framework Run's ``n = max(1, 0)`` behaviour). + """ + valid = [o for o in outcomes if o.normalized_result in VALID_RESULTS] + if not valid: + return 0.0 + wins = sum(1 for o in valid if o.normalized_result == WIN) + return wins / len(valid) + + +def compute_h2h(outcomes: Sequence[GameOutcome]) -> Dict[str, Dict[str, float]]: + """``h2h[row][col]`` = fraction of valid games between row and col that row + won (SKILL.md h2h方向: 行策略战胜列策略的胜率). + + Error games and games without a decisive winner are excluded. With draws, + ``h2h[row][col] + h2h[col][row]`` need not equal 1 (draws in denominator). + For Miracle (Judge never draws) the two entries over a pair sum to 1. + """ + games: Dict[Tuple[str, str], int] = {} + row_wins: Dict[Tuple[str, str], int] = {} + for o in outcomes: + if o.normalized_result == ERROR: + continue + a0, a1 = o.agent_at_camp(0), o.agent_at_camp(1) + # every valid game (win/loss/draw) counts toward the denominator + for pair in ((a0, a1), (a1, a0)): + games[pair] = games.get(pair, 0) + 1 + if o.raw_winner in (0, 1): # draws add to denominator but not to wins + winner = o.agent_at_camp(o.raw_winner) + loser = a1 if winner == a0 else a0 + row_wins[(winner, loser)] = row_wins.get((winner, loser), 0) + 1 + h2h: Dict[str, Dict[str, float]] = {} + for (row, col), n in games.items(): + h2h.setdefault(row, {})[col] = row_wins.get((row, col), 0) / n + return h2h + + +def outcome_counts(outcomes: Sequence[GameOutcome]) -> Dict[str, int]: + """Tally {win, loss, draw, error} counts (every key always present).""" + c = Counter(o.normalized_result for o in outcomes) + return {k: int(c.get(k, 0)) for k in (WIN, LOSS, DRAW, ERROR)} + + +# --------------------------------------------------------------------------- # +# run-level statistics (written into summary.json by MiracleEvalRunner) +# --------------------------------------------------------------------------- # +#: No deterministic seed support this round: the Judge draws map_type/day_time +#: via random.randint and reads no external seed. map_type/day_time are the +#: realized random environment parameters, NOT a seed. +DETERMINISTIC_SEED_SUPPORTED = False + + +def build_seed_provenance(realized_randomization: Optional[Dict[str, int]]) -> Dict[str, Any]: + """Build the per-game seed-provenance record. The Judge supports no seed, so + requested/effective seed are null; realized_randomization records the actual + random environment parameters (map_type/day_time) from the replay.""" + return { + "requested_seed": None, + "effective_seed": None, + "deterministic_seed_supported": DETERMINISTIC_SEED_SUPPORTED, + "reproducible_from_seed": False, + "realized_randomization": realized_randomization, + } + + +def compute_run_stats(outcomes: Sequence[GameOutcome]) -> Dict[str, Any]: + """Run-level statistics for summary.json. + + * attempted_games : every game whose attempt was recorded + * valid_games : games with a decisive, evidence-consistent result + * win_rate_denominator == valid_games; win_rate = wins / valid_games + * total_steps counts only valid games; attempted_steps counts all attempts + * win_rate is None and evaluation_status is NO_VALID_GAMES when valid_games == 0 + (we never report a fabricated 0% strength conclusion). + """ + attempted = len(outcomes) + valid = [o for o in outcomes if o.normalized_result in VALID_RESULTS] + valid_games = len(valid) + wins = sum(1 for o in valid if o.normalized_result == WIN) + losses = sum(1 for o in valid if o.normalized_result == LOSS) + draws = sum(1 for o in valid if o.normalized_result == DRAW) + attempted_steps = sum(int(getattr(o, "steps", 0) or 0) for o in outcomes) + total_steps = sum(int(getattr(o, "steps", 0) or 0) for o in valid) + return { + "attempted_games": attempted, + "valid_games": valid_games, + "invalid_games": attempted - valid_games, + "wins": wins, + "losses": losses, + "draws": draws, + "win_rate_denominator": valid_games, + "attempted_steps": attempted_steps, + "total_steps": total_steps, + "win_rate": (wins / valid_games) if valid_games > 0 else None, + "evaluation_status": ("COMPLETE" if valid_games > 0 else "NO_VALID_GAMES"), + } + + +# --------------------------------------------------------------------------- # +# replay / file hashing +# --------------------------------------------------------------------------- # +def read_replay_header(path) -> Optional[Dict[str, int]]: + """Read the actual random environment parameters from a Miracle replay file. + + The Judge writes the replay as big-endian signed int32s; the first 7 ints + are ``[0, 0, 0, map_type, day_time, 0, 0]`` (main.py:310-313). The Judge + draws ``map_type``/``day_time`` via ``random.randint`` (main.py:85-86) and + reads no external seed, so these are NOT a seed — they are the realized + random environment parameters recorded in the replay. Returns None if the + file is missing or too short. + """ + try: + with open(path, "rb") as f: + head = f.read(28) + except OSError: + return None + if len(head) < 28: + return None + try: + vals = struct.unpack(">7i", head) + except struct.error: + return None + return {"map_type": int(vals[3]), "day_time": int(vals[4])} + + +def sha256_file(path, chunk: int = 1 << 20) -> Optional[str]: + """SHA-256 of a file, or None if unreadable.""" + try: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(chunk), b""): + h.update(block) + return h.hexdigest() + except OSError: + return None + + +# --------------------------------------------------------------------------- # +# resumability (SKILL.md: 可恢复但不重复成功对局) +# --------------------------------------------------------------------------- # +def select_games_to_run(planned_ids: Sequence[str], + completed_valid_ids: Sequence[str]) -> List[str]: + """Return planned game ids that still need to run, i.e. those without an + existing valid (successful) recorded result. Order preserved.""" + done = set(completed_valid_ids) + return [gid for gid in planned_ids if gid not in done] + + +def would_rerun_successful(game_id: str, completed_valid_ids: Sequence[str]) -> bool: + """True if ``game_id`` already has a valid result and would be re-run.""" + return game_id in set(completed_valid_ids) + + +# --------------------------------------------------------------------------- # +# event-record builder (SKILL.md events.jsonl per-game fields) +# --------------------------------------------------------------------------- # +#: fields SKILL.md requires in each per-game event. Used by tests to assert +#: the record is contract-complete. +REQUIRED_EVENT_FIELDS = ( + "game_id", "seed", "policy_ids", "policy_source_sha256", "camps", + "raw_winner", "winner_agent", "normalized_result", "scores", "draw", + "started_at", "finished_at", "duration", "judge_exit", "ai0_exit", "ai1_exit", + "process_cleanup", "timeout_s", "exception", "replay_path", "replay_sha256", + "valid", "is_resume", "is_rerun", +) + + +def to_event_record(o: GameOutcome) -> Dict: + """Build the per-game event dict for ``Run.write("game", **...)``. + + Covers every SKILL.md events.jsonl per-game field; ``REQUIRED_EVENT_FIELDS`` + lists the contract keys so tests can assert completeness. + """ + return { + "event": "game", + "game_id": o.game_id, + "seed": build_seed_provenance(o.realized_randomization), + "score_tie": o.score_tie, + "judge_tiebreak_applied": o.judge_tiebreak_applied, + "evaluated_agent": o.evaluated_agent, + "opponent": o.opponent, + "evaluated_agent_camp": o.evaluated_agent_camp, + "policy_ids": [o.evaluated_agent, o.opponent], + "policy_source_sha256": [o.evaluated_source_sha256, o.opponent_source_sha256], + "camps": [0, 1], + "raw_winner": o.raw_winner, + "winner_agent": o.winner_agent, + "normalized_result": o.normalized_result, + "scores": {"0": o.score0, "1": o.score1}, + "draw": o.draw, + "valid": o.valid, + "started_at": o.started_at, + "finished_at": o.finished_at, + "duration": o.duration_s, + "judge_exit": o.judge_exit, + "ai0_exit": o.ai0_exit, + "ai1_exit": o.ai1_exit, + "process_cleanup": o.process_cleanup, + "timeout_s": o.timeout_s, + "timeout": o.timeout, + "wrapper_exception": o.wrapper_exception, + "vendor_exception": o.vendor_exception, + "run_match_returncode": o.run_match_returncode, + "wrapper_timeout": o.wrapper_timeout, + "exception": o.exception, + "ai_error_player": o.ai_error_player, + "ai_timeout_player": o.ai_timeout_player, + "judge_ok": o.judge_ok, + "replay_ok": o.replay_ok, + "steps": o.steps, + "replay_path": o.replay_path, + "replay_sha256": o.replay_sha256, + "reason": o.reason, + "error_type": o.error_type, + "result_json_status": o.result_json_status, + "vendor_returncode": o.vendor_returncode, + "evidence_paths": o.evidence_paths, + "is_resume": o.is_resume, + "is_rerun": o.is_rerun, + } diff --git a/src/agentbench_frame/games/miracle/runner.py b/src/agentbench_frame/games/miracle/runner.py new file mode 100644 index 0000000..ef9e8e8 --- /dev/null +++ b/src/agentbench_frame/games/miracle/runner.py @@ -0,0 +1,214 @@ +"""MiracleEvalRunner — top-level evaluation driver for 24_miracle. + +Does NOT inherit ``BaseRunner`` (that path has the win-attribution / run_type / +summary-ordering bugs — risks #2/#3/#4/#5/#6). Instead it drives ``Run`` directly: + + Run.start(game="24_miracle", agent=…, run_type="eval", data_dir=) + → run each game via match_runner.run_match_attempt (side-swapped) + → map each MatchAttempt to a GameOutcome (match_runner's classification wins) + → feed_outcomes_to_run (events for ALL attempts; log_episode for valid only) + → log_h2h + → Run.finish() + → atomically enrich summary.json with run-level statistics + → re-read disk summary + independently recompute from events.jsonl, assert equal + +When ``valid_games == 0`` the persisted ``win_rate`` is ``null`` and +``evaluation_status`` is ``NO_VALID_GAMES`` (never a fabricated 0% conclusion). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from agentbench_frame.games.miracle.driver import feed_outcomes_to_run +from agentbench_frame.games.miracle.match_runner import MatchAttempt, run_match_attempt +from agentbench_frame.games.miracle.result import ( + DRAW, + VALID_RESULTS, + GameOutcome, + compute_h2h, + compute_run_stats, +) +from agentbench_frame.tracking.run import Run + +GAME_ID = "24_miracle" + + +def attempt_to_outcome(att: MatchAttempt) -> GameOutcome: + """Carry a MatchAttempt into a GameOutcome. match_runner.classify sees the + full timeline (end_info order, cleanup vs crash, evidence consistency) so its + normalized_result/valid are authoritative here — we do not re-run normalize.""" + scores = att.scores or {} + o = GameOutcome( + game_id=att.game_id, + evaluated_agent=att.evaluated_agent, + opponent=att.opponent, + evaluated_agent_camp=att.evaluated_agent_camp, + raw_winner=att.raw_winner, + score0=scores.get("0") if isinstance(scores, dict) else None, + score1=scores.get("1") if isinstance(scores, dict) else None, + ai_error_player=att.ai_crash_player, + ai_timeout_player=att.ai_timeout_player, + judge_ok=(att.result_json_status == "ok" and not att.judge_crash + and not att.wrapper_timeout), + replay_ok=(att.realized_randomization is not None), + realized_randomization=att.realized_randomization, + steps=att.steps, + duration_s=att.duration_s, + started_at=att.started_at, + finished_at=att.finished_at, + replay_path=att.evidence_paths.get("replay"), + is_resume=att.collision_detected, + judge_exit=att.judge_exit, + ai0_exit=att.ai0_exit, + ai1_exit=att.ai1_exit, + process_cleanup=att.process_cleanup, + # Section 4: compat ``exception`` carries REAL exceptions only + # (wrapper_exception OR vendor_exception), NEVER the classification + # ``reason``. The rule lives in match_runner._build_attempt_from_files; + # here we just forward the deterministic value verbatim. + exception=att.exception, + timeout=getattr(att, "timeout", None), + timeout_s=getattr(att, "timeout_s", None), + wrapper_exception=getattr(att, "wrapper_exception", None), + vendor_exception=getattr(att, "vendor_exception", None), + run_match_returncode=getattr(att, "run_match_returncode", None), + wrapper_timeout=getattr(att, "wrapper_timeout", None), + ) + o.normalized_result = att.normalized_result + o.valid = att.valid + o.draw = (att.normalized_result == DRAW) + o.winner_agent = att.winner_agent + o.reason = att.reason + o.error_type = att.error_type + o.result_json_status = att.result_json_status + o.vendor_returncode = att.vendor_returncode + o.evidence_paths = att.evidence_paths + o.replay_sha256 = getattr(att, "replay_sha256", None) + o.score_tie = (o.score0 is not None and o.score1 is not None and o.score0 == o.score1) + o.judge_tiebreak_applied = o.score_tie + return o + + +def enrich_summary_atomically(run_dir: Path, outcomes: List[GameOutcome]) -> Dict[str, Any]: + """Merge run-level statistics into summary.json with an atomic temp+replace.""" + summary_path = run_dir / "summary.json" + summary = json.loads(summary_path.read_text(encoding="utf-8")) + stats = compute_run_stats(outcomes) + summary.update(stats) + summary["win_rate"] = stats["win_rate"] # None when valid_games == 0 + summary["win_rate_available"] = (stats["valid_games"] > 0) + summary["h2h"] = compute_h2h(outcomes) + from agentbench_frame.games.miracle.atomicio import atomic_write_json + atomic_write_json(summary_path, summary) + return summary + + +def recompute_from_events(run_dir: Path) -> Dict[str, Any]: + """Independently recompute win_rate / counts straight from events.jsonl.""" + games: List[dict] = [] + p = run_dir / "events.jsonl" + if p.exists(): + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + if e.get("event") == "game": + games.append(e) + valid = [g for g in games if g.get("normalized_result") in VALID_RESULTS] + wins = sum(1 for g in valid if g.get("normalized_result") == "win") + return { + "attempted_games": len(games), + "valid_games": len(valid), + "win_rate": (wins / len(valid)) if valid else None, + } + + +class MiracleEvalRunner: + def __init__(self, *, agent: str, data_dir: str, judge_dir, vendor_script, + framework_src, evaluated_dir, opponent_dir, n_games: int, + opponent: str = "opponent", + timeout: float = 12.0, wrapper_timeout_s: float = 60.0, work_dir, + config: Optional[Dict[str, Any]] = None, + attempt_fn: Optional[Callable] = None, prefix: str = "smoke"): + self.agent = agent + self.data_dir = data_dir + self.judge_dir = judge_dir + self.vendor_script = vendor_script + self.framework_src = framework_src + self.evaluated_dir = evaluated_dir + self.opponent_dir = opponent_dir + self.n_games = n_games + self.opponent = opponent + self.timeout = timeout + self.wrapper_timeout_s = wrapper_timeout_s + self.work_dir = work_dir + self.config = config or {} + self.prefix = prefix + self.attempt_fn = attempt_fn or self._default_attempt_fn + + def _default_attempt_fn(self, *, game_id, evaluated_agent_camp, + evaluated_agent, opponent, **_): + # TRUE side-swap: the evaluated agent is player0 on camp0 games and + # player1 on camp1 games, so the Judge sees both camp assignments. + if evaluated_agent_camp == 0: + p0_dir, p1_dir = self.evaluated_dir, self.opponent_dir + p0_name, p1_name = evaluated_agent, opponent + else: + p0_dir, p1_dir = self.opponent_dir, self.evaluated_dir + p0_name, p1_name = opponent, evaluated_agent + return run_match_attempt( + game_id=game_id, p0_dir=p0_dir, p1_dir=p1_dir, + p0_name=p0_name, p1_name=p1_name, + judge_dir=self.judge_dir, work_dir=self.work_dir, + vendor_script=self.vendor_script, framework_src=self.framework_src, + timeout=self.timeout, wrapper_timeout_s=self.wrapper_timeout_s, + evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent=evaluated_agent, opponent=opponent, + ) + + def run(self) -> Dict[str, Any]: + run = Run.start(game=GAME_ID, agent=self.agent, run_type="eval", + data_dir=self.data_dir, config=self.config) + outcomes: List[GameOutcome] = [] + attempts: List[MatchAttempt] = [] + for i in range(self.n_games): + eval_camp = i % 2 # alternate sides + game_id = f"{self.prefix}_{i:02d}_camp{eval_camp}" + att = self.attempt_fn( + game_id=game_id, evaluated_agent_camp=eval_camp, + evaluated_agent=self.agent, opponent=self.opponent, + ) + attempts.append(att) + outcomes.append(attempt_to_outcome(att)) + self.attempts = attempts + + feed_outcomes_to_run(run, outcomes) + run.finish() + + run_dir = Path(run.run_dir) + summary = enrich_summary_atomically(run_dir, outcomes) + + # cross-check: re-read disk summary and recompute from events independently + disk = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + recompute = recompute_from_events(run_dir) + wr_disk, wr_recompute = disk.get("win_rate"), recompute["win_rate"] + if wr_disk is None or wr_recompute is None: + assert wr_disk is wr_recompute, f"win_rate None-mismatch: {wr_disk} vs {wr_recompute}" + else: + assert abs(wr_disk - wr_recompute) < 1e-9, \ + f"win_rate mismatch: disk={wr_disk} recompute={wr_recompute}" + assert disk["total_episodes"] == recompute["valid_games"] + # guard against the double-runs path bug (risk #6): run_dir must be + # /runs/24_miracle//, never .../runs/runs/... + rel = run_dir.relative_to(self.data_dir) + assert rel.parts == ("runs", GAME_ID, self.agent, run.run_id), \ + f"unexpected run path shape (risk #6): {rel}" + summary["_recompute_check"] = recompute + return summary diff --git a/src/agentbench_frame/games/miracle/smoke_audit.py b/src/agentbench_frame/games/miracle/smoke_audit.py new file mode 100644 index 0000000..13dcd20 --- /dev/null +++ b/src/agentbench_frame/games/miracle/smoke_audit.py @@ -0,0 +1,197 @@ +"""Evidence-safety helpers for the 24_miracle smoke driver. + +Pure + unit-testable: session identity (never delete/overwrite), a STRICT +Group-1 gate, an INDEPENDENT residual-process check (exact PID + psutil +create_time — never trusts ``cleanup_succeeded``, never kills by name, never +touches unrelated processes), and manifest building. No subprocess execution +lives here; that stays in ``tools/miracle_smoke.py``. +""" +from __future__ import annotations + +import hashlib +import json +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +try: + import psutil +except ImportError as _exc: # pragma: no cover + raise ImportError( + "Miracle smoke residual check requires psutil.\n" + "Install with: uv sync --extra miracle (or: pip install psutil)" + ) from _exc + +IDENTITY_TOL_S = 1.0 + + +@dataclass +class ManagedProc: + pid: int + started_at: float # psutil create_time captured when the process was spawned + role: str + + +# --------------------------------------------------------------------------- # +# session identity +# --------------------------------------------------------------------------- # +def make_session_id() -> str: + """A fresh, effectively-unique session id (timestamp + token).""" + import datetime + import secrets + return datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "_" + secrets.token_hex(3) + + +def session_exists(root, session_id: str) -> bool: + return (Path(root) / session_id).exists() + + +def ensure_fresh_session(root, session_id: str) -> Path: + """Create and return a brand-new session dir. REFUSE (raise) if it already + exists — this module NEVER deletes or overwrites a prior session.""" + root = Path(root) + sd = root / session_id + if sd.exists(): + raise FileExistsError(f"session already exists; refusing to overwrite: {sd}") + root.mkdir(parents=True, exist_ok=True) + sd.mkdir(parents=True) + return sd + + +# --------------------------------------------------------------------------- # +# hashing + manifest +# --------------------------------------------------------------------------- # +def sha256_file(path) -> Optional[str]: + try: + h = hashlib.sha256() + with open(path, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + except OSError: + return None + + +def build_manifest(*, session_id: str, auth_cap: int, python_executable: str, + python_version: str, code_files, asset_files, + groups_planned: List[Dict[str, Any]], + notes: Optional[List[str]] = None) -> Dict[str, Any]: + return { + "session_id": session_id, + "created_unix": time.time(), + "auth_cap_games": auth_cap, + "python_executable": python_executable, + "python_version": python_version, + "code_hashes": {str(p): sha256_file(p) for p in code_files}, + "asset_hashes": {str(p): sha256_file(p) for p in asset_files}, + "groups_planned": groups_planned, + "notes": notes or [], + } + + +def write_manifest_atomic(session_dir, manifest: Dict[str, Any]) -> Path: + p = Path(session_dir) / "manifest.json" + from agentbench_frame.games.miracle.atomicio import atomic_write_json + return atomic_write_json(p, manifest) + + +# --------------------------------------------------------------------------- # +# INDEPENDENT residual-process check (exact PID + create_time identity) +# --------------------------------------------------------------------------- # +def load_managed_procs_from_result_json(path) -> List[ManagedProc]: + """Independently read a result-json and extract Judge/AI0/AI1 with their + PID + create_time identity. Returns [] if the file is missing/corrupt or has + no usable identity (callers must treat [] as 'cannot verify', never 'clean').""" + try: + rj = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return [] + out: List[ManagedProc] = [] + for role in ("judge", "ai0", "ai1"): + p = rj.get(role) or {} + pid = p.get("pid") + started = p.get("started_at") + if isinstance(pid, int) and pid > 0 and isinstance(started, (int, float)): + out.append(ManagedProc(pid=pid, started_at=float(started), role=role)) + return out + + +def check_residual_procs(procs: List[ManagedProc], + tol: float = IDENTITY_TOL_S) -> Dict[str, List[ManagedProc]]: + """For each recorded proc: + pid gone -> clean + pid exists, create_time matches -> RESIDUAL (real leftover) + pid exists, create_time differs -> reused (PID reuse; DO NOT kill) + No name matching, no batch kill, unrelated processes are never touched.""" + clean, residual, reused = [], [], [] + for mp in procs: + if not psutil.pid_exists(mp.pid): + clean.append(mp) + continue + try: + ct = psutil.Process(mp.pid).create_time() + except psutil.NoSuchProcess: + clean.append(mp) + continue + if abs(ct - mp.started_at) < tol: + residual.append(mp) + else: + reused.append(mp) + return {"clean": clean, "residual": residual, "reused": reused} + + +# --------------------------------------------------------------------------- # +# STRICT Group-1 gate +# --------------------------------------------------------------------------- # +def _att_reasons(att) -> List[str]: + r = [] + gid = getattr(att, "game_id", "?") + if not getattr(att, "valid", False): + r.append(f"{gid}: valid!=true (normalized={getattr(att,'normalized_result',None)})") + if getattr(att, "error_type", None): + r.append(f"{gid}: error_type={att.error_type} reason={getattr(att,'reason','')}") + if getattr(att, "wrapper_timeout", False): + r.append(f"{gid}: wrapper_timeout") + if getattr(att, "result_json_status", None) != "ok": + r.append(f"{gid}: result_json_status={att.result_json_status}") + if getattr(att, "discrepancies", None): + r.append(f"{gid}: evidence discrepancies={att.discrepancies}") + if not getattr(att, "realized_randomization", None): + r.append(f"{gid}: realized_randomization missing (replay not parseable)") + if getattr(att, "raw_winner", None) not in (0, 1): + r.append(f"{gid}: raw_winner not decisive ({att.raw_winner})") + return r + + +def group1_strict_clean(attempts, summary) -> Tuple[bool, List[str]]: + """Strict Group-1 gate. ANY anomaly blocks Group 2. An empty/missing invalid + set or missing process-identity can NEVER pass this gate.""" + reasons: List[str] = [] + if len(attempts) != 2: + reasons.append(f"attempt_count={len(attempts)} != 2") + for att in attempts: + reasons.extend(_att_reasons(att)) + # independent residual check — requires non-empty identity, never an empty free-pass + rj_path = getattr(att, "evidence_paths", {}).get("result_json") if hasattr(att, "evidence_paths") else None + procs = load_managed_procs_from_result_json(rj_path) if rj_path else [] + if not procs: + reasons.append(f"{att.game_id}: no managed-proc identity in result-json (cannot verify by empty set)") + else: + res = check_residual_procs(procs) + if res["residual"]: + reasons.append(f"{att.game_id}: RESIDUAL pids={[(p.pid, p.role) for p in res['residual']]}") + s = summary or {} + for k, want in (("attempted_games", 2), ("valid_games", 2), ("invalid_games", 0)): + if s.get(k) != want: + reasons.append(f"summary.{k}={s.get(k)} != {want}") + # events/summary recompute consistency (summary carries the runner's own check) + if s.get("evaluation_status") != "COMPLETE": + reasons.append(f"summary.evaluation_status={s.get('evaluation_status')} != COMPLETE") + return (len(reasons) == 0, reasons) + + +def should_run_group2(g1_ok: bool) -> bool: + """Group 2 is run ONLY when Group 1 is fully clean.""" + return bool(g1_ok) diff --git a/src/agentbench_frame/tracking/run.py b/src/agentbench_frame/tracking/run.py index 6329d54..e8e03cc 100644 --- a/src/agentbench_frame/tracking/run.py +++ b/src/agentbench_frame/tracking/run.py @@ -60,7 +60,12 @@ class Run: def start(cls, game: str, agent: str, run_type: str = "eval", data_dir: Optional[str] = None, - config: Optional[Dict[str, Any]] = None) -> "Run": + config: Optional[Dict[str, Any]] = None, + run_id: Optional[str] = None, + append: bool = True, + created: Optional[str] = None, + git_commit: Optional[str] = None, + started_at: Optional[float] = None) -> "Run": """Create a Run. Args: @@ -69,9 +74,36 @@ def start(cls, game: str, agent: str, run_type: "rl" | "rule_iter" | "eval" data_dir: Override data root (default: $AGENTBENCH_DATA or ./agentbench_data) config: Arbitrary config dict written to run.toml [config] section + run_id: Optional caller-supplied run_id (review #1 backcompat). When + provided (e.g. ``20260722-001929_52bd14`` for matrix resume or + offline migration), the run directory reuses that id verbatim + — no second run directory is ever created. When omitted, the + framework auto-generates one via :meth:`_make_run_id`. + append: If True (default), the events.jsonl writer opens in + append mode so multiple ``Run`` objects on the same run_id + preserve history. If False, the events.jsonl is truncated at + start — used by idempotent rebuild paths (review #1 matrix + ``write_run_compatible_output``) so re-invocations do not + create duplicate event lines. + created: Optional ISO-8601 timestamp string (e.g. + ``2026-07-21T16:19:30Z``) for offline migration. When + provided, the run's ``created`` metadata (and the persisted + run.toml + summary.json ``created`` field) reflect this + authoritative evidence timestamp instead of the framework's + "now". When omitted, the current wall clock is used + (default behaviour). Must be a valid ISO-8601 string. + git_commit: Optional git commit short-SHA, overriding the + framework's auto-detected ``git rev-parse --short HEAD``. + Used in offline migration where the migrated run should carry + the original commit identity. """ data_dir = data_dir or _data_root() - run_id = cls._make_run_id() + if run_id is None or not isinstance(run_id, str) or not run_id: + run_id = cls._make_run_id() + if created is None or not isinstance(created, str) or not created: + created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + if git_commit is None or not isinstance(git_commit, str): + git_commit = _git_commit() # CI-expected structure: runs/{game}/{agent}/{run_id}/ run_path = os.path.join(data_dir, "runs", game, agent, run_id) os.makedirs(run_path, exist_ok=True) @@ -79,13 +111,13 @@ def start(cls, game: str, agent: str, meta = RunMeta( run_id=run_id, game=game, agent=agent, run_type=run_type, - created=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - git_commit=_git_commit(), - started_at=time.time(), + created=created, + git_commit=git_commit, + started_at=(started_at if started_at is not None else time.time()), config=config or {}, ) - writer = JSONLWriter(os.path.join(run_path, "events.jsonl")) + writer = JSONLWriter(os.path.join(run_path, "events.jsonl"), append=append) return cls(run_id=run_id, run_dir=run_path, meta=meta, writer=writer, config=config or {}) @staticmethod @@ -415,16 +447,94 @@ def log_h2h(self, h2h: Dict[str, Dict[str, float]]): # ---- finish ---- - def finish(self, extra_summary: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + def recompute_totals_from_events(self, *, + game_event_type: str = "game") -> Dict[str, Any]: + """Recompute ``_episodes`` / ``_total_steps`` / win-loss-draw counts + from the live ``game`` events written to disk by callers that bypass + :meth:`log_episode` (e.g. matrix re-emit, offline migration). Returns + the recalculated counters as a dict. + + Why this is needed (review #1 §5): matrix records ARE per-game events + emitted via ``run.write("game", ...)``; framework counters + (``self._episodes`` / ``self._total_steps``) are only incremented by + :meth:`log_episode`. Without this recomputation, ``_write_toml`` / + ``_build_summary`` would report zero totals even though the events + file on disk has the real per-game records. We refuse to fabricate + episode events; matrix events ARE the events. + + Semantics: + + - reads only events of ``event_type == game_event_type`` (default + ``"game"``) — game records that carry ``valid``, + ``normalized_result`` (win/loss/draw/error), ``steps``; + - increments ``_episodes`` only for valid game records (matches + :meth:`log_episode`'s ``valid=True`` semantics + matches the + Miracle aggregate's ``valid_games`` count); + - increments ``_total_steps`` by per-game steps for valid games; + - DOES NOT mutate ``_episode_rewards`` / ``_episode_winners`` / + ``_episode_agent_players`` — these bookkeeping lists are only + populated by :meth:`log_episode`, and matrix migration has no + meaningful reward / per-episode winner representation. summary + ``win_rate`` / ``wins`` / ``losses`` / ``draws`` are still set + by ``extra_summary`` passed to :meth:`finish` for matrix paths. + + Returns a dict with the recomputed counters so callers (matrix + migration) can sanity-check totals are consistent with the hard + anchors BEFORE the run is committed/promoted. + """ + # Flush buffered writes before reading. + self.writer.flush() + path = os.path.join(self.run_dir, "events.jsonl") + recomputed = dict(episodes=0, total_steps=0, wins=0, losses=0, + draws=0, valid_games=0) + if not os.path.exists(path): + return recomputed + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if not isinstance(rec, dict): + continue + if rec.get("event_type", rec.get("event")) != game_event_type: + continue + valid = bool(rec.get("valid")) + if not valid: + continue + recomputed["valid_games"] += 1 + recomputed["episodes"] += 1 + recomputed["total_steps"] += int(rec.get("steps") or 0) + nr = rec.get("normalized_result") + if nr == "win": + recomputed["wins"] += 1 + elif nr == "loss": + recomputed["losses"] += 1 + elif nr == "draw": + recomputed["draws"] += 1 + # Mutate framework counters in place so _build_summary / _write_toml + # see the recomputed totals. + self._episodes = recomputed["episodes"] + self._total_steps = recomputed["total_steps"] + return recomputed + + def finish(self, extra_summary: Optional[Dict[str, Any]] = None, + finished_at: Optional[float] = None) -> Dict[str, Any]: """Stop tracking and persist the final run summary. ``extra_summary`` lets runners add workflow-specific metrics before the JSON file is written, so the returned and persisted summaries stay consistent. + + ``finished_at`` lets offline migration preserve the original execution + wall-clock time instead of stamping the migration's own time. """ if self._sampler: self._sampler.stop() self.writer.flush() - self.meta.finished_at = time.time() + self.meta.finished_at = (finished_at if finished_at is not None else time.time()) summary = self._build_summary() summary["budget"] = self._budget.snapshot() if extra_summary: @@ -490,12 +600,12 @@ def _resource_summary(self) -> Dict: def _write_toml(self, path: str): lines = ["[run]", - f'run_id = "{self.run_id}"', - f'game = "{self.meta.game}"', - f'agent = "{self.meta.agent}"', - f'type = "{self.meta.run_type}"', - f'created = "{self.meta.created}"', - f'git_commit = "{self.meta.git_commit}"', + f'run_id = "{self._toml_escape(self.run_id)}"', + f'game = "{self._toml_escape(self.meta.game)}"', + f'agent = "{self._toml_escape(self.meta.agent)}"', + f'type = "{self._toml_escape(self.meta.run_type)}"', + f'created = "{self._toml_escape(self.meta.created)}"', + f'git_commit = "{self._toml_escape(self.meta.git_commit)}"', f"started_at = {self.meta.started_at}"] if self.meta.finished_at: lines.append(f"finished_at = {self.meta.finished_at}") @@ -504,8 +614,13 @@ def _write_toml(self, path: str): if self.config: lines.append(""); lines.append("[config]") for k, v in self.config.items(): - if isinstance(v, str): lines.append(f'{k} = "{v}"') + if isinstance(v, str): lines.append(f'{k} = "{self._toml_escape(v)}"') elif isinstance(v, bool): lines.append(f"{k} = {str(v).lower()}") elif isinstance(v, (int, float)): lines.append(f"{k} = {v}") - with open(path, "w") as f: + with open(path, "w", encoding="utf-8") as f: f.write("\n".join(lines) + "\n") + + @staticmethod + def _toml_escape(s: str) -> str: + """Escape a string for TOML double-quoted value: backslash first, then quote.""" + return str(s).replace("\\", "\\\\").replace('"', '\\"') diff --git a/tests/miracle/_fake_vendor.py b/tests/miracle/_fake_vendor.py new file mode 100644 index 0000000..3c949ae --- /dev/null +++ b/tests/miracle/_fake_vendor.py @@ -0,0 +1,117 @@ +"""Fake vendor runner for match_runner tests. Simulates vendor run_match.py +behaviours WITHOUT the real Judge, so the integration tests are fast, hermetic, +and need none of the Miracle assets. + +Accepts the same standard args as the real vendor runner (--p0-dir, --p1-dir, +--p0-name, --p1-name, --timeout, --out, --tag, --result-json) plus a --fake-mode +that selects the simulated behaviour: + normal write clean result-json + trace (3 ai_operations) + valid replay, exit 0 + no_result exit 0 but write NO result-json + corrupt write truncated/invalid result-json, exit 0 + nonzero exit 2 immediately + hang spawn a long-lived child then hang the parent (wrapper-timeout test) + bigio write several MiB to stdout AND stderr, exit 0 +""" +import argparse +import json +import pathlib +import struct +import subprocess +import sys +import time + + +def _write_replay(path: pathlib.Path, map_type=0, day_time=1): + path.write_bytes(struct.pack(">7i", 0, 0, 0, map_type, day_time, 0, 0)) + + +def _write_trace(path: pathlib.Path, events): + with path.open("w", encoding="utf-8") as f: + for e in events: + f.write(json.dumps(e) + "\n") + + +def _proc(role, cleanup=True, natural=False, term_req=False, rc=0): + return {"role": role, "cleanup_succeeded": cleanup, + "natural_exit": natural, "termination_requested": term_req, + "final_returncode": rc, "forced_kill": False, + "identity_confirmed": True, "started_at": 0.0, "pid": 0} + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--p0-dir"); p.add_argument("--p1-dir") + p.add_argument("--p0-name", default="p0"); p.add_argument("--p1-name", default="p1") + p.add_argument("--timeout", type=float, default=12.0) + p.add_argument("--out", type=pathlib.Path, default=pathlib.Path(".")) + p.add_argument("--tag", default="fake") + p.add_argument("--result-json", type=pathlib.Path, default=None) + p.add_argument("--fake-mode", default="normal") + a = p.parse_args() + + a.out.mkdir(parents=True, exist_ok=True) + trace = a.out / f"{a.tag}.jsonl" + replay = a.out / f"{a.tag}.replay" + + def write_result(d): + if a.result_json is not None: + a.result_json.parent.mkdir(parents=True, exist_ok=True) + a.result_json.write_text(json.dumps(d)) + + if a.fake_mode == "normal": + _write_trace(trace, [ + {"kind": "match_start", "players": [a.p0_name, a.p1_name]}, + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_operation", "player": 1}, + {"kind": "ai_operation", "player": 0}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + _write_replay(replay, 0, 1) + write_result({ + "schema_version": 1, "tag": a.tag, + "end_info_received": True, "end_info": {"0": 5, "1": 2}, + "scores": {"0": 5, "1": 2}, "raw_winner": 0, + "score_tie": False, "judge_tiebreak_applied": False, + "timeout": {"ai0": False, "ai1": False}, + "ai_error": {"ai0": False, "ai1": False}, + "trace_path": str(trace), "replay_path": str(replay), + "cleanup_all_succeeded": True, "exception": None, + "run_match_returncode": 0, + "judge": _proc("judge"), + "ai0": _proc("ai0", cleanup=True, natural=True, term_req=False, rc=0), + "ai1": _proc("ai1", cleanup=True, natural=True, term_req=False, rc=0), + }) + sys.stdout.write("ok\n") + return 0 + + if a.fake_mode == "no_result": + return 0 # exit clean, write nothing + + if a.fake_mode == "corrupt": + if a.result_json is not None: + a.result_json.parent.mkdir(parents=True, exist_ok=True) + a.result_json.write_text('{"schema_version": 1, "end_info": {') # truncated + return 0 + + if a.fake_mode == "nonzero": + return 2 + + if a.fake_mode == "hang": + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(3600)"]) + sys.stdout.write(f"CHILD {child.pid}\n") + sys.stdout.flush() + time.sleep(3600) + return 0 + + if a.fake_mode == "bigio": + chunk = b"x" * (1024 * 1024) + for _ in range(3): + sys.stdout.buffer.write(chunk); sys.stdout.buffer.flush() + sys.stderr.buffer.write(chunk); sys.stderr.buffer.flush() + return 0 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/miracle/_proc_helper.py b/tests/miracle/_proc_helper.py new file mode 100644 index 0000000..d77bf5e --- /dev/null +++ b/tests/miracle/_proc_helper.py @@ -0,0 +1,56 @@ +"""Harmless helper process for the process-tree tests. NOT the real Judge. + +Modes (argv[1]): + stay sleep then exit 0 (or until killed) + stay_forever sleep until killed + spawn_child + spawn `stay ` as a detached child, print + "READY spawn_child parent= child=" (flush), + then sleep and exit + +The tests use these to build real parent/child trees without touching the +Miracle Judge or any AI binary. +""" +import os +import subprocess +import sys +import time + + +def _main() -> int: + mode = sys.argv[1] + py = sys.executable + me = os.path.abspath(__file__) + + if mode == "stay": + secs = float(sys.argv[2]) if len(sys.argv) > 2 else 60.0 + print(f"READY stay pid={os.getpid()} secs={secs}", flush=True) + time.sleep(secs) + return 0 + + if mode == "stay_forever": + print(f"READY stay_forever pid={os.getpid()}", flush=True) + while True: + time.sleep(3600) + return 0 + + if mode == "spawn_child": + child_secs = float(sys.argv[2]) + parent_secs = float(sys.argv[3]) + child = subprocess.Popen( + [py, me, "stay", str(child_secs)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + print( + f"READY spawn_child parent={os.getpid()} child={child.pid}", + flush=True, + ) + time.sleep(parent_secs) + return 0 + + return 2 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/tests/miracle/test_dependencies.py b/tests/miracle/test_dependencies.py new file mode 100644 index 0000000..7700c21 --- /dev/null +++ b/tests/miracle/test_dependencies.py @@ -0,0 +1,34 @@ +"""Dependency-declaration test: psutil is required by the Miracle adapter and a +missing install must produce an actionable error (not a bare ModuleNotFoundError).""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[2] +_SRC = _REPO / "src" + + +def test_missing_psutil_gives_actionable_error(): + code = ( + "import sys\n" + "sys.modules['psutil'] = None # force import psutil to fail\n" + "import agentbench_frame.games.miracle.proctree\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = str(_SRC) + os.pathsep + env.get("PYTHONPATH", "") + env.pop("PYTHONHOME", None) + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env) + assert r.returncode != 0 + combined = r.stderr + r.stdout + assert "psutil" in combined + assert "uv sync --extra miracle" in combined, combined + + +def test_pyproject_declares_miracle_extra(): + text = (_REPO / "pyproject.toml").read_text(encoding="utf-8") + assert 'miracle = ["psutil"]' in text diff --git a/tests/miracle/test_driver.py b/tests/miracle/test_driver.py new file mode 100644 index 0000000..b8478cb --- /dev/null +++ b/tests/miracle/test_driver.py @@ -0,0 +1,218 @@ +"""Run-contract tests for the 24_miracle driver (SKILL.md 测试门槛 items 11 & 12, +plus the disk-level side-swap correctness that defeats framework risks #2/#3/#5). + +These instantiate the real framework ``Run`` (no Judge, no subprocess) and +assert that the persisted ``run.toml`` / ``summary.json`` are contract-complete +and that win_rate is correct on disk after side-swapping. + + py -3.13 -m pytest tests/miracle/test_driver.py -v +""" +from __future__ import annotations + +import json +import tomllib +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.driver import feed_outcomes_to_run +from agentbench_frame.games.miracle.result import ( + GameOutcome, + compute_h2h, + compute_win_rate, + finalize, +) +from agentbench_frame.tracking.run import Run + + +# ---- helpers ------------------------------------------------------------- # +def _outcome(gid, eval_camp, raw_winner, steps=40, score0=None, score1=None, **kw): + s0 = score0 if score0 is not None else (5 if raw_winner == 0 else 2) + s1 = score1 if score1 is not None else (2 if raw_winner == 0 else 5) + return finalize( + GameOutcome( + game_id=gid, + evaluated_agent="ifelse", + opponent="rank04", + evaluated_agent_camp=eval_camp, + raw_winner=raw_winner, + score0=s0, + score1=s1, + steps=steps, + replay_path=f"/tmp/{gid}.replay", + replay_sha256="x" * 64, + **kw, + ) + ) + + +def _build_run(tmp_path: Path, outcomes, agent="ifelse", run_type="eval"): + run = Run.start( + game="24_miracle", + agent=agent, + run_type=run_type, + data_dir=str(tmp_path), + config={"opponent_set": "smoke", "n_games": str(len(outcomes))}, + ) + feed_outcomes_to_run(run, outcomes) + summary = run.finish() + run_dir = tmp_path / "runs" / "24_miracle" / agent / run.run_id + return run, summary, run_dir + + +# ---- item 11: run.toml fields -------------------------------------------- # +def test_run_toml_has_required_fields(tmp_path): + _, _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + meta = tomllib.loads((run_dir / "run.toml").read_text()) + run_section = meta["run"] + for field in ("run_id", "game", "agent", "type", "created", + "started_at", "total_steps", "total_episodes"): + assert field in run_section, f"run.toml missing [run].{field}" + assert run_section["game"] == "24_miracle" + assert run_section["agent"] == "ifelse" + assert run_section["type"] == "eval" + assert meta["config"]["opponent_set"] == "smoke" + + +def test_run_toml_finished_at_written(tmp_path): + _, _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + meta = tomllib.loads((run_dir / "run.toml").read_text()) + assert "finished_at" in meta["run"] # only after finish() + + +# ---- item 12: summary.json fields ---------------------------------------- # +REQUIRED_SUMMARY_FIELDS = ( + "run_id", "game", "agent", "run_type", "created", "git_commit", + "wall_hours", "total_episodes", "total_steps", "win_rate", + "best_elo", "final_elo", "elo_history", "h2h", "resource_summary", "config", +) + + +def test_summary_json_has_required_fields(tmp_path): + _, summary, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + disk = json.loads((run_dir / "summary.json").read_text()) + for f in REQUIRED_SUMMARY_FIELDS: + assert f in disk, f"summary.json missing {f}" + # eval run: Elo fields present but null, NOT omitted (SKILL.md) + assert disk["best_elo"] is None + assert disk["final_elo"] is None + assert disk["elo_history"] == [] + + +# ---- disk-level side-swap correctness (risks #2/#3/#5) ------------------- # +def test_persisted_win_rate_correct_under_side_swap(tmp_path): + outcomes = [ + _outcome("g1", 0, 0, steps=40), # win as camp0 + _outcome("g2", 1, 1, steps=38), # win as camp1 (swapped) + _outcome("g3", 0, 1, steps=50), # loss + ] + _, summary, run_dir = _build_run(tmp_path, outcomes) + expected = compute_win_rate(outcomes) # 2/3 + # in-memory return is correct... + assert summary["win_rate"] == pytest.approx(expected) + # ...AND the value actually persisted to disk (this is the risk-#5 check) + disk = json.loads((run_dir / "summary.json").read_text()) + assert disk["win_rate"] == pytest.approx(expected) + assert disk["total_episodes"] == 3 + assert disk["total_steps"] == 128 # 40 + 38 + 50 (valid games only) + + +def test_error_games_excluded_from_persisted_counts(tmp_path): + outcomes = [ + _outcome("g1", 0, 0, steps=40), # win + _outcome("g2", 0, 0, steps=10, ai_error_player=1), # error (opp crash) + ] + _, _, run_dir = _build_run(tmp_path, outcomes) + disk = json.loads((run_dir / "summary.json").read_text()) + # only the valid game counts toward episodes/steps/win_rate + assert disk["total_episodes"] == 1 + assert disk["total_steps"] == 40 + assert disk["win_rate"] == pytest.approx(1.0) + # but the error game IS still in events.jsonl as an audit record + events = [json.loads(l) for l in (run_dir / "events.jsonl").read_text().splitlines() if l.strip()] + game_events = [e for e in events if e.get("event") == "game"] + assert len(game_events) == 2 + assert any(e["normalized_result"] == "error" for e in game_events) + + +def test_no_runs_runs_double_dir(tmp_path): + # risk #6: Run.start must write /runs/... NOT /runs/runs/... + _, _, run_dir = _build_run(tmp_path, [_outcome("g1", 0, 0)]) + assert run_dir.exists() + # the forbidden double-runs path must NOT exist + assert not (tmp_path / "runs" / "runs").exists() + + +# ---- persisted h2h (item 13 integration) --------------------------------- # +def test_summary_h2h_matches_compute(tmp_path): + outcomes = [ + _outcome("g1", 0, 0), + _outcome("g2", 1, 1), + _outcome("g3", 0, 1), + ] + _, _, run_dir = _build_run(tmp_path, outcomes) + disk = json.loads((run_dir / "summary.json").read_text()) + expected = compute_h2h(outcomes) + assert disk["h2h"] == expected + assert disk["h2h"]["ifelse"]["rank04"] == pytest.approx(2 / 3) + + +# ---- NEW framework defect: Run._write_toml must escape backslashes -------- # +def test_run_toml_valid_with_windows_path_in_config(tmp_path): + """A config value containing a Windows path (backslashes) must NOT produce + invalid TOML. This is the root cause of the smoke `data check` failure + (run.toml 'Invalid hex value' on judge_dir_resolved). Run._write_toml must + escape backslashes and quotes in string values.""" + windows_path = "C:" + "\\\\Users\\\\" + "example\\\\judge_dev_logic" + run = Run.start( + game="24_miracle", agent="x", run_type="eval", data_dir=str(tmp_path), + config={"judge_dir_resolved": windows_path}, + ) + run.log_episode(reward=1.0, steps=5, winner=0) + run.finish() + run_dir = tmp_path / "runs" / "24_miracle" / "x" / run.run_id + # run.toml must parse cleanly and round-trip the path + meta = tomllib.loads((run_dir / "run.toml").read_text(encoding="utf-8")) + assert meta["config"]["judge_dir_resolved"] == windows_path + + +# ---- _write_toml escape regression: quotes / backslashes / mixed types ---- # +def _run_with_config(tmp_path, config): + run = Run.start(game="24_miracle", agent="c", run_type="eval", + data_dir=str(tmp_path), config=config) + run.log_episode(reward=1.0, steps=1, winner=0) + run.finish() + run_dir = tmp_path / "runs" / "24_miracle" / "c" / run.run_id + return tomllib.loads((run_dir / "run.toml").read_text(encoding="utf-8")) + + +def test_run_toml_config_double_quotes_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {"note": 'he said "hi"'}) + assert meta["config"]["note"] == 'he said "hi"' + + +def test_run_toml_config_backslashes_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {"p": "a\\b\\c"}) + assert meta["config"]["p"] == "a\\b\\c" + + +def test_run_toml_config_mixed_types_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {"s": "plain", "b": True, "i": 7, "f": 1.5, + "path": r"C:\x\y"}) + c = meta["config"] + assert c["s"] == "plain" and c["b"] is True and c["i"] == 7 and c["f"] == 1.5 + assert c["path"] == r"C:\x\y" + + +def test_run_toml_normal_string_not_over_escaped(tmp_path): + # a plain ASCII value must round-trip unchanged (no double-escaping) + meta = _run_with_config(tmp_path, {"group": "GROUP1"}) + assert meta["config"]["group"] == "GROUP1" + + +def test_run_toml_run_section_roundtrip(tmp_path): + meta = _run_with_config(tmp_path, {}) + r = meta["run"] + assert r["game"] == "24_miracle" and r["agent"] == "c" and r["type"] == "eval" + assert r["total_steps"] == 1 and r["total_episodes"] == 1 + assert r["started_at"] == float(r["started_at"]) # numeric, not a quoted string diff --git a/tests/miracle/test_entry.py b/tests/miracle/test_entry.py new file mode 100644 index 0000000..f8acb91 --- /dev/null +++ b/tests/miracle/test_entry.py @@ -0,0 +1,105 @@ +"""Tests for the cross-platform AI entry resolver (阶段9A §2). + +The executable is returned as an ABSOLUTE path (Windows CreateProcess does not +search the cwd= argument for a bare name), so Popen(cmd, cwd=ai_dir) works on +every platform. + +Covers: Windows main.exe / POSIX main / main.py; explicit priority; paths with +spaces not split; main.exe+main coexistence (deterministic); missing entry; +resolver does not modify the dir; returned exe path is absolute. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from agentbench_frame.games.miracle.entry import resolve_ai_command + + +def _set_platform(monkeypatch, name): + monkeypatch.setattr("agentbench_frame.games.miracle.entry._IS_NT", name == "nt") + + +def _abs(tmp_path, name): + return str(Path(tmp_path).resolve() / name) + + +def test_windows_recognises_main_exe(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main.exe")] + + +def test_posix_recognises_main(tmp_path, monkeypatch): + _set_platform(monkeypatch, "posix") + (tmp_path / "main").write_bytes(b"\x7fELF") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main")] + + +def test_python_main_py(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.py").write_text("print('ai')") + assert resolve_ai_command(tmp_path) == [sys.executable, _abs(tmp_path, "main.py")] + + +def test_python_main_py_posix(tmp_path, monkeypatch): + _set_platform(monkeypatch, "posix") + (tmp_path / "main.py").write_text("print('ai')") + assert resolve_ai_command(tmp_path) == [sys.executable, _abs(tmp_path, "main.py")] + + +def test_explicit_list_takes_priority(tmp_path): + (tmp_path / "main.exe").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path, explicit=["my", "args"]) == ["my", "args"] + + +def test_explicit_str_with_spaces_not_split(tmp_path): + explicit = r"C:\Program Files\Some Dir\main.exe" + assert resolve_ai_command(tmp_path, explicit=explicit) == [explicit] + assert len(resolve_ai_command(tmp_path, explicit=explicit)) == 1 + + +def test_explicit_empty_falls_through_to_autodetect(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path, explicit="") == [_abs(tmp_path, "main.exe")] + assert resolve_ai_command(tmp_path, explicit=None) == [_abs(tmp_path, "main.exe")] + + +def test_coexistence_windows_picks_main_exe(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + (tmp_path / "main").write_bytes(b"MZ") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main.exe")] + + +def test_coexistence_posix_picks_main(tmp_path, monkeypatch): + _set_platform(monkeypatch, "posix") + (tmp_path / "main.exe").write_bytes(b"MZ") + (tmp_path / "main").write_bytes(b"\x7fELF") + assert resolve_ai_command(tmp_path) == [_abs(tmp_path, "main")] + + +def test_missing_entry_clear_error(tmp_path): + with pytest.raises(FileNotFoundError) as ei: + resolve_ai_command(tmp_path) + assert str(Path(tmp_path).resolve()) in str(ei.value) + + +def test_resolver_does_not_modify_strategy_dir(tmp_path, monkeypatch): + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + before = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + resolve_ai_command(tmp_path) + after = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + assert before == after + + +def test_returned_exe_path_is_absolute(tmp_path, monkeypatch): + # Windows CreateProcess does not search cwd= for a bare name -> must be absolute + _set_platform(monkeypatch, "nt") + (tmp_path / "main.exe").write_bytes(b"MZ") + cmd = resolve_ai_command(tmp_path) + assert Path(cmd[0]).is_absolute() diff --git a/tests/miracle/test_match_runner.py b/tests/miracle/test_match_runner.py new file mode 100644 index 0000000..8fb98dd --- /dev/null +++ b/tests/miracle/test_match_runner.py @@ -0,0 +1,421 @@ +"""match_runner contract tests (阶段4b-4 spec, 20 required behaviours). + +RED LIGHT FIRST: the implementation match_runner.py does not exist yet, so the +whole module fails to import; the failure is saved to +docs/games/evidence/match_runner_redlight.txt, THEN the implementation is +written to turn these green. + +Pure-logic behaviours (1, 5-16, 18) drive synthesized result-json / trace / +Replay files through match_runner's reader + cross-validation + classifier. +Integration behaviours (2, 3, 4, 17, 20) drive the harmless _fake_vendor.py +through run_match_attempt. Behaviour 19 checks cwd-independent import. + + py -3.13 -m pytest tests/miracle/test_match_runner.py -v +""" +from __future__ import annotations + +import json +import os +import struct +import subprocess +import sys +import time +from pathlib import Path + +import psutil +import pytest + +from agentbench_frame.games.miracle import match_runner +from agentbench_frame.games.miracle.match_runner import ( + Classification, + MatchAttempt, + ReplayInfo, + TraceStats, + classify, + cross_validate, + load_result_json, + read_replay_info, + run_match_attempt, + stream_trace, +) + +FAKE_VENDOR = Path(__file__).parent / "_fake_vendor.py" +REPO = Path(__file__).resolve().parents[2] +SRC = REPO / "src" + + +# ---- synthesized-data builders ------------------------------------------- # +def mk_rj(**kw): + base = { + "schema_version": 1, "tag": "g", + "end_info_received": True, "end_info": {"0": 5, "1": 2}, + "scores": {"0": 5, "1": 2}, "raw_winner": 0, + "score_tie": False, "judge_tiebreak_applied": False, + "timeout": {"ai0": False, "ai1": False}, + "ai_error": {"ai0": False, "ai1": False}, + "cleanup_all_succeeded": True, "exception": None, + "run_match_returncode": 0, + "judge": {"cleanup_succeeded": True, "natural_exit": True, "termination_requested": False, "final_returncode": 0}, + "ai0": {"cleanup_succeeded": True, "natural_exit": True, "termination_requested": False, "final_returncode": 0}, + "ai1": {"cleanup_succeeded": True, "natural_exit": True, "termination_requested": False, "final_returncode": 0}, + } + base.update(kw) + return base + + +def write_json(p: Path, d): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(d)) + + +def write_trace(p: Path, events): + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("w", encoding="utf-8") as f: + for e in events: + f.write(json.dumps(e) + "\n") + + +def write_replay(p: Path, map_type=0, day_time=1): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(struct.pack(">7i", 0, 0, 0, map_type, day_time, 0, 0)) + + +def _classify_for(rj, ts, ri, *, wrapper_timeout=False, discrepancies=None, + evaluated_agent_camp=0, vendor_returncode=0): + status, data = ("ok", rj) if rj is not None else ("missing", None) + return classify(rj_status=status, rj=data, ts=ts, ri=ri, + discrepancies=discrepancies or [], vendor_returncode=vendor_returncode, + wrapper_timeout=wrapper_timeout, evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent="ifelse", opponent="rank04") + + +# ========================================================================== # +# 1. vendor runner returns a complete result-json (integration, fake vendor) # +# ========================================================================== # +def test_01_normal_complete_result_json(tmp_path): + att = run_match_attempt( + game_id="g1", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="sampleA", p1_name="sampleB", judge_dir=tmp_path / "judge", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + evaluated_agent="sampleA", opponent="sampleB", evaluated_agent_camp=0, + extra_vendor_args=["--fake-mode", "normal"], + ) + assert att.result_json_status == "ok" + assert att.valid is True + assert att.normalized_result == "win" # raw_winner 0, evaluated camp 0 + assert att.raw_winner == 0 + assert att.steps == 3 # 3 ai_operation events + assert att.realized_randomization == {"map_type": 0, "day_time": 1} + + +# ===== 2. result-json missing ===== # +def test_02_result_json_missing(tmp_path): + att = run_match_attempt( + game_id="g2", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "no_result"], + ) + assert att.result_json_status == "missing" + assert att.valid is False + assert att.error_type == "result_json_missing" + + +# ===== 3. result-json corrupt / truncated ===== # +def test_03_result_json_corrupt(tmp_path): + att = run_match_attempt( + game_id="g3", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "corrupt"], + ) + assert att.result_json_status == "corrupt" + assert att.valid is False + assert att.error_type == "result_json_corrupt" + + +# ===== 4. vendor runner nonzero exit ===== # +def test_04_vendor_nonzero_exit(tmp_path): + att = run_match_attempt( + game_id="g4", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "nonzero"], + ) + assert att.vendor_returncode != 0 + assert att.valid is False + assert att.error_type == "result_json_missing" # no result-json produced + + +# ===== 5. result-json & trace end_info consistent ===== # +def test_05_end_info_consistent(tmp_path): + rj = mk_rj() + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_operation", "player": 0}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + ri = read_replay_info(tmp_path / "absent.replay") + assert cross_validate(rj, ts, ri) == [] + + +# ===== 6. result-json & trace scores conflict ===== # +def test_06_scores_conflict(tmp_path): + rj = mk_rj(end_info={"0": 5, "1": 2}, scores={"0": 5, "1": 2}, raw_winner=0) + trace = tmp_path / "t.jsonl" + write_trace(trace, [{"kind": "match_end", "end_info": json.dumps({"0": 2, "1": 5})}]) + ts = stream_trace(trace) + ri = read_replay_info(tmp_path / "absent.replay") + discs = cross_validate(rj, ts, ri) + assert any("score" in d.lower() or "end_info" in d.lower() for d in discs) + c = _classify_for(rj, ts, ri, discrepancies=discs) + assert c.error_type == "evidence_mismatch" and not c.valid + + +# ===== 7. derived raw_winner conflicts with score rule ===== # +def test_07_raw_winner_conflicts(tmp_path): + rj = mk_rj(end_info={"0": 2, "1": 5}, scores={"0": 2, "1": 5}, raw_winner=0) # 0 says p0 won but s1>s0 + ri = read_replay_info(tmp_path / "absent.replay") + ts = TraceStats() # empty + discs = cross_validate(rj, ts, ri) + assert any("raw_winner" in d.lower() or "winner" in d.lower() for d in discs) + c = _classify_for(rj, ts, ri, discrepancies=discs) + assert c.error_type == "evidence_mismatch" and not c.valid + + +# ===== 8. trace has ai_error ===== # +def test_08_trace_ai_error(tmp_path): + rj = mk_rj() + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_error", "player": 1, "state": 3}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + assert 1 in ts.ai_error_players + ri = read_replay_info(tmp_path / "absent.replay") + c = _classify_for(rj, ts, ri) + assert c.error_type == "ai_crash" and c.ai_crash_player == 1 and not c.valid + + +# ===== 9. trace has ai_timeout ===== # +def test_09_trace_ai_timeout(tmp_path): + rj = mk_rj() + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_timeout", "player": 0, "state": 2}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + assert 0 in ts.ai_timeout_players + ri = read_replay_info(tmp_path / "absent.replay") + c = _classify_for(rj, ts, ri) + assert c.error_type == "ai_timeout" and c.ai_timeout_player == 0 and not c.valid + + +# ===== 10. AI natural exit before end_info ===== # +def test_10_ai_natural_exit_before_end_info(tmp_path): + rj = mk_rj(end_info_received=False, end_info=None, scores=None, raw_winner=None, + ai0={"cleanup_succeeded": True, "natural_exit": True, + "termination_requested": False, "final_returncode": 1}) + ri = read_replay_info(tmp_path / "absent.replay") + ts = TraceStats() + c = _classify_for(rj, ts, ri) + assert c.error_type == "ai_crash" and c.ai_crash_player == 0 and not c.valid + + +# ===== 11. AI cleanup-killed after end_info, nonzero returncode -> NOT crash # +def test_11_ai_cleanup_nonzero_after_end_info_is_normal(tmp_path): + rj = mk_rj(ai0={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": True, "final_returncode": 1}, + ai1={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": True, "final_returncode": 1}) + trace = tmp_path / "t.jsonl" + write_trace(trace, [{"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}]) + ts = stream_trace(trace) + replay = tmp_path / "r.replay"; write_replay(replay) + ri = read_replay_info(replay) + c = _classify_for(rj, ts, ri) + assert c.valid is True + assert c.normal_cleanup_nonzero is True + assert c.error_type is None + assert c.normalized_result == "win" + + +# ===== 12. Judge exits before end_info ===== # +def test_12_judge_exit_before_end_info(tmp_path): + rj = mk_rj(end_info_received=False, end_info=None, scores=None, raw_winner=None, + judge={"cleanup_succeeded": True, "natural_exit": True, + "termination_requested": False, "final_returncode": 1}, + ai0={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": False, "final_returncode": None}, + ai1={"cleanup_succeeded": True, "natural_exit": False, + "termination_requested": False, "final_returncode": None}) + ri = read_replay_info(tmp_path / "absent.replay") + ts = TraceStats() + c = _classify_for(rj, ts, ri) + assert c.error_type == "judge_crash" and c.judge_crash and not c.valid + + +# ===== 13. Replay missing ===== # +def test_13_replay_missing(tmp_path): + rj = mk_rj() + ts = TraceStats() + ri = read_replay_info(tmp_path / "absent.replay") + assert ri.exists is False + c = _classify_for(rj, ts, ri) + assert c.error_type == "replay_missing" and not c.valid + + +# ===== 14. Replay corrupt / header too short ===== # +def test_14_replay_short_header(tmp_path): + rj = mk_rj() + ts = TraceStats() + replay = tmp_path / "r.replay" + replay.write_bytes(b"\x00" * 10) # too short for 7 int32 + ri = read_replay_info(replay) + assert ri.exists is True and ri.header_valid is False + c = _classify_for(rj, ts, ri) + assert c.error_type == "replay_corrupt" and not c.valid + + +# ===== 15. Replay map_type/day_time parseable ===== # +def test_15_replay_header_parseable(tmp_path): + replay = tmp_path / "r.replay" + write_replay(replay, map_type=1, day_time=0) + ri = read_replay_info(replay) + assert ri.header_valid is True + assert ri.map_type == 1 and ri.day_time == 0 + assert ri.sha256 and len(ri.sha256) == 64 + + +# ===== 16. ai_operation streaming count ===== # +def test_16_ai_operation_streaming_count(tmp_path): + trace = tmp_path / "t.jsonl" + write_trace(trace, [ + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_operation", "player": 1}, + {"kind": "ai_operation", "player": 0}, + {"kind": "ai_operation", "player": 1}, + {"kind": "ai_operation", "player": 0}, + {"kind": "match_end", "end_info": json.dumps({"0": 5, "1": 2})}, + ]) + ts = stream_trace(trace) + assert ts.n_ai_operation == 5 + assert ts.end_info_seen is True + + +# ===== 17. wrapper timeout cleans vendor + child tree ===== # +def test_17_wrapper_timeout_kills_vendor_tree(tmp_path): + import os as _os + att = run_match_attempt( + game_id="g17", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + wrapper_timeout_s=2.0, + extra_vendor_args=["--fake-mode", "hang"], + ) + assert att.wrapper_timeout is True + assert att.valid is False + assert att.error_type == "wrapper_timeout" + # the fake vendor printed "CHILD " to its stdout (captured to a file) + stdout_file = Path(att.evidence_paths["stdout"]) + child_pid = None + for line in stdout_file.read_text(errors="replace").splitlines(): + if line.startswith("CHILD "): + child_pid = int(line.split()[1]) + assert child_pid is not None, "fake vendor did not report a child PID" + # grace for the OS to reap + deadline = time.time() + 5 + while time.time() < deadline and psutil.pid_exists(child_pid): + time.sleep(0.1) + assert not psutil.pid_exists(child_pid), "orphan child survived wrapper-timeout cleanup" + + +# ===== 18. duplicate game_id / output-dir collision ===== # +def test_18_duplicate_game_id_collision(tmp_path): + common = dict( + p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", p0_name="a", p1_name="b", + judge_dir=tmp_path / "j", work_dir=tmp_path / "work", + vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "normal"], + ) + first = run_match_attempt(game_id="dup", evaluated_agent_camp=0, **common) + assert first.valid is True + first_json = Path(first.evidence_paths["result_json"]) + first_content = first_json.read_text() + second = run_match_attempt(game_id="dup", evaluated_agent_camp=0, **common) + assert second.collision_detected is True + # the first attempt's result-json must NOT have been clobbered + assert first_json.read_text() == first_content + + +# ===== 19. import works from any cwd ===== # +def test_19_import_from_any_cwd(tmp_path): + code = ( + "import sys\n" + f"sys.path.insert(0, {str(SRC)!r})\n" + "from agentbench_frame.games.miracle import match_runner\n" + "print('IMPORT_OK')\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = str(SRC) + os.pathsep + env.get("PYTHONPATH", "") + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, + env=env, cwd=str(tmp_path)) + assert r.returncode == 0, r.stderr + assert "IMPORT_OK" in r.stdout + + +# ===== 20. large stdout/stderr does not deadlock ===== # +def test_20_large_stdout_stderr_no_deadlock(tmp_path): + t0 = time.time() + att = run_match_attempt( + game_id="g20", p0_dir=tmp_path / "p0", p1_dir=tmp_path / "p1", + p0_name="a", p1_name="b", judge_dir=tmp_path / "j", + work_dir=tmp_path / "work", vendor_script=FAKE_VENDOR, framework_src=SRC, + extra_vendor_args=["--fake-mode", "bigio"], + ) + elapsed = time.time() - t0 + assert elapsed < 30, f"wrapper deadlocked or hung: {elapsed:.1f}s" + stdout_size = Path(att.evidence_paths["stdout"]).stat().st_size + stderr_size = Path(att.evidence_paths["stderr"]).stat().st_size + assert stdout_size >= 3 * 1024 * 1024 - 1024 + assert stderr_size >= 3 * 1024 * 1024 - 1024 + + +def test_relative_work_paths_are_absolute_before_vendor_spawn(tmp_path, monkeypatch): + """Judge cwd may differ from framework cwd; vendor gets absolute artifacts.""" + from agentbench_frame.games.miracle import match_runner as mod + monkeypatch.chdir(tmp_path) + real_popen = mod.subprocess.Popen + seen = {} + + def checked_popen(cmd, *args, **kwargs): + for flag in ("--out", "--result-json"): + value = Path(cmd[cmd.index(flag) + 1]) + assert value.is_absolute(), f"{flag} was relative: {value}" + seen[flag] = value + assert seen["--out"].exists(), "replay/trace parent absent before process start" + return real_popen(cmd, *args, **kwargs) + + monkeypatch.setattr(mod.subprocess, "Popen", checked_popen) + att = run_match_attempt( + game_id="relative", p0_dir=Path("p0"), p1_dir=Path("p1"), + p0_name="a", p1_name="b", judge_dir=Path("judge-cwd"), + work_dir=Path("relative") / "work", vendor_script=FAKE_VENDOR, + framework_src=Path("framework-src"), extra_vendor_args=["--fake-mode", "normal"], + ) + assert att.valid is True + assert Path(att.evidence_paths["replay"]).is_absolute() + assert Path(att.evidence_paths["trace"]).is_absolute() + + +def test_judge_start_path_error_is_not_collapsed_to_vendor_eof(tmp_path): + rj = mk_rj(end_info_received=False, end_info=None, scores=None, raw_winner=None) + rj["exception"] = "FileNotFoundError: replay parent missing" + c = _classify_for(rj, TraceStats(), read_replay_info(tmp_path / "absent.replay"), + vendor_returncode=1) + assert c.error_type == "judge_start_path_error" + assert "startup path" in c.reason diff --git a/tests/miracle/test_proctree.py b/tests/miracle/test_proctree.py new file mode 100644 index 0000000..8280257 --- /dev/null +++ b/tests/miracle/test_proctree.py @@ -0,0 +1,201 @@ +"""Process-tree manager contract tests (SKILL.md 测试门槛 item 17 + the 8 +process-safety requirements from the 阶段4b spec). + +These run FIRST as a RED LIGHT (the implementation proctree.py does not exist +yet), the failure is saved as evidence, THEN the implementation is written to +turn them green. + +Validated behaviours: + 1. normal child exits naturally (natural_exit, no kill) + 2. parent exits but child still survives -> child is reachable + cleanable + 3. timeout -> the whole recorded tree is cleaned + 4. cleanup is idempotent + 5. an already-exited PID is skipped, never a false kill of others + 6. only this run's recorded exact PIDs are cleaned + 7. after cleanup neither parent nor child PID remains + 8. NO name-based / fuzzy kill (structural scan of the implementation) + + PID identity (create_time) must match before any kill; a stale/reused PID + is never killed. + + py -3.13 -m pytest tests/miracle/test_proctree.py -v +""" +from __future__ import annotations + +import pathlib +import re +import subprocess +import sys +import time + +import psutil +import pytest + +from agentbench_frame.games.miracle import proctree # noqa: F401 (red-light import) +from agentbench_frame.games.miracle.proctree import ManagedProcess, ProcessTreeManager + +HELPER = pathlib.Path(__file__).parent / "_proc_helper.py" +PY = sys.executable + + +# ---- spawn helpers ------------------------------------------------------- # +def _spawn(args, **kw): + return subprocess.Popen( + [PY, str(HELPER), *args], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **kw, + ) + + +def _ready(proc, timeout=5.0): + """Read the single READY line the helper prints on startup.""" + line = proc.stdout.readline() + assert line.startswith("READY"), f"unexpected helper output: {line!r}" + return line + + +def _child_pid_from(line: str) -> int: + return int(line.split("child=")[1].strip()) + + +def _alive(pid: int, started_at: float, tol: float = 1.0) -> bool: + """Identity-checked liveness: PID exists AND its create_time matches.""" + if not psutil.pid_exists(pid): + return False + try: + return abs(psutil.Process(pid).create_time() - started_at) < tol + except psutil.NoSuchProcess: + return False + + +# ---- 1: natural exit ----------------------------------------------------- # +def test_normal_child_exits_naturally(): + mgr = ProcessTreeManager() + p = _spawn(["stay", "0.3"]) + try: + _ready(p) + mp = mgr.register_popen(p, "child") + p.wait(timeout=5) + mgr.poll() + assert mp.natural_exit is True + assert mp.natural_returncode == 0 + assert mp.termination_requested is False + mgr.cleanup_all("test-done") + assert mp.cleanup_succeeded is True + finally: + if p.poll() is None: + p.kill() + + +# ---- 2: parent exits, child survives, then cleaned ----------------------- # +def test_parent_exits_child_survives_then_cleaned(): + mgr = ProcessTreeManager() + p = _spawn(["spawn_child", "30", "0.2"]) # child lives 30s, parent 0.2s + line = _ready(p) + child_pid = _child_pid_from(line) + p.wait(timeout=5) # parent exits naturally + cmp = mgr.register_pid(child_pid, "grandchild") + assert _alive(cmp.pid, cmp.started_at) is True # child still alive + mgr.cleanup_all("parent-gone") + assert _alive(cmp.pid, cmp.started_at) is False # child now cleaned + assert cmp.cleanup_succeeded is True + + +# ---- 3: timeout cleans the whole recorded tree --------------------------- # +def test_timeout_kills_whole_tree(): + mgr = ProcessTreeManager() + p = _spawn(["spawn_child", "30", "30"]) # both long-lived + line = _ready(p) + child_pid = _child_pid_from(line) + mp = mgr.register_popen(p, "parent") + cmp = mgr.register_pid(child_pid, "child") + assert _alive(mp.pid, mp.started_at) and _alive(cmp.pid, cmp.started_at) + mgr.cleanup_all("timeout") + assert not _alive(mp.pid, mp.started_at) + assert not _alive(cmp.pid, cmp.started_at) + assert mp.cleanup_succeeded and cmp.cleanup_succeeded + + +# ---- 4: idempotent ------------------------------------------------------- # +def test_cleanup_is_idempotent(): + mgr = ProcessTreeManager() + p = _spawn(["stay", "30"]) + _ready(p) + mp = mgr.register_popen(p, "x") + mgr.cleanup_all("first") + assert not _alive(mp.pid, mp.started_at) + # second cleanup must be a no-op without raising + mgr.cleanup_all("second") + mgr.cleanup_one(mp, "third") + assert not _alive(mp.pid, mp.started_at) + + +# ---- 5: already-exited PID is skipped, no false kill --------------------- # +def test_exited_pid_does_not_cause_false_kill(): + mgr = ProcessTreeManager() + dead = _spawn(["stay", "0.2"]) + _ready(dead) + dead.wait(timeout=5) + dmp = mgr.register_popen(dead, "dead") # already exited + other = _spawn(["stay", "30"]) + _ready(other) + other_mp = mgr.register_popen(other, "other") + mgr.cleanup_one(dmp, "noop") # must skip the dead one + assert _alive(other_mp.pid, other_mp.started_at) is True # other untouched + mgr.cleanup_all("final") + assert not _alive(other_mp.pid, other_mp.started_at) + + +# ---- 6: only recorded exact PIDs are cleaned ----------------------------- # +def test_only_recorded_pids_are_cleaned(): + mgr = ProcessTreeManager() + a = _spawn(["stay", "30"]); _ready(a) + b = _spawn(["stay", "30"]); _ready(b) + b_ct = psutil.Process(b.pid).create_time() + ma = mgr.register_popen(a, "a") # register ONLY a + mgr.cleanup_all("partial") # should clean a only + assert not _alive(ma.pid, ma.started_at) + # b was never registered -> must still be alive + assert _alive(b.pid, b_ct) is True + b.terminate(); b.wait(timeout=5) + + +# ---- 7: no residual parent nor child PID --------------------------------- # +def test_cleanup_leaves_neither_parent_nor_child_pid(): + mgr = ProcessTreeManager() + p = _spawn(["spawn_child", "30", "30"]) + line = _ready(p) + child_pid = _child_pid_from(line) + mp = mgr.register_popen(p, "parent") + cmp = mgr.register_pid(child_pid, "child") + mgr.cleanup_all("done") + assert not _alive(mp.pid, mp.started_at) + assert not _alive(cmp.pid, cmp.started_at) + + +# ---- 8: no name-based / fuzzy kill (structural) -------------------------- # +def test_no_name_based_or_fuzzy_kill(): + src = pathlib.Path(proctree.__file__).read_text(encoding="utf-8") + forbidden = ["/im ", "/im\"", "/IM ", "/IM\"", "tskill ", "imagename", + "Get-Process", "wmic process where name"] + hits = [f for f in forbidden if f in src] + assert not hits, f"fuzzy/name-based kill patterns found in proctree.py: {hits}" + # every taskkill must be /PID-based (exact PID), never by image name + for m in re.finditer(r"taskkill", src, re.IGNORECASE): + ctx = src[m.start(): m.start() + 160] + assert "/PID" in ctx or "/pid" in ctx, f"non-PID taskkill: {ctx!r}" + + +# ---- +: PID identity (create_time) guards against PID reuse -------------- # +def test_identity_mismatch_does_not_kill(): + mgr = ProcessTreeManager() + p = _spawn(["stay", "30"]) + _ready(p) + mp = mgr.register_popen(p, "x") + mp.started_at = 1.0 # forge a stale start time -> simulates PID reuse + mgr.cleanup_one(mp, "stale") + assert mp.identity_confirmed is False + assert _alive(p.pid, psutil.Process(p.pid).create_time()) is True # NOT killed + # restore real identity and clean up properly + mp.started_at = psutil.Process(p.pid).create_time() + mgr.cleanup_one(mp, "real") + assert mp.identity_confirmed is True + assert _alive(mp.pid, mp.started_at) is False diff --git a/tests/miracle/test_result.py b/tests/miracle/test_result.py new file mode 100644 index 0000000..243e746 --- /dev/null +++ b/tests/miracle/test_result.py @@ -0,0 +1,363 @@ +"""Contract tests for the 24_miracle pure result module. + +Covers SKILL.md 测试门槛 items that are pure logic (no subprocess, no Judge): + 1. Agent wins at camp0 + 2. Agent wins at camp1 + 3. side-swap tallying (raw_winner==0 is NOT always the evaluated agent's win) + 4. draw recorded separately, not a win + 5. opponent crash -> error (not a capability win) + 6. agent crash -> error + 7. judge crash -> error + 8. timeout -> error + 9. replay missing -> error + 10. duplicate game_id resumability + 13. h2h direction +plus end_info parsing, replay-header parsing, file hashing, and event-record +contract completeness. + +Run with the 3.11+ interpreter: + py -3.13 -m pytest tests/miracle/test_result.py -v +""" +from __future__ import annotations + +import json +import struct + +import pytest + +from agentbench_frame.games.miracle.result import ( + DRAW, + ERROR, + GameOutcome, + LOSS, + VALID_RESULTS, + WIN, + build_seed_provenance, + compute_h2h, + compute_run_stats, + compute_win_rate, + derive_raw_winner, + finalize, + normalize, + outcome_counts, + read_replay_header, + scores_from_end_info, + select_games_to_run, + sha256_file, + to_event_record, + would_rerun_successful, +) + + +# ---- helpers ------------------------------------------------------------- # +def outcome(eval_camp=0, raw_winner=None, **kw): + """Build a finalized GameOutcome with sane defaults for tests.""" + o = GameOutcome( + game_id=kw.pop("game_id", "g1"), + evaluated_agent=kw.pop("evaluated_agent", "ifelse"), + opponent=kw.pop("opponent", "rank04"), + evaluated_agent_camp=eval_camp, + raw_winner=raw_winner, + **kw, + ) + return finalize(o) + + +# ---- 1 & 2: win at camp0 / camp1 ----------------------------------------- # +def test_win_at_camp0(): + o = outcome(eval_camp=0, raw_winner=0) + assert o.normalized_result == WIN + assert o.winner_agent == "ifelse" + + +def test_win_at_camp1(): + # evaluated agent on camp 1, Judge says camp 1 won -> evaluated agent wins + o = outcome(eval_camp=1, raw_winner=1) + assert o.normalized_result == WIN + assert o.winner_agent == "ifelse" + + +def test_loss_at_camp1_when_raw_winner_zero(): + # side-swapped: evaluated on camp1, raw_winner==0 (opponent camp) -> LOSS, + # NOT a win. This is the exact bug the framework's Match has. + o = outcome(eval_camp=1, raw_winner=0) + assert o.normalized_result == LOSS + assert o.winner_agent == "rank04" + + +# ---- 3: side-swap tallying ----------------------------------------------- # +def test_side_swap_win_rate_counts_both_sides(): + # two wins, one as camp0, one as camp1 (swapped) -> 100% + games = [ + outcome(game_id="a", eval_camp=0, raw_winner=0), + outcome(game_id="b", eval_camp=1, raw_winner=1), + ] + assert compute_win_rate(games) == 1.0 + + +def test_side_swap_raw_winner_zero_is_not_always_eval_win(): + # raw_winner==0 once means eval win (camp0) and once means eval loss (camp1) + games = [ + outcome(game_id="a", eval_camp=0, raw_winner=0), # win + outcome(game_id="b", eval_camp=1, raw_winner=0), # loss (swapped) + ] + assert compute_win_rate(games) == pytest.approx(0.5) + # naive "raw_winner==0 => win" would wrongly give 1.0 + + +# ---- 4: draw ------------------------------------------------------------- # +def test_draw_recorded_not_a_win(): + # Miracle Judge never draws; simulate a hypothetical draw (raw_winner=-1). + games = [ + outcome(game_id="a", eval_camp=0, raw_winner=0), # win + finalize(GameOutcome(game_id="b", evaluated_agent="ifelse", + opponent="rank04", evaluated_agent_camp=0, + raw_winner=-1)), # draw + ] + counts = outcome_counts(games) + assert counts["win"] == 1 and counts["draw"] == 1 + # draw is a valid game but not a win -> win_rate = 1/2 + assert compute_win_rate(games) == pytest.approx(0.5) + + +# ---- 5-9: anomaly -> error, never a capability win ----------------------- # +def test_opponent_crash_is_error_not_win(): + # Judge gave camp0 the win (raw_winner=0, evaluated on camp0), but the + # OPPONENT crashed -> invalid evidence, must NOT count as a capability win. + o = outcome(eval_camp=0, raw_winner=0, ai_error_player=1) + assert o.normalized_result == ERROR + assert o.valid is False + + +def test_agent_crash_is_error(): + o = outcome(eval_camp=0, raw_winner=1, ai_error_player=0) + assert o.normalized_result == ERROR + + +def test_judge_crash_is_error(): + o = outcome(eval_camp=0, judge_ok=False) + assert o.normalized_result == ERROR + assert o.raw_winner is None or o.normalized_result == ERROR + + +def test_timeout_is_error(): + o = outcome(eval_camp=0, raw_winner=0, ai_timeout_player=0) + assert o.normalized_result == ERROR + o2 = outcome(eval_camp=0, raw_winner=1, ai_timeout_player=1) + assert o2.normalized_result == ERROR + + +def test_replay_missing_is_error(): + o = outcome(eval_camp=0, raw_winner=0, replay_ok=False) + assert o.normalized_result == ERROR + + +def test_error_games_excluded_from_win_rate(): + games = [ + outcome(game_id="ok", eval_camp=0, raw_winner=0), # win + outcome(game_id="crash", eval_camp=0, raw_winner=0, + ai_error_player=1), # error + ] + # only 1 valid game (the win) -> win_rate 1.0, not 0.5 + assert compute_win_rate(games) == 1.0 + assert outcome_counts(games)["error"] == 1 + + +# ---- 10: duplicate game_id resumability ---------------------------------- # +def test_select_games_skips_completed_successful(): + planned = ["g1", "g2", "g3", "g4"] + done_valid = ["g2"] # g2 already has a valid result + assert select_games_to_run(planned, done_valid) == ["g1", "g3", "g4"] + + +def test_would_rerun_successful_detected(): + done = ["g2"] + assert would_rerun_successful("g2", done) is True + assert would_rerun_successful("g1", done) is False + + +def test_no_valid_games_means_rerun_all(): + planned = ["g1", "g2"] + assert select_games_to_run(planned, []) == ["g1", "g2"] + + +# ---- 13: h2h direction --------------------------------------------------- # +def test_h2h_direction_no_draws_sums_to_one(): + # ifelse vs rank04: ifelse won 3, rank04 won 1, across side-swaps + games = [ + outcome(game_id="1", eval_camp=0, raw_winner=0), # ifelse win + outcome(game_id="2", eval_camp=1, raw_winner=1), # ifelse win (swapped) + outcome(game_id="3", eval_camp=0, raw_winner=0), # ifelse win + outcome(game_id="4", eval_camp=0, raw_winner=1), # rank04 win + ] + h2h = compute_h2h(games) + assert h2h["ifelse"]["rank04"] == pytest.approx(0.75) + assert h2h["rank04"]["ifelse"] == pytest.approx(0.25) + # no draws -> symmetric pair sums to 1 + assert h2h["ifelse"]["rank04"] + h2h["rank04"]["ifelse"] == pytest.approx(1.0) + + +def test_h2h_with_draw_does_not_sum_to_one(): + games = [ + outcome(game_id="1", eval_camp=0, raw_winner=0), # win + finalize(GameOutcome(game_id="2", evaluated_agent="ifelse", + opponent="rank04", evaluated_agent_camp=0, + raw_winner=-1)), # draw + ] + h2h = compute_h2h(games) + assert h2h["ifelse"]["rank04"] == pytest.approx(0.5) # 1 win / 2 valid + assert h2h["rank04"]["ifelse"] == pytest.approx(0.0) + assert h2h["ifelse"]["rank04"] + h2h["rank04"]["ifelse"] == pytest.approx(0.5) + + +def test_h2h_excludes_error_games(): + games = [ + outcome(game_id="1", eval_camp=0, raw_winner=0), # win + outcome(game_id="2", eval_camp=0, raw_winner=0, ai_error_player=1), # error + ] + h2h = compute_h2h(games) + assert h2h["ifelse"]["rank04"] == pytest.approx(1.0) # only the valid game counts + + +# ---- end_info parsing (Judge main.py semantics) -------------------------- # +def test_derive_raw_winner_player0_wins(): + assert derive_raw_winner({"0": 5, "1": 2}) == 0 + + +def test_derive_raw_winner_player1_wins_on_tie(): + # Judge breaks ties toward player1 (main.py:448-449) + assert derive_raw_winner({"0": 3, "1": 3}) == 1 + assert derive_raw_winner({"0": 2, "1": 5}) == 1 + + +def test_derive_raw_winner_none_when_missing(): + assert derive_raw_winner(None) is None + assert derive_raw_winner({}) is None + assert derive_raw_winner({"0": 1}) is None + assert derive_raw_winner({"0": "x", "1": "y"}) is None + + +def test_scores_from_end_info(): + assert scores_from_end_info({"0": 7, "1": 4}) == (7, 4) + assert scores_from_end_info(None) == (None, None) + + +# ---- replay header + file hashing ---------------------------------------- # +def test_read_replay_header(tmp_path): + # [0,0,0,map_type,day_time,0,0] as big-endian signed int32 + blob = struct.pack(">7i", 0, 0, 0, 1, 0, 0, 0) + p = tmp_path / "g.replay" + p.write_bytes(blob) + assert read_replay_header(p) == {"map_type": 1, "day_time": 0} + + +def test_read_replay_header_missing(tmp_path): + assert read_replay_header(tmp_path / "nope.replay") is None + + +def test_read_replay_header_too_short(tmp_path): + p = tmp_path / "short.replay" + p.write_bytes(b"\x00" * 10) + assert read_replay_header(p) is None + + +def test_sha256_file(tmp_path): + p = tmp_path / "x.bin" + p.write_bytes(b"hello") + # known sha256 of "hello" + assert sha256_file(p) == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + + +def test_sha256_file_missing(tmp_path): + assert sha256_file(tmp_path / "nope") is None + + +# ---- event-record contract completeness ---------------------------------- # +def test_event_record_has_all_required_fields(): + o = outcome(game_id="g1", eval_camp=0, raw_winner=0, + replay_path="/tmp/g1.replay", replay_sha256="abc", + evaluated_source_sha256="d1", opponent_source_sha256="e1", + judge_exit=0, ai0_exit=0, ai1_exit=0, steps=42, + started_at=1.0, finished_at=2.0, duration_s=1.0) + rec = to_event_record(o) + missing = [f for f in ( + "game_id", "seed", "policy_ids", "policy_source_sha256", "camps", + "raw_winner", "winner_agent", "normalized_result", "scores", "draw", + "started_at", "finished_at", "duration", "judge_exit", "ai0_exit", + "ai1_exit", "timeout_s", "exception", "replay_path", "replay_sha256", + "valid", "is_resume", "is_rerun") if f not in rec] + assert missing == [] + # direction-critical fields must be present and correct + assert rec["raw_winner"] == 0 + assert rec["winner_agent"] == "ifelse" + assert rec["normalized_result"] == WIN + assert rec["evaluated_agent_camp"] == 0 + assert rec["valid"] is True + + +def test_event_record_seed_provenance_and_serializable(): + o = outcome(game_id="g1", eval_camp=0, raw_winner=0, + realized_randomization={"map_type": 1, "day_time": 0}) + rec = to_event_record(o) + # seed field is the structured provenance, NOT map_type/day_time called a seed + assert rec["seed"] == { + "requested_seed": None, "effective_seed": None, + "deterministic_seed_supported": False, "reproducible_from_seed": False, + "realized_randomization": {"map_type": 1, "day_time": 0}, + } + # must round-trip through JSON (events.jsonl is newline-delimited JSON) + s = json.dumps(rec, default=str) + assert json.loads(s)["game_id"] == "g1" + + +# ---- run-level statistics ------------------------------------------------ # +def test_compute_run_stats_counts_and_steps(): + outcomes = [ + outcome(game_id="1", eval_camp=0, raw_winner=0, steps=40), # win + outcome(game_id="2", eval_camp=1, raw_winner=1, steps=38), # win (swapped) + outcome(game_id="3", eval_camp=0, raw_winner=1, steps=50), # loss + outcome(game_id="4", eval_camp=0, raw_winner=0, steps=10, ai_error_player=1), # error + ] + stats = compute_run_stats(outcomes) + assert stats["attempted_games"] == 4 + assert stats["valid_games"] == 3 + assert stats["invalid_games"] == 1 + assert (stats["wins"], stats["losses"], stats["draws"]) == (2, 1, 0) + assert stats["win_rate_denominator"] == 3 + assert stats["win_rate"] == pytest.approx(2 / 3) + assert stats["attempted_steps"] == 138 # 40+38+50+10 + assert stats["total_steps"] == 128 # 40+38+50 (valid only) + assert stats["evaluation_status"] == "COMPLETE" + + +def test_compute_run_stats_no_valid_games_win_rate_null(): + outcomes = [ + outcome(game_id="1", eval_camp=0, raw_winner=0, steps=10, ai_error_player=1), + outcome(game_id="2", eval_camp=0, judge_ok=False), + ] + stats = compute_run_stats(outcomes) + assert stats["valid_games"] == 0 + assert stats["win_rate"] is None + assert stats["evaluation_status"] == "NO_VALID_GAMES" + assert stats["win_rate_denominator"] == 0 + + +# ---- tie semantics: Judge resolves ties to player1, NOT a draw ----------- # +def test_tie_resolved_to_player1_not_draw(): + o_win = outcome(eval_camp=1, raw_winner=1, score0=4, score1=4) # evaluated camp1 wins the tiebreak + assert o_win.score_tie is True and o_win.judge_tiebreak_applied is True + assert o_win.normalized_result == "win" + assert o_win.draw is False + o_loss = outcome(eval_camp=0, raw_winner=1, score0=4, score1=4) # evaluated camp0 loses the tiebreak + assert o_loss.normalized_result == "loss" + assert o_loss.score_tie is True + assert o_loss.draw is False + + +def test_build_seed_provenance_structure(): + p = build_seed_provenance({"map_type": 0, "day_time": 1}) + assert p == { + "requested_seed": None, "effective_seed": None, + "deterministic_seed_supported": False, "reproducible_from_seed": False, + "realized_randomization": {"map_type": 0, "day_time": 1}, + } diff --git a/tests/miracle/test_runner.py b/tests/miracle/test_runner.py new file mode 100644 index 0000000..ec1c59f --- /dev/null +++ b/tests/miracle/test_runner.py @@ -0,0 +1,129 @@ +"""MiracleEvalRunner tests (阶段4b-6). Uses an injected fake attempt_fn so the +runner logic is exercised without running real matches (real-match behaviour is +covered by match_runner's own tests + smoke). + +Verifies: + * run-level statistics land in the persisted summary (attempted/valid/invalid/ + wins/losses/draws/win_rate_denominator/attempted_steps/total_steps/evaluation_status); + * win_rate == wins/valid_games and matches an independent recompute from events; + * valid_games == 0 -> win_rate is null, evaluation_status NO_VALID_GAMES; + * side-swapped wins both count (raw_winner differs across games); + * no runs/runs double-dir (risk #6); disk summary (not just in-memory) is correct; + * Run is driven directly (no BaseRunner). +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from agentbench_frame.games.miracle.match_runner import MatchAttempt +from agentbench_frame.games.miracle.runner import MiracleEvalRunner + +SRC = Path(__file__).resolve().parents[2] / "src" + + +def _mk_att(*, game_id, evaluated_agent_camp, evaluated_agent="ifelse", + opponent="rank04", normalized="error", error_type="ai_crash", + raw_winner=None, scores=None, steps=30, + realized=None, reason="synthetic") -> MatchAttempt: + valid = normalized in ("win", "loss", "draw") + if raw_winner is None and valid: + raw_winner = evaluated_agent_camp if normalized == "win" else (1 - evaluated_agent_camp) + winner = None + if valid and raw_winner in (0, 1): + winner = evaluated_agent if raw_winner == evaluated_agent_camp else opponent + return MatchAttempt( + game_id=game_id, evaluated_agent=evaluated_agent, opponent=opponent, + evaluated_agent_camp=evaluated_agent_camp, + valid=valid, normalized_result=normalized, + error_type=(None if valid else error_type), reason=reason, + raw_winner=raw_winner, winner_agent=winner, scores=scores, steps=steps, + realized_randomization=realized, + result_json_status=("ok" if error_type not in ("result_json_missing", "result_json_corrupt") else "missing"), + discrepancies=[], ai_crash_player=(0 if error_type == "ai_crash" else None), + ai_timeout_player=(0 if error_type == "ai_timeout" else None), + judge_crash=(error_type == "judge_crash"), + wrapper_timeout=(error_type == "wrapper_timeout"), + normal_cleanup_nonzero=False, + evidence_paths={"stdout": "", "stderr": "", "trace": "", "replay": "", "result_json": ""}, + collision_detected=False, process_cleanup=[], vendor_returncode=0, + started_at=0.0, finished_at=1.0, duration_s=1.0, exception=None, + ) + + +def _runner_with_plan(tmp_path: Path, plan: List[Dict[str, Any]], *, n_games=None): + counter = {"i": 0} + n = n_games if n_games is not None else len(plan) + + def fake(*, game_id, evaluated_agent_camp, evaluated_agent, opponent, **_): + i = counter["i"] + counter["i"] += 1 + p = plan[i % len(plan)] + return _mk_att( + game_id=game_id, evaluated_agent_camp=evaluated_agent_camp, + evaluated_agent=evaluated_agent, opponent=opponent, **p, + ) + + return MiracleEvalRunner( + agent="ifelse", data_dir=str(tmp_path), judge_dir=tmp_path / "judge", + vendor_script=tmp_path / "vendor.py", framework_src=str(SRC), + evaluated_dir=tmp_path / "eval", opponent_dir=tmp_path / "opp", n_games=n, + opponent="rank04", work_dir=tmp_path / "work", attempt_fn=fake, + config={"source": "test"}, + ) + + +def test_mixed_run_statistics_and_consistency(tmp_path): + plan = [ + {"normalized": "win", "scores": {"0": 5, "1": 2}, "steps": 40, "realized": {"map_type": 0, "day_time": 1}}, + {"normalized": "win", "scores": {"0": 2, "1": 5}, "steps": 38, "realized": {"map_type": 1, "day_time": 0}}, + {"normalized": "loss", "scores": {"0": 2, "1": 5}, "steps": 50, "realized": {"map_type": 0, "day_time": 0}}, + {"normalized": "error", "error_type": "ai_crash", "reason": "ai crash", "steps": 10}, + ] + runner = _runner_with_plan(tmp_path, plan) + summary = runner.run() + # persisted on disk + run_dir = tmp_path / "runs" / "24_miracle" / "ifelse" / runner_run_id(runner, summary) + disk = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) + for k in ("attempted_games", "valid_games", "invalid_games", "wins", "losses", + "draws", "win_rate_denominator", "attempted_steps", "total_steps", + "evaluation_status", "win_rate_available", "h2h"): + assert k in disk, f"summary missing {k}" + assert disk["attempted_games"] == 4 + assert disk["valid_games"] == 3 and disk["invalid_games"] == 1 + assert (disk["wins"], disk["losses"], disk["draws"]) == (2, 1, 0) + assert disk["win_rate_denominator"] == 3 + assert disk["win_rate"] == pytest.approx(2 / 3) + assert disk["attempted_steps"] == 138 and disk["total_steps"] == 128 + assert disk["evaluation_status"] == "COMPLETE" and disk["win_rate_available"] is True + assert disk["total_episodes"] == 3 # == valid_games + assert disk["win_rate"] == pytest.approx(summary["_recompute_check"]["win_rate"]) + # no double-runs + assert not (tmp_path / "runs" / "runs").exists() + + +def test_no_valid_games_win_rate_null(tmp_path): + plan = [{"normalized": "error", "error_type": "ai_crash", "reason": "x", "steps": 5}] + runner = _runner_with_plan(tmp_path, plan, n_games=2) + summary = runner.run() + assert summary["valid_games"] == 0 + assert summary["win_rate"] is None + assert summary["evaluation_status"] == "NO_VALID_GAMES" + assert summary["win_rate_available"] is False + assert summary["_recompute_check"]["win_rate"] is None + + +def test_side_swap_both_wins_count(tmp_path): + plan = [{"normalized": "win", "scores": {"0": 5, "1": 2}, "steps": 40, + "realized": {"map_type": 0, "day_time": 0}}] + runner = _runner_with_plan(tmp_path, plan, n_games=2) # camp0 then camp1 + summary = runner.run() + assert summary["wins"] == 2 + assert summary["win_rate"] == pytest.approx(1.0) + + +def runner_run_id(runner, summary) -> str: + return summary["run_id"] diff --git a/tests/miracle/test_smoke_audit.py b/tests/miracle/test_smoke_audit.py new file mode 100644 index 0000000..3b89a79 --- /dev/null +++ b/tests/miracle/test_smoke_audit.py @@ -0,0 +1,182 @@ +"""Tests for the smoke evidence-safety helpers (阶段4b smoke driver fixes).""" +from __future__ import annotations + +import json +import subprocess +import sys +import time +from pathlib import Path + +import psutil +import pytest + +from agentbench_frame.games.miracle.smoke_audit import ( + ManagedProc, + build_manifest, + check_residual_procs, + ensure_fresh_session, + group1_strict_clean, + load_managed_procs_from_result_json, + make_session_id, + session_exists, + should_run_group2, + write_manifest_atomic, +) + + +# ---- 1. existing session is rejected, never deleted ---- # +def test_existing_session_rejected_and_not_deleted(tmp_path): + sid = make_session_id() + sd = ensure_fresh_session(tmp_path, sid) + (sd / "marker").write_text("prior evidence") + # a second ensure with the SAME id must refuse + with pytest.raises(FileExistsError): + ensure_fresh_session(tmp_path, sid) + # and the prior evidence must still be there + assert (sd / "marker").read_text() == "prior evidence" + + +def test_session_ids_are_unique(): + ids = {make_session_id() for _ in range(50)} + assert len(ids) == 50 + + +# ---- a fake MatchAttempt-shaped object for gate tests ---- # +class FakeAtt: + def __init__(self, *, game_id="g1_00_camp0", valid=True, normalized_result="win", + error_type=None, wrapper_timeout=False, result_json_status="ok", + discrepancies=None, realized_randomization=None, raw_winner=0, + reason="", evidence_paths=None): + self.game_id = game_id + self.valid = valid + self.normalized_result = normalized_result + self.error_type = error_type + self.wrapper_timeout = wrapper_timeout + self.result_json_status = result_json_status + self.discrepancies = discrepancies or [] + self.realized_randomization = realized_randomization or {"map_type": 0, "day_time": 1} + self.raw_winner = raw_winner + self.reason = reason + self.evidence_paths = evidence_paths or {"result_json": ""} + + +def _good_summary(): + return {"attempted_games": 2, "valid_games": 2, "invalid_games": 0, + "evaluation_status": "COMPLETE"} + + +def _two_good_attempts_with_procs(tmp_path): + # give each attempt a result-json with judge/ai0/ai1 identity for the residual check + rj = tmp_path / "rj.json" + rj.write_text(json.dumps({ + "judge": {"pid": 999999, "started_at": 1.0, "role": "judge"}, + "ai0": {"pid": 999998, "started_at": 1.0, "role": "ai0"}, + "ai1": {"pid": 999997, "started_at": 1.0, "role": "ai1"}, + })) + a0 = FakeAtt(game_id="g1_00_camp0", raw_winner=0, + evidence_paths={"result_json": str(rj)}) + a1 = FakeAtt(game_id="g1_01_camp1", raw_winner=1, + evidence_paths={"result_json": str(rj)}) + return [a0, a1] + + +# ---- 2. invalid attempt blocks Group 2 ---- # +def test_invalid_attempt_blocks_group2(tmp_path): + attempts = _two_good_attempts_with_procs(tmp_path) + attempts[0].valid = False + attempts[0].error_type = "ai_crash" + ok, reasons = group1_strict_clean(attempts, _good_summary()) + assert ok is False + assert should_run_group2(ok) is False + + +# ---- 3. attempt count < or > 2 blocks ---- # +def test_attempt_count_not_two_blocks(tmp_path): + one = _two_good_attempts_with_procs(tmp_path)[:1] + ok, reasons = group1_strict_clean(one, _good_summary()) + assert ok is False and any("attempt_count" in r for r in reasons) + three = _two_good_attempts_with_procs(tmp_path) + [FakeAtt()] + ok3, _ = group1_strict_clean(three, _good_summary()) + assert ok3 is False + + +# ---- 4. summary count mismatch blocks ---- # +def test_summary_count_mismatch_blocks(tmp_path): + attempts = _two_good_attempts_with_procs(tmp_path) + bad = {"attempted_games": 2, "valid_games": 1, "invalid_games": 1, + "evaluation_status": "COMPLETE"} + ok, reasons = group1_strict_clean(attempts, bad) + assert ok is False + assert any("valid_games" in r for r in reasons) + + +# ---- 5. PID alive + same create_time -> residual ---- # +def test_residual_detected_for_live_pid_same_createtime(): + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + ct = psutil.Process(p.pid).create_time() + res = check_residual_procs([ManagedProc(pid=p.pid, started_at=ct, role="judge")]) + assert res["residual"] and res["residual"][0].pid == p.pid + assert res["clean"] == [] and res["reused"] == [] + finally: + p.terminate(); p.wait(timeout=5) + + +# ---- 6. PID reuse (wrong create_time) -> NOT residual, NOT killed ---- # +def test_pid_reuse_not_flagged_as_residual(): + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + res = check_residual_procs([ManagedProc(pid=p.pid, started_at=1.0, role="ai0")]) + assert res["residual"] == [] # not flagged + assert res["reused"] and res["reused"][0].pid == p.pid + assert psutil.pid_exists(p.pid) # still alive — we did NOT kill it + finally: + p.terminate(); p.wait(timeout=5) + + +# ---- 7. missing process identity -> cannot pass via empty set ---- # +def test_missing_proc_identity_blocks_gate(tmp_path): + a0 = FakeAtt(game_id="g1_00_camp0", evidence_paths={"result_json": ""}) # no identity + a1 = FakeAtt(game_id="g1_01_camp1", evidence_paths={"result_json": ""}) + ok, reasons = group1_strict_clean([a0, a1], _good_summary()) + assert ok is False + assert any("no managed-proc identity" in r for r in reasons) + + +def test_load_managed_procs_handles_missing_and_identity(): + assert load_managed_procs_from_result_json("/no/such/file.json") == [] + import tempfile + f = Path(tempfile.mkdtemp()) / "r.json" + f.write_text(json.dumps({"judge": {"pid": 123, "started_at": 9.0}})) + procs = load_managed_procs_from_result_json(f) + assert len(procs) == 1 and procs[0].pid == 123 and procs[0].role == "judge" + + +# ---- 8. full manifest + log file generated ---- # +def test_manifest_written_atomically(tmp_path): + code = tmp_path / "code.py"; code.write_text("print(1)") + asset = tmp_path / "asset.bin"; asset.write_bytes(b"xyz") + m = build_manifest(session_id="SID", auth_cap=4, python_executable="py", + python_version="3.13.5", code_files=[code], asset_files=[asset], + groups_planned=[{"group": "GROUP1", "n_games": 2}], + notes=["prior 429 was a tool rejection, not a game result"]) + p = write_manifest_atomic(tmp_path, m) + loaded = json.loads(p.read_text(encoding="utf-8")) + assert loaded["session_id"] == "SID" + assert loaded["auth_cap_games"] == 4 + assert loaded["code_hashes"][str(code)] + assert loaded["notes"][0].startswith("prior 429") + # atomic temp must be gone + assert not (tmp_path / "manifest.json.tmp").exists() + + +# ---- 9. Group 2 never called when Group 1 not fully passing ---- # +def test_group2_only_when_group1_clean(): + assert should_run_group2(True) is True + assert should_run_group2(False) is False + + +def test_strict_clean_passes_for_two_clean_attempts(tmp_path): + attempts = _two_good_attempts_with_procs(tmp_path) + ok, reasons = group1_strict_clean(attempts, _good_summary()) + assert ok is True, reasons diff --git a/tools/miracle_precheck.py b/tools/miracle_precheck.py new file mode 100644 index 0000000..8383704 --- /dev/null +++ b/tools/miracle_precheck.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""24_miracle 阶段9A 预检：隔离编译 13 个 C++ 策略 + 无对局启动预检。 + +绝不启动 Judge / server / 真实对局。对每个 C++ 策略： + 1. 从受保护 extracted 源目录复制到唯一全新 session 目录（不动原策略）。 + 2. 用策略自带 makefile 原样编译（不改逻辑/优化）。 + 3. 记录源哈希、编译命令、完整 stdout/stderr/exit、产物类型/大小/SHA256、PE/DLL 静态检查。 + 4. 用 entry.resolve_ai_command 构造命令（与正式 runner 一致），短时 Popen（stdin=DEVNULL， + 不接 Judge），按 PID+create_time 精确清理。 + 5. 分类：COMPILE_PASS / OS_SPAWN_PASS / COMMAND_RESOLUTION_PASS / PROTOCOL_NOT_VALIDATED。 + +无 Judge 启动 → 协议/比赛可用性未验证（PROTOCOL_NOT_VALIDATED），不得表述为"策略已成功完成比赛"。 +rank03 同样编译记录，但保留"运行时崩溃风险"（编译/启动成功 ≠ 比赛可用）。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.entry import resolve_ai_command # noqa: E402 +from agentbench_frame.games.miracle.proctree import ProcessTreeManager # noqa: E402 +from agentbench_frame.games.miracle.smoke_audit import ( # noqa: E402 + ensure_fresh_session, make_session_id, write_manifest_atomic, +) + +from agentbench_frame.games.miracle.paths import extracted_dir +EXTRACTED = extracted_dir() +ROSTER = REPO / "docs" / "games" / "24_miracle_roster_manifest.json" +PRECHECK_ROOT = REPO / ".smoke" / "precheck" +LOG = None + + +def log(msg=""): + print(msg, flush=True) + if LOG: + LOG.write(str(msg) + "\n"); LOG.flush() + + +def sha256_file(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + + +def is_pe(p: Path) -> bool: + try: + return p.read_bytes()[:2] == b"MZ" + except OSError: + return False + + +def dll_deps(p: Path): + """Static DLL-dependency check via objdump (no execution of the AI).""" + try: + r = subprocess.run(["objdump", "-p", str(p)], capture_output=True, text=True, timeout=15) + deps = [ln.split(":", 1)[1].strip() for ln in r.stdout.splitlines() + if ln.strip().startswith("DLL Name")] + return deps + except Exception as e: + return [f""] + + +def find_binary(d: Path): + for name in ("main.exe", "main"): + if (d / name).exists(): + return d / name + return None + + +def cpp_strategies(): + roster = json.loads(ROSTER.read_text(encoding="utf-8")) + out = [] + for s in roster["strategies"]: + if s.get("type") == "cpp_source": + rank = s["rank"] + cand = list(EXTRACTED.glob(f"rank{rank:02d}__*")) + out.append({"rank": rank, "name": s["entity"], "extracted": cand[0] if cand else None, + "archive_sha256": s["archive_sha256"], "invalid_now": s.get("invalid_now", False)}) + return out + + +def compile_one(copy_dir: Path): + """Run the strategy's own makefile. Record everything.""" + t0 = time.time() + cmd = ["make"] + try: + r = subprocess.run(cmd, cwd=str(copy_dir), capture_output=True, text=True, timeout=180) + return {"command": cmd, "stdout": r.stdout, "stderr": r.stderr, + "returncode": r.returncode, "duration_s": round(time.time() - t0, 2), + "exception": None} + except subprocess.TimeoutExpired as e: + return {"command": cmd, "stdout": e.stdout or "", "stderr": e.stderr or "", + "returncode": None, "duration_s": round(time.time() - t0, 2), + "exception": "timeout"} + except Exception as e: + return {"command": cmd, "stdout": "", "stderr": str(e), + "returncode": None, "duration_s": round(time.time() - t0, 2), + "exception": repr(e)} + + +def spawn_check(copy_dir: Path, timeout_s=2.0): + """Short isolated process-spawn check. NO Judge. stdin=DEVNULL so the AI + sees EOF and exits fast; we only verify the OS can create the process.""" + result = {"command_resolution": None, "command_resolution_pass": False, + "os_spawn_pass": None, "returncode": None, "stderr_head": None, + "cleanup_all_succeeded": None, "exception": None, "protocol_validated": False} + # 1. command resolution (same path the real runner uses) + try: + cmd = resolve_ai_command(copy_dir) + result["command_resolution"] = cmd + result["command_resolution_pass"] = True + except FileNotFoundError as e: + result["command_resolution"] = f"FileNotFoundError: {e}" + result["command_resolution_pass"] = False + return result + # 2. OS spawn (NO judge; stdin closed) + mgr = ProcessTreeManager() + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0 + try: + proc = subprocess.Popen(cmd, cwd=str(copy_dir), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + creationflags=creationflags) + mgr.register_popen(proc, f"ai_spawn") + result["os_spawn_pass"] = True + try: + proc.communicate(timeout=timeout_s) + result["returncode"] = proc.returncode + except subprocess.TimeoutExpired: + result["returncode"] = "timeout" + except (OSError, FileNotFoundError) as e: + result["os_spawn_pass"] = False + result["exception"] = repr(e) + return result + finally: + # precise PID + create_time cleanup + procs = mgr.cleanup_all("spawn-check") + result["cleanup_all_succeeded"] = all(p.cleanup_succeeded for p in procs) + result["cleanup_detail"] = mgr.status() # list[dict], JSON-serializable + # capture a short stderr head if any + return result + + +def main() -> int: + global LOG + session_id = make_session_id() + session_dir = ensure_fresh_session(PRECHECK_ROOT, session_id) + LOG = open(session_dir / "precheck.full.log", "w", encoding="utf-8") + log(f"precheck session: {session_dir}") + log(f"compiler: g++ {subprocess.run(['g++','--version'],capture_output=True,text=True).stdout.splitlines()[0]}") + log(f"make: {subprocess.run(['make','--version'],capture_output=True,text=True).stdout.splitlines()[0]}") + log("NOTE: 无 Judge / 无对局启动；仅编译 + 短时进程创建预检。PROTOCOL_NOT_VALIDATED。") + + targets = cpp_strategies() + reports = [] + for t in targets: + rank = t["rank"]; name = t["name"] + log(f"\n===== rank{rank:02d} ({name}) =====") + if t["extracted"] is None: + log(f" EXTRACTED_DIR_MISSING"); reports.append({"rank": rank, "status": "MISSING_SOURCE"}); continue + copy_dir = session_dir / "strategies" / f"rank{rank:02d}" + shutil.copytree(t["extracted"], copy_dir) + # source hashes (record, do not modify) + src_files = sorted(p for p in copy_dir.iterdir() if p.suffix in (".cpp", ".c", ".h", ".hpp", ".json") or p.name == "makefile") + source_hashes = {p.name: sha256_file(p) for p in src_files} + log(f" copied {len(src_files)} source files from {t['extracted'].name}") + # compile + comp = compile_one(copy_dir) + binary = find_binary(copy_dir) + compile_pass = (comp["returncode"] == 0 and binary is not None) + log(f" compile: returncode={comp['returncode']} binary={'main.exe' if binary and binary.name=='main.exe' else binary.name if binary else None} -> {'COMPILE_PASS' if compile_pass else 'COMPILE_FAIL'}") + if comp["stderr"]: + log(f" [stderr tail] {comp['stderr'][-300:]!r}") + bin_info = None + if binary: + bin_info = {"name": binary.name, "size": binary.stat().st_size, + "sha256": sha256_file(binary), "is_pe": is_pe(binary), + "dll_deps": dll_deps(binary)} + log(f" binary: size={bin_info['size']} pe={bin_info['is_pe']} sha256={bin_info['sha256'][:16]}… dlls={bin_info['dll_deps']}") + # spawn pre-check (only if compiled) + spawn = None + if compile_pass: + spawn = spawn_check(copy_dir) + log(f" spawn: cmd_res={spawn['command_resolution_pass']} os_spawn={spawn['os_spawn_pass']} rc={spawn['returncode']} cleanup={spawn['cleanup_all_succeeded']}") + report = { + "rank": rank, "name": name, "invalid_now": t["invalid_now"], + "source_dir_original": str(t["extracted"]), + "source_dir_copy": str(copy_dir), + "archive_sha256": t["archive_sha256"], + "source_hashes": source_hashes, + "compile": comp, "binary": bin_info, + "compile_pass": compile_pass, + "spawn_precheck": spawn, + "classification": { + "COMPILE_PASS": compile_pass, + "OS_SPAWN_PASS": bool(spawn and spawn["os_spawn_pass"]), + "COMMAND_RESOLUTION_PASS": bool(spawn and spawn["command_resolution_pass"]), + "PROTOCOL_NOT_VALIDATED": True, + }, + "verification_note": "仅静态编译 + 短时进程创建预检；未启动 Judge/对局，不证明协议或比赛可用。" + + (" rank03 历史运行时崩溃风险保留，编译/启动成功不等于比赛可用。" if rank == 3 else ""), + } + reports.append(report) + (session_dir / "reports" ).mkdir(exist_ok=True) + (session_dir / "reports" / f"rank{rank:02d}.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + + manifest = { + "session_id": session_id, "kind": "stage9A_precheck", + "created_unix": time.time(), "python": sys.executable, + "python_version": platform.python_version(), + "compiler": "g++ 15.2.0 (MinGW)", "make": "GNU Make 4.4.1", + "no_judge_no_match": True, + "strategy_count": len(reports), + "summary": { + "compile_pass": sum(1 for r in reports if r.get("compile_pass")), + "compile_fail": sum(1 for r in reports if not r.get("compile_pass") and r.get("status") != "MISSING_SOURCE"), + "missing_source": sum(1 for r in reports if r.get("status") == "MISSING_SOURCE"), + }, + } + write_manifest_atomic(session_dir, manifest) + (session_dir / "summary.json").write_text(json.dumps({"reports": reports}, ensure_ascii=False, indent=2), encoding="utf-8") + log(f"\n===== SUMMARY =====") + log(f"compile_pass={manifest['summary']['compile_pass']}/{len(reports)} compile_fail={manifest['summary']['compile_fail']} missing={manifest['summary']['missing_source']}") + for r in reports: + c = r.get("classification", {}) + log(f" rank{r['rank']:02d}: compile={c.get('COMPILE_PASS')} os_spawn={c.get('OS_SPAWN_PASS')} cmd_res={c.get('COMMAND_RESOLUTION_PASS')} protocol=NOT_VALIDATED{' [invalid_now]' if r.get('invalid_now') else ''}") + log(f"\nsession dir: {session_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/miracle_rank16_build.py b/tools/miracle_rank16_build.py new file mode 100644 index 0000000..504bef1 --- /dev/null +++ b/tools/miracle_rank16_build.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""阶段9B：rank16 隔离构建修复 + 无对局启动预检。 + +用户决定：允许只在 rank16 的隔离构建副本中预先创建空 ``build/`` 目录（构建环境 +准备，非策略修复）。不改源码 / Makefile / 编译参数 / 优化。 + +步骤： + 1. 全新唯一 session；从受保护 extracted 源复制 rank16。 + 2. 记录编译前全部源文件 + Makefile SHA256。 + 3. 仅在副本中创建 Makefile 预期的空 build/。 + 4. 用原 Makefile 编译一次。 + 5. 编译后再次核对源文件 + Makefile 哈希 == 编译前（必须一致）。 + 6. 记录新增文件清单（仅构建产物 + 空目录准备）。 + 7. 若成功：PE/大小/SHA256/依赖 + 绝对路径命令解析 + 短时无 Judge 启动 + PID+create_time 清理。 +不删除此前 rank16 失败证据。不启动 Judge / 真实对局。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.entry import resolve_ai_command # noqa: E402 +from agentbench_frame.games.miracle.proctree import ProcessTreeManager # noqa: E402 +from agentbench_frame.games.miracle.smoke_audit import ( # noqa: E402 + ensure_fresh_session, make_session_id, write_manifest_atomic, +) + +from agentbench_frame.games.miracle.paths import extracted_dir +EXTRACTED = extracted_dir() +RANK16_SRC = next(EXTRACTED.glob("rank16__*")) +SESSION_ROOT = REPO / ".smoke" / "rank16build" +LOG = None + +SOURCE_SUFFIXES = (".cpp", ".c", ".h", ".hpp", ".json") + + +def log(msg=""): + print(msg, flush=True) + if LOG: + LOG.write(str(msg) + "\n"); LOG.flush() + + +def sha256_file(p: Path) -> str: + h = hashlib.sha256() + with open(p, "rb") as f: + for b in iter(lambda: f.read(1 << 20), b""): + h.update(b) + return h.hexdigest() + + +def is_pe(p: Path) -> bool: + try: + return p.read_bytes()[:2] == b"MZ" + except OSError: + return False + + +def dll_deps(p: Path): + try: + r = subprocess.run(["objdump", "-p", str(p)], capture_output=True, text=True, timeout=15) + return [ln.split(":", 1)[1].strip() for ln in r.stdout.splitlines() + if ln.strip().startswith("DLL Name")] + except Exception as e: + return [f""] + + +def source_hashes(d: Path): + """Hash strategy SOURCE files only (not build/ artifacts, not main exe).""" + out = {} + for p in sorted(d.iterdir()): + if p.is_dir(): + continue + if p.suffix in SOURCE_SUFFIXES or p.name == "makefile": + out[p.name] = sha256_file(p) + return out + + +def spawn_check(copy_dir: Path, timeout_s=2.0): + res = {"command_resolution": None, "command_resolution_pass": False, + "os_spawn_pass": None, "returncode": None, "cleanup_all_succeeded": None, + "exception": None} + try: + cmd = resolve_ai_command(copy_dir) + res["command_resolution"] = cmd + res["command_resolution_pass"] = True + except FileNotFoundError as e: + res["command_resolution"] = f"FileNotFoundError: {e}" + return res + mgr = ProcessTreeManager() + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0 + try: + proc = subprocess.Popen(cmd, cwd=str(copy_dir), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + creationflags=creationflags) + mgr.register_popen(proc, "rank16_spawn") + res["os_spawn_pass"] = True + try: + proc.communicate(timeout=timeout_s) + res["returncode"] = proc.returncode + except subprocess.TimeoutExpired: + res["returncode"] = "timeout" + except (OSError, FileNotFoundError) as e: + res["os_spawn_pass"] = False + res["exception"] = repr(e) + return res + finally: + procs = mgr.cleanup_all("spawn-check") + res["cleanup_all_succeeded"] = all(p.cleanup_succeeded for p in procs) + res["cleanup_detail"] = mgr.status() + return res + + +def main() -> int: + global LOG + session_id = make_session_id() + session_dir = ensure_fresh_session(SESSION_ROOT, session_id) + LOG = open(session_dir / "rank16_build.full.log", "w", encoding="utf-8") + log(f"rank16 build session: {session_dir}") + log(f"source (protected, read-only): {RANK16_SRC}") + log("用户决定：仅在隔离副本创建空 build/（构建环境准备，非策略修复）；不改源码/Makefile。") + + copy_dir = session_dir / "rank16_copy" + shutil.copytree(RANK16_SRC, copy_dir) + log(f"copied to: {copy_dir}") + + hashes_before = source_hashes(copy_dir) + log(f"source files before: {len(hashes_before)} files hashed") + + # build-env prep ONLY in the copy: empty build/ dir the Makefile assumes + (copy_dir / "build").mkdir(exist_ok=False) + log("created empty build/ in copy (Makefile assumes it exists)") + + files_before_make = {p.name for p in copy_dir.iterdir() if p.is_file()} | {"build"} + + # compile with the ORIGINAL makefile (no parameter/optimization change) + cmd = ["make"] + t0 = time.time() + comp = subprocess.run(cmd, cwd=str(copy_dir), capture_output=True, text=True, timeout=180) + comp_rec = {"command": cmd, "stdout": comp.stdout, "stderr": comp.stderr, + "returncode": comp.returncode, "duration_s": round(time.time() - t0, 2)} + + hashes_after = source_hashes(copy_dir) + source_integrity_ok = (hashes_before == hashes_after) + + # new files after make (build artifacts + exe only) + files_after = set() + for p in copy_dir.rglob("*"): + if p.is_file(): + files_after.add(str(p.relative_to(copy_dir)).replace("\\", "/")) + files_before_set = set() + for p in copy_dir.rglob("*"): + pass + # compute new files = files present now that are NOT original source files + original_source_names = set(hashes_before.keys()) | {"Data.json"} if "Data.json" in hashes_before else set(hashes_before.keys()) + new_files = sorted(f for f in files_after if Path(f).name not in hashes_before) + + binary = None + for name in ("main.exe", "main"): + if (copy_dir / name).exists(): + binary = copy_dir / name + break + compile_pass = (comp_rec["returncode"] == 0 and binary is not None) + + log(f"compile: returncode={comp_rec['returncode']} binary={binary.name if binary else None} -> {'COMPILE_PASS' if compile_pass else 'COMPILE_FAIL'}") + log(f"source_integrity (before==after): {source_integrity_ok}") + log(f"new files after make: {new_files}") + if comp_rec["stderr"]: + log(f"[stderr tail] {comp_rec['stderr'][-400:]!r}") + + binary_info = None + spawn = None + if binary: + binary_info = {"name": binary.name, "size": binary.stat().st_size, + "sha256": sha256_file(binary), "is_pe": is_pe(binary), + "dll_deps": dll_deps(binary)} + log(f"binary: size={binary_info['size']} pe={binary_info['is_pe']} sha256={binary_info['sha256']} dlls={binary_info['dll_deps']}") + if compile_pass: + spawn = spawn_check(copy_dir) + log(f"spawn: cmd_res={spawn['command_resolution_pass']} os_spawn={spawn['os_spawn_pass']} rc={spawn['returncode']} cleanup={spawn['cleanup_all_succeeded']}") + + report = { + "session_id": session_id, + "rank": 16, + "source_dir_original": str(RANK16_SRC), + "source_dir_copy": str(copy_dir), + "build_env_prep": "created empty build/ in copy ONLY (Makefile assumes it); no source/Makefile/param/optimization change", + "source_hashes_before": hashes_before, + "source_hashes_after": hashes_after, + "source_integrity_ok": source_integrity_ok, + "new_files_after_make": new_files, + "make": comp_rec, + "binary": binary_info, + "compile_pass": compile_pass, + "spawn_precheck": spawn, + "classification": { + "COMPILE_PASS": compile_pass, + "OS_SPAWN_PASS": bool(spawn and spawn["os_spawn_pass"]), + "COMMAND_RESOLUTION_PASS": bool(spawn and spawn["command_resolution_pass"]), + "PROTOCOL_NOT_VALIDATED": True, + }, + "verification_note": "仅隔离编译（副本内建空 build/）+ 短时无 Judge 进程创建预检；未启动 Judge/对局，不证明协议或比赛可用。", + "prior_failure_evidence_preserved": ".smoke/precheck/20260721-184652_da899d/ (rank16 COMPILE_FAIL 历史保留)", + } + (session_dir / "rank16_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + manifest = { + "session_id": session_id, "kind": "stage9B_rank16_isolated_build", + "created_unix": time.time(), "python": sys.executable, + "compiler": subprocess.run(["g++", "--version"], capture_output=True, text=True).stdout.splitlines()[0], + "make": subprocess.run(["make", "--version"], capture_output=True, text=True).stdout.splitlines()[0], + "compile_pass": compile_pass, + "source_integrity_ok": source_integrity_ok, + "no_source_or_makefile_modification": source_integrity_ok, + } + write_manifest_atomic(session_dir, manifest) + log(f"\nRESULT: compile_pass={compile_pass} source_integrity_ok={source_integrity_ok}") + log(f"session dir: {session_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/miracle_smoke.py b/tools/miracle_smoke.py new file mode 100644 index 0000000..0e58fa3 --- /dev/null +++ b/tools/miracle_smoke.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""24_miracle local smoke (阶段6, pre-authorized, AT MOST 4 games total). + +Evidence-safety design (this round's fixes): + * Every run gets a unique session dir under .smoke/sessions//. + A pre-existing session id is REFUSED — this script NEVER deletes or + overwrites prior sessions, and NEVER re-runs a successful game. + * sampleA/sampleB are ASSET dirs, kept separate from session products. + * A manifest (time, code+asset hashes, python version, auth cap=4) is written + atomically before any match starts. + * Group 1 (sampleA vs sampleB, 2 games, camps swapped) must pass a STRICT gate + before Group 2 (if-else vs sampleB, 2 games, camps swapped) is even considered. + * Residual processes are checked INDEPENDENTLY by exact PID + psutil + create_time (never trusting cleanup_succeeded, never killing by name). + * Full stdout/stderr are written to UTF-8 files in the session dir; the console + shows a summary. Tool rejections / exceptions are recorded separately and are + NEVER written as game results. + +Run from the repo root with a 3.11+ interpreter, UTF-8 mode: + PYTHONUTF8=1 py -3.13 tools/miracle_smoke.py +""" +from __future__ import annotations + +import json +import platform +import sys +import traceback +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from agentbench_frame.games.miracle.runner import MiracleEvalRunner # noqa: E402 +from agentbench_frame.games.miracle.smoke_audit import ( # noqa: E402 + build_manifest, + check_residual_procs, + ensure_fresh_session, + group1_strict_clean, + load_managed_procs_from_result_json, + make_session_id, + should_run_group2, + write_manifest_atomic, +) + +SESSIONS_ROOT = REPO / ".smoke" / "sessions" +from agentbench_frame.games.miracle.paths import judge_dir, ifelse_dir, sample_ai_dir +JUDGE = judge_dir() +SAMPLE_AI_SRC = sample_ai_dir() +VENDOR = REPO / "vendor" / "miracle_local" / "run_match.py" +FW_SRC = REPO / "src" +SAMPLE_A = REPO / ".smoke" / "sampleA" +SAMPLE_B = REPO / ".smoke" / "sampleB" +IFELSE = ifelse_dir() +AUTH_CAP = 4 + +_LOG = None + + +def log(msg=""): + print(msg, flush=True) + if _LOG is not None: + _LOG.write(str(msg) + "\n") + _LOG.flush() + + +def _attempt_report(att) -> dict: + rr = att.realized_randomization or {} + rj_path = att.evidence_paths.get("result_json") + procs = load_managed_procs_from_result_json(rj_path) if rj_path else [] + res = check_residual_procs(procs) if procs else {"clean": [], "residual": [], "reused": []} + rec = { + "game_id": att.game_id, + "valid": att.valid, + "normalized_result": att.normalized_result, + "raw_winner": att.raw_winner, + "winner_agent": att.winner_agent, + "scores": att.scores, + "evaluated_agent_camp": att.evaluated_agent_camp, + "steps": att.steps, + "seed": {"requested_seed": None, "effective_seed": None, + "reproducible_from_seed": False, + "realized_randomization": rr}, + "error_type": att.error_type, + "reason": att.reason, + "wrapper_timeout": att.wrapper_timeout, + "normal_cleanup_nonzero": att.normal_cleanup_nonzero, + "result_json_status": att.result_json_status, + "discrepancies": att.discrepancies, + "evidence_paths": att.evidence_paths, + "residual_check": { + "n_identity_procs": len(procs), + "clean": [(p.pid, p.role) for p in res["clean"]], + "residual": [(p.pid, p.role) for p in res["residual"]], + "reused": [(p.pid, p.role) for p in res["reused"]], + }, + } + return rec + + +def _run_group(name, *, agent, opponent, evaluated_dir, opponent_dir, data_root, work_dir, + timeout, wrapper_timeout_s, prefix): + log(f"\n===== {name}: {agent} vs {opponent} (2 games, camps swapped) =====") + runner = MiracleEvalRunner( + agent=agent, opponent=opponent, + evaluated_dir=evaluated_dir, opponent_dir=opponent_dir, + n_games=2, data_dir=str(data_root), judge_dir=JUDGE, + vendor_script=VENDOR, framework_src=FW_SRC, work_dir=work_dir, + timeout=timeout, wrapper_timeout_s=wrapper_timeout_s, prefix=prefix, + config={"group": name, "judge_dir_resolved": str(JUDGE.resolve())}, + ) + summary = runner.run() + reports = [_attempt_report(att) for att in runner.attempts] + for rec in reports: + rr = rec["seed"]["realized_randomization"] + log(f" {rec['game_id']}: valid={rec['valid']} result={rec['normalized_result']} " + f"raw_winner={rec['raw_winner']} camp={rec['evaluated_agent_camp']} " + f"scores={rec['scores']} steps={rec['steps']} " + f"map=(mt={rr.get('map_type')},dt={rr.get('day_time')}) " + f"err={rec['error_type']} residual={rec['residual_check']['residual']}") + s = {k: summary.get(k) for k in + ("attempted_games", "valid_games", "invalid_games", "wins", "losses", "draws", + "win_rate_denominator", "win_rate", "evaluation_status", "total_steps", "attempted_steps")} + log(f" summary[{agent}]: {json.dumps(s, ensure_ascii=False)}") + ok, reasons = group1_strict_clean(runner.attempts, summary) + log(f" {name} strict gate: {'PASS' if ok else 'FAIL'}" + ("" if ok else f" -> {reasons}")) + return {"ok": ok, "reasons": reasons, "summary": s, "reports": reports, + "run_dir": f"{data_root}/runs/24_miracle/{agent}//"} + + +def main() -> int: + global _LOG + session_id = make_session_id() + session_dir = ensure_fresh_session(SESSIONS_ROOT, session_id) + _LOG = open(session_dir / "smoke.full.log", "w", encoding="utf-8") + data_root = session_dir / "data" + work_dir = session_dir / "work" + data_root.mkdir(parents=True, exist_ok=True) + work_dir.mkdir(parents=True, exist_ok=True) + + log(f"session_id: {session_id}") + log(f"session_dir: {session_dir}") + log(f"Judge (authoritative): {JUDGE.resolve()}") + log(f"vendor runner: {VENDOR}") + log(f"sample A / B: {SAMPLE_A} / {SAMPLE_B}") + log(f"if-else bot: {IFELSE}") + log(f"auth_cap_games: {AUTH_CAP}") + log("NOTE: the previous smoke attempt was rejected by a 429/tool error before " + "any subprocess launched; that is NOT a game result.") + + manifest = build_manifest( + session_id=session_id, auth_cap=AUTH_CAP, + python_executable=sys.executable, python_version=platform.python_version(), + code_files=[REPO / "tools" / "miracle_smoke.py", REPO / "src" / "agentbench_frame" / "games" / "miracle" / "runner.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "match_runner.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "proctree.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "result.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "driver.py", + REPO / "src" / "agentbench_frame" / "games" / "miracle" / "smoke_audit.py", + VENDOR], + asset_files=[JUDGE / "main.py", JUDGE / "Data.json", + SAMPLE_AI_SRC / "main.py", + IFELSE / "main.py", SAMPLE_A / "main.py", SAMPLE_B / "main.py"], + groups_planned=[{"group": "GROUP1", "agent": "sampleA", "opponent": "sampleB", "n_games": 2}, + {"group": "GROUP2", "agent": "miracle_ifelse", "opponent": "sampleB", "n_games": 2, + "conditional_on": "GROUP1 strict pass"}], + notes=["429/tool rejection on prior attempt was not a game result", + "no session is ever deleted or overwritten; successful games are never re-run"], + ) + mp = write_manifest_atomic(session_dir, manifest) + log(f"manifest: {mp}") + + try: + g1 = _run_group("GROUP1", agent="sampleA", opponent="sampleB", + evaluated_dir=SAMPLE_A, opponent_dir=SAMPLE_B, + data_root=data_root, work_dir=work_dir, + timeout=8.0, wrapper_timeout_s=120.0, prefix="g1") + (session_dir / "group1.json").write_text( + json.dumps(g1, ensure_ascii=False, indent=2), encoding="utf-8") + + g2 = None + if should_run_group2(g1["ok"]): + log("\n>> GROUP 1 strict pass — proceeding to GROUP 2.") + g2 = _run_group("GROUP2", agent="miracle_ifelse", opponent="sampleB", + evaluated_dir=IFELSE, opponent_dir=SAMPLE_B, + data_root=data_root, work_dir=work_dir, + timeout=10.0, wrapper_timeout_s=180.0, prefix="g2") + (session_dir / "group2.json").write_text( + json.dumps(g2, ensure_ascii=False, indent=2), encoding="utf-8") + else: + log("\n>> GROUP 1 strict gate FAILED — GROUP 2 NOT started (per spec).") + + log("\n===== residual process check (independent, exact PID + create_time) =====") + for grp_name, grp in (("GROUP1", g1), ("GROUP2", g2)): + if grp is None: + continue + for rec in grp["reports"]: + rc = rec["residual_check"] + log(f" {rec['game_id']}: identity_procs={rc['n_identity_procs']} " + f"clean={len(rc['clean'])} residual={rc['residual']} reused={len(rc['reused'])}") + + overall = g1["ok"] and (g2 is None or g2["ok"]) and (g2 is not None) + # overall green only if BOTH groups ran and passed; G2-not-started is not green + log(f"\nSMOKE RESULT: {'BOTH_GROUPS_PASS' if (g1['ok'] and g2 and g2['ok']) else 'NOT_FULLY_PASSING'}") + return 0 if (g1["ok"] and g2 is not None and g2["ok"]) else 1 + except BaseException as exc: # noqa: BLE001 + # record the exception SEPARATELY from game results; preserve the session + tb = traceback.format_exc() + (session_dir / "exception.log").write_text(tb, encoding="utf-8") + log(f"\nEXCEPTION (not a game result): {exc!r}") + log(tb) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/miracle_local/run_match.py b/vendor/miracle_local/run_match.py new file mode 100644 index 0000000..0743724 --- /dev/null +++ b/vendor/miracle_local/run_match.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""LOCAL VERIFICATION COPY of 高翔's ``tools/miracle/run_match.py``. + +VENDOR PROVENANCE + Original source : external_asset:AgentBench-gaoxiang/tools/miracle/run_match.py + Original SHA256 : 91d5693651402a3579146181ee6ada5586c103f38ba5f321b92c23a17a7a8ae2 + Authoritative Judge : external_asset:24_miracle/judge_dev_logic + (高翔's in-tree Judge copy is byte-identical to it, diff verified.) + Role : local Windows verification only; the original stays frozen, this is a + minimally-patched vendored copy used by match_runner.py. + +MODIFICATIONS (minimal, cross-platform portability; NO protocol/contract change) + 1. JUDGE_DIR overridable via env MIRACLE_JUDGE_DIR (default unchanged). Formal + smoke sets it to the authoritative Judge path. + 2. ``terminate()`` (Unix-only ``os.killpg``) replaced by + ``agentbench_frame.games.miracle.proctree.ProcessTreeManager`` — graceful + terminate -> grace window -> force-kill tree, by EXACT pid with psutil + create_time identity check. Works on Windows and POSIX, no name-based kill. + 3. ``spawn`` flags: POSIX ``start_new_session=True``; Windows + ``CREATE_NEW_PROCESS_GROUP``. + 4. ``read_ai_operation`` no longer uses ``selectors`` (broken on Windows pipes); + it uses a daemon reader thread + queue with timeout — cross-platform. + 5. NEW ``--result-json ``: writes a machine-readable structured result + (process metadata, end_info, derived raw_winner, scores, tie flags, + per-AI timeout/error, trace/replay paths, cleanup status, timings, any + exception). match_runner.py reads THIS file, not the human-readable stdout. + 6. Per-process metadata (pid, started_at, natural_exit, natural_returncode, + termination_requested, termination_reason, final_returncode, forced_kill, + cleanup_succeeded, identity_confirmed) is captured so the adapter can tell a + post-end_info cleanup-kill of an idling AI apart from a strategy crash. + +Everything else is byte-faithful to the original protocol bridge. +""" + +from __future__ import annotations + +import argparse +import json +import os +import queue +import struct +import subprocess +import sys +import threading +import time +from pathlib import Path + +# --- make the framework's proctree importable when launched as a subprocess --- # +_FW_SRC = os.environ.get("MIRACLE_FRAMEWORK_SRC") +if _FW_SRC and _FW_SRC not in sys.path: + sys.path.insert(0, _FW_SRC) +from agentbench_frame.games.miracle.proctree import ProcessTreeManager # noqa: E402 +from agentbench_frame.games.miracle.entry import resolve_ai_command # noqa: E402 +from agentbench_frame.games.miracle.atomicio import atomic_write_json # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_JUDGE_DIR = ROOT / "backend_sources/corpus/24_miracle/logic/judge_dev_logic" +JUDGE_DIR = Path(os.environ.get("MIRACLE_JUDGE_DIR") or DEFAULT_JUDGE_DIR) + + +class ProtocolError(RuntimeError): + pass + + +# ---- low-level framed I/O (unchanged protocol) --------------------------- # +def read_exact(stream, n: int) -> bytes: + chunks = [] + remaining = n + while remaining: + chunk = stream.read(remaining) + if not chunk: + raise EOFError("unexpected EOF") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def write_judge_input(proc: subprocess.Popen, obj: dict) -> None: + payload = json.dumps(obj, separators=(",", ":")).encode("utf-8") + proc.stdin.write(struct.pack(">i", len(payload)) + payload) + proc.stdin.flush() + + +def read_judge_frame(proc: subprocess.Popen): + header = read_exact(proc.stdout, 8) + length, goal = struct.unpack(">ii", header) + if length < 0: + raise ProtocolError(f"negative judge frame length: {length}") + payload = read_exact(proc.stdout, length) + return goal, json.loads(payload.decode("utf-8")) + + +def write_ai_state(proc: subprocess.Popen, payload: str) -> None: + proc.stdin.write(payload.encode("utf-8")) + proc.stdin.flush() + + +def read_ai_operation(proc: subprocess.Popen, timeout: float): + """Read one AI operation with a timeout. Cross-platform: a daemon reader + thread + queue (the original used ``selectors`` which does not work on + Windows pipes). Raises ``TimeoutError`` on timeout.""" + q: "queue.Queue[tuple]" = queue.Queue() + + def _read(): + try: + header = read_exact(proc.stdout, 4) + length = struct.unpack(">i", header)[0] + if length < 0: + q.put(("error", ProtocolError(f"negative AI frame length: {length}"))) + return + payload = read_exact(proc.stdout, length) + q.put(("ok", json.loads(payload.decode("utf-8")))) + except BaseException as exc: # noqa: BLE001 - report any read failure upstream + q.put(("error", exc)) + + t = threading.Thread(target=_read, daemon=True) + t.start() + try: + kind, val = q.get(timeout=timeout) + except queue.Empty: + raise TimeoutError("AI operation timed out") + if kind == "error": + raise val + return val + + +# resolve_ai_command is imported from agentbench_frame.games.miracle.entry +# (cross-platform: Windows main.exe / POSIX ./main / main.py; explicit priority; +# paths with spaces kept whole). See tests/miracle/test_entry.py. + + +def _creation_flags(): + """Platform-appropriate process-group flag so the tree can be managed.""" + if os.name == "nt": + return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + return {"start_new_session": True} + + +def spawn(cmd, cwd: Path) -> subprocess.Popen: + return subprocess.Popen( + cmd, + cwd=str(cwd), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **_creation_flags(), + ) + + +def drain_stderr(proc: subprocess.Popen) -> str: + if proc.stderr is None: + return "" + try: + return proc.stderr.read().decode("utf-8", errors="replace") + except Exception: + return "" + + +def player_error(player: int, state: int, error: int = 0) -> dict: + return { + "player": -1, + "content": json.dumps({"error": error, "player": player, "state": state}), + } + + +def derive_raw_winner(end_info): + """Judge end_info = {"0": score0, "1": score1}; winner = 0 if s0>s1 else 1. + Ties are broken toward player1 (Judge main.py:448-449). Returns (winner, s0, s1, tie).""" + if not isinstance(end_info, dict) or "0" not in end_info or "1" not in end_info: + return None, None, None, False + try: + s0, s1 = int(end_info["0"]), int(end_info["1"]) + except (TypeError, ValueError): + return None, None, None, False + tie = (s0 == s1) + return (0 if s0 > s1 else 1), s0, s1, tie + + +# ---- match entry point --------------------------------------------------- # +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--p0-dir", required=True, type=Path) + parser.add_argument("--p1-dir", required=True, type=Path) + parser.add_argument("--p0-cmd") + parser.add_argument("--p1-cmd") + parser.add_argument("--p0-name", default="player0") + parser.add_argument("--p1-name", default="player1") + parser.add_argument("--timeout", type=float, default=12.0) + parser.add_argument("--out", type=Path, default=ROOT / "reports/miracle_rollouts") + parser.add_argument("--tag", default=None) + parser.add_argument("--result-json", type=Path, default=None, + help="machine-readable structured result (used by match_runner)") + args = parser.parse_args() + + args.out.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") + tag = args.tag or f"{args.p0_name}_vs_{args.p1_name}_{stamp}" + replay = args.out / f"{tag}.replay" + trace = args.out / f"{tag}.jsonl" + + mgr = ProcessTreeManager() + match_started = time.time() + end_info_received = False + end_info = None + timeout_flag = {"ai0": False, "ai1": False} + ai_error_flag = {"ai0": False, "ai1": False} + last_state = 0 + run_exc = None + + def log(obj: dict) -> None: + with trace.open("a", encoding="utf-8") as f: + f.write(json.dumps(obj, ensure_ascii=False, separators=(",", ":")) + "\n") + + try: + judge_p = spawn([sys.executable, "main.py"], JUDGE_DIR) + judge_mp = mgr.register_popen(judge_p, "judge") + ai0_p = spawn(resolve_ai_command(args.p0_dir, args.p0_cmd), args.p0_dir) + ai1_p = spawn(resolve_ai_command(args.p1_dir, args.p1_cmd), args.p1_dir) + ai0_mp = mgr.register_popen(ai0_p, "ai0") + ai1_mp = mgr.register_popen(ai1_p, "ai1") + procs = [ai0_p, ai1_p] + mps = {"ai0": ai0_mp, "ai1": ai1_mp} + names = [args.p0_name, args.p1_name] + + write_judge_input(judge_p, {"replay": str(replay), "player_list": [1, 1]}) + log({"kind": "match_start", "players": names, "replay": str(replay), + "judge_dir": str(JUDGE_DIR.resolve())}) + + while True: + goal, frame = read_judge_frame(judge_p) + last_state = frame.get("state", last_state) + log({"kind": "judge_frame", "goal": goal, "frame": frame}) + + if frame.get("state") == -1: + end_info_received = True + try: + end_info = json.loads(frame.get("end_info", "{}")) + except Exception: + end_info = None + log({"kind": "match_end", "end_info": frame.get("end_info")}) + break + + players = frame.get("player") or [] + contents = frame.get("content") or [] + if not players or not contents: + continue + + for idx, player in enumerate(players): + payload = contents[idx] + write_ai_state(procs[player], payload) + try: + operation = read_ai_operation(procs[player], args.timeout) + except TimeoutError: + timeout_flag[f"ai{player}"] = True + log({"kind": "ai_timeout", "player": player, "state": last_state}) + write_judge_input(judge_p, player_error(player, last_state, error=1)) + continue + except Exception as exc: + ai_error_flag[f"ai{player}"] = True + log({"kind": "ai_error", "player": player, "state": last_state, "error": repr(exc)}) + write_judge_input(judge_p, player_error(player, last_state, error=0)) + continue + + log({"kind": "ai_operation", "player": player, "name": names[player], "operation": operation}) + write_judge_input( + judge_p, + {"player": operation["player"], "content": json.dumps(operation, separators=(",", ":"))}, + ) + + returncode = 0 + except BaseException as exc: # noqa: BLE001 - capture ANY failure (e.g. Judge crash) + run_exc = repr(exc) + returncode = 1 + finally: + # clean up judge + AIs (graceful -> force tree, exact PID, identity-checked) + mgr.cleanup_all("match-end") + match_finished = time.time() + + # drain stderr after processes are dead (safe, non-blocking once EOF) + for idx, p in enumerate([judge_p] + [mp.popen for mp in [ai0_mp, ai1_mp]]): + err = drain_stderr(p) + if err: + who = "judge" if idx == 0 else f"ai{idx-1}" + log({"kind": "stderr", "who": who, "stderr": err[-12000:]}) + + status = {s["role"]: s for s in mgr.status()} + raw_winner, s0, s1, tie = derive_raw_winner(end_info) + result = { + "schema_version": 1, + "tag": tag, + "started_at": match_started, + "finished_at": match_finished, + "duration_s": round(match_finished - match_started, 3), + "judge_dir_resolved": str(JUDGE_DIR.resolve()), + "p0": {"name": args.p0_name, "dir": str(args.p0_dir.resolve())}, + "p1": {"name": args.p1_name, "dir": str(args.p1_dir.resolve())}, + "judge": status.get("judge"), + "ai0": status.get("ai0"), + "ai1": status.get("ai1"), + "end_info_received": end_info_received, + "end_info": end_info, + "scores": ({"0": s0, "1": s1} if s0 is not None else None), + "raw_winner": raw_winner, + "score_tie": bool(tie), + "judge_tiebreak_applied": bool(tie), + "timeout": timeout_flag, + "ai_error": ai_error_flag, + "trace_path": str(trace), + "replay_path": str(replay), + "cleanup_all_succeeded": bool(mgr.status() and all(s["cleanup_succeeded"] for s in mgr.status())), + "exception": run_exc, + "run_match_returncode": returncode, + } + if args.result_json is not None: + atomic_write_json(args.result_json, result) # temp + fsync + os.replace (atomic) + # human-readable stdout kept for back-compat (match_runner uses result-json) + print(json.dumps({"trace": str(trace), "replay": str(replay), + "end_info_received": end_info_received}, ensure_ascii=False)) + + return returncode + + +if __name__ == "__main__": + raise SystemExit(main())