From 65ba0012a681223df9944dfe4b3057be82272c69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=9A=E6=81=92?= Date: Mon, 3 Aug 2026 14:01:10 +0800 Subject: [PATCH] feat(miracle): define explicit policy state and formal IG contract --- docs/games/24_miracle_decision_space_kl.v2.md | 82 + .../24_miracle_ifelse_policy_state.v1.md | 122 ++ .../24_miracle_kl_contract_authority.v2.md | 105 + docs/games/24_miracle_research_protocol.v2.md | 370 ++++ docs/integration/trajectory-kl.md | 48 +- docs/research/information-gain-design.md | 43 +- docs/research/research-methodology-summary.md | 36 +- src/agentbench_frame/eval/__init__.py | 34 + src/agentbench_frame/eval/information_gain.py | 141 +- src/agentbench_frame/eval/trajectory_kl.py | 465 ++++- .../games/miracle/__init__.py | 16 + .../games/miracle/decision_kl_v1.py | 132 +- .../games/miracle/ifelse_policy_state_v1.py | 1738 +++++++++++++++++ .../games/miracle/iteration_protocol.py | 201 +- .../games/miracle/research_protocol.py | 31 +- src/agentbench_frame/report/builder.py | 165 +- .../report/templates/index.html | 6 +- src/agentbench_frame/tracking/run.py | 99 +- tests/miracle/test_decision_kl_core_v1.py | 143 +- tests/miracle/test_ifelse_policy_state_v1.py | 502 +++++ .../test_information_gain_formal_contract.py | 278 +++ tests/miracle/test_iteration_acceptance.py | 91 +- tests/miracle/test_research_protocol.py | 16 +- tests/test_local_report_research.py | 103 +- tests/test_tracking_contracts.py | 167 +- tests/test_trajectory_kl_runtime.py | 190 +- 26 files changed, 4825 insertions(+), 499 deletions(-) create mode 100644 docs/games/24_miracle_decision_space_kl.v2.md create mode 100644 docs/games/24_miracle_ifelse_policy_state.v1.md create mode 100644 docs/games/24_miracle_kl_contract_authority.v2.md create mode 100644 docs/games/24_miracle_research_protocol.v2.md create mode 100644 src/agentbench_frame/games/miracle/ifelse_policy_state_v1.py create mode 100644 tests/miracle/test_ifelse_policy_state_v1.py create mode 100644 tests/miracle/test_information_gain_formal_contract.py diff --git a/docs/games/24_miracle_decision_space_kl.v2.md b/docs/games/24_miracle_decision_space_kl.v2.md new file mode 100644 index 0000000..4704295 --- /dev/null +++ b/docs/games/24_miracle_decision_space_kl.v2.md @@ -0,0 +1,82 @@ +# 24_miracle Decision Space + policy-KL Core v2 + +Authority: [24_miracle policy information-gain contract](24_miracle_kl_contract_authority.v2.md). + +## Scope + +The core is a pure, synthetic/fake-only calculator. It never runs a policy, +Judge, environment, opponent, provider, match or session, and does not claim +real old/new policy provenance or authoritative readiness. + +One Miracle decision is one legal atomic Judge operation at a target-agent +pre-action observation. Public families are `init`, `move`, `attack`, +`summon`, `use`, `endround` and `surrender`. Parser-internal operations are not +decisions. The complete support is rebuilt from the observation, canonicalized, +assigned stable action IDs, sorted and bound to `support_id`. WindBlessing +continues to fail closed where its authoritative position support cannot be +finitely enumerated. + +## Strict evidence boundary + +`DecisionKLEvidence` contains only: + +- continuous `decision_step`; +- frozen `state_before`; +- frozen complete support identity; +- complete old and new action-ID probability mappings. + +Nested caller-owned data is copied and frozen. The calculator regenerates the +support, checks schema/support/action order, rejects missing or extra actions, +rejects bool/string/NaN/Infinity/negative probabilities, and requires unit +mass without normalization or implicit zero filling. A submitted local KL or +summary is not accepted as input. + +Issuer-only decision records and summaries are bound to closure-owned snapshots. +Construction, copying, replacement or post-issue mutation cannot be reused as +trusted aggregation evidence. Expected support/distribution unavailability is +represented as a structured `incomplete` record with a null scalar. API type, +identity and ordering violations fail closed. + +## Formal calculation + +For complete current support `A(s)` and fixed `epsilon=0.01`: + +```text +pi_epsilon(a|s) = 0.99 * pi(a|s) + 0.01 / |A(s)| + +local_policy_kl(s) + = sum_a new_epsilon(a|s) * ln(new_epsilon(a|s) / old_epsilon(a|s)) +``` + +The logarithm is natural. Both policies use the same support and identical +smoothing channel. Equal deterministic actions produce zero; changed +deterministic actions produce a finite positive value. Epsilon affects the +measurement only and never the executed policy. + +The summary retains ordered `decision_records` and `trace`. For a complete, +non-empty trace: + +```text +information_gain = trajectory_kl = arithmetic_mean(trace) +sum_local_kl = sum(trace) +``` + +`information_gain`/`trajectory_kl` use `nats / decision`; `sum_local_kl` uses +`nats / episode`. Maximum and percentiles remain diagnostics. There is no +policy-change acceptance threshold; the deprecated compatibility fields +`acceptance_threshold` and `threshold_passed` are always null. Empty or partly +incomplete traces never expose an episode scalar. + +Every machine-readable summary remains labeled +`evidence_scope=synthetic_fake_only`, `authoritative_readiness=false`, +`verified_rollout_source=null` and `policy_binding_verified=false`. +`rollout_source_contract=new_policy` states the target contract only; it does +not turn synthetic states into verified occupancy evidence. + +## Remaining real-evidence gap + +The current synthetic input records visible state only. If a real HL policy is +stateful, authoritative comparison requires canonical `z=(s,m)` evidence. +Until internal memory `m` is captured and bound, real stateful policy KL is +incomplete even when visible-state support and distributions are otherwise +valid. diff --git a/docs/games/24_miracle_ifelse_policy_state.v1.md b/docs/games/24_miracle_ifelse_policy_state.v1.md new file mode 100644 index 0000000..eb465fa --- /dev/null +++ b/docs/games/24_miracle_ifelse_policy_state.v1.md @@ -0,0 +1,122 @@ +# 24_miracle explicit if-else policy state v1 + +Status: **local deterministic adapter candidate; H08 v2 regeneration still +required**. + +This adapter closes the implementation gap between the stateful legacy +`IfElseAI.play()` control flow and the policy-information-gain contract. It +does not approve a replay, change an allowlist, or turn historical post-hoc +data into formal experiment evidence. + +## Decision boundary + +One decision is one atomic Judge operation. `summon`, `move`, `attack`, and +`use` send an operation and then read a fresh Judge observation before the +legacy Python function continues. A single `play()` can therefore adaptively +emit artifact, repeated pre-move attacks, moves, post-move attacks, summons, +and end-round operations. The complete turn-level command sequence is not a +finite action known at turn start, so a turn-level macro-action cannot supply +the required complete canonical `A(s)`. + +The formal comparison context is consequently `z=(s,m)`, where `s` is the +pre-operation observation and `m` is `PolicyMemoryV1`. + +## Frozen policy identity + +The policy identity binds: + +```text +verified six-file source identity ++ canonical 62-input configuration identity ++ 24-miracle-ifelse-explicit-state-machine-v1 ++ provider version +``` + +The 62 inputs are 59 strict booleans plus `MIRACLE_CAMP1_OPENING`, +`MIRACLE_ARTIFACT`, and `MIRACLE_DECK`. Opening is one of `FF/SF/IF`; +artifact and creature names must be members of the Judge `Data.json` +enumerations. An explicit configuration must contain all 62 inputs. A +historical unknown is retained as the literal `unknown` and makes formal +provider construction incomplete. + +The source reader binds `Data.json`, `ai_client.py`, `calculator.py`, +`card.py`, `gameunit.py`, and `main.py`; all must be non-empty strict UTF-8, +BOM-free regular files. It revalidates the source before every selection and +loads with bytecode writes disabled. + +## Serializable memory + +`PolicyMemoryV1` contains only canonical serializable values: + +```text +schema_version, state_machine_version +policy_identity, policy_config_identity +episode_id, decision_step, lifecycle +phase, instruction_label, attack_pass, preserve_for_move, acted_iteration +ordered_unit_ids, ordered_unit_snapshots, current_unit_cursor +ordered_target_ids, current_target_cursor +ordered_positions, position_cursor +remaining_capacities, local_mana, local_unit_counts +camp, rng_mode, rng_state, previous_transition_sha256 +``` + +The move-phase snapshots are necessary because legacy `move_phase()` freezes +its sorted unit-object iteration at phase entry. Recomputing that order from a +later observation changes behavior. No Python frame, iterator, policy object, +address, file handle, clock, environment lookup, or RNG object is retained. +The only supported RNG identity is `rng_mode=none`, `rng_state=null`. + +Memory and pair-decision evidence are issuer-bound immutable snapshots. Every +episode starts from one explicit reset state. The machine keeps an issuer-side +transition tip only to reject replayed, skipped, or spliced evidence; all +policy decision state remains serialized in `m`. Old and new providers are +recomputed on the same exact `m_before`, observation, and regenerated complete +ActionSupport. They return unsmoothed one-hot base distributions. Epsilon +regularization belongs exclusively to the formal IG measurement layer. + +## Local replay evidence + +The H08 trace with SHA-256 +`e0e280338ae3b06818a923733fb4a04a3730d931ea800e144ecd0aadf4241948` +was replayed sequentially from one reset under an explicit all-false/default +configuration assumption: + +```text +399 / 399 chosen operations reproduced +399 / 399 complete ActionSupport values rebuilt +399 / 399 recorded operations were in support +399 / 399 old/new evaluations shared m_before +``` + +The trace exercised init, opening, artifact, repeated pre-move attack, move, +post-move attack, summon, and endround behavior; the longest consecutive attack +run was five operations. + +All eight retained historical v0/v1 traces also replayed completely for both +camps and rank04/rank09 when evaluated under explicit default assumptions. The +observed action counts were 403, 621, 342, and 375 for each version. This is a +behavior-equivalence diagnostic only: those sessions did not record all 62 +environment values, so their configuration evidence remains unknown and +cannot be promoted to formal IG evidence. + +The verified H08/v0 source has canonical tree SHA-256 +`c27a8b646e57b902f527061b4f041b24e000420c7e9bfeacd9d7563ab0de7254`, +legacy tree SHA-256 +`209f182637e1abaee4ff50de6b1a37777fbba9fbab5f611aae58a06a109c0a3b`, +and full `main.py` SHA-256 +`98199fae8875de63b41d2eacd92ad5c58b4d5aa95b01b05aeacedd0b55402f4b`. +The shorter value ending in `...402f4` is not a valid SHA-256. + +A deterministic v0/v1 diagnostic on the fixed camp-0 H08 episode used +different provider identities and explicit FF/SF configurations. Both chose +the same 399 operations, so the ordered fixed-epsilon trace contained 399 +zeros, with mean `0.0 nats / decision` and sum `0.0 nats / episode`. This is +not a formal curve or performance result. + +## Remaining gate + +`CURRENT_INTERNAL_MEMORY_EVIDENCE` remains `not_collected`. A new H08 v2 run +must explicitly record all 62 configuration values and their canonical digest, +the verified source/provider identities, `m_before` and transition evidence at +every decision, and the actual new-policy rollout identity. Only that new +evidence can bind `internal_memory_evidence` and permit a formal IG preflight. diff --git a/docs/games/24_miracle_kl_contract_authority.v2.md b/docs/games/24_miracle_kl_contract_authority.v2.md new file mode 100644 index 0000000..52f8dc4 --- /dev/null +++ b/docs/games/24_miracle_kl_contract_authority.v2.md @@ -0,0 +1,105 @@ +# 24_miracle policy information-gain contract authority v2 + +Status: **measurement contract migrated locally; authoritative execution remains blocked**. + +This document records the group-approved scientific definition for 24_miracle. +It governs measurement semantics, not production approval, real replay status, +benchmark completion, or permission to start an experiment. + +## Fixed scientific identity + +```text +ACTION_BOUNDARY = one legal atomic Judge operation +SUPPORT = same complete canonical state-local ActionSupport A(s) +KL_DIRECTION = new||old +LOGARITHM = natural +SMOOTHING = symmetric epsilon mixture with uniform(A(s)) +MAIN_EPSILON = 0.01 +SENSITIVITY_EPSILONS = 0.001, 0.01, 0.05 +OCCUPANCY_SOURCE = actual new-policy rollout +PRIMARY_EPISODE_AGGREGATE = arithmetic mean +PRIMARY_UNIT = nats / decision +OPTIONAL_SUM_UNIT = nats / episode +TERMINAL_STATE_INCLUDED = false +``` + +For `v_{k-1} -> v_k`, at each actual target-agent decision context `z_t`: + +```text +pi_epsilon(a|z_t) = (1-epsilon) * pi(a|z_t) + epsilon / |A(s_t)| + +local_policy_kl_k(z_t) + = KL(pi_k,epsilon(.|z_t) || pi_k-1,epsilon(.|z_t)) +``` + +The same epsilon and the same ordered, complete support are used for both +policies. Smoothing is a measurement layer only; it never changes the action +executed by the new policy. The main result always uses epsilon `0.01`. +Sensitivity values use the fixed panel above and cannot replace or relabel the +main result. + +Each episode retains the ordered source trace: + +```text +local_policy_kl_trace = [k_0, ..., k_(T-1)] +information_gain = mean(local_policy_kl_trace) # nats / decision +local_policy_kl_sum = sum(local_policy_kl_trace) # nats / episode +``` + +The terminal state is excluded because it produces no action. Empty or +incomplete evidence produces `null/incomplete`, never zero. There is no KL +magnitude acceptance threshold: epsilon `0.01` is not a performance gate, and +large policy change does not imply score improvement. + +## Action and probability boundary + +For Miracle, the action-mask concept is a state-local, complete, ordered and +verifiable `ActionSupport + support_id`, not a fixed global Boolean vector. +The support is regenerated from the visible observation where the synthetic +core can do so. Old/new action IDs must match it exactly. Missing, extra, +duplicated or drifted identities fail closed. + +Probabilities must be exact numeric values (`int` or `float`, never `bool` or +strings), finite, non-negative and unit mass. The framework does not normalize, +zero-fill or coerce submitted distributions. Uploaded local KL, trace summary, +episode mean or sum is never authoritative: validators recompute the values +from the ordered distributions and reject disagreement. + +## Occupancy and interpretation + +`occupancy_shift` is an independent measurement of state visitation change. +It is never added to local policy KL or episode information gain. Policy KL is +behavioral/policy information gain, not epistemic information gain and not a +performance score. Performance remains governed by the frozen evaluation +win-rate/Elo contract. + +The occupancy source for the main episode measure is the actual rollout of the +unsmoothed new policy. Consequently the saved local trace is an +epsilon-regularized policy-change measurement under new-policy occupancy; the +optional sum is not presented as a third independent KL. + +## Decision context and current data gap + +If action choice depends on internal memory, the decision context is +`z=(s,m)`. The current Miracle adapters can advance stateful callbacks through +`reset()` and `observe_transition()`, but the evidence record only proves the +visible observation/state identity. It does not yet serialize a canonical +policy-memory identity. A stateful real run therefore has an explicit +`internal_memory_evidence=not_collected` gap and cannot claim authoritative +policy KL until that evidence is supplied. Stateless fake tests do not close +this real-data gap. + +## Security and lifecycle boundary + +The migration preserves support completeness, chosen-in-support checks, +ordered decision records, strict probability validation, immutable evidence +snapshots, issuer/provenance checks, replay/path safety, revocation checks and +lifecycle revalidation. Evaluation-case `information_gain` must equal the mean +derived from its authoritative ordered trajectory evidence. Acceptance derives +its aggregate from those validated per-case means rather than self-reported +case fields. + +Production approval tables remain empty. The bootstrap file and approved +bootstrap SHA are unchanged. Historical v0/v1 data remains `information_gain = +null` because it excluded replay, decision trace, complete action support and +old/new policy distributions; no formal IG can be reconstructed from it. diff --git a/docs/games/24_miracle_research_protocol.v2.md b/docs/games/24_miracle_research_protocol.v2.md new file mode 100644 index 0000000..e6d7ecb --- /dev/null +++ b/docs/games/24_miracle_research_protocol.v2.md @@ -0,0 +1,370 @@ +# 24_miracle research protocol v2 + +Target-contract authority: +[24_miracle KL contract authority v2](24_miracle_kl_contract_authority.v2.md). +The migrated contract is `KL(new||old)`, natural logarithm, symmetric uniform +measurement smoothing with fixed main `epsilon=0.01`, actual new-policy +occupancy, ordered local trace, and arithmetic-mean episode information gain in +`nats / decision`. The distinct optional sum uses `nats / episode`. There is no +KL acceptance threshold. + +The research/measurement contract has protocol version +`24-miracle-research-v2`. The separately versioned frozen 72-case evaluation +schema has benchmark version `24m-frozen-v1`. The manifest emits both values: +changing measurement semantics requires a protocol-version change, while +changing the frozen population or case identities requires a benchmark-version +change. + +`tools/miracle_research_manifest.py` emits deterministic canonical UTF-8 JSON +(sorted keys, compact separators, one trailing LF). A future Results research +run must copy those exact bytes to `research_manifest.json` in the run +directory and record their SHA-256, `protocol_version`, and +`benchmark_version` in `summary.json`. Results validates the submitted file; +it does not recreate the Framework seed algorithm. + +The run-local file and summary reference are integrity evidence, not approval. +Results maintains a separate, version-controlled allowlist from benchmark +version to approved canonical manifest digests. A run cannot add its own digest +to that allowlist, and run data must never update it automatically. The +production allowlist is currently empty. + +The current generated manifest remains `BLOCKED_NOT_AUTHORITATIVE` with +`authoritative_ready=false`, non-empty blockers, and unqualified rank02, +rank10, and rank16 audits. Canonical serialization does not change that status. + +An authoritative approval requires this ordered process: + +1. Implement and review the real old/new-policy binding and Judge seed + integration, then qualify rank02, rank10, and rank16. +2. Generate a new canonical manifest whose internal status is + `AUTHORITATIVE_READY` and whose blockers and audits support that claim. +3. Human-review the relevant source code, asset hashes, random controls, case + identities, seed bundles, and exact canonical manifest bytes. +4. Through a separate reviewed Results code/configuration change, register the + canonical digest in the approved allowlist for its benchmark version. +5. Only after registration may Results accept a `complete` run, and only when + its remaining schema, case, seed, and KL evidence also passes. + +The canonical blocked digest changes with this protocol migration. It is for +blocked reporting only and must not be placed in the complete allowlist. Fake +qualified manifests used by tests are likewise test-only and must never be +registered as production approvals. + +Status: **pre-registered, blocked for authoritative execution**. + +This protocol replaces neither the historical Plan A run nor its evidence. It +defines the next no-backpropagation heuristic-learning experiment and its final +frozen evaluation. + +## Population + +- train (10): rank01, rank03, rank04, rank06, rank07, rank08, rank11, rank12, + rank14, rank15 +- validation (3): rank05, rank09, rank13 +- test (3): rank02, rank10, rank16 + +Test opponent source, behavior, replay, intermediate scores, and artifacts must +not be exposed to the coding agent during learning or validation. + +## Frozen test matrix + +One case is identified by: + +```text +opponent version × evaluated-agent camp × map type × day time × repeat +``` + +The final matrix contains: + +```text +3 test opponents × 2 camps × 2 maps × 2 day states × 3 repeats = 72 cases +``` + +`research_protocol.py` deterministically derives a seed bundle for every case. +The logic seed is selected so that the Judge's first two `random.randint(0, 1)` +draws equal the frozen map and day values. An authoritative run must also prove +that the evaluated agent and opponent either consume their assigned seeds or +are deterministic. An opaque unseeded process is not eligible for the frozen +test matrix. + +Static audit on 2026-07-27 found that the current real match bridge still +starts the Judge as `python main.py`; it does not invoke +`seeded_python_entry.py`. More importantly, the compiled rank02 policy seeds +`rand()` from `clock()+time(0)`, while rank10 terminates search according to the +process CPU clock. The Python launcher cannot control either C++ source. +Rank16 contains no active RNG call found by the audit, but its deterministic +qualification remains open because its retained compiler/runtime risk has not +been closed. Consequently, the 72-case manifest is a preregistration only; it +is not yet an executable authoritative frozen matrix. + +The complete JSON-ready case list can be inspected without starting a runtime: + +```text +python tools/miracle_research_manifest.py +``` + +## Heuristic learning boundary + +HL optimization in this experiment performs no backpropagation or gradient +updates. Rule editing, scoring, path search, planning, symbolic reasoning, and +controlled random heuristics are permitted. Every coding-agent act produces a +retained version; lower-performing versions are not silently rolled back. + +### Versioned iteration protocol + +The frozen benchmark case identity remains unchanged. The separate +iteration mechanism has version `24-miracle-iteration-v4`; its canonical +description is produced by `iteration_protocol_manifest()` and records the +version and SHA-256 of the checked-in `24m-minimal-bootstrap-v1` template. + +The fake-only lifecycle is explicit and ordered: `PLANNED`, `MATCH_READY`, +`BASELINE_STORED`, +`REPLAY_CAPTURED_UNAPPROVED`, `REPLAY_APPROVED`, `LEARNING_READY`, +`CANDIDATE_STORED`, `EVALUATION_READY`, and finally `COMPLETED` or +`INCOMPLETE`. A canonical `MatchPlanManifest` is independently approved before +a runner factory can be opened. It binds the champion, human replay-reading +Skill, evaluated policy and source digest, bootstrap, protocol/research +identities, and every planned case role, camp/map/day/repeat value and seed +bundle. It deliberately contains no future replay. + +After MatchPlan approval, the fixed bootstrap is materialized as +`strategy-v0`, written once to the immutable store, loaded back, and checked +byte-for-byte and by source SHA against the plan before the match runner +factory is called. The factory receives the stored version, source SHA, store +identity, and frozen cases. A missing, changed, or differently approved +baseline prevents the factory from opening; saving v0 after a match cannot +satisfy this gate. + +After a fake match stage, captured artifacts form a separate canonical +`ReplayEvidenceManifest`. It references the approved match-plan digest and +must cover every planned case exactly once. Artifact path and bytes, case, +role, seeds, champion, human Skill, and evaluated policy must still match the +plan. Missing, extra, duplicate, or drifted cases fail closed. Replay evidence +receives a second independent approval before any learning/controller factory +may read it. Both production approval sets are empty. + +New research defaults to a from-scratch v0 created from that template. The +template contains only the observation/action adapter, complete legal-action +handling, and a minimal legal fallback. It contains no copied if-else tactics +or opponent knowledge. The previous v0/v1 smoke entry is available only as +the explicit `--legacy-smoke` compatibility mode; its observed 0.5 to 0.5 +score was not an improvement and is not authoritative evidence. + +Before any runner, session, or log is created, normal preflight requires an +explicit frozen human-champion descriptor with logical ID, version, artifact +SHA-256, provenance, qualification evidence, and compatible protocol and +benchmark versions. There is no rank04, rank09, filename-based, or other bot +fallback. The full canonical descriptor digest, rather than its self-declared +qualification field or artifact digest alone, must equal the single current +digest in the version-controlled production approval boundary. That value is +currently `None`, so production execution remains blocked. + +Human replay reading and Agent-authored experience are separate schemas. A +canonical human replay-reading Skill must receive independent digest approval; +the experience Skill can only reference that digest and cannot replace it with +a `human_authored=true` flag. The approved human replay-Skill, match-plan, and +replay-evidence sets are also currently empty. Tests exercise future success only by +temporarily monkeypatching these module constants; no fake digest is production +approval state, and no CLI, run file, manifest, or ordinary Python function can +override an approval table. + +The default CLI is read-only and has separate `plan`, `replay`, and `learning` +preflight commands. It never starts a Judge, opponent, provider, match, or +session. With the empty production approvals each command returns a fatal +blocked result without creating output. The opt-in `--legacy-smoke` route +remains deprecated and non-authoritative. Only evidence whose frozen role is +`train` can enter a prompt, change plan, or experience Skill; renaming +validation evidence cannot change its role. + +One HL/IG decision is exactly one legal atomic Judge command. Multi-command +macro-actions and illegal commands are outside ActionSupport. This iteration +protocol now validates the v2 formal profile: fixed epsilon `0.01`, +`KL(new || old)`, natural logarithm, symmetric uniform smoothing, actual +new-policy rollout, ordered decision records, and the arithmetic-mean episode +IG. Generic research KL remains separately available but cannot acquire the +formal profile or populate formal IG fields. + +Every strategy version is a canonical immutable UTF-8 JSON record containing +the complete source snapshot, entrypoint, dependency list, source SHA-256, +structured change plan, interpretability category, complexity metrics, and +read-only probability-query capability. Allowed interpretable forms include +rule tables, finite-state machines, deterministic heuristic planners, scoring +functions, and structured combinations. Network dependencies, unregistered +external state, symlinks, and opaque binary models fail closed. + +Dependencies are individually registered against a small module allowlist; +`python-standard-library` is not an authorization. Dynamic import, `eval`/ +`exec`/`compile`, unregistered file reads, process or system launch, and unsafe +modules fail static validation. Strategy, Skill, version, iteration, and replay +logical IDs are path-safe, and every resolved read/store target must remain +inside its declared root. This static validation is not a runtime sandbox and +does not authorize an authoritative run; a separately audited sandbox remains +required before real execution. + +Change plans distinguish `add`, `replace`, `merge`, `delete`, and `simplify`. +Complexity records effective nodes, source lines, duplicate or shadowed rules, +and maximum decision depth and is deterministically computed from the complete +source snapshot. Candidate-reported values must match. Before any store path is +created, every non-v0 version requires independently validated train evidence, +loads its exact parent, validates its real structured operations, compares +computed metrics, and applies the growth gate. Complexity growth additionally +requires an explicit reason. +Validation replay, validation score, and final test conclusions are never +included in the modification payload. `strategy-v0` has no parent and must +exactly materialize the fixed bootstrap as `main.py`. + +The Miracle experience Skill uses schema +`24-miracle-experience-skill-v2`, separate from both the generic `generals` +replay Skill and the human replay-reading Skill. It distinguishes frozen facts, +training observations, supported heuristics, failed/revoked heuristics, +unverified hypotheses, approved train-evidence digests, strategy references, +case identities, authors, and an immutable version SHA. `experience-v0` is the +fixed empty baseline: it records no observations, heuristics, hypotheses, or +training evidence and binds the approved human reader plus the same complete +iteration identity as the bootstrap. Every later Skill requires an existing +same-identity parent and non-empty case/evidence references drawn only from the +currently approved train replay. +Candidate readiness binds a same-generation pair such as `strategy-v1` and +`experience-v1`, including both canonical SHA-256 values and their common +parents and control identity. The Experience Skill must reference that exact +candidate strategy and the approved training evidence. A strategy without its +corresponding non-v0 Skill is not a candidate bundle. +Each round reads the last approved version and may append a candidate version; +in-place replacement is forbidden. The required human-authored Miracle replay +reader has not been supplied and remains explicitly +`HUMAN_AUTHORED_CONTENT_REQUIRED`. Agent-generated text cannot satisfy it. + +Rollback is append-only and the target must be an ancestor of the source; a +sibling or unrelated branch cannot be selected. It first revalidates the source and target version +bytes/SHA, bootstrap, champion, human replay Skill, match plan, replay-evidence +manifest, artifact bytes, and all current approval tables, then writes a new rollback +iteration record containing source, target, reason, and operator. It never +deletes or changes later versions. Missing, tampered, or incompatible inputs +fail before a rollback record or runtime resource is created. + +Candidate evaluation is a separate lifecycle, not reuse of the initial v0 +MatchPlan. A canonical `CandidateEvaluationPlan` binds the baseline and +candidate policy versions/source digests, Experience Skill, champion, human +replay-reading Skill, training provenance, research/iteration identities, +purpose, and frozen evaluation case/seed bundles. Its production approval set +is empty. Fake-only tests may temporarily approve a digest with pytest +monkeypatch; that state is restored after the test. + +Evaluation produces a separate canonical `EvaluationEvidenceManifest` with +exactly one baseline/candidate replay-result pair for every planned evaluation +case. It binds artifact bytes, outcomes and scores, policy and Skill identity, +strict trajectory KL evidence, information gain, and failure reasons. Missing, +extra, duplicate, seed-drifted, policy-drifted, or tampered evidence fails +closed. Evaluation cases and artifacts never enter the training prompt, +change plan, or Experience Skill. + +There is no boolean completion API. `IterationAcceptanceManifest` derives its +status from the independently approved evaluation plan and evidence: all cases +must be complete, and the validator requires the exact formal profile, +epsilon `0.01`, `KL(new || old)`, new-policy rollout and full ActionSupport +distributions. Every local value, mean and sum is recomputed. IG must be +present, score gain must be positive, and blockers must be empty. The +acceptance manifest itself +also needs independent digest approval. Complete fake evidence is reported as +`fake_only=true`; all three production approval sets are empty, so production +`COMPLETED` remains unreachable. Its Results summary mapping retains raw/evo/ +gain, IG, incomplete reasons, protocol/research/iteration identities, policy, +Skill, training/evaluation plan, evidence, KL-evidence, and acceptance SHA +fields without changing Results validation. + +All mutable factories and stores perform their own stage preflight or fully +revalidate an internally issued context immediately before side effects. A +publicly constructed or forged dataclass is not authorization. Contexts bind +the original manifest and asset paths, canonical digests, and approval state; +post-preflight replacement of the champion, human Skill, match plan, replay +manifest, replay artifact, or bootstrap therefore fails with zero new writes. + +Current status is deliberately split as follows: + +- mechanism implemented: yes; +- fake-only verified: yes, by temporary fixtures only; +- human input required: champion asset and human-authored replay Skill; +- authoritative execution blocked: yes; +- authoritative completed: no. + +The final HL checklist cannot be marked complete until a separately approved +real experiment retains complete replays, auditable strategy changes, old and +new immutable versions, at least one valid score improvement, complete real +KL/IG, and score-iteration and IG-iteration curves. + +## Policy information gain + +- measurement profile: `24_miracle_policy_information_gain_v2` +- epsilon: `0.01` +- direction: `new||old` +- smoothing: symmetric uniform mixture on the complete current support +- rollout: actual new-policy rollout +- primary episode aggregate: arithmetic mean, `nats / decision` +- optional local-KL sum: `nats / episode` +- decision: one atomic Judge command (`init`, `move`, `attack`, `summon`, `use`, + `endround`, or `surrender`) +- decision-change rate: not collected + +The game adapter must provide the complete legal atomic-command support and +strict new/old distributions on the same visible state. A deterministic HL +policy reports a true one-hot distribution. Random behavior must report its +actual categorical distribution and must not be disguised as one-hot. +Framework applies the same epsilon channel to both distributions only for +measurement; it does not change the new policy action executed by the Judge. + +If support completeness or either distribution is unavailable, the episode is +`incomplete`; trace-derived information gain and sum remain missing rather than +zero. Uploaded local or episode summaries are checked against values recomputed +from the ordered distributions and cannot override them. `occupancy_shift` +remains separate and is never added to information gain. + +If a selector depends on internal memory, authoritative context is `z=(s,m)`. +Current evidence records visible-state identity but not canonical `m`, so a +stateful real policy has +`internal_memory_evidence=not_collected`. The production iteration preflight +therefore derives an incomplete formal IG until an H08 integration binds +`m` to the policy/context identity. Tests can inject the separately named +test-only bound state; uploaded run data cannot override this control-plane +state. + +`Run.log_policy_kl_trace()` remains readable for old events, including records +with `epsilon=None`, but writes `measurement_profile=legacy_policy_kl_trace`, +`information_gain=null`, and `information_gain_status=unverified`. Only a rich +v2 payload whose frozen decision distributions and all derived fields reverify +can populate formal IG in tracking and reports. + +The authoritative observation frame is six ASCII decimal length bytes followed +by UTF-8 JSON. `decode_ai_observation()` validates that frame and +`enumerate_legal_commands()` derives finite legal support from the decoded +dynamic state plus fixed map geometry. The authoritative Judge accepts +WindBlessing at any cube coordinate without an in-map check; that support is +unbounded, so the enumerator deliberately fails closed for such a state. The +frozen if-else configuration selects InfernoFlame unless explicitly overridden. + +`IsolatedDeterministicPolicy` is a fake-testable deep-copy boundary for a pure +deterministic selector. It is not yet a binding to the frozen +`ifelse_bot/main.py`: that program's `AiClient` methods write a command to +stdout and then consume the next observation. A separately reviewed adapter is +still required to intercept exactly one command from an isolated copy without +submitting it or advancing either old/new policy session. Until that binding is +implemented and verified, real-policy trajectory KL remains incomplete. + +The Judge preserves the submitted `creatures` order in the player's capacity +list and exposes that order in later observations. Init support therefore +contains all `4 * P(7, 3) = 840` ordered legal card selections; permutations are +distinct canonical actions. + +## Results integration boundary + +Framework's Results pipeline checks load the repository named by +`AGENTBENCH_RESULTS`. In a two-worktree integration environment they exercise +that exact aggregate/report implementation. In an independent Framework +checkout without the environment variable and Results files, these optional +cross-repository checks skip with an explicit dependency reason; zero skips is +not an independent-checkout guarantee. + +## Execution boundary + +This document and its tests authorize no Judge, opponent, Provider, real match, +matrix, or authoritative session. Runtime execution requires separate approval +after the adapter and Results CI pass fake-only acceptance. diff --git a/docs/integration/trajectory-kl.md b/docs/integration/trajectory-kl.md index eb0489f..3d6e634 100644 --- a/docs/integration/trajectory-kl.md +++ b/docs/integration/trajectory-kl.md @@ -1,15 +1,11 @@ # Trajectory KL 下游接入指南 -> **24_miracle 作用域与迁移状态。** 本文记录的是当前 generic legacy -> trajectory-KL 接口：`KL(new||old)`、epsilon regularization 和 episode -> local-KL sum。它继续适用于仍采用该接口的通用/其他游戏接入，但不再是 -> 24_miracle 的目标合同。24_miracle 已裁决为 atomic Judge operation、状态 -> 局部完整有序的 `ActionSupport + support_id`，以及 `KL(old||new)`、自然 -> 对数、无 smoothing、new-policy occupancy、trajectory arithmetic mean、 -> `nats / decision`、阈值 `0.01`。参见 -> [24_miracle KL contract authority v1](../games/24_miracle_kl_contract_authority.v1.md)。 -> 当前 generic runtime、tracking 和 downstream integration 尚未迁移；本文 -> 不构成迁移完成声明。 +> **24_miracle 作用域。** 当前接口已统一为 atomic Judge operation、状态 +> 局部完整有序的 `ActionSupport + support_id`、`KL(new||old)`、自然对数、 +> 双方固定 `epsilon=0.01` 均匀 smoothing、new-policy occupancy、episode +> arithmetic mean 与 `nats / decision`。不存在 KL 大小阈值。参见 +> [24_miracle KL contract authority v2](../games/24_miracle_kl_contract_authority.v2.md)。 +> 真实运行与权威批准仍是独立边界。 本文面向接入具体 Saiblo 游戏、RL agent、HL（heuristic learning， rule-based agent iteration）agent 或自定义对局 runner 的开发者。目标是让 @@ -25,8 +21,8 @@ KL，也不是 replay-based KL。数学背景和设计取舍见 一次接入会为每个目标 agent episode 产生： - 每个目标决策点的新旧原始策略分布、合法动作 ID、实际动作 ID 和 local KL； -- 主指标 `trajectory_kl_episode`，单位为 `nats / episode`； -- 辅助指标 `mean_local_policy_kl`，单位为 `nats / decision`； +- 主指标 `information_gain`（等于 `mean_local_policy_kl`），单位为 `nats / decision`； +- 独立派生 `local_policy_kl_sum`（兼容字段 `trajectory_kl_episode`），单位为 `nats / episode`； - episode 的版本、epsilon、对手、seed、先后手和错误信息； - `events.jsonl` 中的追加式一手记录； - 本地和 CI 中按 episode 展示且不跨越缺失点的折线图。 @@ -481,10 +477,9 @@ measured_agent = TrajectoryKLAgent( active_policy=new_policy_adapter, reference_policy=old_policy_adapter, support_provider=support_provider, - config=TrajectoryKLConfig( + config=TrajectoryKLConfig.for_policy_information_gain( version_before="artifact-sha256:old", version_after="artifact-sha256:new", - epsilon=0.01, metadata={ "evaluation_suite": "fixed-suite-v1", }, @@ -714,8 +709,8 @@ on_episode_complete=run.log_trajectory_kl_result | `epsilon` | 本次固定 regularization 参数 | | `trace` | 与 decisions 一一对齐的 local KL 序列 | | `decision_steps` | 目标 agent 决策次数 | -| `trajectory_kl_episode` | 主指标，`nats / episode` | -| `mean_local_policy_kl` | 辅助指标，`nats / decision` | +| `information_gain` / `mean_local_policy_kl` | 主 episode IG（trace 算术平均），`nats / decision` | +| `local_policy_kl_sum` / `trajectory_kl_episode` | 可选 trace 总和，`nats / episode` | | `direction` | 固定为 `new||old` | | `log_base` | 固定为自然对数 `e` | | `rollout_source` | rich 在线测量为 `new_policy` | @@ -723,11 +718,13 @@ on_episode_complete=run.log_trajectory_kl_result | `metadata` | seed、对手、先后手和实验协议等 | | `errors` | episode 级错误列表 | -报告会从 `trace` 重新派生总和与均值，不盲信可能陈旧的预计算标量。畸形、 -不对齐、非有限或 incomplete 事件在图中保留为缺口。 +报告会从有序 decision records 的 old/new distributions 重新计算每个 local +KL、trace、总和与均值，不盲信上传的 local 或预计算 summary。畸形、不对齐、 +非有限或 incomplete 事件在图中保留为缺口。 旧的 `Run.log_policy_kl_trace()` 只用于兼容已经计算好的 legacy trace，无法 -证明其 rollout 来源；新接入应使用 `log_trajectory_kl_result()`。 +证明其 rollout 来源；它保留原 trace，但正式 `information_gain` 必须为 null。 +新接入应使用 `log_trajectory_kl_result()`。 ## 13. 固定实验协议 @@ -758,12 +755,16 @@ trajectory KL 具有统一单位，但其数值仍受游戏、动作支持规模 不能把整条曲线的变化全部归因于 learning。更稳妥的做法是每轮迭代重复同一 固定对手序列，并按对手或 case 分层比较。 -epsilon 由 `TrajectoryKLConfig` 固定，必须满足： +24_miracle 正式主 epsilon 由无 epsilon 参数的 +`TrajectoryKLConfig.for_policy_information_gain()` 固定，必须满足： ```text -0 < epsilon < 1 +epsilon == 0.01 ``` +通用研究工具仍可用 `TrajectoryKLConfig(..., epsilon=...)` 选择合法的其他 +epsilon，但其 `measurement_profile=generic_trajectory_kl`，不能填充正式 IG。 + 不要让 adapter 自行平滑后又让 framework 二次平滑。 ## 14. 错误语义与排障 @@ -835,8 +836,9 @@ rollout；但该 episode 不再产生主标量。下游不能从已有 local KL - [ ] 一个正常 episode 的 decision 数与 trace 长度严格相等。 - [ ] 每个正常 local KL 有限且非负。 -- [ ] `trajectory_kl_episode` 等于完整 trace 之和。 -- [ ] 单位明确为主 `nats / episode`、辅 `nats / decision`。 +- [ ] `information_gain` 等于完整 trace 的算术平均。 +- [ ] `local_policy_kl_sum`（及兼容 sum 字段）等于完整 trace 之和。 +- [ ] 单位明确为主 `nats / decision`、可选 sum `nats / episode`。 - [ ] 分布/查询失败会保留当前决策已取得的 raw evidence，并使 episode incomplete。 - [ ] support、active 类型或非法动作的前置失败只要求保留先前 decisions 和 abort error。 - [ ] incomplete episode 的总和与均值为缺失，不是 0。 diff --git a/docs/research/information-gain-design.md b/docs/research/information-gain-design.md index e034ad1..488cbf6 100644 --- a/docs/research/information-gain-design.md +++ b/docs/research/information-gain-design.md @@ -12,13 +12,11 @@ 动作表示为状态局部、完整、有序、可验证的 `ActionSupport + support_id`，而 不是固定全局 Boolean/0-1 mask。 -本文其余 `KL(new||old)`、epsilon smoothing 和 episode local-KL sum 内容是 -当前 generic implementation 的设计记录，现对 24_miracle 标记为 legacy。 -24_miracle 的目标合同为 `KL(old||new)`、自然对数、无 smoothing、new-policy -occupancy、trajectory arithmetic mean、`nats / decision`、阈值 `0.01`；参见 -[24_miracle KL contract authority v1](../games/24_miracle_kl_contract_authority.v1.md)。 -这项 game-specific 裁决不改变其他游戏，也不表示 framework、runtime、 -tracking 或 report 已经迁移。 +24_miracle 的合同为 `KL(new||old)`、自然对数、双方固定 `epsilon=0.01` +均匀 smoothing、new-policy occupancy、episode trace arithmetic mean 和 +`nats / decision`；sum 仅为 `nats / episode` 的独立派生量，不存在 KL 阈值。参见 +[24_miracle KL contract authority v2](../games/24_miracle_kl_contract_authority.v2.md)。 +这项 game-specific 裁决不授权真实执行或生产批准。 ## 1. 目标与术语 @@ -143,7 +141,7 @@ epsilon-regularized 局部 KL 和”，不是原始策略 path distribution 的 $KL(P_k\|P_{k-1})$。字段名 `trajectory_kl_episode` 为数据格式稳定性保留， 事件通过 `estimand` 明确其严格定义。 -长度归一化辅助量为： +主 episode 信息增益为： $$ MeanLocalKL_{k,e} @@ -151,19 +149,19 @@ MeanLocalKL_{k,e} \frac{TrajectoryKL_{k,e}}{T_{k,e}}. $$ -主要图像应为： +主要图像应为每个 episode 的该算术平均： $$ x=\text{policy iteration }k, -\qquad y=TrajectoryKL_{k,e}. +\qquad y=MeanLocalKL_{k,e}. $$ 如果每轮有多个 episode，则保留散点，同时可绘制该轮均值/中位数和置信区间： $$ -\overline{TrajectoryKL}_k +\overline{MeanLocalKL}_k = -\frac{1}{M_k}\sum_{e=1}^{M_k}TrajectoryKL_{k,e}. +\frac{1}{M_k}\sum_{e=1}^{M_k}MeanLocalKL_{k,e}. $$ 这里的 (M_k) 只是同一轮的重复实验数量，不是学习曲线的主指标。 @@ -468,16 +466,17 @@ local_policy_kl_k(z) local_policy_kl_trace = [local_policy_kl_k(z_0), ..., local_policy_kl_k(z_{T-1})] ``` -这里的 `z_t` 只包括该 episode 中真实到达的目标 agent 决策点，不需要构造全局测量域，也不把 coding agent 的 act 次数当作策略决策点。该 trace 是策略变化的原始测量数据。当前主派生量按 episode 求和： +这里的 `z_t` 只包括该 episode 中真实到达的目标 agent 决策点，不需要构造全局测量域，也不把 coding agent 的 act 次数当作策略决策点。该 trace 是策略变化的原始测量数据。主 episode 派生量是算术平均： ```text -trajectory_kl_episode = Σ_t local_policy_kl_k(z_t) +information_gain = mean_t local_policy_kl_k(z_t) +local_policy_kl_sum = Σ_t local_policy_kl_k(z_t) ``` -`trajectory_kl_episode` 的单位是 `nats / episode`。长度归一化的 -`mean_local_policy_kl = trajectory_kl_episode / T` 可以作为辅助统计， -其单位是 `nats / decision`，不能替代 trajectory KL。CI 保留每个 -episode 的点并按 episode/迭代顺序绘图，不先压成整个实验的单一平均值。 +`information_gain`（兼容均值字段 `mean_local_policy_kl`）的单位是 +`nats / decision`。`local_policy_kl_sum`（兼容 sum 字段 +`trajectory_kl_episode`）是单位为 `nats / episode` 的可选派生量。CI 保留 +每个 episode 的主 IG 点并按 episode/迭代顺序绘图。 第二个核心对象是同一评测上下文下的状态访问变化 `occupancy_shift`。它可以按时间步保存状态分布差异，或者保存由 rollout 得到的规范化 state-ID 直方图；它描述策略变化通过环境动力学和对手交互后造成的访问分布变化，不能和局部策略 KL 直接相加当作一个“总信息增益”。 @@ -496,7 +495,7 @@ KL(q_k || q_{k-1}) + E[s ~ d_k] KL(π_k(·|s) || π_{k-1}(·|s)) ``` -因此不能把任意固定参考分布下的 `policy_kl`、状态 occupancy KL 和 trajectory KL 当成三个可独立相加的指标。RL/HL 必须使用同一合法动作集和概率分布；occupancy 必须使用可比较的规范化 state ID；HL 自己提供 one-hot 分布，framework 不推断其内部逻辑；之后 RL/HL 统一使用 benchmark 固定的 epsilon smoothing。原始 trace 和 occupancy 数据优先保存，trajectory KL 是主 episode 派生量。 +因此不能把任意固定参考分布下的 `policy_kl`、状态 occupancy KL 和 trajectory KL 当成三个可独立相加的指标。RL/HL 必须使用同一合法动作集和概率分布；occupancy 必须使用可比较的规范化 state ID；HL 自己提供 one-hot 分布，framework 不推断其内部逻辑；之后 RL/HL 统一使用 benchmark 固定的 epsilon smoothing。原始 trace 和 occupancy 数据优先保存，trace mean 是主 episode IG。 ### 10.13.1 Replay-based KL 讨论结论 @@ -509,8 +508,8 @@ stateful agent 的历史恢复、跨语言只读概率查询、replay/parser/pol artifact 身份以及失败完整性语义。当前不新增 replay-based KL 接口，也不 把历史动作频率解释为策略概率。 -当前生效方案仍是实际 rollout 上的 `local_policy_kl_trace` 和按 episode -求和得到的 `trajectory_kl_episode`。完整讨论与重新启动条件见 +当前生效方案仍是实际 rollout 上的 `local_policy_kl_trace`、主 episode +算术平均和独立 sum。完整讨论与重新启动条件见 `docs/superpowers/specs/2026-07-25-trajectory-kl-replay-decision.md`。 ### 10.13.2 在线测量接口 @@ -604,7 +603,7 @@ created_at - canonical state ID、严格 `ActionSupport`/`PolicyDecision` 契约、在线 `TrajectoryKLAgent` 对照 session； - 完整和 incomplete trajectory-KL 一手 JSONL 记录； -- 以 episode trace 求和为主值、局部均值为辅助值的本地报告，以及保留缺口 +- 以 episode trace 算术平均为主 IG、sum 为独立派生量的本地报告，以及保留缺口 的 episode 折线图。 仍由接入方决定的部分：具体 benchmark 测试集内容，以及具体环境 runtime diff --git a/docs/research/research-methodology-summary.md b/docs/research/research-methodology-summary.md index 0573b35..c888202 100644 --- a/docs/research/research-methodology-summary.md +++ b/docs/research/research-methodology-summary.md @@ -12,13 +12,12 @@ 不同层级；后两者当前对被测策略一一对应，双方合计的 `ai_operation` 数不能 替代被测策略 decision 数。 -本文其余 `KL(new||old)`、epsilon smoothing 和 episode local-KL sum 方法是 -当前 generic legacy 口径。24_miracle 的目标合同改为 `KL(old||new)`、自然 -对数、无 smoothing、new-policy occupancy、trajectory arithmetic mean、 -`nats / decision`、阈值 `0.01`；其动作表示是状态局部、完整、有序的 +24_miracle 的合同为 `KL(new||old)`、自然对数、双方固定 `epsilon=0.01` +均匀 smoothing、new-policy occupancy、episode trace arithmetic mean、 +`nats / decision`，且没有 KL 阈值；其动作表示是状态局部、完整、有序的 `ActionSupport + support_id`，不是固定全局 Boolean mask。权威裁决见 -[24_miracle KL contract authority v1](../games/24_miracle_kl_contract_authority.v1.md)。 -这不表示 schema、runtime、tracking、report 或真实执行已完成迁移。 +[24_miracle KL contract authority v2](../games/24_miracle_kl_contract_authority.v2.md)。 +这不授权真实执行或生产批准。 ## 1. 研究对象与术语 @@ -183,7 +182,7 @@ epsilon-regularized 的 $\tilde\pi$。因此严格 estimand 是 策略的精确 path KL。只有 rollout 也从同一个 $\tilde\pi^k$ 采样时，才能 使用 regularized 策略 forward trajectory KL 的链式分解解释。 -长度归一化的辅助量为： +主 episode 信息增益为： $$ MeanLocalKL_{k,i} @@ -191,9 +190,9 @@ MeanLocalKL_{k,i} \frac{TrajectoryKL_{k,i}}{T_{k,i}}. $$ -主分析保留每个 episode 的 $TrajectoryKL_{k,i}$；如果一轮有多个 episode， -可以绘制散点、均值/中位数和置信区间。`MeanLocalKL` 单独展示，不与 -trajectory KL 共用名称或单位。不要把所有迭代过程先压成一个全局均值。 +主分析保留每个 episode 的 $MeanLocalKL_{k,i}$；如果一轮有多个 episode， +可以绘制散点、均值/中位数和置信区间。trajectory sum 是单位不同的可选 +派生量。不要把所有迭代过程先压成一个全局均值。 ## 5. 时间权重与 $M$ 的含义 @@ -417,13 +416,14 @@ local_policy_kl_trace = [local_policy_kl_k(z_0), ..., local_policy_kl_k(z_{T-1}) `z_t` 只包括该 episode 中真实到达的目标 agent 决策点；不构造不可行的全局测量域，也不把 coding agent 的 act 次数当作策略决策点。主 episode 派生量定义为： ```text -trajectory_kl_episode = Σ_t local_policy_kl_k(z_t) +information_gain = mean_t local_policy_kl_k(z_t) +local_policy_kl_sum = Σ_t local_policy_kl_k(z_t) ``` -其单位为 `nats / episode`。`mean_local_policy_kl = -trajectory_kl_episode / T` 只作为 `nats / decision` 的长度归一化辅助量。 -CI 保留每个 episode 的 trajectory KL 并按 episode/迭代顺序绘图，不先 -压成整个实验的单一平均值。 +`information_gain`（兼容均值字段 `mean_local_policy_kl`）单位为 +`nats / decision`。`local_policy_kl_sum`（兼容 sum 字段 +`trajectory_kl_episode`）单位为 `nats / episode`。CI 保留每个 episode 的 +主 IG 并按 episode/迭代顺序绘图。 `occupancy_shift` 描述同一评测上下文下的状态访问分布变化，可以按时间步保存分布差异，也可以保存 rollout 得到的规范化 state-ID 直方图。它反映策略变化经环境动力学和对手交互后的访问变化，不能与局部策略 KL 直接相加。 @@ -442,7 +442,7 @@ KL(q_k || q_{k-1}) + E[s ~ d_k] KL(π_k(·|s) || π_{k-1}(·|s)) ``` -因此，任意固定参考分布下的 `policy_kl`、occupancy KL 和 trajectory KL 不能被当成三个可独立相加的指标。RL/HL 使用同一合法动作集和概率分布；occupancy 使用规范化 state ID；HL 自己返回 one-hot 分布，framework 再对 RL/HL 统一使用固定 epsilon。优先保存原始 trace 和 occupancy 数据，trajectory KL 是主 episode 派生量。 +因此，任意固定参考分布下的 `policy_kl`、occupancy KL 和 trajectory KL 不能被当成三个可独立相加的指标。RL/HL 使用同一合法动作集和概率分布；occupancy 使用规范化 state ID；HL 自己返回 one-hot 分布，framework 再对 RL/HL 统一使用固定 epsilon。优先保存原始 trace 和 occupancy 数据，trace mean 是主 episode IG。 ## 15.1 Replay-based KL 暂缓决策 @@ -452,7 +452,7 @@ KL(q_k || q_{k-1}) 冻结 replay/adapter/policy 身份，并提供跨语言只读查询协议。 本轮不实现 replay-based KL，也不从历史动作推断概率。当前继续使用实际 -rollout 上的局部 KL trace，并按 episode 求和得到 trajectory KL。完整决策 +rollout 上的局部 KL trace、主 episode 算术平均和独立 sum。完整决策 记录见 `docs/superpowers/specs/2026-07-25-trajectory-kl-replay-decision.md`。 ## 15.2 在线 trajectory-KL 接口 @@ -497,7 +497,7 @@ created_at - provider-neutral 的 `ProviderAdapter`、`CodingAgentController` 和本地 workspace manifest/hash/diff snapshotter； - canonical state ID、严格 action-ID 支持集和在线新/旧策略对照 session； - 完整与 incomplete trajectory-KL 一手事件； -- 正确区分 `nats / episode` 主值和 `nats / decision` 辅助值的报告，以及 +- 正确区分 `nats / decision` 主 IG 和 `nats / episode` 可选 sum 的报告，以及 保留 incomplete 缺口的 episode 折线图。 仍由接入方决定的部分：具体 benchmark 测试集内容，以及具体游戏 runtime diff --git a/src/agentbench_frame/eval/__init__.py b/src/agentbench_frame/eval/__init__.py index 3a823f1..85cf196 100644 --- a/src/agentbench_frame/eval/__init__.py +++ b/src/agentbench_frame/eval/__init__.py @@ -17,11 +17,28 @@ ) from agentbench_frame.eval.curves import trapezoid_auc from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION, + FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND, + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + FORMAL_POLICY_INFORMATION_GAIN_UNIT, + FORMAL_POLICY_KL_DIRECTION, + FORMAL_POLICY_KL_LOG_BASE, + FORMAL_POLICY_KL_ROLLOUT_SOURCE, + FORMAL_POLICY_KL_SMOOTHING, + FORMAL_POLICY_KL_SUM_ESTIMAND, + FORMAL_POLICY_KL_SUM_UNIT, + GENERIC_TRAJECTORY_KL_PROFILE, + LEGACY_POLICY_KL_TRACE_PROFILE, + MAIN_POLICY_KL_EPSILON, + POLICY_KL_SENSITIVITY_EPSILONS, + episode_information_gain_from_trace, epsilon_regularize, episode_policy_kl_trace, occupancy_histogram, occupancy_shift, policy_kl, + formal_policy_kl, + policy_kl_sensitivity, trajectory_kl_from_trace, validate_policy_distribution, ) @@ -51,11 +68,28 @@ "evaluate_benchmark", "trapezoid_auc", "epsilon_regularize", + "episode_information_gain_from_trace", "episode_policy_kl_trace", "occupancy_histogram", "occupancy_shift", "policy_kl", + "formal_policy_kl", + "policy_kl_sensitivity", "trajectory_kl_from_trace", + "MAIN_POLICY_KL_EPSILON", + "POLICY_KL_SENSITIVITY_EPSILONS", + "GENERIC_TRAJECTORY_KL_PROFILE", + "LEGACY_POLICY_KL_TRACE_PROFILE", + "FORMAL_POLICY_INFORMATION_GAIN_PROFILE", + "FORMAL_POLICY_KL_DIRECTION", + "FORMAL_POLICY_KL_LOG_BASE", + "FORMAL_POLICY_KL_SMOOTHING", + "FORMAL_POLICY_KL_ROLLOUT_SOURCE", + "FORMAL_POLICY_KL_SUM_ESTIMAND", + "FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND", + "FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION", + "FORMAL_POLICY_INFORMATION_GAIN_UNIT", + "FORMAL_POLICY_KL_SUM_UNIT", "validate_policy_distribution", "ActionCandidate", "ActionSupport", diff --git a/src/agentbench_frame/eval/information_gain.py b/src/agentbench_frame/eval/information_gain.py index 2029242..7ce96fe 100644 --- a/src/agentbench_frame/eval/information_gain.py +++ b/src/agentbench_frame/eval/information_gain.py @@ -13,6 +13,52 @@ POLICY_SUM_TOLERANCE = 1e-9 +MAIN_POLICY_KL_EPSILON = 0.01 +POLICY_KL_SENSITIVITY_EPSILONS = (0.001, 0.01, 0.05) +GENERIC_TRAJECTORY_KL_PROFILE = "generic_trajectory_kl" +LEGACY_POLICY_KL_TRACE_PROFILE = "legacy_policy_kl_trace" +FORMAL_POLICY_INFORMATION_GAIN_PROFILE = ( + "24_miracle_policy_information_gain_v2" +) +FORMAL_POLICY_KL_DIRECTION = "new||old" +FORMAL_POLICY_KL_LOG_BASE = "e" +FORMAL_POLICY_KL_SMOOTHING = "symmetric_epsilon_uniform_full_support" +FORMAL_POLICY_KL_ROLLOUT_SOURCE = "new_policy" +FORMAL_POLICY_KL_SUM_ESTIMAND = ( + "epsilon_regularized_local_kl_sum_under_new_policy_occupancy" +) +FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND = ( + "epsilon_regularized_mean_local_policy_kl_under_new_policy_occupancy" +) +FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION = "arithmetic_mean" +FORMAL_POLICY_INFORMATION_GAIN_UNIT = "nats / decision" +FORMAL_POLICY_KL_SUM_UNIT = "nats / episode" + + +def _strict_number(value: Any, label: str) -> float: + if type(value) not in {int, float}: + raise TypeError(f"{label} must be an int or float, not bool") + number = float(value) + if not math.isfinite(number) or number < 0.0: + raise ValueError(f"{label} must be finite and non-negative") + return number + + +def _strict_probability_sequence(values: Sequence[float]) -> List[float]: + if not values: + raise ValueError("probability support cannot be empty") + converted = [ + _strict_number(value, f"probability[{index}]") + for index, value in enumerate(values) + ] + if not math.isclose( + math.fsum(converted), + 1.0, + rel_tol=0.0, + abs_tol=POLICY_SUM_TOLERANCE, + ): + raise ValueError("policy probabilities must sum to 1") + return converted def validate_policy_distribution( @@ -31,26 +77,19 @@ def validate_policy_distribution( raise ValueError( f"policy distribution support mismatch; missing={missing}, extra={extra}" ) - values = [float(distribution[action_id]) for action_id in support.action_ids] - if any(not math.isfinite(value) or value < 0.0 for value in values): - raise ValueError("probabilities must be finite and non-negative") - if not math.isclose( - sum(values), - 1.0, - rel_tol=0.0, - abs_tol=POLICY_SUM_TOLERANCE, - ): - raise ValueError("policy probabilities must sum to 1") - return values + return _strict_probability_sequence( + [distribution[action_id] for action_id in support.action_ids] + ) def _normalize(values: Sequence[float]) -> List[float]: if not values: raise ValueError("probability support cannot be empty") - converted = [float(value) for value in values] - if any(not math.isfinite(value) or value < 0.0 for value in converted): - raise ValueError("probabilities must be finite and non-negative") - total = sum(converted) + converted = [ + _strict_number(value, f"mass[{index}]") + for index, value in enumerate(values) + ] + total = math.fsum(converted) if total <= 0.0: raise ValueError("probability mass must be positive") return [value / total for value in converted] @@ -61,14 +100,17 @@ def epsilon_regularize( ) -> List[float]: """Mix a distribution with uniform mass over the common legal support.""" - if not 0.0 <= epsilon <= 1.0: + if type(epsilon) not in {int, float}: + raise TypeError("epsilon must be an int or float, not bool") + epsilon = float(epsilon) + if not math.isfinite(epsilon) or not 0.0 <= epsilon <= 1.0: raise ValueError("epsilon must be in [0, 1]") - normalized = _normalize(probs) - support_size = legal_count if legal_count is not None else len(normalized) - if support_size != len(normalized) or support_size <= 0: + validated = _strict_probability_sequence(probs) + support_size = legal_count if legal_count is not None else len(validated) + if type(support_size) is not int or support_size != len(validated) or support_size <= 0: raise ValueError("legal_count must match the probability support") uniform = 1.0 / support_size - return [(1.0 - epsilon) * value + epsilon * uniform for value in normalized] + return [(1.0 - epsilon) * value + epsilon * uniform for value in validated] def policy_kl( @@ -81,8 +123,8 @@ def policy_kl( if len(new_probs) != len(old_probs) or not new_probs: raise ValueError("new and old distributions must share a non-empty support") if epsilon is None: - new = _normalize(new_probs) - old = _normalize(old_probs) + new = _strict_probability_sequence(new_probs) + old = _strict_probability_sequence(old_probs) else: new = epsilon_regularize(new_probs, epsilon) old = epsilon_regularize(old_probs, epsilon) @@ -97,10 +139,42 @@ def policy_kl( return value +def formal_policy_kl( + new_probs: Sequence[float], + old_probs: Sequence[float], +) -> float: + """Compute the formal 24_miracle local IG primitive. + + This entry point intentionally has no epsilon argument. Research callers + that need another epsilon (or no smoothing) must use :func:`policy_kl`, + whose result does not carry the formal measurement profile. + """ + + return policy_kl( + new_probs, + old_probs, + epsilon=MAIN_POLICY_KL_EPSILON, + ) + + +def policy_kl_sensitivity( + new_probs: Sequence[float], + old_probs: Sequence[float], +) -> dict[float, float]: + """Derive the fixed sensitivity panel without changing the main identity.""" + + return { + epsilon: policy_kl(new_probs, old_probs, epsilon=epsilon) + for epsilon in POLICY_KL_SENSITIVITY_EPSILONS + } + + def _distribution_for_context(policy: Callable[[Any], Any], context: Any, actions: List[Any]) -> List[float]: raw = policy(context) if isinstance(raw, Mapping): - return [float(raw.get(action, 0.0)) for action in actions] + if set(raw) != set(actions): + raise ValueError("policy distribution must exactly match legal actions") + return [raw[action] for action in actions] values = list(raw) if len(values) != len(actions): raise ValueError("policy distribution length does not match legal actions") @@ -164,7 +238,20 @@ def occupancy_shift( def trajectory_kl_from_trace(trace: Iterable[float]) -> float: """Return the per-rollout trajectory-KL contribution derived from a trace.""" - values = [float(value) for value in trace] - if any(not math.isfinite(value) or value < 0.0 for value in values): - raise ValueError("KL trace values must be finite and non-negative") - return sum(values) + values = [ + _strict_number(value, f"KL trace[{index}]") + for index, value in enumerate(trace) + ] + total = math.fsum(values) + if not math.isfinite(total): + raise ValueError("KL trace sum must be finite") + return total + + +def episode_information_gain_from_trace(trace: Iterable[float]) -> float: + """Return the primary episode IG: the unweighted local-KL trace mean.""" + + values = tuple(trace) + if not values: + raise ValueError("episode information gain requires a non-empty trace") + return trajectory_kl_from_trace(values) / len(values) diff --git a/src/agentbench_frame/eval/trajectory_kl.py b/src/agentbench_frame/eval/trajectory_kl.py index d3a714f..08aaf38 100644 --- a/src/agentbench_frame/eval/trajectory_kl.py +++ b/src/agentbench_frame/eval/trajectory_kl.py @@ -6,10 +6,25 @@ import math from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from types import MappingProxyType from typing import Any, Optional from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION, + FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND, + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + FORMAL_POLICY_INFORMATION_GAIN_UNIT, + FORMAL_POLICY_KL_DIRECTION, + FORMAL_POLICY_KL_LOG_BASE, + FORMAL_POLICY_KL_ROLLOUT_SOURCE, + FORMAL_POLICY_KL_SUM_ESTIMAND, + FORMAL_POLICY_KL_SUM_UNIT, + GENERIC_TRAJECTORY_KL_PROFILE, + MAIN_POLICY_KL_EPSILON, + episode_information_gain_from_trace, + formal_policy_kl, policy_kl, + trajectory_kl_from_trace, validate_policy_distribution, ) from agentbench_frame.eval.measurement import ( @@ -19,6 +34,57 @@ ) +def _freeze_value(value: Any) -> Any: + if isinstance(value, Mapping): + return MappingProxyType( + {copy.deepcopy(key): _freeze_value(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_value(item) for item in value) + return copy.deepcopy(value) + + +def _thaw_value(value: Any) -> Any: + if isinstance(value, Mapping): + return {key: _thaw_value(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw_value(item) for item in value] + return copy.deepcopy(value) + + +def _strict_text(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} must be a non-empty string") + return value + + +def _strict_nonnegative(value: Any, label: str) -> float: + if type(value) not in {int, float}: + raise TypeError(f"{label} must be an int or float, not bool") + number = float(value) + if not math.isfinite(number) or number < 0.0: + raise ValueError(f"{label} must be finite and non-negative") + return number + + +def _same_number(left: Any, right: Any) -> bool: + if left is None or right is None: + return left is right + if type(left) not in {int, float} or type(right) not in {int, float}: + return False + + left_number = float(left) + right_number = float(right) + return ( + math.isfinite(left_number) + and math.isfinite(right_number) + and math.isclose( + left_number, + right_number, + rel_tol=0.0, + abs_tol=1e-12, + ) + ) @dataclass(frozen=True) class TrajectoryKLConfig: """Immutable identities and measurement-channel parameters.""" @@ -27,19 +93,51 @@ class TrajectoryKLConfig: version_after: str epsilon: float metadata: Mapping[str, Any] = field(default_factory=dict) + measurement_profile: str = GENERIC_TRAJECTORY_KL_PROFILE def __post_init__(self) -> None: - if not isinstance(self.version_before, str) or not self.version_before: - raise ValueError("version_before must be a non-empty string") - if not isinstance(self.version_after, str) or not self.version_after: - raise ValueError("version_after must be a non-empty string") + _strict_text(self.version_before, "version_before") + _strict_text(self.version_after, "version_after") + if type(self.epsilon) not in {int, float}: + raise TypeError("epsilon must be an int or float, not bool") epsilon = float(self.epsilon) - if not math.isfinite(epsilon) or not 0.0 < epsilon < 1.0: - raise ValueError("epsilon must be finite and strictly between 0 and 1") + if not math.isfinite(epsilon) or not 0.0 <= epsilon <= 1.0: + raise ValueError("epsilon must be in [0, 1]") + if self.measurement_profile not in { + GENERIC_TRAJECTORY_KL_PROFILE, + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + }: + raise ValueError("measurement_profile is not supported") + if ( + self.measurement_profile == FORMAL_POLICY_INFORMATION_GAIN_PROFILE + and epsilon != MAIN_POLICY_KL_EPSILON + ): + raise ValueError("formal policy information gain epsilon must be exactly 0.01") if not isinstance(self.metadata, Mapping): raise TypeError("metadata must be a mapping") object.__setattr__(self, "epsilon", epsilon) - object.__setattr__(self, "metadata", dict(self.metadata)) + object.__setattr__(self, "metadata", _freeze_value(self.metadata)) + + @classmethod + def for_policy_information_gain( + cls, + version_before: str, + version_after: str, + metadata: Mapping[str, Any] | None = None, + ) -> "TrajectoryKLConfig": + """Build the fixed formal profile without an epsilon override.""" + + return cls( + version_before=version_before, + version_after=version_after, + epsilon=MAIN_POLICY_KL_EPSILON, + metadata={} if metadata is None else metadata, + measurement_profile=FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + ) + + @property + def is_formal_policy_information_gain(self) -> bool: + return self.measurement_profile == FORMAL_POLICY_INFORMATION_GAIN_PROFILE @dataclass(frozen=True) @@ -59,6 +157,61 @@ class TrajectoryKLDecisionRecord: local_policy_kl: Optional[float] errors: tuple[str, ...] = () + def __post_init__(self) -> None: + if type(self.decision_step) is not int or self.decision_step <= 0: + raise ValueError("decision_step must be a positive integer") + for label in ( + "context_ref", + "action_schema_version", + "support_id", + "selected_action_id", + ): + _strict_text(getattr(self, label), label) + legal = tuple(self.legal_action_ids) + if ( + not legal + or any(not isinstance(item, str) or not item for item in legal) + or len(set(legal)) != len(legal) + ): + raise ValueError("legal_action_ids must be non-empty and unique") + if self.selected_action_id not in legal: + raise ValueError("selected_action_id must belong to legal_action_ids") + if not isinstance(self.new_distribution, Mapping): + raise TypeError("new_distribution must be a mapping") + if self.old_distribution is not None and not isinstance( + self.old_distribution, Mapping + ): + raise TypeError("old_distribution must be a mapping or None") + for label in ("new_probabilities", "old_probabilities"): + values = getattr(self, label) + if values is not None: + values = tuple( + _strict_nonnegative(item, f"{label}[{index}]") + for index, item in enumerate(values) + ) + object.__setattr__(self, label, values) + if self.local_policy_kl is not None: + object.__setattr__( + self, + "local_policy_kl", + _strict_nonnegative(self.local_policy_kl, "local_policy_kl"), + ) + errors = tuple(self.errors) + if any(not isinstance(item, str) for item in errors): + raise TypeError("decision errors must be strings") + object.__setattr__(self, "legal_action_ids", legal) + object.__setattr__(self, "errors", errors) + object.__setattr__( + self, "new_distribution", _freeze_value(self.new_distribution) + ) + object.__setattr__( + self, + "old_distribution", + None + if self.old_distribution is None + else _freeze_value(self.old_distribution), + ) + def to_dict(self) -> dict[str, Any]: return { "decision_step": self.decision_step, @@ -103,17 +256,146 @@ class TrajectoryKLEpisodeResult: mean_local_policy_kl: Optional[float] errors: tuple[str, ...] metadata: Mapping[str, Any] - direction: str = "new||old" - log_base: str = "e" - rollout_source: str = "new_policy" - estimand: str = ( - "epsilon_regularized_local_kl_sum_under_new_policy_occupancy" - ) + measurement_profile: str = GENERIC_TRAJECTORY_KL_PROFILE + direction: str = FORMAL_POLICY_KL_DIRECTION + log_base: str = FORMAL_POLICY_KL_LOG_BASE + rollout_source: str = FORMAL_POLICY_KL_ROLLOUT_SOURCE + estimand: str = FORMAL_POLICY_KL_SUM_ESTIMAND + information_gain_estimand: str = FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND + aggregation: str = FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION + information_gain_unit: str = FORMAL_POLICY_INFORMATION_GAIN_UNIT + local_policy_kl_sum_unit: str = FORMAL_POLICY_KL_SUM_UNIT + + def __post_init__(self) -> None: + if type(self.episode) is not int or self.episode <= 0: + raise ValueError("episode must be a positive integer") + _strict_text(self.version_before, "version_before") + _strict_text(self.version_after, "version_after") + if type(self.epsilon) not in {int, float}: + raise TypeError("epsilon must be an int or float, not bool") + epsilon = float(self.epsilon) + if not math.isfinite(epsilon) or not 0.0 <= epsilon <= 1.0: + raise ValueError("epsilon must be in [0, 1]") + if self.measurement_profile not in { + GENERIC_TRAJECTORY_KL_PROFILE, + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + }: + raise ValueError("measurement profile is not supported") + if self.measurement_profile == FORMAL_POLICY_INFORMATION_GAIN_PROFILE: + if epsilon != MAIN_POLICY_KL_EPSILON: + raise ValueError("formal policy information gain epsilon must be 0.01") + identity = { + "direction": FORMAL_POLICY_KL_DIRECTION, + "log_base": FORMAL_POLICY_KL_LOG_BASE, + "rollout_source": FORMAL_POLICY_KL_ROLLOUT_SOURCE, + "estimand": FORMAL_POLICY_KL_SUM_ESTIMAND, + "information_gain_estimand": FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND, + "aggregation": FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION, + "information_gain_unit": FORMAL_POLICY_INFORMATION_GAIN_UNIT, + "local_policy_kl_sum_unit": FORMAL_POLICY_KL_SUM_UNIT, + } + for label, expected in identity.items(): + if getattr(self, label) != expected: + raise ValueError(f"formal trajectory KL {label} identity mismatch") + if self.status not in {"complete", "incomplete"}: + raise ValueError("status must be complete or incomplete") + decisions = tuple(self.decisions) + if any(type(item) is not TrajectoryKLDecisionRecord for item in decisions): + raise TypeError("decisions must contain TrajectoryKLDecisionRecord values") + if tuple(item.decision_step for item in decisions) != tuple( + range(1, len(decisions) + 1) + ): + raise ValueError("decision steps must be strict and continuous") + trace = tuple(self.trace) + if len(trace) != len(decisions): + raise ValueError("trace must align one-to-one with decisions") + for index, (value, decision) in enumerate( + zip(trace, decisions, strict=True), start=1 + ): + if value is not None: + value = _strict_nonnegative(value, f"trace[{index}]") + if not _same_number(value, decision.local_policy_kl): + raise ValueError("trace must be derived from decision records") + errors = tuple(self.errors) + if any(not isinstance(item, str) for item in errors): + raise TypeError("episode errors must be strings") + if self.status == "complete": + if not decisions or errors: + raise ValueError("complete trajectory KL requires decisions and no errors") + recomputed: list[float] = [] + for decision in decisions: + if decision.errors or decision.old_distribution is None: + raise ValueError("complete decision cannot contain errors or missing policy") + if ( + decision.new_probabilities is None + or decision.old_probabilities is None + or decision.local_policy_kl is None + ): + raise ValueError("complete decision requires probability evidence") + legal = decision.legal_action_ids + if ( + set(decision.new_distribution) != set(legal) + or set(decision.old_distribution) != set(legal) + ): + raise ValueError("complete decision distribution support mismatch") + new = [decision.new_distribution[action_id] for action_id in legal] + old = [decision.old_distribution[action_id] for action_id in legal] + if any( + not _same_number(left, right) + for left, right in zip( + decision.new_probabilities, new, strict=True + ) + ) or any( + not _same_number(left, right) + for left, right in zip( + decision.old_probabilities, old, strict=True + ) + ): + raise ValueError("probability vectors disagree with distributions") + local = ( + formal_policy_kl(new, old) + if self.measurement_profile + == FORMAL_POLICY_INFORMATION_GAIN_PROFILE + else policy_kl(new, old, epsilon=epsilon) + ) + if not math.isfinite(local) or not _same_number( + local, decision.local_policy_kl + ): + raise ValueError("local policy KL disagrees with distributions") + recomputed.append(local) + total = trajectory_kl_from_trace(recomputed) + mean = episode_information_gain_from_trace(recomputed) + if not _same_number(self.trajectory_kl_episode, total): + raise ValueError("trajectory KL aggregate must match decision records") + if not _same_number(self.mean_local_policy_kl, mean): + raise ValueError("mean local policy KL must match decision records") + else: + if self.trajectory_kl_episode is not None or self.mean_local_policy_kl is not None: + raise ValueError("incomplete trajectory KL cannot expose aggregates") + if not errors and not any(item.errors for item in decisions): + raise ValueError("incomplete trajectory KL requires a structured error") + if not isinstance(self.metadata, Mapping): + raise TypeError("metadata must be a mapping") + object.__setattr__(self, "epsilon", epsilon) + object.__setattr__(self, "decisions", decisions) + object.__setattr__(self, "trace", trace) + object.__setattr__(self, "errors", errors) + object.__setattr__(self, "metadata", _freeze_value(self.metadata)) @property def decision_steps(self) -> int: return len(self.decisions) + @property + def information_gain(self) -> Optional[float]: + if self.measurement_profile != FORMAL_POLICY_INFORMATION_GAIN_PROFILE: + return None + return self.mean_local_policy_kl + + @property + def local_policy_kl_sum(self) -> Optional[float]: + return self.trajectory_kl_episode + def to_dict(self) -> dict[str, Any]: return { "episode": self.episode, @@ -122,20 +404,138 @@ def to_dict(self) -> dict[str, Any]: "epsilon": self.epsilon, "status": self.status, "measurement_status": self.status, + "measurement_profile": self.measurement_profile, "direction": self.direction, "log_base": self.log_base, "rollout_source": self.rollout_source, "estimand": self.estimand, + "information_gain_estimand": self.information_gain_estimand, + "aggregation": self.aggregation, "decision_steps": self.decision_steps, "trace": list(self.trace), "trajectory_kl_episode": self.trajectory_kl_episode, "mean_local_policy_kl": self.mean_local_policy_kl, + "information_gain": self.information_gain, + "local_policy_kl_sum": self.local_policy_kl_sum, + "information_gain_unit": self.information_gain_unit, + "local_policy_kl_sum_unit": self.local_policy_kl_sum_unit, "errors": list(self.errors), - "metadata": dict(self.metadata), + "metadata": _thaw_value(self.metadata), "decisions": [decision.to_dict() for decision in self.decisions], } +def trajectory_kl_result_from_payload( + payload: Mapping[str, Any], + *, + require_formal: bool = False, +) -> TrajectoryKLEpisodeResult: + """Rebuild and verify a rich result instead of trusting its summary.""" + + if not isinstance(payload, Mapping): + raise TypeError("trajectory KL result must be a mapping") + required = { + "episode", + "version_before", + "version_after", + "epsilon", + "status", + "measurement_status", + "measurement_profile", + "direction", + "log_base", + "rollout_source", + "estimand", + "information_gain_estimand", + "aggregation", + "information_gain_unit", + "local_policy_kl_sum_unit", + "decision_steps", + "trace", + "trajectory_kl_episode", + "mean_local_policy_kl", + "information_gain", + "local_policy_kl_sum", + "errors", + "metadata", + "decisions", + } + missing = sorted(required - set(payload)) + if missing: + if "measurement_profile" in missing: + raise ValueError("trajectory KL measurement profile is required") + raise ValueError(f"trajectory KL result is missing fields: {missing}") + if payload["status"] != payload["measurement_status"]: + raise ValueError("trajectory KL status identity mismatch") + if require_formal and ( + payload["measurement_profile"] != FORMAL_POLICY_INFORMATION_GAIN_PROFILE + ): + raise ValueError("formal trajectory KL measurement profile is required") + raw_decisions = payload["decisions"] + if not isinstance(raw_decisions, list): + raise TypeError("trajectory KL decisions must be a list") + decisions = [] + for raw in raw_decisions: + if not isinstance(raw, Mapping): + raise TypeError("trajectory KL decisions must be objects") + decisions.append( + TrajectoryKLDecisionRecord( + decision_step=raw.get("decision_step"), + context_ref=raw.get("context_ref"), + action_schema_version=raw.get("action_schema_version"), + support_id=raw.get("support_id"), + legal_action_ids=tuple(raw.get("legal_action_ids", ())), + selected_action_id=raw.get("selected_action_id"), + new_distribution=raw.get("new_distribution"), + old_distribution=raw.get("old_distribution"), + new_probabilities=( + None + if raw.get("new_probabilities") is None + else tuple(raw.get("new_probabilities")) + ), + old_probabilities=( + None + if raw.get("old_probabilities") is None + else tuple(raw.get("old_probabilities")) + ), + local_policy_kl=raw.get("local_policy_kl"), + errors=tuple(raw.get("errors", ())), + ) + ) + result = TrajectoryKLEpisodeResult( + episode=payload["episode"], + version_before=payload["version_before"], + version_after=payload["version_after"], + epsilon=payload["epsilon"], + status=payload["status"], + decisions=tuple(decisions), + trace=tuple(payload["trace"]), + trajectory_kl_episode=payload["trajectory_kl_episode"], + mean_local_policy_kl=payload["mean_local_policy_kl"], + errors=tuple(payload["errors"]), + metadata=payload["metadata"], + measurement_profile=payload["measurement_profile"], + direction=payload["direction"], + log_base=payload["log_base"], + rollout_source=payload["rollout_source"], + estimand=payload["estimand"], + information_gain_estimand=payload["information_gain_estimand"], + aggregation=payload["aggregation"], + information_gain_unit=payload["information_gain_unit"], + local_policy_kl_sum_unit=payload["local_policy_kl_sum_unit"], + ) + if ( + type(payload["decision_steps"]) is not int + or payload["decision_steps"] != result.decision_steps + ): + raise ValueError("trajectory KL decision count mismatch") + if not _same_number(payload["information_gain"], result.information_gain): + raise ValueError("information gain summary must match decision records") + if not _same_number(payload["local_policy_kl_sum"], result.local_policy_kl_sum): + raise ValueError("local policy KL sum must match decision records") + return result + + class TrajectoryKLAgent: """Agent-compatible online wrapper for strict trajectory-KL measurement. @@ -237,10 +637,14 @@ def act(self, observation: Any) -> Any: local_kl = None if new_probabilities is not None and old_probabilities is not None: - local_kl = policy_kl( - new_probabilities, - old_probabilities, - epsilon=self.config.epsilon, + local_kl = ( + formal_policy_kl(new_probabilities, old_probabilities) + if self.config.is_formal_policy_information_gain + else policy_kl( + new_probabilities, + old_probabilities, + epsilon=self.config.epsilon, + ) ) if not math.isfinite(local_kl) or local_kl < 0.0: decision_errors.append( @@ -316,17 +720,21 @@ def _finish_episode(self) -> TrajectoryKLEpisodeResult: and not self._errors and all(value is not None for value in trace) ) - trajectory_kl_episode = ( - sum(value for value in trace if value is not None) - if complete - else None - ) - mean_local_policy_kl = ( - trajectory_kl_episode / len(self._decisions) - if trajectory_kl_episode is not None - else None - ) errors = list(self._errors) + trajectory_kl_episode = None + mean_local_policy_kl = None + if complete: + try: + complete_trace = tuple( + value for value in trace if value is not None + ) + trajectory_kl_episode = trajectory_kl_from_trace(complete_trace) + mean_local_policy_kl = episode_information_gain_from_trace( + complete_trace + ) + except (OverflowError, TypeError, ValueError) as exc: + complete = False + errors.append(f"episode KL aggregate is invalid: {exc}") if not self._decisions: errors.append("episode contains no target-agent decisions") result = TrajectoryKLEpisodeResult( @@ -341,6 +749,7 @@ def _finish_episode(self) -> TrajectoryKLEpisodeResult: mean_local_policy_kl=mean_local_policy_kl, errors=tuple(errors), metadata=dict(self._episode_metadata), + measurement_profile=self.config.measurement_profile, ) self._latest_result = result if self.on_episode_complete is not None: diff --git a/src/agentbench_frame/games/miracle/__init__.py b/src/agentbench_frame/games/miracle/__init__.py index ebc5849..76ad79d 100644 --- a/src/agentbench_frame/games/miracle/__init__.py +++ b/src/agentbench_frame/games/miracle/__init__.py @@ -131,6 +131,18 @@ preflight_replay_reading, render_replay_timeline, ) +from agentbench_frame.games.miracle.ifelse_policy_state_v1 import ( + ExplicitIfElseStateMachineV1, + IncompletePolicyEvidenceError, + LegacyPolicySourceV1, + PolicyComparisonIdentityV1, + PolicyConfigV1, + PolicyMemoryV1, + PolicyPairDecisionV1, + ProviderIdentityV1, + SequentialReplayAuditV1, + audit_sequential_trace, +) __all__ = [ "GameOutcome", "WIN", "LOSS", "DRAW", "ERROR", "VALID_RESULTS", @@ -177,4 +189,8 @@ "validate_distribution", "DecisionFrame", "ReplayPacket", "ReplayReadingContext", "open_replay_reading", "preflight_replay_reading", "render_replay_timeline", + "ExplicitIfElseStateMachineV1", "IncompletePolicyEvidenceError", + "LegacyPolicySourceV1", "PolicyComparisonIdentityV1", "PolicyConfigV1", + "PolicyMemoryV1", "PolicyPairDecisionV1", "ProviderIdentityV1", + "SequentialReplayAuditV1", "audit_sequential_trace", ] diff --git a/src/agentbench_frame/games/miracle/decision_kl_v1.py b/src/agentbench_frame/games/miracle/decision_kl_v1.py index 638c0b3..d5a23cd 100644 --- a/src/agentbench_frame/games/miracle/decision_kl_v1.py +++ b/src/agentbench_frame/games/miracle/decision_kl_v1.py @@ -14,6 +14,15 @@ from types import MappingProxyType from typing import Any +from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_UNIT, + FORMAL_POLICY_KL_DIRECTION, + FORMAL_POLICY_KL_ROLLOUT_SOURCE, + FORMAL_POLICY_KL_SMOOTHING, + FORMAL_POLICY_KL_SUM_UNIT, + MAIN_POLICY_KL_EPSILON, + formal_policy_kl, +) from agentbench_frame.eval.measurement import ActionSupport from agentbench_frame.games.miracle.research_protocol import ( IncompleteActionSupportError, @@ -22,11 +31,13 @@ ) -ACCEPTANCE_THRESHOLD = 0.01 -DIRECTION = "old||new" -ROLLOUT_SOURCE = "new_policy" -SMOOTHING = "none" -UNIT = "nats / decision" +ACCEPTANCE_THRESHOLD = None +DIRECTION = FORMAL_POLICY_KL_DIRECTION +ROLLOUT_SOURCE = FORMAL_POLICY_KL_ROLLOUT_SOURCE +EPSILON = MAIN_POLICY_KL_EPSILON +SMOOTHING = FORMAL_POLICY_KL_SMOOTHING +UNIT = FORMAL_POLICY_INFORMATION_GAIN_UNIT +SUM_UNIT = FORMAL_POLICY_KL_SUM_UNIT _IDENTITY_KEYS = {"schema_version", "support_id", "action_ids"} _DISTRIBUTION_SUM_TOLERANCE = 1e-9 _STRICT_MASS_ROUNDOFF = 8 * math.ulp(1.0) @@ -54,10 +65,6 @@ def __init__(self, code: str, message: str) -> None: self.code = code -def _passes_acceptance_threshold(value: float) -> bool: - return value <= ACCEPTANCE_THRESHOLD - - def _strict_step(value: Any) -> int: if type(value) is not int or value <= 0: raise ValueError("decision_step must be a positive strict integer") @@ -204,6 +211,7 @@ class DecisionKLRecord: local_kl: float | None reason: str | None = None direction: str = DIRECTION + epsilon: float = EPSILON smoothing: str = SMOOTHING log_base: str = "e" @@ -226,7 +234,7 @@ def __post_init__(self) -> None: object.__setattr__(self, "action_ids", action_ids) if ( type(self.status) is not str - or self.status not in {"complete", "incomplete", "threshold_failed"} + or self.status not in {"complete", "incomplete"} ): raise ValueError("local KL status is invalid") if self.status == "complete": @@ -238,20 +246,17 @@ def __post_init__(self) -> None: if self.reason is not None: raise ValueError("complete local KL cannot have a failure reason") elif self.local_kl is not None: - raise ValueError("failed or incomplete local KL must not expose a scalar") + raise ValueError("incomplete local KL must not expose a scalar") if self.reason is not None and type(self.reason) is not str: raise ValueError("local KL reason must be an exact string or null") if self.status == "incomplete" and self.reason not in _INCOMPLETE_REASONS: raise ValueError("incomplete local KL reason is invalid") - if ( - self.status == "threshold_failed" - and self.reason != "old_positive_new_zero" - ): - raise ValueError("threshold-failed local KL reason is invalid") if ( type(self.direction) is not str or type(self.smoothing) is not str or self.direction != DIRECTION + or type(self.epsilon) is not float + or self.epsilon != EPSILON or self.smoothing != SMOOTHING ): raise ValueError("local KL contract identity is invalid") @@ -270,6 +275,7 @@ def to_dict(self) -> dict[str, Any]: "local_kl": self.local_kl, "reason": self.reason, "direction": self.direction, + "epsilon": self.epsilon, "smoothing": self.smoothing, "log_base": self.log_base, } @@ -285,6 +291,7 @@ class _IncompleteDecisionRecord: support_id: None = None action_ids: tuple[()] = () direction: str = DIRECTION + epsilon: float = EPSILON smoothing: str = SMOOTHING log_base: str = "e" @@ -306,9 +313,11 @@ def __post_init__(self) -> None: if ( type(self.status) is not str or type(self.direction) is not str + or type(self.epsilon) is not float or type(self.smoothing) is not str or type(self.log_base) is not str or self.direction != DIRECTION + or self.epsilon != EPSILON or self.smoothing != SMOOTHING or self.log_base != "e" ): @@ -326,6 +335,7 @@ def to_dict(self) -> dict[str, Any]: "local_kl": self.local_kl, "reason": self.reason, "direction": self.direction, + "epsilon": self.epsilon, "smoothing": self.smoothing, "log_base": self.log_base, } @@ -344,6 +354,7 @@ def _decision_record_snapshot( record.local_kl, record.reason, record.direction, + record.epsilon, record.smoothing, record.log_base, ) @@ -371,6 +382,7 @@ def issue_supported( "local_kl": local_kl, "reason": reason, "direction": DIRECTION, + "epsilon": EPSILON, "smoothing": SMOOTHING, "log_base": "e", } @@ -391,6 +403,7 @@ def issue_without_support(step: int, reason: str) -> _IncompleteDecisionRecord: "support_id": None, "action_ids": (), "direction": DIRECTION, + "epsilon": EPSILON, "smoothing": SMOOTHING, "log_base": "e", } @@ -458,18 +471,6 @@ def _derived_trajectory_fields( "reason": "empty_trajectory", **missing, } - failed = next( - (record for record in records if record.status == "threshold_failed"), - None, - ) - if failed is not None: - return { - "status": "threshold_failed", - "trace": trace, - "threshold_passed": False, - "reason": failed.reason or "threshold_failed", - **missing, - } incomplete = next( (record for record in records if record.status == "incomplete"), None, @@ -493,13 +494,12 @@ def _derived_trajectory_fields( mean = total / len(values) if not math.isfinite(total) or not math.isfinite(mean): raise ValueError("trajectory aggregates must remain finite") - passed = _passes_acceptance_threshold(mean) return { - "status": "complete" if passed else "threshold_failed", + "status": "complete", "trajectory_kl": mean, "trace": values, - "threshold_passed": passed, - "reason": None if passed else "trajectory_kl_above_threshold", + "threshold_passed": None, + "reason": None, "sum_local_kl": total, "max_local_kl": max(values), "p50_local_kl": _percentile(values, 0.50), @@ -521,8 +521,9 @@ class TrajectoryKLSummary: max_local_kl: float | None p50_local_kl: float | None p95_local_kl: float | None - acceptance_threshold: float = ACCEPTANCE_THRESHOLD + acceptance_threshold: None = ACCEPTANCE_THRESHOLD direction: str = DIRECTION + epsilon: float = EPSILON smoothing: str = SMOOTHING log_base: str = "e" unit: str = UNIT @@ -532,6 +533,8 @@ class TrajectoryKLSummary: verified_rollout_source: None = None policy_binding_verified: bool = False aggregation: str = "arithmetic_mean" + information_gain_unit: str = UNIT + sum_local_kl_unit: str = SUM_UNIT def __new__(cls, *_args: Any, **_kwargs: Any): raise TypeError("TrajectoryKLSummary can only be issued by the KL calculator") @@ -550,6 +553,7 @@ def __post_init__(self) -> None: contract = { "acceptance_threshold": ACCEPTANCE_THRESHOLD, "direction": DIRECTION, + "epsilon": EPSILON, "smoothing": SMOOTHING, "log_base": "e", "unit": UNIT, @@ -559,6 +563,8 @@ def __post_init__(self) -> None: "verified_rollout_source": None, "policy_binding_verified": False, "aggregation": "arithmetic_mean", + "information_gain_unit": UNIT, + "sum_local_kl_unit": SUM_UNIT, } for field_name, expected in contract.items(): if not _exact_scientific_value(getattr(self, field_name), expected): @@ -583,6 +589,7 @@ def to_dict(self) -> dict[str, Any]: return { "status": self.status, "trajectory_kl": self.trajectory_kl, + "information_gain": self.information_gain, "trace": list(self.trace), "decision_records": [record.to_dict() for record in self.decision_records], "threshold_passed": self.threshold_passed, @@ -593,6 +600,7 @@ def to_dict(self) -> dict[str, Any]: "p95_local_kl": self.p95_local_kl, "acceptance_threshold": self.acceptance_threshold, "direction": self.direction, + "epsilon": self.epsilon, "smoothing": self.smoothing, "log_base": self.log_base, "unit": self.unit, @@ -602,8 +610,14 @@ def to_dict(self) -> dict[str, Any]: "verified_rollout_source": self.verified_rollout_source, "policy_binding_verified": self.policy_binding_verified, "aggregation": self.aggregation, + "information_gain_unit": self.information_gain_unit, + "sum_local_kl_unit": self.sum_local_kl_unit, } + @property + def information_gain(self) -> float | None: + return self.trajectory_kl + def _trajectory_summary_snapshot(summary: TrajectoryKLSummary) -> tuple[Any, ...]: summary.__post_init__() @@ -620,6 +634,7 @@ def _trajectory_summary_snapshot(summary: TrajectoryKLSummary) -> tuple[Any, ... summary.p95_local_kl, summary.acceptance_threshold, summary.direction, + summary.epsilon, summary.smoothing, summary.log_base, summary.unit, @@ -629,6 +644,8 @@ def _trajectory_summary_snapshot(summary: TrajectoryKLSummary) -> tuple[Any, ... summary.verified_rollout_source, summary.policy_binding_verified, summary.aggregation, + summary.information_gain_unit, + summary.sum_local_kl_unit, ) @@ -665,6 +682,7 @@ def issue( "p95_local_kl": p95_local_kl, "acceptance_threshold": ACCEPTANCE_THRESHOLD, "direction": DIRECTION, + "epsilon": EPSILON, "smoothing": SMOOTHING, "log_base": "e", "unit": UNIT, @@ -674,6 +692,8 @@ def issue( "verified_rollout_source": None, "policy_binding_verified": False, "aggregation": "arithmetic_mean", + "information_gain_unit": UNIT, + "sum_local_kl_unit": SUM_UNIT, } for field_name, value in values.items(): object.__setattr__(summary, field_name, value) @@ -828,7 +848,7 @@ def _compute_local_kl( *, decision_step: int, ) -> DecisionKLRecord: - """Compute strict unsmoothed local ``D_KL(old || new)``.""" + """Compute epsilon-regularized local ``D_KL(new || old)``.""" step = _strict_step(decision_step) _validate_support_identity(support_identity, support) @@ -854,25 +874,11 @@ def _compute_local_kl( return _incomplete_local_record( step, support, "new_distribution_mass_not_strict" ) - for action_id in support.action_ids: - old_probability = old[action_id] - new_probability = new[action_id] - if old_probability > 0.0 and new_probability == 0.0: - return _issue_supported_decision_record( - step, - support, - "threshold_failed", - None, - "old_positive_new_zero", - ) - terms = [ - old_probability - * (math.log(old_probability) - math.log(new[action_id])) - for action_id in support.action_ids - if (old_probability := old[action_id]) > 0.0 - ] - value = math.fsum(terms) - negative_roundoff = 8 * math.ulp(1.0) * max(1, len(terms)) + value = formal_policy_kl( + [new[action_id] for action_id in support.action_ids], + [old[action_id] for action_id in support.action_ids], + ) + negative_roundoff = 8 * math.ulp(1.0) * max(1, len(support.action_ids)) if value < 0.0: if abs(value) <= negative_roundoff: value = 0.0 @@ -948,17 +954,6 @@ def compute_trajectory_kl( ) trace = tuple(record.local_kl for record in records) frozen_records = tuple(records) - failed = next( - (record for record in records if record.status == "threshold_failed"), None - ) - if failed is not None: - return _missing_summary( - "threshold_failed", - trace, - frozen_records, - failed.reason or "threshold_failed", - False, - ) incomplete = next( (record for record in records if record.status == "incomplete"), None ) @@ -976,14 +971,13 @@ def compute_trajectory_kl( finite = [float(value) for value in values if value is not None] total = math.fsum(finite) mean = total / len(finite) - passed = _passes_acceptance_threshold(mean) return _issue_trajectory_summary( - "complete" if passed else "threshold_failed", + "complete", mean, tuple(finite), frozen_records, - passed, - None if passed else "trajectory_kl_above_threshold", + None, + None, total, max(finite), _percentile(finite, 0.50), @@ -994,9 +988,11 @@ def compute_trajectory_kl( __all__ = [ "ACCEPTANCE_THRESHOLD", "DIRECTION", + "EPSILON", "ROLLOUT_SOURCE", "SMOOTHING", "UNIT", + "SUM_UNIT", "DecisionKLEvidence", "DecisionKLRecord", "TrajectoryKLSummary", diff --git a/src/agentbench_frame/games/miracle/ifelse_policy_state_v1.py b/src/agentbench_frame/games/miracle/ifelse_policy_state_v1.py new file mode 100644 index 0000000..d776d67 --- /dev/null +++ b/src/agentbench_frame/games/miracle/ifelse_policy_state_v1.py @@ -0,0 +1,1738 @@ +"""Explicit state and deterministic distribution providers for Miracle HL. + +The legacy ``miracle_ifelse`` process emits several atomic Judge operations +from one ``play()`` call. Every operation is followed by a fresh Judge +observation while Python continues inside the active phase. This module makes +that continuation explicit and serializable so formal policy KL can compare +old and new policies on the same ``(observation, memory)`` context. + +No epsilon smoothing is performed here. Providers return strict one-hot base +distributions; the formal information-gain layer owns the shared measurement +channel. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import math +import os +import stat +import sys +import weakref +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from agentbench_frame.eval.measurement import ( + ActionSupport, + canonical_state_id, +) +from agentbench_frame.games.miracle.decision_kl_v1 import ( + build_trusted_action_support, +) +from agentbench_frame.games.miracle.research_protocol import ( + canonical_command, + command_action_id, + one_hot, +) + + +POLICY_CONFIG_SCHEMA_VERSION = "24-miracle-ifelse-policy-config-v1" +POLICY_MEMORY_SCHEMA_VERSION = "24-miracle-ifelse-policy-memory-v1" +STATE_MACHINE_VERSION = "24-miracle-ifelse-explicit-state-machine-v1" +PROVIDER_SCHEMA_VERSION = "24-miracle-ifelse-distribution-provider-v1" +POLICY_COMPARISON_SCHEMA_VERSION = "24-miracle-ifelse-policy-comparison-v1" +POLICY_PAIR_DECISION_SCHEMA_VERSION = "24-miracle-ifelse-policy-pair-decision-v1" +UNKNOWN_CONFIG_VALUE = "unknown" +LEGACY_SOURCE_FILES = ( + "Data.json", + "ai_client.py", + "calculator.py", + "card.py", + "gameunit.py", + "main.py", +) +MAX_POLICY_SOURCE_FILE_BYTES = 2_000_000 +MAX_REPLAY_TRACE_BYTES = 64_000_000 +VALID_ARTIFACTS = frozenset( + {"HolyLight", "SalamanderShield", "InfernoFlame", "WindBlessing"} +) +VALID_CREATURES = frozenset( + { + "Archer", + "Swordsman", + "BlackBat", + "Priest", + "VolcanoDragon", + "Inferno", + "FrostDragon", + } +) + + +class IncompletePolicyEvidenceError(ValueError): + """A formal policy identity or decision context cannot be proved.""" + + +class PolicySourceError(ValueError): + """A legacy policy source tree is missing, replaced, or unapproved.""" + + +def _canonical_bytes(value: Any) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _strict_text(value: Any, label: str) -> str: + if type(value) is not str or not value: + raise TypeError(f"{label} must be a non-empty string") + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise ValueError(f"{label} contains invalid Unicode") from exc + return value + + +def _strict_sha256(value: Any, label: str) -> str: + value = _strict_text(value, label) + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{label} must be a lowercase SHA-256") + return value + + +def _freeze_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: + frozen: dict[str, Any] = {} + for key, item in value.items(): + if type(key) is not str: + raise TypeError("mapping keys must be strings") + if isinstance(item, Mapping): + frozen[key] = _freeze_mapping(item) + elif type(item) in {list, tuple}: + frozen[key] = tuple(item) + else: + frozen[key] = item + return MappingProxyType(frozen) + + +def _freeze_sequence(value: Any) -> Any: + if type(value) in {list, tuple}: + return tuple(_freeze_sequence(item) for item in value) + return value + + +@dataclass(frozen=True, slots=True) +class MiracleConfigSpec: + name: str + attribute: str | None + kind: str + default_v0: Any + default_v1: Any + + +_BOOLEAN_CONFIG = ( + ("MIRACLE_GATE_DEFENSE", "gate_defense_enabled"), + ("MIRACLE_ANTIDECOY", "antidecoy_enabled"), + ("MIRACLE_RACE_LANE", "race_lane_enabled"), + ("MIRACLE_FRONT_SWORD3", "front_sword3_enabled"), + ("MIRACLE_FRONT_ARCHER3", "front_archer3_enabled"), + ("MIRACLE_PAIRED_CLOCK", "paired_clock_enabled"), + ("MIRACLE_FRONT_ARCHER_LANE", "front_archer_lane_enabled"), + ("MIRACLE_INFERNO_CLOCK", "inferno_clock_enabled"), + ("MIRACLE_LANE_MELEE_GUARD", "lane_melee_guard_enabled"), + ("MIRACLE_LANE_RANGED_GUARD", "lane_ranged_guard_enabled"), + ("MIRACLE_CLOCK_CONDITIONAL_GUARD", "clock_conditional_guard_enabled"), + ("MIRACLE_CLOCK_STATE_TABLE", "clock_state_table_enabled"), + ("MIRACLE_SECOND_ARCHER_LANE", "second_archer_lane_enabled"), + ("MIRACLE_CONDITIONAL_SECOND_LANE", "conditional_second_lane_enabled"), + ("MIRACLE_CONDITIONAL_SECOND_LANE_STRICT", "conditional_second_lane_strict"), + ("MIRACLE_SWORD_CLOCK_SUPPORT", "sword_clock_support_enabled"), + ("MIRACLE_PREGATE_SWORD_INTERCEPT", "pregate_sword_intercept_enabled"), + ("MIRACLE_EARLY_SWORD_INTERCEPT", "early_sword_intercept_enabled"), + ("MIRACLE_MELEE_ARCHER_DISRUPT", "melee_archer_disrupt_enabled"), + ("MIRACLE_LANE_BODYGUARD", "lane_bodyguard_enabled"), + ("MIRACLE_LANE_KILL_GUARD", "lane_kill_guard_enabled"), + ("MIRACLE_LANE_ARTIFACT_GUARD", "lane_artifact_guard_enabled"), + ("MIRACLE_LATE_SWORD_KILL_GUARD", "late_sword_kill_guard_enabled"), + ("MIRACLE_LATE_SWORD_ARTIFACT_GUARD", "late_sword_artifact_guard_enabled"), + ("MIRACLE_LATE_PREGATE_SWORD_FOCUS", "late_pregate_sword_focus_enabled"), + ("MIRACLE_LATE_INFERNO_GUARD", "late_inferno_guard_enabled"), + ("MIRACLE_LATE_INFERNO_ARTIFACT_GUARD", "late_inferno_artifact_guard_enabled"), + ("MIRACLE_LATE_CLOCK_ARTIFACT_TABLE", "late_clock_artifact_table_enabled"), + ("MIRACLE_EARLY_MIXED_GATE_GUARD", "early_mixed_gate_guard_enabled"), + ("MIRACLE_EARLY_MIXED_ARTIFACT_GUARD", "early_mixed_artifact_guard_enabled"), + ("MIRACLE_EARLY_GATE_ANCHOR", "early_gate_anchor_enabled"), + ("MIRACLE_EARLY_NONLANE_GATE_BLOCKER", "early_nonlane_gate_blocker_enabled"), + ("MIRACLE_EARLY_GATE_KILL_ONLY", "early_gate_kill_only_enabled"), + ("MIRACLE_PREGATE_ARCHER_SUPPORT", "pregate_archer_support_enabled"), + ("MIRACLE_ARCHER_BURST_GUARD", "archer_burst_guard_enabled"), + ("MIRACLE_PREARCHER_REPLACEMENT_GUARD", "prearcher_replacement_guard_enabled"), + ("MIRACLE_PREARCHER_ID_GUARD", "prearcher_id_guard_enabled"), + ("MIRACLE_DIRECT_ARCHER_LOCK", "direct_archer_lock_enabled"), + ("MIRACLE_OFFENSIVE_LANE_LOCK", "offensive_lane_lock_enabled"), + ("MIRACLE_SAFE_ARCHER_ROUTE", "safe_archer_route_enabled"), + ("MIRACLE_RELAY_ARCHER_ROUTE", "relay_archer_route_enabled"), + ("MIRACLE_SECOND_WAVE_ARCHER_ROUTE", "second_wave_archer_route_enabled"), + ("MIRACLE_FRONT_SUMMON_VACANCY", "front_summon_vacancy_enabled"), + ("MIRACLE_ANTI_BLACKBAT", "anti_blackbat_enabled"), + ("MIRACLE_AIR_OPENING_ARCHER_RUSH", "air_opening_archer_rush_enabled"), + ("MIRACLE_AIR_POSTRUSH_BLACKBAT_GUARD", "air_postrush_blackbat_guard_enabled"), + ("MIRACLE_LATE_AIR_RACE_RELEASE", "late_air_race_release_enabled"), + ("MIRACLE_LATE_FROST_ARTIFACT", "late_frost_artifact_enabled"), + ("MIRACLE_SAVE_ARTIFACT_FOR_FROST", "save_artifact_for_frost_enabled"), + ("MIRACLE_LATE_SHELL_DRAGON_PRESSURE", "late_shell_dragon_pressure_enabled"), + ("MIRACLE_LATE_SHELL_DRAGON_PRESSURE_SHORT", "late_shell_dragon_pressure_short"), + ("MIRACLE_LATE_BLACKBAT_GATE_GUARD", "late_blackbat_gate_guard_enabled"), + ("MIRACLE_SWORD_OFFSET_DECOY", "sword_offset_decoy_enabled"), + ("MIRACLE_SECOND_OFFSET_DECOY", "second_offset_decoy_enabled"), + ("MIRACLE_PRIEST_BAT_DECOY", "priest_bat_decoy_enabled"), + ("MIRACLE_SOUTH_BAT_SCREEN", "south_bat_screen_enabled"), + ("MIRACLE_MID_ARCHER_LATTICE_BLOCK", "mid_archer_lattice_block_enabled"), + ("MIRACLE_RANK13_ARCHER90_FOCUS", "rank13_archer90_focus_enabled"), + ("MIRACLE_RANK13_SWORD103_PRIEST_AURA", "rank13_sword103_priest_aura_enabled"), +) + +MIRACLE_CONFIG_SPECS: tuple[MiracleConfigSpec, ...] = tuple( + MiracleConfigSpec(name, attribute, "bool", False, False) + for name, attribute in _BOOLEAN_CONFIG +) + ( + MiracleConfigSpec("MIRACLE_CAMP1_OPENING", "camp1_opening", "opening", "FF", "SF"), + MiracleConfigSpec("MIRACLE_ARTIFACT", None, "artifact", "InfernoFlame", "InfernoFlame"), + MiracleConfigSpec( + "MIRACLE_DECK", + None, + "deck", + ("Priest", "Archer", "Swordsman"), + ("Priest", "Archer", "Swordsman"), + ), +) +_CONFIG_BY_NAME = {spec.name: spec for spec in MIRACLE_CONFIG_SPECS} +if len(MIRACLE_CONFIG_SPECS) != 62 or len(_CONFIG_BY_NAME) != 62: + raise RuntimeError("Miracle policy configuration inventory must contain 62 unique inputs") + + +def _validate_config_value(spec: MiracleConfigSpec, value: Any) -> Any: + if value == UNKNOWN_CONFIG_VALUE: + return value + if spec.kind == "bool": + if type(value) is not bool: + raise TypeError(f"{spec.name} must be a strict boolean") + return value + if spec.kind == "opening": + if type(value) is not str: + raise TypeError(f"{spec.name} must be a string") + if value not in {"FF", "SF", "IF"}: + raise ValueError(f"{spec.name} must be FF, SF, or IF") + return value + if spec.kind == "artifact": + normalized = _strict_text(value, spec.name) + if normalized not in VALID_ARTIFACTS: + raise ValueError(f"{spec.name} is not a known Judge artifact") + return normalized + if spec.kind == "deck": + if type(value) not in {list, tuple} or len(value) != 3: + raise TypeError(f"{spec.name} must contain exactly three entries") + normalized = tuple(_strict_text(item, spec.name) for item in value) + if len(set(normalized)) != 3: + raise ValueError(f"{spec.name} must contain three distinct creatures") + if any(item not in VALID_CREATURES for item in normalized): + raise ValueError(f"{spec.name} contains an unknown Judge creature") + return normalized + raise RuntimeError(f"unknown config kind: {spec.kind}") + + +@dataclass(frozen=True, slots=True) +class PolicyConfigV1: + values: Mapping[str, Any] + evidence: str + + def __post_init__(self) -> None: + if set(self.values) != set(_CONFIG_BY_NAME): + raise ValueError("policy config must contain exactly all 62 inputs") + normalized = { + name: _validate_config_value(_CONFIG_BY_NAME[name], self.values[name]) + for name in sorted(self.values) + } + object.__setattr__(self, "values", _freeze_mapping(normalized)) + _strict_text(self.evidence, "config evidence") + + @classmethod + def from_explicit(cls, values: Mapping[str, Any]) -> "PolicyConfigV1": + if not isinstance(values, Mapping): + raise TypeError("explicit policy config must be a mapping") + return cls(dict(values), "explicit_complete") + + @classmethod + def historical_unknown( + cls, *, observed: Mapping[str, Any] | None = None + ) -> "PolicyConfigV1": + observed = {} if observed is None else dict(observed) + extra = set(observed) - set(_CONFIG_BY_NAME) + if extra: + raise ValueError(f"unknown historical config inputs: {sorted(extra)}") + values = { + name: observed.get(name, UNKNOWN_CONFIG_VALUE) + for name in _CONFIG_BY_NAME + } + return cls(values, "historical_partial_unknown") + + @property + def complete(self) -> bool: + return all(value != UNKNOWN_CONFIG_VALUE for value in self.values.values()) + + def require_complete(self) -> None: + if not self.complete: + missing = sorted( + name + for name, value in self.values.items() + if value == UNKNOWN_CONFIG_VALUE + ) + raise IncompletePolicyEvidenceError( + f"policy config contains unknown values: {missing}" + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": POLICY_CONFIG_SCHEMA_VERSION, + "evidence": self.evidence, + "complete": self.complete, + "values": { + name: list(value) if type(value) is tuple else value + for name, value in self.values.items() + }, + } + + @property + def canonical_bytes(self) -> bytes: + return _canonical_bytes(self.to_dict()) + + @property + def sha256(self) -> str: + return hashlib.sha256(self.canonical_bytes).hexdigest() + + +@dataclass(frozen=True, slots=True) +class ProviderIdentityV1: + version: str + source_identity: str + config_identity: str + state_machine_identity: str = STATE_MACHINE_VERSION + schema_version: str = PROVIDER_SCHEMA_VERSION + + def __post_init__(self) -> None: + _strict_text(self.version, "policy version") + _strict_sha256(self.source_identity, "policy source identity") + _strict_sha256(self.config_identity, "policy config identity") + if self.state_machine_identity != STATE_MACHINE_VERSION: + raise ValueError("policy state-machine identity mismatch") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "version": self.version, + "source_identity": self.source_identity, + "config_identity": self.config_identity, + "state_machine_identity": self.state_machine_identity, + } + + @property + def sha256(self) -> str: + return _digest(self.to_dict()) + + +@dataclass(frozen=True, slots=True) +class LegacyPolicySourceV1: + """Verified six-file source identity for the legacy if-else bot.""" + + root: Path + version: str + main_sha256: str + canonical_tree_sha256: str + legacy_tree_sha256: str + files: tuple[tuple[str, int, str], ...] + + @classmethod + def open( + cls, + root: str | os.PathLike[str], + *, + version: str, + expected_main_sha256: str | None = None, + expected_canonical_tree_sha256: str | None = None, + expected_legacy_tree_sha256: str | None = None, + ) -> "LegacyPolicySourceV1": + path = Path(root).resolve(strict=True) + if not path.is_dir() or path.is_symlink(): + raise PolicySourceError("policy source root must be a real directory") + _strict_text(version, "policy version") + entries: list[tuple[str, int, str]] = [] + payloads: dict[str, bytes] = {} + for name in LEGACY_SOURCE_FILES: + candidate = path / name + if ( + not candidate.is_file() + or candidate.is_symlink() + or candidate.parent.resolve(strict=True) != path + ): + raise PolicySourceError(f"policy source file is unavailable: {name}") + payload = candidate.read_bytes() + if not payload or len(payload) > MAX_POLICY_SOURCE_FILE_BYTES: + raise PolicySourceError(f"policy source file size is invalid: {name}") + if payload.startswith(b"\xef\xbb\xbf"): + raise PolicySourceError(f"policy source has a BOM: {name}") + try: + payload.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise PolicySourceError( + f"policy source is not strict UTF-8: {name}" + ) from exc + payloads[name] = payload + entries.append((name, len(payload), hashlib.sha256(payload).hexdigest())) + # The captured H08 and historical v0/v1 manifests were produced on + # Windows with case-insensitive path ordering. Bind that ordering + # explicitly so the identity is portable to POSIX. + entries.sort(key=lambda item: (item[0].casefold(), item[0])) + main_sha = next(item[2] for item in entries if item[0] == "main.py") + canonical_payload = "".join( + f"{name}\t{digest}\n" for name, _size, digest in entries + ).encode("utf-8") + canonical_tree = hashlib.sha256(canonical_payload).hexdigest() + legacy_hasher = hashlib.sha256() + for name, _size, _digest_value in entries: + legacy_hasher.update(name.encode("utf-8")) + legacy_hasher.update(payloads[name]) + legacy_tree = legacy_hasher.hexdigest() + for actual, expected, label in ( + (main_sha, expected_main_sha256, "main.py"), + (canonical_tree, expected_canonical_tree_sha256, "canonical tree"), + (legacy_tree, expected_legacy_tree_sha256, "legacy tree"), + ): + if expected is not None: + _strict_sha256(expected, f"expected {label} SHA") + if actual != expected: + raise PolicySourceError(f"policy {label} identity mismatch") + return cls(path, version, main_sha, canonical_tree, legacy_tree, tuple(entries)) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": "24-miracle-ifelse-policy-source-v1", + "version": self.version, + "main_sha256": self.main_sha256, + "canonical_tree_sha256": self.canonical_tree_sha256, + "legacy_tree_sha256": self.legacy_tree_sha256, + "files": [ + {"path": name, "bytes": size, "sha256": digest} + for name, size, digest in self.files + ], + } + + @property + def sha256(self) -> str: + return _digest(self.to_dict()) + + def revalidate(self) -> None: + current = LegacyPolicySourceV1.open( + self.root, + version=self.version, + expected_main_sha256=self.main_sha256, + expected_canonical_tree_sha256=self.canonical_tree_sha256, + expected_legacy_tree_sha256=self.legacy_tree_sha256, + ) + if current.to_dict() != self.to_dict(): + raise PolicySourceError("policy source identity changed") + + +@dataclass(frozen=True, slots=True) +class PolicyComparisonIdentityV1: + old_provider: ProviderIdentityV1 + new_provider: ProviderIdentityV1 + old_config: PolicyConfigV1 + new_config: PolicyConfigV1 + schema_version: str = POLICY_COMPARISON_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.old_provider.config_identity != self.old_config.sha256: + raise ValueError("old provider config identity mismatch") + if self.new_provider.config_identity != self.new_config.sha256: + raise ValueError("new provider config identity mismatch") + + @classmethod + def for_test( + cls, + *, + old_version: str, + new_version: str, + config: PolicyConfigV1, + ) -> "PolicyComparisonIdentityV1": + old_source = hashlib.sha256(("test:" + old_version).encode()).hexdigest() + new_source = hashlib.sha256(("test:" + new_version).encode()).hexdigest() + return cls( + ProviderIdentityV1(old_version, old_source, config.sha256), + ProviderIdentityV1(new_version, new_source, config.sha256), + config, + config, + ) + + @classmethod + def from_sources( + cls, + *, + old_source: LegacyPolicySourceV1, + new_source: LegacyPolicySourceV1, + old_config: PolicyConfigV1, + new_config: PolicyConfigV1, + ) -> "PolicyComparisonIdentityV1": + old_source.revalidate() + new_source.revalidate() + old_config.require_complete() + new_config.require_complete() + return cls( + ProviderIdentityV1( + old_source.version, old_source.sha256, old_config.sha256 + ), + ProviderIdentityV1( + new_source.version, new_source.sha256, new_config.sha256 + ), + old_config, + new_config, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "old_provider": self.old_provider.to_dict(), + "new_provider": self.new_provider.to_dict(), + "old_config_sha256": self.old_config.sha256, + "new_config_sha256": self.new_config.sha256, + "state_machine_identity": STATE_MACHINE_VERSION, + } + + @property + def sha256(self) -> str: + return _digest(self.to_dict()) + + @property + def config_pair_sha256(self) -> str: + return _digest( + { + "old": self.old_config.sha256, + "new": self.new_config.sha256, + } + ) + + +@dataclass(frozen=True, slots=True, init=False, eq=False, weakref_slot=True) +class PolicyMemoryV1: + schema_version: str + state_machine_version: str + policy_identity: str + policy_config_identity: str + episode_id: str + decision_step: int + lifecycle: str + phase: str + instruction_label: str + attack_pass: str | None + preserve_for_move: bool + acted_iteration: int + ordered_unit_ids: tuple[int, ...] + ordered_unit_snapshots: tuple[tuple[Any, ...], ...] + current_unit_cursor: int + ordered_target_ids: tuple[int, ...] + current_target_cursor: int + ordered_positions: tuple[tuple[int, int, int], ...] + position_cursor: int + remaining_capacities: tuple[tuple[str, int], ...] + local_mana: int | None + local_unit_counts: tuple[tuple[str, int], ...] + camp: int + rng_mode: str + rng_state: None + previous_transition_sha256: str | None + + def __new__(cls, *_args: Any, **_kwargs: Any): + raise TypeError("PolicyMemoryV1 can only be issued by the explicit state machine") + + def __copy__(self) -> "PolicyMemoryV1": + copied = object.__new__(PolicyMemoryV1) + for name in self.__slots__: + if name != "__weakref__": + object.__setattr__(copied, name, getattr(self, name)) + return copied + + def __deepcopy__(self, _memo: dict[int, Any]) -> "PolicyMemoryV1": + return self.__copy__() + + def _unsigned_dict(self) -> dict[str, Any]: + return { + field: ( + [list(item) if type(item) is tuple else item for item in value] + if field in { + "ordered_positions", + "ordered_unit_snapshots", + "remaining_capacities", + "local_unit_counts", + } + else list(value) + if type(value) is tuple + else value + ) + for field, value in ( + ("schema_version", self.schema_version), + ("state_machine_version", self.state_machine_version), + ("policy_identity", self.policy_identity), + ("policy_config_identity", self.policy_config_identity), + ("episode_id", self.episode_id), + ("decision_step", self.decision_step), + ("lifecycle", self.lifecycle), + ("phase", self.phase), + ("instruction_label", self.instruction_label), + ("attack_pass", self.attack_pass), + ("preserve_for_move", self.preserve_for_move), + ("acted_iteration", self.acted_iteration), + ("ordered_unit_ids", self.ordered_unit_ids), + ("ordered_unit_snapshots", self.ordered_unit_snapshots), + ("current_unit_cursor", self.current_unit_cursor), + ("ordered_target_ids", self.ordered_target_ids), + ("current_target_cursor", self.current_target_cursor), + ("ordered_positions", self.ordered_positions), + ("position_cursor", self.position_cursor), + ("remaining_capacities", self.remaining_capacities), + ("local_mana", self.local_mana), + ("local_unit_counts", self.local_unit_counts), + ("camp", self.camp), + ("rng_mode", self.rng_mode), + ("rng_state", self.rng_state), + ("previous_transition_sha256", self.previous_transition_sha256), + ) + } + + def to_dict(self) -> dict[str, Any]: + _validate_issued_memory(self) + return self._unsigned_dict() + + @property + def sha256(self) -> str: + _validate_issued_memory(self) + return _digest(self._unsigned_dict()) + + +_MEMORY_SNAPSHOTS: "weakref.WeakKeyDictionary[PolicyMemoryV1, str]" = ( + weakref.WeakKeyDictionary() +) + + +def _issue_memory(**values: Any) -> PolicyMemoryV1: + memory = object.__new__(PolicyMemoryV1) + for name, value in values.items(): + object.__setattr__(memory, name, value) + _validate_memory_schema(memory) + snapshot = _digest(memory._unsigned_dict()) + _MEMORY_SNAPSHOTS[memory] = snapshot + return memory + + +def _validate_issued_memory(memory: PolicyMemoryV1) -> None: + if type(memory) is not PolicyMemoryV1: + raise TypeError("memory must be an issued PolicyMemoryV1") + expected = _MEMORY_SNAPSHOTS.get(memory) + try: + _validate_memory_schema(memory) + actual = _digest(memory._unsigned_dict()) + except (AttributeError, TypeError, ValueError) as exc: + raise ValueError("PolicyMemoryV1 is not an intact issued memory snapshot") from exc + if expected is None or expected != actual: + raise ValueError("PolicyMemoryV1 is not an intact issued memory snapshot") + + +def _validate_nonnegative_int(value: Any, label: str) -> None: + if type(value) is not int or value < 0: + raise ValueError(f"{label} must be a non-negative strict integer") + + +def _validate_id_sequence(value: Any, label: str) -> None: + if type(value) is not tuple: + raise ValueError(f"{label} must be an immutable tuple") + if any(type(item) is not int or item < 0 for item in value): + raise ValueError(f"{label} must contain non-negative strict integer IDs") + if len(set(value)) != len(value): + raise ValueError(f"{label} must not contain duplicate IDs") + + +def _validate_memory_schema(memory: PolicyMemoryV1) -> None: + if memory.schema_version != POLICY_MEMORY_SCHEMA_VERSION: + raise ValueError("memory schema identity mismatch") + if memory.state_machine_version != STATE_MACHINE_VERSION: + raise ValueError("memory state-machine identity mismatch") + _strict_sha256(memory.policy_identity, "memory policy identity") + _strict_sha256(memory.policy_config_identity, "memory policy config identity") + _strict_text(memory.episode_id, "memory episode ID") + if type(memory.decision_step) is not int or memory.decision_step <= 0: + raise ValueError("memory decision step must be a positive strict integer") + if memory.lifecycle != "active": + raise ValueError("memory lifecycle must be active") + phases = { + "turn_start", + "opening", + "opening_endround", + "artifact", + "attack_pre_move", + "move", + "attack_post_move", + "summon", + "endround", + } + if memory.phase not in phases: + raise ValueError("memory phase is not defined by the state machine") + _strict_text(memory.instruction_label, "memory instruction label") + if memory.attack_pass not in {None, "pre_move", "post_move"}: + raise ValueError("memory attack_pass is invalid") + if type(memory.preserve_for_move) is not bool: + raise ValueError("memory preserve_for_move must be a strict boolean") + _validate_nonnegative_int(memory.acted_iteration, "acted_iteration") + _validate_id_sequence(memory.ordered_unit_ids, "ordered_unit_ids") + if type(memory.ordered_unit_snapshots) is not tuple: + raise ValueError("ordered_unit_snapshots must be an immutable tuple") + snapshot_ids: list[int] = [] + for snapshot in memory.ordered_unit_snapshots: + if type(snapshot) is not tuple or not snapshot or type(snapshot[0]) is not int: + raise ValueError("ordered_unit_snapshots contains an invalid unit snapshot") + snapshot_ids.append(snapshot[0]) + if memory.ordered_unit_snapshots and tuple(snapshot_ids) != memory.ordered_unit_ids: + raise ValueError("ordered_unit_snapshots disagree with ordered_unit_ids") + _validate_nonnegative_int(memory.current_unit_cursor, "current_unit_cursor") + if memory.current_unit_cursor > len(memory.ordered_unit_ids): + raise ValueError("current_unit_cursor exceeds ordered_unit_ids") + _validate_id_sequence(memory.ordered_target_ids, "ordered_target_ids") + _validate_nonnegative_int(memory.current_target_cursor, "current_target_cursor") + if memory.current_target_cursor > len(memory.ordered_target_ids): + raise ValueError("current_target_cursor exceeds ordered_target_ids") + if type(memory.ordered_positions) is not tuple: + raise ValueError("ordered_positions must be an immutable tuple") + for position in memory.ordered_positions: + if ( + type(position) is not tuple + or len(position) != 3 + or any(type(axis) is not int for axis in position) + or sum(position) != 0 + ): + raise ValueError("ordered_positions contains an invalid cube coordinate") + _validate_nonnegative_int(memory.position_cursor, "position_cursor") + if memory.position_cursor > len(memory.ordered_positions): + raise ValueError("position_cursor exceeds ordered_positions") + for field, entries in ( + ("remaining_capacities", memory.remaining_capacities), + ("local_unit_counts", memory.local_unit_counts), + ): + if type(entries) is not tuple: + raise ValueError(f"{field} must be an immutable tuple") + names: list[str] = [] + for entry in entries: + if type(entry) is not tuple or len(entry) != 2: + raise ValueError(f"{field} contains an invalid entry") + name, count = entry + _strict_text(name, f"{field} name") + _validate_nonnegative_int(count, f"{field} count") + names.append(name) + if len(set(names)) != len(names): + raise ValueError(f"{field} contains duplicate names") + if memory.local_mana is not None: + _validate_nonnegative_int(memory.local_mana, "local_mana") + if type(memory.camp) is not int or memory.camp not in {0, 1}: + raise ValueError("memory camp must be 0 or 1") + if memory.rng_mode != "none" or memory.rng_state is not None: + raise ValueError("deterministic provider cannot carry RNG state") + if memory.previous_transition_sha256 is not None: + _strict_sha256( + memory.previous_transition_sha256, "previous transition identity" + ) + + +@dataclass(frozen=True, slots=True) +class SelectedCommandV1: + command: Mapping[str, Any] + next_phase: str + instruction_label: str = "selected" + attack_pass: str | None = None + preserve_for_move: bool = False + acted_increment: int = 0 + memory_updates: Mapping[str, Any] | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "command", _freeze_mapping(canonical_command(self.command))) + _strict_text(self.next_phase, "next phase") + _strict_text(self.instruction_label, "instruction label") + updates = {} if self.memory_updates is None else dict(self.memory_updates) + allowed = { + "ordered_unit_ids", + "ordered_unit_snapshots", + "current_unit_cursor", + "ordered_target_ids", + "current_target_cursor", + "ordered_positions", + "position_cursor", + "remaining_capacities", + "local_mana", + "local_unit_counts", + } + if set(updates) - allowed: + raise ValueError("selected command contains unknown memory updates") + object.__setattr__(self, "memory_updates", _freeze_mapping(updates)) + + +@dataclass(frozen=True, slots=True, init=False, eq=False, weakref_slot=True) +class PolicyPairDecisionV1: + schema_version: str + policy_identity: str + episode_id: str + decision_step: int + observation_id: str + old_provider_identity: str + new_provider_identity: str + old_action_id: str + new_action_id: str + old_distribution: Mapping[str, float] + new_distribution: Mapping[str, float] + memory_before_sha256: str + m_after: PolicyMemoryV1 + transition_sha256: str + support_id: str + + def __new__(cls, *_args: Any, **_kwargs: Any): + raise TypeError( + "PolicyPairDecisionV1 can only be issued by the explicit state machine" + ) + + def __copy__(self) -> "PolicyPairDecisionV1": + copied = object.__new__(PolicyPairDecisionV1) + for name in self.__slots__: + if name != "__weakref__": + object.__setattr__(copied, name, getattr(self, name)) + return copied + + def __deepcopy__(self, _memo: dict[int, Any]) -> "PolicyPairDecisionV1": + return self.__copy__() + + def _unsigned_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "policy_identity": self.policy_identity, + "episode_id": self.episode_id, + "decision_step": self.decision_step, + "observation_id": self.observation_id, + "old_provider_identity": self.old_provider_identity, + "new_provider_identity": self.new_provider_identity, + "old_action_id": self.old_action_id, + "new_action_id": self.new_action_id, + "old_distribution": dict(self.old_distribution), + "new_distribution": dict(self.new_distribution), + "memory_before_sha256": self.memory_before_sha256, + "m_after": self.m_after._unsigned_dict(), + "transition_sha256": self.transition_sha256, + "support_id": self.support_id, + } + + def to_dict(self) -> dict[str, Any]: + _validate_issued_decision(self) + return self._unsigned_dict() + + @property + def sha256(self) -> str: + _validate_issued_decision(self) + return _digest(self._unsigned_dict()) + + +_DECISION_SNAPSHOTS: "weakref.WeakKeyDictionary[PolicyPairDecisionV1, str]" = ( + weakref.WeakKeyDictionary() +) + + +def _issue_decision(**values: Any) -> PolicyPairDecisionV1: + decision = object.__new__(PolicyPairDecisionV1) + for name, value in values.items(): + if name in {"old_distribution", "new_distribution"}: + value = MappingProxyType(dict(value)) + object.__setattr__(decision, name, value) + for distribution in (decision.old_distribution, decision.new_distribution): + if any( + type(value) is not float or not math.isfinite(value) + for value in distribution.values() + ): + raise ValueError("provider distribution must contain finite floats") + if math.fsum(distribution.values()) != 1.0: + raise ValueError("provider distribution must have exact unit mass") + _validate_issued_memory(decision.m_after) + _DECISION_SNAPSHOTS[decision] = _digest(decision._unsigned_dict()) + return decision + + +def _validate_issued_decision(decision: PolicyPairDecisionV1) -> None: + if type(decision) is not PolicyPairDecisionV1: + raise TypeError("decision must be an issued PolicyPairDecisionV1") + expected = _DECISION_SNAPSHOTS.get(decision) + try: + _validate_issued_memory(decision.m_after) + actual = _digest(decision._unsigned_dict()) + except (AttributeError, TypeError, ValueError) as exc: + raise ValueError("PolicyPairDecisionV1 is not an intact issued decision") from exc + if expected is None or expected != actual: + raise ValueError("PolicyPairDecisionV1 is not an intact issued decision") + + +class _SelectedLegacyCommand(RuntimeError): + def __init__(self, command: Mapping[str, Any]) -> None: + super().__init__("legacy policy selected one atomic command") + self.command = canonical_command(command) + + +class _LegacyIfElseRuntime: + """Read-only loader queried through fresh objects with no retained frame.""" + + def __init__(self, source: LegacyPolicySourceV1) -> None: + source.revalidate() + self.source = source + self.module = self._load_module(source) + if not hasattr(self.module, "IfElseAI"): + raise PolicySourceError("legacy main.py does not define IfElseAI") + + @staticmethod + def _load_module(source: LegacyPolicySourceV1) -> Any: + root = source.root + module_name = f"_agentbench_miracle_ifelse_{source.sha256[:20]}" + saved_modules = { + name: sys.modules.get(name) + for name in ("calculator", "gameunit", "ai_client", "card") + } + original_cwd = Path.cwd() + original_path = list(sys.path) + original_dont_write_bytecode = sys.dont_write_bytecode + try: + sys.dont_write_bytecode = True + for name in saved_modules: + sys.modules.pop(name, None) + os.chdir(root) + sys.path.insert(0, str(root)) + spec = importlib.util.spec_from_file_location(module_name, root / "main.py") + if spec is None or spec.loader is None: + raise PolicySourceError("cannot load legacy main.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + sys.dont_write_bytecode = original_dont_write_bytecode + os.chdir(original_cwd) + sys.path[:] = original_path + for name, saved in saved_modules.items(): + sys.modules.pop(name, None) + if saved is not None: + sys.modules[name] = saved + + def _new_policy( + self, observation: Mapping[str, Any], config: PolicyConfigV1 + ) -> Any: + config.require_complete() + module = self.module + base = module.IfElseAI + + class CapturingIfElseAI(base): + def _emit(self, operation_type: str, **parameters: Any) -> None: + raise _SelectedLegacyCommand( + { + "player": self.my_camp, + "round": self.round, + "operation_type": operation_type, + "operation_parameters": parameters, + } + ) + + def attack(self, attacker: int, target: int) -> None: + self._emit("attack", attacker=attacker, target=target) + + def move(self, mover: int, position: Sequence[int]) -> None: + self._emit("move", mover=mover, position=list(position)) + + def summon( + self, unit_type: str, level: int, position: Sequence[int] + ) -> None: + self._emit( + "summon", type=unit_type, level=level, position=list(position) + ) + + def use(self, artifact: int, target: Any) -> None: + normalized = list(target) if type(target) is tuple else target + self._emit("use", card=artifact, target=normalized) + + def end_round(self) -> None: + self._emit("endround") + + policy = object.__new__(CapturingIfElseAI) + camp = observation.get("camp") + if type(camp) is not int or camp not in {0, 1}: + raise ValueError("policy observation camp must be 0 or 1") + policy.my_camp = camp + policy.round = observation.get("round", 0) + if type(policy.round) is not int: + raise ValueError("policy observation round must be an integer") + policy.map = module.gameunit.Map() + policy.players = [module.gameunit.Player(0), module.gameunit.Player(1)] + if "map" in observation or "players" in observation: + if set(observation) < {"camp", "round", "map", "players"}: + raise ValueError("policy observation is incomplete") + policy.map.update(observation["map"]) + policy.players = [ + module.gameunit.Player(0, observation["players"][0]), + module.gameunit.Player(1, observation["players"][1]), + ] + policy.my_miracle = module.MY_MIRACLES[camp] + policy.enemy_miracle = module.MY_MIRACLES[camp ^ 1] + for spec in MIRACLE_CONFIG_SPECS: + if spec.attribute is not None: + setattr(policy, spec.attribute, config.values[spec.name]) + policy.artifacts = [config.values["MIRACLE_ARTIFACT"]] + policy.creatures = list(config.values["MIRACLE_DECK"]) + return policy + + @staticmethod + def _direct_command( + observation: Mapping[str, Any], operation_type: str + ) -> Mapping[str, Any]: + return canonical_command( + { + "player": observation["camp"], + "round": observation.get("round", 0), + "operation_type": operation_type, + "operation_parameters": {}, + } + ) + + def select( + self, + observation: Mapping[str, Any], + memory: PolicyMemoryV1, + config: PolicyConfigV1, + ) -> SelectedCommandV1: + self.source.revalidate() + if set(observation) == {"camp"}: + command = canonical_command( + { + "player": observation["camp"], + "round": 0, + "operation_type": "init", + "operation_parameters": { + "artifacts": [config.values["MIRACLE_ARTIFACT"]], + "creatures": list(config.values["MIRACLE_DECK"]), + }, + } + ) + return SelectedCommandV1(command, "turn_start", "init") + + policy = self._new_policy(observation, config) + policy.refresh_static_positions() + phase = memory.phase + phase_memory_updates: dict[str, Any] = {} + for _advance in range(8): + try: + if phase == "turn_start": + phase = "opening" if policy.round in {0, 1} else "artifact" + continue + if phase == "opening": + policy.play() + raise RuntimeError("legacy opening did not emit an action") + if phase == "opening_endround": + return SelectedCommandV1( + self._direct_command(observation, "endround"), + "turn_start", + "opening.endround", + ) + if phase == "artifact": + policy.use_artifact() + phase = "attack_pre_move" + continue + if phase == "attack_pre_move": + policy.attack_phase(preserve_for_move=True) + phase = "move" + continue + if phase == "move": + snapshots = memory.ordered_unit_snapshots + cursor = memory.current_unit_cursor + if not snapshots: + ordered = sorted( + policy.allies(), + key=lambda unit: (unit.type != "Swordsman", unit.id), + ) + raw_by_id = { + raw[0]: raw + for raw in observation["map"]["units"] + if raw[1] == policy.my_camp + } + snapshots = tuple( + tuple(raw_by_id[unit.id]) for unit in ordered + ) + cursor = 0 + while cursor < len(snapshots): + unit = self.module.gameunit.Unit(list(snapshots[cursor])) + cursor += 1 + if not unit.can_move: + continue + position = policy.best_move_for(unit) + if position and position != unit.pos: + command = canonical_command( + { + "player": policy.my_camp, + "round": policy.round, + "operation_type": "move", + "operation_parameters": { + "mover": unit.id, + "position": list(position), + }, + } + ) + return SelectedCommandV1( + command, + "move", + "move.move", + memory_updates={ + "ordered_unit_ids": tuple( + snapshot[0] for snapshot in snapshots + ), + "ordered_unit_snapshots": snapshots, + "current_unit_cursor": cursor, + }, + ) + phase_memory_updates = { + "ordered_unit_ids": (), + "ordered_unit_snapshots": (), + "current_unit_cursor": 0, + } + phase = "attack_post_move" + continue + if phase == "attack_post_move": + policy.attack_phase() + phase = "summon" + continue + if phase == "summon": + policy.summon_phase() + phase = "endround" + continue + if phase == "endround": + return SelectedCommandV1( + self._direct_command(observation, "endround"), + "turn_start", + "play.endround", + memory_updates=phase_memory_updates, + ) + raise ValueError(f"unknown policy memory phase: {phase}") + except _SelectedLegacyCommand as selected: + operation = selected.command["operation_type"] + if phase == "opening": + next_phase = ( + "opening_endround" + if operation == "summon" + else "turn_start" + ) + elif phase == "artifact": + next_phase = "attack_pre_move" + else: + next_phase = phase + attack_pass = ( + "pre_move" + if phase == "attack_pre_move" + else "post_move" + if phase == "attack_post_move" + else None + ) + return SelectedCommandV1( + selected.command, + next_phase, + f"{phase}.{operation}", + attack_pass=attack_pass, + preserve_for_move=phase == "attack_pre_move", + acted_increment=int(operation == "attack"), + memory_updates=phase_memory_updates, + ) + raise RuntimeError("policy phase advancement did not produce an action") + + +_RUNTIMES: dict[str, _LegacyIfElseRuntime] = {} + + +def _select_legacy_command( + provider_identity: str, + observation: Mapping[str, Any], + memory: PolicyMemoryV1, + config: PolicyConfigV1, +) -> SelectedCommandV1: + runtime = _RUNTIMES.get(provider_identity) + if runtime is None: + raise IncompletePolicyEvidenceError( + f"no verified legacy runtime is registered for provider {provider_identity}" + ) + return runtime.select(observation, memory, config) + + +class ExplicitIfElseStateMachineV1: + """Issuer for one comparison's explicit, replayable policy continuation.""" + + def __init__(self, identity: PolicyComparisonIdentityV1) -> None: + if not isinstance(identity, PolicyComparisonIdentityV1): + raise TypeError("identity must be a PolicyComparisonIdentityV1") + self.identity = identity + self._episode_tips: dict[str, PolicyMemoryV1] = {} + + @classmethod + def from_sources( + cls, + *, + old_source: LegacyPolicySourceV1, + new_source: LegacyPolicySourceV1, + old_config: PolicyConfigV1, + new_config: PolicyConfigV1, + ) -> "ExplicitIfElseStateMachineV1": + identity = PolicyComparisonIdentityV1.from_sources( + old_source=old_source, + new_source=new_source, + old_config=old_config, + new_config=new_config, + ) + machine = cls(identity) + _RUNTIMES[identity.old_provider.sha256] = _LegacyIfElseRuntime(old_source) + _RUNTIMES[identity.new_provider.sha256] = _LegacyIfElseRuntime(new_source) + return machine + + def reset(self, *, camp: int, episode_id: str) -> PolicyMemoryV1: + self.identity.old_config.require_complete() + self.identity.new_config.require_complete() + if type(camp) is not int or camp not in {0, 1}: + raise ValueError("camp must be 0 or 1") + _strict_text(episode_id, "episode ID") + memory = _issue_memory( + schema_version=POLICY_MEMORY_SCHEMA_VERSION, + state_machine_version=STATE_MACHINE_VERSION, + policy_identity=self.identity.sha256, + policy_config_identity=self.identity.config_pair_sha256, + episode_id=episode_id, + decision_step=1, + lifecycle="active", + phase="turn_start", + instruction_label="reset", + attack_pass=None, + preserve_for_move=False, + acted_iteration=0, + ordered_unit_ids=(), + ordered_unit_snapshots=(), + current_unit_cursor=0, + ordered_target_ids=(), + current_target_cursor=0, + ordered_positions=(), + position_cursor=0, + remaining_capacities=(), + local_mana=None, + local_unit_counts=(), + camp=camp, + rng_mode="none", + rng_state=None, + previous_transition_sha256=None, + ) + self._episode_tips[episode_id] = memory + return memory + + def validate_memory( + self, memory: PolicyMemoryV1, *, episode_id: str | None = None + ) -> None: + _validate_issued_memory(memory) + if memory.schema_version != POLICY_MEMORY_SCHEMA_VERSION: + raise ValueError("memory schema identity mismatch") + if memory.state_machine_version != STATE_MACHINE_VERSION: + raise ValueError("memory state-machine identity mismatch") + if memory.policy_identity != self.identity.sha256: + raise ValueError("memory policy identity mismatch") + if memory.policy_config_identity != self.identity.config_pair_sha256: + raise ValueError("memory policy config identity mismatch") + if episode_id is not None and memory.episode_id != episode_id: + raise ValueError("memory episode identity mismatch") + if type(memory.decision_step) is not int or memory.decision_step <= 0: + raise ValueError("memory decision step is invalid") + if memory.rng_mode != "none" or memory.rng_state is not None: + raise ValueError("deterministic provider cannot carry RNG state") + + def _validate_current_tip(self, memory: PolicyMemoryV1) -> None: + current = self._episode_tips.get(memory.episode_id) + if current is not memory: + raise ValueError( + "memory is not the current transition tip for its episode" + ) + + @staticmethod + def _validate_support( + observation: Mapping[str, Any], support: ActionSupport + ) -> None: + if not isinstance(support, ActionSupport): + raise TypeError("support must be an ActionSupport") + trusted = build_trusted_action_support(dict(observation)) + if ( + support.schema_version != trusted.schema_version + or support.support_id != trusted.support_id + or support.action_ids != trusted.action_ids + or tuple(candidate.action for candidate in support.actions) + != tuple(candidate.action for candidate in trusted.actions) + ): + raise ValueError("supplied support does not match trusted complete support") + + def evaluate_pair( + self, + observation: Mapping[str, Any], + memory: PolicyMemoryV1, + complete_action_support: ActionSupport, + ) -> PolicyPairDecisionV1: + if not isinstance(observation, Mapping): + raise TypeError("observation must be a mapping") + self.validate_memory(memory) + self._validate_current_tip(memory) + if observation.get("camp") != memory.camp: + raise ValueError("observation camp disagrees with policy memory") + self._validate_support(observation, complete_action_support) + + old_selected = _select_legacy_command( + self.identity.old_provider.sha256, + dict(observation), + memory, + self.identity.old_config, + ) + new_selected = _select_legacy_command( + self.identity.new_provider.sha256, + dict(observation), + memory, + self.identity.new_config, + ) + old_action_id = command_action_id(old_selected.command) + new_action_id = command_action_id(new_selected.command) + old_distribution = one_hot(old_action_id, complete_action_support) + new_distribution = one_hot(new_action_id, complete_action_support) + + transition_payload = { + "schema_version": POLICY_MEMORY_SCHEMA_VERSION, + "state_machine_version": STATE_MACHINE_VERSION, + "policy_identity": self.identity.sha256, + "episode_id": memory.episode_id, + "decision_step": memory.decision_step, + "memory_before_sha256": memory.sha256, + "observation_id": canonical_state_id(observation), + "support_id": complete_action_support.support_id, + "old_action_id": old_action_id, + "new_action_id": new_action_id, + "next_phase": new_selected.next_phase, + "instruction_label": new_selected.instruction_label, + } + transition_sha256 = _digest(transition_payload) + updates = new_selected.memory_updates + after = _issue_memory( + schema_version=POLICY_MEMORY_SCHEMA_VERSION, + state_machine_version=STATE_MACHINE_VERSION, + policy_identity=self.identity.sha256, + policy_config_identity=self.identity.config_pair_sha256, + episode_id=memory.episode_id, + decision_step=memory.decision_step + 1, + lifecycle="active", + phase=new_selected.next_phase, + instruction_label=new_selected.instruction_label, + attack_pass=new_selected.attack_pass, + preserve_for_move=new_selected.preserve_for_move, + acted_iteration=memory.acted_iteration + new_selected.acted_increment, + ordered_unit_ids=tuple( + updates.get("ordered_unit_ids", memory.ordered_unit_ids) + ), + ordered_unit_snapshots=tuple( + _freeze_sequence(item) + for item in updates.get( + "ordered_unit_snapshots", memory.ordered_unit_snapshots + ) + ), + current_unit_cursor=updates.get( + "current_unit_cursor", memory.current_unit_cursor + ), + ordered_target_ids=tuple( + updates.get("ordered_target_ids", memory.ordered_target_ids) + ), + current_target_cursor=updates.get( + "current_target_cursor", memory.current_target_cursor + ), + ordered_positions=tuple( + tuple(item) + for item in updates.get( + "ordered_positions", memory.ordered_positions + ) + ), + position_cursor=updates.get("position_cursor", memory.position_cursor), + remaining_capacities=tuple( + tuple(item) + for item in updates.get( + "remaining_capacities", memory.remaining_capacities + ) + ), + local_mana=updates.get("local_mana", memory.local_mana), + local_unit_counts=tuple( + tuple(item) + for item in updates.get( + "local_unit_counts", memory.local_unit_counts + ) + ), + camp=memory.camp, + rng_mode="none", + rng_state=None, + previous_transition_sha256=transition_sha256, + ) + decision = _issue_decision( + schema_version=POLICY_PAIR_DECISION_SCHEMA_VERSION, + policy_identity=self.identity.sha256, + episode_id=memory.episode_id, + decision_step=memory.decision_step, + observation_id=transition_payload["observation_id"], + old_provider_identity=self.identity.old_provider.sha256, + new_provider_identity=self.identity.new_provider.sha256, + old_action_id=old_action_id, + new_action_id=new_action_id, + old_distribution=old_distribution, + new_distribution=new_distribution, + memory_before_sha256=memory.sha256, + m_after=after, + transition_sha256=transition_sha256, + support_id=complete_action_support.support_id, + ) + self.validate_decision( + decision, + episode_id=memory.episode_id, + decision_step=memory.decision_step, + observation=observation, + complete_action_support=complete_action_support, + ) + self._episode_tips[memory.episode_id] = after + return decision + + def validate_decision( + self, + decision: PolicyPairDecisionV1, + *, + episode_id: str, + decision_step: int, + observation: Mapping[str, Any], + complete_action_support: ActionSupport, + ) -> None: + _validate_issued_decision(decision) + _strict_text(episode_id, "episode ID") + if type(decision_step) is not int or decision_step <= 0: + raise ValueError("decision step must be a positive integer") + if decision.schema_version != POLICY_PAIR_DECISION_SCHEMA_VERSION: + raise ValueError("decision schema identity mismatch") + if decision.policy_identity != self.identity.sha256: + raise ValueError("decision policy identity mismatch") + if decision.episode_id != episode_id: + raise ValueError("decision episode identity mismatch") + if decision.decision_step != decision_step: + raise ValueError("decision step identity mismatch") + observation_id = canonical_state_id(observation) + if decision.observation_id != observation_id: + raise ValueError("decision observation identity mismatch") + self._validate_support(observation, complete_action_support) + if decision.support_id != complete_action_support.support_id: + raise ValueError("decision support identity mismatch") + if decision.old_provider_identity != self.identity.old_provider.sha256: + raise ValueError("old provider identity mismatch") + if decision.new_provider_identity != self.identity.new_provider.sha256: + raise ValueError("new provider identity mismatch") + expected_actions = set(complete_action_support.action_ids) + for action_id, distribution, label in ( + (decision.old_action_id, decision.old_distribution, "old"), + (decision.new_action_id, decision.new_distribution, "new"), + ): + if set(distribution) != expected_actions: + raise ValueError(f"{label} distribution support mismatch") + if action_id not in expected_actions or distribution[action_id] != 1.0: + raise ValueError(f"{label} chosen action is not the one-hot mass") + after = decision.m_after + self.validate_memory(after, episode_id=episode_id) + if after.decision_step != decision_step + 1: + raise ValueError("decision output memory step mismatch") + if after.previous_transition_sha256 != decision.transition_sha256: + raise ValueError("decision transition chain mismatch") + transition_payload = { + "schema_version": POLICY_MEMORY_SCHEMA_VERSION, + "state_machine_version": STATE_MACHINE_VERSION, + "policy_identity": self.identity.sha256, + "episode_id": episode_id, + "decision_step": decision_step, + "memory_before_sha256": decision.memory_before_sha256, + "observation_id": observation_id, + "support_id": complete_action_support.support_id, + "old_action_id": decision.old_action_id, + "new_action_id": decision.new_action_id, + "next_phase": after.phase, + "instruction_label": after.instruction_label, + } + if _digest(transition_payload) != decision.transition_sha256: + raise ValueError("decision transition evidence mismatch") + + +@dataclass(frozen=True, slots=True) +class SequentialReplayAuditV1: + trace_sha256: str + episode_id: str + evaluated_camp: int + decision_frames: int + chosen_reproduced: int + action_support_complete: int + chosen_in_support: int + first_mismatch_step: int | None + final_memory_sha256: str | None + old_new_same_context: int + memory_before: tuple[PolicyMemoryV1, ...] + decisions: tuple[PolicyPairDecisionV1, ...] + recorded_action_ids: tuple[str, ...] + + @property + def complete(self) -> bool: + return ( + self.decision_frames > 0 + and self.chosen_reproduced == self.decision_frames + and self.action_support_complete == self.decision_frames + and self.chosen_in_support == self.decision_frames + and self.old_new_same_context == self.decision_frames + and self.first_mismatch_step is None + and len(self.memory_before) == self.decision_frames + and len(self.decisions) == self.decision_frames + and len(self.recorded_action_ids) == self.decision_frames + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": "24-miracle-ifelse-sequential-replay-audit-v1", + "trace_sha256": self.trace_sha256, + "episode_id": self.episode_id, + "evaluated_camp": self.evaluated_camp, + "decision_frames": self.decision_frames, + "chosen_reproduced": self.chosen_reproduced, + "action_support_complete": self.action_support_complete, + "chosen_in_support": self.chosen_in_support, + "old_new_same_context": self.old_new_same_context, + "first_mismatch_step": self.first_mismatch_step, + "final_memory_sha256": self.final_memory_sha256, + "decision_records": [ + { + "decision_step": step, + "recorded_action_id": recorded_action_id, + "m_before": memory.to_dict(), + "pair_decision": decision.to_dict(), + } + for step, (recorded_action_id, memory, decision) in enumerate( + zip( + self.recorded_action_ids, + self.memory_before, + self.decisions, + strict=True, + ), + start=1, + ) + ], + "complete": self.complete, + } + + +def _trace_decisions( + payload: bytes, *, evaluated_camp: int +) -> list[tuple[dict[str, Any], dict[str, Any]]]: + try: + text = payload.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ValueError("trace must be strict UTF-8") from exc + pending: dict[str, Any] | None = None + pairs: list[tuple[dict[str, Any], dict[str, Any]]] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"trace line {line_number} is invalid JSON") from exc + if not isinstance(record, dict): + raise ValueError(f"trace line {line_number} must be an object") + if record.get("kind") == "judge_frame": + frame = record.get("frame") + if not isinstance(frame, dict): + continue + if ( + evaluated_camp in frame.get("listen", []) + and evaluated_camp in frame.get("player", []) + and isinstance(frame.get("content"), list) + and frame["content"] + ): + framed = frame["content"][0] + if type(framed) is not str or len(framed) < 6: + raise ValueError("evaluated Judge observation framing is invalid") + try: + declared = int(framed[:6]) + except ValueError as exc: + raise ValueError("evaluated Judge observation length is invalid") from exc + encoded = framed[6:].encode("utf-8") + if declared != len(encoded): + raise ValueError("evaluated Judge observation length mismatch") + observation = json.loads(encoded) + if not isinstance(observation, dict): + raise ValueError("evaluated Judge observation must be an object") + pending = observation + elif record.get("kind") == "ai_operation" and record.get("player") == evaluated_camp: + if pending is None: + raise ValueError("evaluated operation has no preceding observation") + operation = record.get("operation") + if not isinstance(operation, dict): + raise ValueError("evaluated operation must be an object") + pairs.append((pending, canonical_command(operation))) + pending = None + return pairs + + +def audit_sequential_trace( + trace_path: str | os.PathLike[str], + *, + machine: ExplicitIfElseStateMachineV1, + evaluated_camp: int, + episode_id: str, +) -> SequentialReplayAuditV1: + """Replay one immutable trace from reset without per-frame reinitialization.""" + + supplied_path = Path(trace_path) + if supplied_path.is_symlink(): + raise ValueError("trace path must not be a symlink") + path = supplied_path.resolve(strict=True) + if not path.is_file() or path.is_symlink(): + raise ValueError("trace path must be a real file") + if type(evaluated_camp) is not int or evaluated_camp not in {0, 1}: + raise ValueError("evaluated camp must be 0 or 1") + lexical_before = path.lstat() + if not stat.S_ISREG(lexical_before.st_mode): + raise ValueError("trace path must remain a regular file") + with path.open("rb") as stream: + opened_before = os.fstat(stream.fileno()) + payload = stream.read() + opened_after = os.fstat(stream.fileno()) + identity_before = ( + opened_before.st_dev, + opened_before.st_ino, + opened_before.st_ctime_ns, + ) + if identity_before != ( + opened_after.st_dev, + opened_after.st_ino, + opened_after.st_ctime_ns, + ): + raise ValueError("trace changed while its frozen snapshot was read") + size_before = opened_before.st_size + if size_before <= 0 or size_before > MAX_REPLAY_TRACE_BYTES: + raise ValueError("trace size is outside the approved bound") + if len(payload) != size_before: + raise ValueError("trace size changed while its frozen snapshot was read") + digest = hashlib.sha256(payload).hexdigest() + pairs = _trace_decisions(payload, evaluated_camp=evaluated_camp) + memory = machine.reset(camp=evaluated_camp, episode_id=episode_id) + reproduced = 0 + complete_support = 0 + chosen_in_support = 0 + same_context = 0 + first_mismatch: int | None = None + memory_records: list[PolicyMemoryV1] = [] + decisions: list[PolicyPairDecisionV1] = [] + recorded_action_ids: list[str] = [] + for step, (observation, recorded) in enumerate(pairs, start=1): + if memory.decision_step != step: + raise ValueError("sequential memory decision step is discontinuous") + try: + support = build_trusted_action_support(observation) + except Exception: + if first_mismatch is None: + first_mismatch = step + break + complete_support += 1 + recorded_id = command_action_id(recorded) + if recorded_id in support.action_ids: + chosen_in_support += 1 + decision = machine.evaluate_pair(observation, memory, support) + memory_records.append(memory) + decisions.append(decision) + recorded_action_ids.append(recorded_id) + if decision.memory_before_sha256 == memory.sha256: + same_context += 1 + if decision.new_action_id == recorded_id: + reproduced += 1 + elif first_mismatch is None: + first_mismatch = step + break + memory = decision.m_after + if path.is_symlink(): + raise ValueError("trace changed during sequential replay audit") + lexical_after = path.lstat() + with path.open("rb") as stream: + reopened = os.fstat(stream.fileno()) + final_payload = stream.read() + identity_after = (reopened.st_dev, reopened.st_ino, reopened.st_ctime_ns) + if ( + not stat.S_ISREG(lexical_after.st_mode) + or identity_after != identity_before + or reopened.st_size != size_before + or hashlib.sha256(final_payload).hexdigest() != digest + ): + raise ValueError("trace changed during sequential replay audit") + return SequentialReplayAuditV1( + trace_sha256=digest, + episode_id=episode_id, + evaluated_camp=evaluated_camp, + decision_frames=len(pairs), + chosen_reproduced=reproduced, + action_support_complete=complete_support, + chosen_in_support=chosen_in_support, + first_mismatch_step=first_mismatch, + final_memory_sha256=memory.sha256 if reproduced else None, + old_new_same_context=same_context, + memory_before=tuple(memory_records), + decisions=tuple(decisions), + recorded_action_ids=tuple(recorded_action_ids), + ) + + +__all__ = [ + "IncompletePolicyEvidenceError", + "LEGACY_SOURCE_FILES", + "LegacyPolicySourceV1", + "MIRACLE_CONFIG_SPECS", + "MiracleConfigSpec", + "POLICY_COMPARISON_SCHEMA_VERSION", + "POLICY_CONFIG_SCHEMA_VERSION", + "POLICY_MEMORY_SCHEMA_VERSION", + "POLICY_PAIR_DECISION_SCHEMA_VERSION", + "PROVIDER_SCHEMA_VERSION", + "PolicyComparisonIdentityV1", + "PolicyConfigV1", + "PolicyMemoryV1", + "PolicyPairDecisionV1", + "ProviderIdentityV1", + "PolicySourceError", + "SequentialReplayAuditV1", + "STATE_MACHINE_VERSION", + "VALID_ARTIFACTS", + "VALID_CREATURES", + "SelectedCommandV1", + "ExplicitIfElseStateMachineV1", + "audit_sequential_trace", +] diff --git a/src/agentbench_frame/games/miracle/iteration_protocol.py b/src/agentbench_frame/games/miracle/iteration_protocol.py index 4282cb7..922576a 100644 --- a/src/agentbench_frame/games/miracle/iteration_protocol.py +++ b/src/agentbench_frame/games/miracle/iteration_protocol.py @@ -19,8 +19,23 @@ from pathlib import Path, PurePosixPath from typing import Any, Callable, Mapping +from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION, + FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND, + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + FORMAL_POLICY_INFORMATION_GAIN_UNIT, + FORMAL_POLICY_KL_DIRECTION, + FORMAL_POLICY_KL_LOG_BASE, + FORMAL_POLICY_KL_ROLLOUT_SOURCE, + FORMAL_POLICY_KL_SMOOTHING, + FORMAL_POLICY_KL_SUM_ESTIMAND, + FORMAL_POLICY_KL_SUM_UNIT, + formal_policy_kl, +) from agentbench_frame.games.miracle.research_protocol import ( BENCHMARK_VERSION, + CURRENT_INTERNAL_MEMORY_EVIDENCE, + INTERNAL_MEMORY_EVIDENCE_BOUND, PROTOCOL_VERSION, SEED_MAX, SEED_MIN, @@ -31,7 +46,7 @@ research_manifest_sha256, ) -ITERATION_PROTOCOL_VERSION = "24-miracle-iteration-v3" +ITERATION_PROTOCOL_VERSION = "24-miracle-iteration-v4" BOOTSTRAP_TEMPLATE_VERSION = "24m-minimal-bootstrap-v1" HUMAN_CHAMPION_SCHEMA_VERSION = "24-miracle-human-champion-v1" HUMAN_REPLAY_SKILL_SCHEMA_VERSION = "24-miracle-human-replay-skill-v1" @@ -44,8 +59,11 @@ STRATEGY_SCHEMA_VERSION = "24-miracle-strategy-v3" ROLLBACK_SCHEMA_VERSION = "24-miracle-rollback-v1" LEGAL_ACTION_UNIT = "one_legal_atomic_judge_command" -KL_DIRECTION = "new||old" -KL_ROLLOUT_SOURCE = "new_policy" +KL_DIRECTION = FORMAL_POLICY_KL_DIRECTION +KL_ROLLOUT_SOURCE = FORMAL_POLICY_KL_ROLLOUT_SOURCE +KL_AGGREGATION = FORMAL_POLICY_INFORMATION_GAIN_AGGREGATION +KL_INFORMATION_GAIN_UNIT = FORMAL_POLICY_INFORMATION_GAIN_UNIT +KL_SUM_UNIT = FORMAL_POLICY_KL_SUM_UNIT DECISION_CHANGE_RATE = "not_collected" HUMAN_AUTHORED_CONTENT_REQUIRED = "HUMAN_AUTHORED_CONTENT_REQUIRED" @@ -1026,9 +1044,19 @@ def from_bytes(cls, payload: bytes) -> "CandidateEvaluationPlan": return plan +@dataclass(frozen=True) +class _ValidatedTrajectoryKL: + complete: bool + information_gain: float | None + local_policy_kl_sum: float | None + + def __bool__(self) -> bool: + return self.complete + + def _validate_trajectory_kl( value: Mapping[str, Any], plan: CandidateEvaluationPlan -) -> bool: +) -> _ValidatedTrajectoryKL: if not isinstance(value, Mapping): raise IterationPreflightError("trajectory KL evidence must be an object") episode = value.get("episode") @@ -1037,12 +1065,16 @@ def _validate_trajectory_kl( raise IterationPreflightError("trajectory KL old-policy identity mismatch") if value.get("version_after") != plan.candidate_strategy_version: raise IterationPreflightError("trajectory KL new-policy identity mismatch") + if value.get("measurement_profile") != FORMAL_POLICY_INFORMATION_GAIN_PROFILE: + raise IterationPreflightError( + "trajectory KL formal measurement profile mismatch" + ) epsilon = value.get("epsilon") if isinstance(epsilon, bool) or not isinstance(epsilon, (int, float)) or epsilon != TRAJECTORY_KL_EPSILON: raise IterationPreflightError("trajectory KL epsilon must be 0.01") if value.get("direction") != KL_DIRECTION or value.get("rollout_source") != KL_ROLLOUT_SOURCE: raise IterationPreflightError("trajectory KL direction/rollout identity mismatch") - if value.get("log_base") != "e" or value.get("estimand") != "epsilon_regularized_local_kl_sum_under_new_policy_occupancy": + if value.get("log_base") != FORMAL_POLICY_KL_LOG_BASE or value.get("estimand") != FORMAL_POLICY_KL_SUM_ESTIMAND: raise IterationPreflightError("trajectory KL estimand mismatch") status = value.get("status") if status not in {"complete", "incomplete", "failed"} or value.get("measurement_status") != status: @@ -1061,7 +1093,7 @@ def _validate_trajectory_kl( complete = status == "complete" if complete and (not decisions or errors): raise IterationPreflightError("complete trajectory KL requires decisions and no errors") - total = 0.0 + local_values: list[float] = [] for index, decision in enumerate(decisions, start=1): if not isinstance(decision, Mapping) or decision.get("decision_step") != index: raise IterationPreflightError("trajectory KL decision steps must be strict and continuous") @@ -1072,38 +1104,120 @@ def _validate_trajectory_kl( raise IterationPreflightError("trajectory KL ActionSupport identity is invalid") for label in ("context_ref", "action_schema_version", "support_id"): _required_text(decision.get(label), f"trajectory KL {label}") - local = decision.get("local_policy_kl") - if complete: + decision_errors = decision.get("errors") + if not isinstance(decision_errors, list) or any( + not isinstance(item, str) for item in decision_errors + ): + raise IterationPreflightError( + "trajectory KL decision errors must be strings" + ) + raw_local = decision.get("local_policy_kl") + if raw_local is None: + if complete: + raise IterationPreflightError( + "complete trajectory KL decision requires local policy KL" + ) + if trace[index - 1] is not None: + raise IterationPreflightError( + "trajectory KL trace disagrees with decisions" + ) + else: + local = raw_local local = _strict_finite_number(local, "local policy KL") if local < 0.0: raise IterationPreflightError("local policy KL cannot be negative") if trace[index - 1] != local: raise IterationPreflightError("trajectory KL trace disagrees with decisions") - total += local + local_values.append(local) + aligned_vectors: dict[str, list[float]] = {} for label in ("new_distribution", "old_distribution"): distribution = decision.get(label) if not isinstance(distribution, Mapping) or set(distribution) != set(legal): raise IterationPreflightError("trajectory KL policy distribution support mismatch") values = [_strict_score(distribution[action], f"{label} probability") for action in legal] - if not math.isclose(sum(values), 1.0, rel_tol=0.0, abs_tol=1e-9): + if not math.isclose(math.fsum(values), 1.0, rel_tol=0.0, abs_tol=1e-9): raise IterationPreflightError("trajectory KL policy distribution is not normalized") + aligned_vectors[label] = values for label in ("new_probabilities", "old_probabilities"): probabilities = decision.get(label) if not isinstance(probabilities, list) or len(probabilities) != len(legal): raise IterationPreflightError("trajectory KL probability vector is incomplete") values = [_strict_score(item, f"{label} probability") for item in probabilities] - if not math.isclose(sum(values), 1.0, rel_tol=0.0, abs_tol=1e-9): + if not math.isclose(math.fsum(values), 1.0, rel_tol=0.0, abs_tol=1e-9): raise IterationPreflightError("trajectory KL probability vector is not normalized") - if decision.get("errors") != []: + distribution_label = label.removesuffix("_probabilities") + "_distribution" + if any( + not math.isclose(left, right, rel_tol=0.0, abs_tol=1e-12) + for left, right in zip( + values, aligned_vectors[distribution_label], strict=True + ) + ): + raise IterationPreflightError( + "trajectory KL probability vector disagrees with distribution" + ) + try: + recomputed_local = formal_policy_kl( + aligned_vectors["new_distribution"], + aligned_vectors["old_distribution"], + ) + except (TypeError, ValueError) as exc: + raise IterationPreflightError( + f"local policy KL inputs are invalid: {exc}" + ) from exc + if not math.isclose(local, recomputed_local, rel_tol=0.0, abs_tol=1e-12): + raise IterationPreflightError( + "local policy KL disagrees with authoritative distributions" + ) + if decision_errors: raise IterationPreflightError("complete trajectory KL decision cannot contain errors") if complete: + try: + total = math.fsum(local_values) + except OverflowError as exc: + raise IterationPreflightError( + "trajectory KL aggregates must be finite" + ) from exc + information_gain = total / len(decisions) + if not math.isfinite(total) or not math.isfinite(information_gain): + raise IterationPreflightError("trajectory KL aggregates must be finite") uploaded_total = _strict_finite_number(value.get("trajectory_kl_episode"), "trajectory KL episode") uploaded_mean = _strict_finite_number(value.get("mean_local_policy_kl"), "mean local policy KL") if not math.isclose(uploaded_total, total, rel_tol=0.0, abs_tol=1e-12): raise IterationPreflightError("trajectory KL episode total mismatch") if not math.isclose(uploaded_mean, total / len(decisions), rel_tol=0.0, abs_tol=1e-12): raise IterationPreflightError("trajectory KL episode mean mismatch") - return complete + uploaded_ig = _strict_finite_number( + value.get("information_gain"), "information gain" + ) + uploaded_sum = _strict_finite_number( + value.get("local_policy_kl_sum"), "local policy KL sum" + ) + if not math.isclose(uploaded_ig, information_gain, rel_tol=0.0, abs_tol=1e-12): + raise IterationPreflightError("information gain disagrees with ordered trace") + if not math.isclose(uploaded_sum, total, rel_tol=0.0, abs_tol=1e-12): + raise IterationPreflightError("local policy KL sum disagrees with ordered trace") + if ( + value.get("aggregation") != KL_AGGREGATION + or value.get("information_gain_unit") != KL_INFORMATION_GAIN_UNIT + or value.get("local_policy_kl_sum_unit") != KL_SUM_UNIT + or value.get("information_gain_estimand") + != FORMAL_POLICY_INFORMATION_GAIN_ESTIMAND + ): + raise IterationPreflightError("information gain aggregate identity mismatch") + if CURRENT_INTERNAL_MEMORY_EVIDENCE != INTERNAL_MEMORY_EVIDENCE_BOUND: + return _ValidatedTrajectoryKL(False, None, None) + return _ValidatedTrajectoryKL(True, information_gain, total) + for label in ( + "trajectory_kl_episode", + "mean_local_policy_kl", + "information_gain", + "local_policy_kl_sum", + ): + if value.get(label) is not None: + raise IterationPreflightError( + "incomplete trajectory KL cannot expose aggregate scalars" + ) + return _ValidatedTrajectoryKL(False, None, None) @dataclass(frozen=True) @@ -1213,12 +1327,27 @@ def validate(self, plan: CandidateEvaluationPlan) -> None: _strict_score(self.candidate_score, "candidate score") if self.terminal_status not in {"complete", "incomplete", "failed"}: raise IterationPreflightError("evaluation terminal status is invalid") - kl_complete = _validate_trajectory_kl(self.trajectory_kl, plan) - if self.information_gain is not None: - _strict_finite_number(self.information_gain, "information gain") + validated_kl = _validate_trajectory_kl(self.trajectory_kl, plan) + if validated_kl.complete: + supplied_information_gain = _strict_finite_number( + self.information_gain, "information gain" + ) + if not math.isclose( + supplied_information_gain, + validated_kl.information_gain, + rel_tol=0.0, + abs_tol=1e-12, + ): + raise IterationPreflightError( + "information gain must match the authoritative ordered KL trace" + ) + elif self.information_gain is not None: + raise IterationPreflightError( + "incomplete trajectory KL cannot expose information gain" + ) if self.failure_reason is not None: _required_text(self.failure_reason, "evaluation failure reason") - if self.terminal_status == "complete" and (not kl_complete or self.information_gain is None): + if self.terminal_status == "complete" and not validated_kl.complete: if self.failure_reason is None: raise IterationPreflightError("incomplete KL/IG requires a failure reason") if self.sha256 and self.sha256 != _digest(self._unsigned_dict()): @@ -1370,13 +1499,22 @@ def _derived( blockers: tuple[str, ...], ) -> dict[str, Any]: evidence.validate(plan) - raw_score = sum(item.baseline_score for item in evidence.evidence) / len(evidence.evidence) - evo_score = sum(item.candidate_score for item in evidence.evidence) / len(evidence.evidence) + raw_score = math.fsum( + item.baseline_score for item in evidence.evidence + ) / len(evidence.evidence) + evo_score = math.fsum( + item.candidate_score for item in evidence.evidence + ) / len(evidence.evidence) gain = evo_score - raw_score - kl_complete = all(_validate_trajectory_kl(item.trajectory_kl, plan) for item in evidence.evidence) - ig_values = [item.information_gain for item in evidence.evidence] + validated_kl = [ + _validate_trajectory_kl(item.trajectory_kl, plan) + for item in evidence.evidence + ] + kl_complete = all(item.complete for item in validated_kl) + ig_values = [item.information_gain for item in validated_kl] information_gain = ( - sum(value for value in ig_values if value is not None) / len(ig_values) + math.fsum(value for value in ig_values if value is not None) + / len(ig_values) if all(value is not None for value in ig_values) else None ) @@ -2440,9 +2578,15 @@ def preflight_learning(config: LearningConfig) -> LearningReadyContext: "benchmark_version": BENCHMARK_VERSION, "legal_action_unit": LEGAL_ACTION_UNIT, "trajectory_kl": { + "measurement_profile": FORMAL_POLICY_INFORMATION_GAIN_PROFILE, "epsilon": TRAJECTORY_KL_EPSILON, "direction": KL_DIRECTION, "rollout_source": KL_ROLLOUT_SOURCE, + "smoothing": FORMAL_POLICY_KL_SMOOTHING, + "information_gain_aggregation": KL_AGGREGATION, + "information_gain_unit": KL_INFORMATION_GAIN_UNIT, + "local_policy_kl_sum_unit": KL_SUM_UNIT, + "internal_memory_evidence": CURRENT_INTERNAL_MEMORY_EVIDENCE, "decision_change_rate": DECISION_CHANGE_RATE, }, }, @@ -3101,6 +3245,19 @@ def iteration_protocol_manifest() -> dict[str, Any]: "sha256": bootstrap.sha256, }, "legal_action_unit": LEGAL_ACTION_UNIT, + "policy_information_gain": { + "measurement_profile": FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + "direction": KL_DIRECTION, + "epsilon": TRAJECTORY_KL_EPSILON, + "smoothing": FORMAL_POLICY_KL_SMOOTHING, + "rollout_source": KL_ROLLOUT_SOURCE, + "aggregation": KL_AGGREGATION, + "unit": KL_INFORMATION_GAIN_UNIT, + "optional_sum_unit": KL_SUM_UNIT, + "terminal_state_included": False, + "occupancy_shift_combined": False, + "internal_memory_evidence": CURRENT_INTERNAL_MEMORY_EVIDENCE, + }, "allowed_change_operations": sorted(_CHANGE_OPERATIONS), "interpretable_strategy_categories": sorted(_INTERPRETABLE_CATEGORIES), "human_champion_schema_version": HUMAN_CHAMPION_SCHEMA_VERSION, diff --git a/src/agentbench_frame/games/miracle/research_protocol.py b/src/agentbench_frame/games/miracle/research_protocol.py index cb9d830..8e91b3d 100644 --- a/src/agentbench_frame/games/miracle/research_protocol.py +++ b/src/agentbench_frame/games/miracle/research_protocol.py @@ -18,13 +18,24 @@ from typing import Any from agentbench_frame.eval import ActionCandidate, ActionSupport, PolicyDecision +from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + FORMAL_POLICY_KL_DIRECTION, + FORMAL_POLICY_KL_ROLLOUT_SOURCE, + FORMAL_POLICY_KL_SMOOTHING, + MAIN_POLICY_KL_EPSILON, +) -PROTOCOL_VERSION = "24-miracle-research-v1" +PROTOCOL_VERSION = "24-miracle-research-v2" BENCHMARK_VERSION = "24m-frozen-v1" -MANIFEST_SCHEMA_VERSION = "24-miracle-research-manifest-v1" +MANIFEST_SCHEMA_VERSION = "24-miracle-research-manifest-v2" ACTION_SCHEMA_VERSION = "24-miracle-command-v1" -TRAJECTORY_KL_EPSILON = 0.01 +TRAJECTORY_KL_EPSILON = MAIN_POLICY_KL_EPSILON +INTERNAL_MEMORY_EVIDENCE_BOUND = ( + "state_and_internal_memory_bound_to_policy_identity" +) +CURRENT_INTERNAL_MEMORY_EVIDENCE = "not_collected" TEST_REPEATS = 3 SEED_MIN = 0 SEED_MAX = 0x7FFF_FFFF @@ -291,9 +302,19 @@ def research_protocol_manifest() -> dict[str, Any]: }, "trajectory_kl": { "required": True, + "measurement_profile": FORMAL_POLICY_INFORMATION_GAIN_PROFILE, "epsilon": TRAJECTORY_KL_EPSILON, - "direction": "new||old", - "rollout_source": "new_policy", + "direction": FORMAL_POLICY_KL_DIRECTION, + "rollout_source": FORMAL_POLICY_KL_ROLLOUT_SOURCE, + "smoothing": FORMAL_POLICY_KL_SMOOTHING, + "local_policy_kl_trace": "required_ordered_target_agent_decisions", + "primary_episode_information_gain": "arithmetic_mean", + "primary_unit": "nats / decision", + "optional_sum_unit": "nats / episode", + "terminal_state_included": False, + "occupancy_shift_combined_with_information_gain": False, + "internal_memory_context": "required_when_policy_stateful", + "current_internal_memory_evidence": CURRENT_INTERNAL_MEMORY_EVIDENCE, "decision_change_rate": "not_collected", "failure_policy": "mark_measurement_incomplete", }, diff --git a/src/agentbench_frame/report/builder.py b/src/agentbench_frame/report/builder.py index 530935a..0571e2f 100644 --- a/src/agentbench_frame/report/builder.py +++ b/src/agentbench_frame/report/builder.py @@ -19,6 +19,11 @@ from typing import Any, Dict, List, Optional from agentbench_frame.eval.curves import multi_axis_auc +from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + GENERIC_TRAJECTORY_KL_PROFILE, +) +from agentbench_frame.eval.trajectory_kl import trajectory_kl_result_from_payload from agentbench_frame.tracking.quality import inspect_event_file try: @@ -199,96 +204,58 @@ def _derive_research(summary: Dict[str, Any], events: List[Dict[str, Any]], even for index, event in enumerate(events, start=1): event_type = event.get("event_type", event.get("event")) if event_type == "policy_kl_trace": - trace = event.get("trace") - values = [] - valid_trace = isinstance(trace, list) and bool(trace) - if valid_trace: - for value in trace: - number = _strict_nonnegative_number(value) - if number is None: - valid_trace = False - break - values.append(number) - declared_status = event.get("measurement_status") - decision_steps = event.get( - "decision_steps", - len(trace) if isinstance(trace, list) else None, - ) - decision_steps_present = "decision_steps" in event - aligned = ( - isinstance(decision_steps, int) - and not isinstance(decision_steps, bool) - and isinstance(trace, list) - and decision_steps == len(trace) - ) - decisions_present = "decisions" in event - decisions = event.get("decisions") - if decisions_present: - aligned = ( - aligned - and decision_steps_present - and isinstance(decisions, list) - and len(decisions) == len(trace) - ) - if aligned: - for decision, value in zip(decisions, values): - local_value = ( - decision.get("local_policy_kl") - if isinstance(decision, dict) - else None - ) - local_number = _strict_nonnegative_number( - local_value - ) - if ( - local_number is None - or not math.isclose( - local_number, - value, - rel_tol=0.0, - abs_tol=1e-12, - ) - ): - aligned = False - break - status_allows_complete = ( - declared_status == "complete" - if decisions_present - else declared_status in {None, "complete"} - ) - complete = valid_trace and aligned and status_allows_complete - trajectory_kl_episode = sum(values) if complete else None - if ( - trajectory_kl_episode is not None - and not math.isfinite(trajectory_kl_episode) - ): - complete = False - trajectory_kl_episode = None - if complete: - display_status = "complete" - elif ( - isinstance(declared_status, str) - and declared_status - and declared_status != "complete" - ): - display_status = declared_status + profile = event.get("measurement_profile") + trajectory_kl_episode = None + mean_local_policy_kl = None + information_gain = None + decision_steps = event.get("decision_steps") + estimand = event.get("estimand", "legacy_unspecified") + if profile in { + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + GENERIC_TRAJECTORY_KL_PROFILE, + }: + candidate = dict(event) + candidate["status"] = candidate.get("measurement_status") + try: + verified = trajectory_kl_result_from_payload( + candidate, + require_formal=( + profile + == FORMAL_POLICY_INFORMATION_GAIN_PROFILE + ), + ) + except (TypeError, ValueError): + display_status = "incomplete" + else: + decision_steps = verified.decision_steps + trajectory_kl_episode = verified.local_policy_kl_sum + mean_local_policy_kl = verified.mean_local_policy_kl + information_gain = verified.information_gain + if verified.status != "complete": + display_status = verified.status + elif information_gain is not None: + display_status = "complete" + else: + display_status = "generic_unverified" else: - display_status = "incomplete" - mean_local_policy_kl = ( - trajectory_kl_episode / len(values) - if trajectory_kl_episode is not None - else None - ) + display_status = "legacy_unverified" ig_history.append({ "episode": event.get("episode", index), "trajectory_kl_episode": trajectory_kl_episode, "mean_local_policy_kl": mean_local_policy_kl, - "ig": trajectory_kl_episode, + "information_gain": information_gain, + "local_policy_kl_sum": trajectory_kl_episode, + "ig": information_gain, + "information_gain_unit": ( + event.get("information_gain_unit") + if information_gain is not None + else None + ), + "local_policy_kl_sum_unit": event.get("local_policy_kl_sum_unit"), "decision_steps": decision_steps, "status": display_status, - "estimand": event.get( - "estimand", "legacy_unspecified" - ), + "estimand": estimand, + "measurement_profile": profile, }) elif event_type == "occupancy": state_ids = event.get("state_ids") @@ -331,9 +298,9 @@ def _trajectory_kl_chart(history: List[Dict[str, Any]]) -> Dict[str, Any]: height = 180 padding = 24 complete_values = [ - float(point["trajectory_kl_episode"]) + float(point["information_gain"]) for point in history - if point.get("trajectory_kl_episode") is not None + if point.get("information_gain") is not None ] y_max = max(complete_values, default=0.0) scale_max = y_max if y_max > 0.0 else 1.0 @@ -341,7 +308,7 @@ def _trajectory_kl_chart(history: List[Dict[str, Any]]) -> Dict[str, Any]: segments = [] current = [] for index, point in enumerate(history): - value = point.get("trajectory_kl_episode") + value = point.get("information_gain") if value is None: if current: segments.append(current) @@ -512,13 +479,13 @@ def _simple_html(self, ctx: Dict[str, Any]) -> str: research = ctx["latest_research"] lines.append("

Information gain

") lines.append( - "

Trajectory KL (nats / episode); " - "Mean local policy KL (nats / decision)

" + "

Episode policy information gain (mean local policy KL, " + "nats / decision); local policy KL sum (nats / episode)

" ) chart = research.get("ig_chart", {}) segments = chart.get("segments", []) lines.append( - '' ) @@ -529,23 +496,23 @@ def _simple_html(self, ctx: Dict[str, Any]) -> str: ) lines.append("") lines.append( - "" - "" + "
EpisodeTrajectory KLMean local policy KLStatus
" + "" ) for point in research.get("ig_history", []): - trajectory_value = point.get("trajectory_kl_episode") - mean_value = point.get("mean_local_policy_kl") - trajectory_text = ( - f"{trajectory_value:.2f}" - if trajectory_value is not None + information_gain = point.get("information_gain") + local_sum = point.get("local_policy_kl_sum") + information_gain_text = ( + f"{information_gain:.2f}" + if information_gain is not None else "missing" ) - mean_text = ( - f"{mean_value:.2f}" if mean_value is not None else "missing" + local_sum_text = ( + f"{local_sum:.2f}" if local_sum is not None else "missing" ) lines.append( f"" - f"" + f"" f"" ) lines.append("
EpisodePolicy IGLocal KL sumStatus
{point.get('episode')}{trajectory_text}{mean_text}{information_gain_text}{local_sum_text}{point.get('status')}
") diff --git a/src/agentbench_frame/report/templates/index.html b/src/agentbench_frame/report/templates/index.html index 33701bc..f702be5 100644 --- a/src/agentbench_frame/report/templates/index.html +++ b/src/agentbench_frame/report/templates/index.html @@ -12,16 +12,16 @@ {% if lr.get('score_history') %}{% for point in lr.get('score_history') %}{% endfor %}
ActScoreAct id
{{ point.x }}{% if point.score is none %}missing{% else %}{{ fmt_pct(point.score) }}{% endif %}{{ point.act_id or '—' }}
{% else %}
No score history yet.
{% endif %}
-

{{ icon('spark') }} Information gain

Per-event estimand and rollout provenance are preserved; incomplete episodes remain gaps

Trajectory KL estimate
nats / episode
+

{{ icon('spark') }} Information gain

Per-event estimand and rollout provenance are preserved; incomplete episodes remain gaps

Policy IG
nats / decision
{% if lr.get('ig_history') %} {% set chart = lr.get('ig_chart', {}) %} - + {% for segment in chart.get('segments', []) %} {% if segment.nodes|length > 1 %}{% endif %} {% for node in segment.nodes %}episode {{ node.episode }}: {{ fmt_num(node.value) }} nats{% endfor %} {% endfor %} - {% for point in lr.get('ig_history') %}{% endfor %}
EpisodeTrajectory KL
nats / episode		Mean local policy KL
nats / decision		Decision stepsEstimandStatus
{{ point.episode }}{% if point.trajectory_kl_episode is none %}missing{% else %}{{ fmt_num(point.trajectory_kl_episode) }}{% endif %}{% if point.mean_local_policy_kl is none %}missing{% else %}{{ fmt_num(point.mean_local_policy_kl) }}{% endif %}{{ point.decision_steps }}{{ point.estimand }}{{ point.status }}
+ {% for point in lr.get('ig_history') %}{% endfor %}
EpisodePolicy IG
nats / decision		Local KL sum
nats / episode		Decision stepsEstimandStatus
{{ point.episode }}{% if point.information_gain is none %}missing{% else %}{{ fmt_num(point.information_gain) }}{% endif %}{% if point.local_policy_kl_sum is none %}missing{% else %}{{ fmt_num(point.local_policy_kl_sum) }}{% endif %}{{ point.decision_steps }}{{ point.estimand }}{{ point.status }}
{% else %}
No policy KL trace events found.
{% endif %}
diff --git a/src/agentbench_frame/tracking/run.py b/src/agentbench_frame/tracking/run.py index e8e03cc..61e6b9f 100644 --- a/src/agentbench_frame/tracking/run.py +++ b/src/agentbench_frame/tracking/run.py @@ -20,7 +20,14 @@ from agentbench_frame.tracking.wrappers import TrackedEnv, TimedAgent from agentbench_frame.tracking.budget import BudgetLedger from agentbench_frame.tracking.iteration import ActRecord, VersionedActRecorder -from agentbench_frame.eval.information_gain import occupancy_shift as derive_occupancy_shift +from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_KL_DIRECTION, + FORMAL_POLICY_KL_LOG_BASE, + FORMAL_POLICY_KL_SUM_UNIT, + LEGACY_POLICY_KL_TRACE_PROFILE, + occupancy_shift as derive_occupancy_shift, +) +from agentbench_frame.eval.trajectory_kl import trajectory_kl_result_from_payload from agentbench_frame.tracking.quality import inspect_event_file @@ -272,18 +279,25 @@ def log_policy_kl_trace( epsilon: Optional[float] = None, ) -> None: """Persist the raw local-policy KL trace for one target-agent episode.""" + if any(type(value) not in {int, float} for value in trace): + raise TypeError("policy KL trace values must be numbers, not bool") values = [float(value) for value in trace] if any(not math.isfinite(value) or value < 0.0 for value in values): raise ValueError("policy KL trace values must be finite and non-negative") if context_refs is not None and len(context_refs) != len(values): raise ValueError("context_refs must align one-to-one with the KL trace") - if epsilon is not None and ( - not math.isfinite(float(epsilon)) - or not 0.0 < float(epsilon) < 1.0 - ): - raise ValueError("epsilon must be finite and strictly between 0 and 1") - trajectory_kl_episode = sum(values) if values else None + if epsilon is not None: + if type(epsilon) not in {int, float}: + raise TypeError("epsilon must be an int or float, not bool") + epsilon = float(epsilon) + if not math.isfinite(epsilon) or not 0.0 <= epsilon <= 1.0: + raise ValueError("epsilon must be in [0, 1]") errors = [] + try: + trajectory_kl_episode = math.fsum(values) if values else None + except OverflowError: + trajectory_kl_episode = None + errors.append("trajectory KL sum is not finite") if ( trajectory_kl_episode is not None and not math.isfinite(trajectory_kl_episode) @@ -305,6 +319,7 @@ def log_policy_kl_trace( decision_steps=len(values), context_refs=context_refs, epsilon=epsilon, + measurement_profile=LEGACY_POLICY_KL_TRACE_PROFILE, measurement_status="complete" if complete else "incomplete", trajectory_kl_episode=trajectory_kl_episode, mean_local_policy_kl=( @@ -312,8 +327,14 @@ def log_policy_kl_trace( if trajectory_kl_episode is not None else None ), - direction="new||old", - log_base="e", + information_gain=None, + information_gain_status="unverified", + local_policy_kl_sum=trajectory_kl_episode, + aggregation=None, + information_gain_unit=None, + local_policy_kl_sum_unit=FORMAL_POLICY_KL_SUM_UNIT, + direction=FORMAL_POLICY_KL_DIRECTION, + log_base=FORMAL_POLICY_KL_LOG_BASE, rollout_source="unspecified", estimand=estimand, errors=errors, @@ -329,66 +350,12 @@ def log_trajectory_kl_result(self, result: Any) -> None: else: raise TypeError("trajectory KL result must be a mapping or expose to_dict()") - status = payload.get("measurement_status", payload.get("status")) - if status not in {"complete", "incomplete"}: - raise ValueError("trajectory KL measurement status must be complete or incomplete") - trace = payload.get("trace") - decisions = payload.get("decisions") - if not isinstance(trace, list) or not isinstance(decisions, list): - raise ValueError("trajectory KL result must contain trace and decisions lists") - if len(trace) != len(decisions): - raise ValueError("trajectory KL decisions must align one-to-one with the trace") - - if status == "complete": - if not trace: - raise ValueError("a complete trajectory KL result cannot be empty") - values = [float(value) for value in trace] - if any(not math.isfinite(value) or value < 0.0 for value in values): - raise ValueError( - "complete trajectory KL trace values must be finite and non-negative" - ) - trajectory_kl_episode = sum(values) - if math.isfinite(trajectory_kl_episode): - mean_local_policy_kl = trajectory_kl_episode / len(values) - else: - status = "incomplete" - trajectory_kl_episode = None - mean_local_policy_kl = None - errors = payload.get("errors") - if not isinstance(errors, list): - errors = [] - payload["errors"] = [ - *errors, - "trajectory KL sum is not finite", - ] - trace = values - else: - for value in trace: - if value is not None: - number = float(value) - if not math.isfinite(number) or number < 0.0: - raise ValueError( - "available trajectory KL trace values must be finite and non-negative" - ) - trajectory_kl_episode = None - mean_local_policy_kl = None - + verified = trajectory_kl_result_from_payload(payload) + payload = verified.to_dict() payload.pop("status", None) - payload["measurement_status"] = status - payload["trace"] = trace - payload["decision_steps"] = len(decisions) - payload["trajectory_kl_episode"] = trajectory_kl_episode - payload["mean_local_policy_kl"] = mean_local_policy_kl - payload.setdefault("direction", "new||old") - payload.setdefault("log_base", "e") - payload.setdefault("rollout_source", "new_policy") - payload.setdefault( - "estimand", - "epsilon_regularized_local_kl_sum_under_new_policy_occupancy", - ) payload.setdefault( "context_refs", - [decision.get("context_ref") for decision in decisions], + [decision.context_ref for decision in verified.decisions], ) self.write("policy_kl_trace", **payload) diff --git a/tests/miracle/test_decision_kl_core_v1.py b/tests/miracle/test_decision_kl_core_v1.py index f3e11de..1b3cbe1 100644 --- a/tests/miracle/test_decision_kl_core_v1.py +++ b/tests/miracle/test_decision_kl_core_v1.py @@ -1,3 +1,4 @@ +import copy import inspect import json import math @@ -90,26 +91,35 @@ def test_wind_blessing_fails_closed_instead_of_claiming_finite_support(): kl.build_trusted_action_support(empty_observation(artifact=wind)) -def test_local_kl_is_strict_unsmoothed_old_new_natural_log(): +def test_local_kl_is_smoothed_new_old_natural_log(): record = local({"a": 0.52, "b": 0.48}, {"a": 0.5, "b": 0.5}) - expected = 0.52 * math.log(0.52 / 0.5) + 0.48 * math.log(0.48 / 0.5) + new = [0.5, 0.5] + old = [(1 - kl.EPSILON) * value + kl.EPSILON / 2 for value in (0.52, 0.48)] + expected = sum(p * math.log(p / q) for p, q in zip(new, old, strict=True)) reverse = local({"a": 0.5, "b": 0.5}, {"a": 0.52, "b": 0.48}) assert record.status == "complete" assert record.trajectory_kl == pytest.approx(expected) assert record.trajectory_kl != pytest.approx(reverse.trajectory_kl) - assert record.direction == "old||new" - assert record.smoothing == "none" + assert record.direction == "new||old" + assert record.epsilon == 0.01 + assert record.smoothing == "symmetric_epsilon_uniform_full_support" assert record.log_base == "e" def test_zero_probability_rules_are_structured_and_json_safe(): zero_old = local({"a": 0.0, "b": 1.0}, {"a": 0.5, "b": 0.5}) - assert zero_old.trajectory_kl == pytest.approx(math.log(2.0)) - infinite = local({"a": 1.0, "b": 0.0}, {"a": 0.0, "b": 1.0}) - assert infinite.status == "threshold_failed" - assert infinite.trajectory_kl is None - assert infinite.reason == "old_positive_new_zero" - json.dumps(infinite.to_dict(), allow_nan=False) + expected = sum( + p * math.log(p / q) + for p, q in zip((0.5, 0.5), (0.005, 0.995), strict=True) + ) + assert zero_old.trajectory_kl == pytest.approx(expected) + changed = local({"a": 1.0, "b": 0.0}, {"a": 0.0, "b": 1.0}) + assert changed.status == "complete" + assert changed.trajectory_kl == pytest.approx( + 0.995 * math.log(0.995 / 0.005) + 0.005 * math.log(0.005 / 0.995) + ) + assert changed.reason is None + json.dumps(changed.to_dict(), allow_nan=False) @pytest.mark.parametrize( @@ -133,7 +143,7 @@ def test_zero_probability_rules_are_structured_and_json_safe(): ], ids=["old-nonstrict", "new-nonstrict", "old-priority"], ) -def test_strict_mass_checks_precede_old_positive_new_zero(old, new, reason): +def test_strict_mass_checks_precede_smoothed_kl(old, new, reason): summary = local(old, new) assert summary.status == "incomplete" assert summary.trajectory_kl is None @@ -189,7 +199,7 @@ def test_multiple_large_finite_probabilities_are_structured_incomplete( json.dumps(summary.to_dict(), allow_nan=False) -def test_strict_zero_failure_keeps_priority_over_mass_incomplete_step(): +def test_mass_incomplete_keeps_priority_over_finite_smoothed_change(): state = empty_observation() support = kl.build_trusted_action_support(state) first, second = support.action_ids @@ -208,11 +218,15 @@ def test_strict_zero_failure_keeps_priority_over_mass_incomplete_step(): new_distribution={first: 0.0, second: 1.0}, ) summary = kl.compute_trajectory_kl([mass_incomplete, strict_zero]) - assert summary.status == "threshold_failed" + assert summary.status == "incomplete" assert summary.trajectory_kl is None - assert summary.threshold_passed is False - assert summary.reason == "old_positive_new_zero" - assert summary.trace == (None, None) + assert summary.threshold_passed is None + assert summary.reason == "old_distribution_mass_not_strict" + assert summary.trace[0] is None + assert summary.trace[1] == pytest.approx( + 0.995 * math.log(0.995 / 0.005) + + 0.005 * math.log(0.005 / 0.995) + ) @pytest.mark.parametrize("forgery", ["schema", "action_id", "illegal_command"]) @@ -275,12 +289,15 @@ def test_tiny_nonzero_new_probability_has_finite_trajectory_kl(): ) ] ) - expected = math.log(1.0) - math.log(5e-324) + expected = ( + 0.005 * math.log(0.005 / 0.995) + + 0.995 * math.log(0.995 / 0.005) + ) assert math.isfinite(summary.trajectory_kl) assert summary.trajectory_kl == pytest.approx(expected) assert summary.trace == pytest.approx((expected,)) - assert summary.status == "threshold_failed" - assert summary.reason == "trajectory_kl_above_threshold" + assert summary.status == "complete" + assert summary.reason is None @pytest.mark.parametrize("invalid", [True, False, "0.5", -0.1, math.nan, math.inf]) @@ -349,11 +366,24 @@ def trajectory_evidence( if old_distribution is None: old_distribution = {first: 0.5, second: 0.5} if new_distribution is None: - first_probability = ( - 0.5 - if target_kl == 0.0 - else (1 - math.sqrt(1 - math.exp(-2 * target_kl))) / 2 - ) + if target_kl == 0.0: + first_probability = 0.5 + else: + lower, upper = 0.0, 0.5 + for _ in range(100): + first_probability = (lower + upper) / 2 + smoothed = ( + (1 - kl.EPSILON) * first_probability + kl.EPSILON / 2 + ) + measured = ( + smoothed * math.log(smoothed / 0.5) + + (1 - smoothed) * math.log((1 - smoothed) / 0.5) + ) + if measured > target_kl: + lower = first_probability + else: + upper = first_probability + first_probability = (lower + upper) / 2 new_distribution = { first: first_probability, second: 1 - first_probability, @@ -614,10 +644,11 @@ def test_exact_unit_mass_zero_kl_remains_complete(): summary = local({"a": 0.5, "b": 0.5}, {"a": 0.5, "b": 0.5}) assert summary.status == "complete" assert summary.trajectory_kl == 0.0 - assert summary.threshold_passed is True + assert summary.threshold_passed is None + assert summary.acceptance_threshold is None -def test_threshold_failure_has_priority_over_other_incomplete_decisions(): +def test_incomplete_decision_has_priority_over_finite_smoothed_change(): state = empty_observation() support = kl.build_trusted_action_support(state) first, second = support.action_ids @@ -640,11 +671,15 @@ def test_threshold_failure_has_priority_over_other_incomplete_decisions(): new_distribution={first: 0.0, second: 1.0}, ) summary = kl.compute_trajectory_kl([incomplete, failed]) - assert summary.status == "threshold_failed" + assert summary.status == "incomplete" assert summary.trajectory_kl is None - assert summary.threshold_passed is False - assert summary.reason == "old_positive_new_zero" - assert summary.trace == (None, None) + assert summary.threshold_passed is None + assert summary.reason == "action_support_not_finitely_enumerable" + assert summary.trace[0] is None + assert summary.trace[1] == pytest.approx( + 0.995 * math.log(0.995 / 0.005) + + 0.005 * math.log(0.005 / 0.995) + ) def test_trajectory_preserves_ordered_per_decision_provenance_and_priority(): @@ -679,9 +714,13 @@ def test_trajectory_preserves_ordered_per_decision_provenance_and_priority(): summary = kl.compute_trajectory_kl( [unavailable, nonstrict_mass, strict_zero] ) - assert summary.status == "threshold_failed" - assert summary.reason == "old_positive_new_zero" - assert summary.trace == (None, None, None) + assert summary.status == "incomplete" + assert summary.reason == "action_support_not_finitely_enumerable" + assert summary.trace[:2] == (None, None) + assert summary.trace[2] == pytest.approx( + 0.995 * math.log(0.995 / 0.005) + + 0.005 * math.log(0.005 / 0.995) + ) assert tuple(record.decision_step for record in summary.decision_records) == (1, 2, 3) records = summary.to_dict()["decision_records"] @@ -715,9 +754,12 @@ def test_trajectory_preserves_ordered_per_decision_provenance_and_priority(): }, { "decision_step": 3, - "status": "threshold_failed", - "local_kl": None, - "reason": "old_positive_new_zero", + "status": "complete", + "local_kl": pytest.approx( + 0.995 * math.log(0.995 / 0.005) + + 0.005 * math.log(0.005 / 0.995) + ), + "reason": None, "schema_version": support.schema_version, "support_id": support.support_id, "action_ids": list(support.action_ids), @@ -768,34 +810,27 @@ def test_summary_fake_only_boundary_cannot_be_overridden(field, value): replace(summary, **{field: value}) -@pytest.mark.parametrize( - ("value", "passed"), - [(0.009999999, True), (0.01, True), (0.010000001, False)], -) -def test_threshold_is_exact_without_hidden_tolerance(value, passed): - assert kl._passes_acceptance_threshold(value) is passed +@pytest.mark.parametrize("value", [0.009999999, 0.01, 0.010000001]) +def test_policy_change_magnitude_does_not_gate_measurement(value): summary = kl.compute_trajectory_kl([trajectory_evidence(value)]) - assert summary.threshold_passed is ( - summary.trajectory_kl <= summary.acceptance_threshold - ) - assert summary.status == ( - "complete" if summary.threshold_passed else "threshold_failed" - ) - assert summary.acceptance_threshold == 0.01 + assert summary.status == "complete" + assert summary.trajectory_kl == pytest.approx(value) + assert summary.threshold_passed is None + assert summary.acceptance_threshold is None -def test_incomplete_or_infinite_decision_cannot_produce_partial_scalar(): +def test_incomplete_decision_cannot_produce_partial_scalar(): state = empty_observation() support = kl.build_trusted_action_support(state) first, second = support.action_ids failed = trajectory_evidence( step=2, state_before=state, - old_distribution={first: 1.0, second: 0.0}, + old_distribution={first: 1.0}, new_distribution={first: 0.0, second: 1.0}, ) summary = kl.compute_trajectory_kl([trajectory_evidence(0.0, step=1), failed]) - assert summary.status == "threshold_failed" + assert summary.status == "incomplete" assert summary.trajectory_kl is None assert summary.trace == (0.0, None) json.dumps(summary.to_dict(), allow_nan=False) @@ -996,6 +1031,8 @@ def test_decision_records_are_issuer_bound_and_cannot_be_replaced_or_forged(): with pytest.raises((TypeError, ValueError)): replace(record, local_kl=999.0) + with pytest.raises(TypeError, match="issued"): + copy.copy(record) forged = object.__new__(kl.DecisionKLRecord) for field_name in ( @@ -1048,7 +1085,7 @@ def test_unavailable_support_record_contract_is_bound_to_issuer_snapshot(): ] ) record = summary.decision_records[0] - object.__setattr__(record, "direction", "new||old") + object.__setattr__(record, "direction", "old||new") with pytest.raises(ValueError, match="issued"): summary.to_dict() diff --git a/tests/miracle/test_ifelse_policy_state_v1.py b/tests/miracle/test_ifelse_policy_state_v1.py new file mode 100644 index 0000000..791735c --- /dev/null +++ b/tests/miracle/test_ifelse_policy_state_v1.py @@ -0,0 +1,502 @@ +import copy +import json +import os + +import pytest + +from agentbench_frame.games.miracle import ifelse_policy_state_v1 as policy_state +from agentbench_frame.games.miracle.decision_kl_v1 import ( + build_trusted_action_support, +) + + +def _explicit_config(*, opening="FF"): + values = {} + for spec in policy_state.MIRACLE_CONFIG_SPECS: + if spec.name == "MIRACLE_CAMP1_OPENING": + values[spec.name] = opening + elif spec.name == "MIRACLE_ARTIFACT": + values[spec.name] = "InfernoFlame" + elif spec.name == "MIRACLE_DECK": + values[spec.name] = ["Priest", "Archer", "Swordsman"] + else: + values[spec.name] = False + return policy_state.PolicyConfigV1.from_explicit(values) + + +def _observation(*, camp=0, round_number=0): + return { + "round": round_number, + "camp": camp, + "map": {"units": [], "barracks": [-1] * 4, "miracles": [30, 30]}, + "players": [ + [[[0, 2, 8, 6, 0, 0, 0, [-1, -1, -1]]], 2, 2, + [[3, 3, []], [0, 3, []], [1, 6, []]], []], + [[[1, 2, 8, 6, 0, 0, 0, [-1, -1, -1]]], 2, 2, + [[3, 3, []], [0, 3, []], [1, 6, []]], []], + ], + } + + +def _init_command(*, camp=0, artifacts=None, creatures=None): + return { + "player": camp, + "round": 0, + "operation_type": "init", + "operation_parameters": { + "artifacts": ["InfernoFlame"] if artifacts is None else artifacts, + "creatures": ( + ["Priest", "Archer", "Swordsman"] + if creatures is None + else creatures + ), + }, + } + + +def _install_init_selector(monkeypatch): + monkeypatch.setattr( + policy_state, + "_select_legacy_command", + lambda *args: policy_state.SelectedCommandV1( + _init_command(), "turn_start", "init" + ), + ) + + +def test_config_contract_freezes_all_62_inputs_and_has_stable_identity(): + assert len(policy_state.MIRACLE_CONFIG_SPECS) == 62 + config = _explicit_config() + assert config.complete is True + assert len(config.values) == 62 + assert config.sha256 == policy_state.PolicyConfigV1.from_explicit( + dict(config.to_dict()["values"]) + ).sha256 + assert json.loads(config.canonical_bytes)["schema_version"] == ( + policy_state.POLICY_CONFIG_SCHEMA_VERSION + ) + + +def test_missing_or_unknown_config_is_not_formal_policy_evidence(): + values = dict(_explicit_config().to_dict()["values"]) + values.pop("MIRACLE_GATE_DEFENSE") + with pytest.raises(ValueError, match="exactly all 62"): + policy_state.PolicyConfigV1.from_explicit(values) + + historical = policy_state.PolicyConfigV1.historical_unknown( + observed={ + "MIRACLE_ARTIFACT": "InfernoFlame", + "MIRACLE_DECK": ["Priest", "Archer", "Swordsman"], + } + ) + assert historical.complete is False + assert historical.values["MIRACLE_GATE_DEFENSE"] == "unknown" + with pytest.raises(policy_state.IncompletePolicyEvidenceError): + historical.require_complete() + + +def test_config_types_and_opening_version_are_strict(): + values = dict(_explicit_config().to_dict()["values"]) + values["MIRACLE_GATE_DEFENSE"] = 1 + with pytest.raises(TypeError, match="MIRACLE_GATE_DEFENSE"): + policy_state.PolicyConfigV1.from_explicit(values) + values = dict(_explicit_config().to_dict()["values"]) + values["MIRACLE_CAMP1_OPENING"] = "XX" + with pytest.raises(ValueError, match="MIRACLE_CAMP1_OPENING"): + policy_state.PolicyConfigV1.from_explicit(values) + + values = dict(_explicit_config().to_dict()["values"]) + values["MIRACLE_ARTIFACT"] = "AttackerInventedArtifact" + with pytest.raises(ValueError, match="MIRACLE_ARTIFACT"): + policy_state.PolicyConfigV1.from_explicit(values) + + values = dict(_explicit_config().to_dict()["values"]) + values["MIRACLE_DECK"] = ["Priest", "Archer", "AttackerInventedUnit"] + with pytest.raises(ValueError, match="MIRACLE_DECK"): + policy_state.PolicyConfigV1.from_explicit(values) + + +def test_memory_is_issuer_bound_serializable_and_reset_is_unique(): + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + first = machine.reset(camp=0, episode_id="episode-1") + second = machine.reset(camp=0, episode_id="episode-1") + assert first.to_dict() == second.to_dict() + assert first.schema_version == policy_state.POLICY_MEMORY_SCHEMA_VERSION + assert first.phase == "turn_start" + assert first.decision_step == 1 + json.dumps(first.to_dict(), allow_nan=False) + + copied = copy.copy(first) + with pytest.raises(ValueError, match="issued"): + copied.to_dict() + with pytest.raises(TypeError, match="issued"): + policy_state.PolicyMemoryV1() + + +def test_mutated_replaced_cross_episode_or_wrong_identity_memory_is_rejected(): + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + object.__setattr__(memory, "decision_step", 9) + with pytest.raises(ValueError, match="issued"): + machine.validate_memory(memory) + + other_episode = machine.reset(camp=0, episode_id="episode-2") + with pytest.raises(ValueError, match="episode"): + machine.validate_memory(other_episode, episode_id="episode-1") + + other_context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v2", config=_explicit_config() + ) + other_machine = policy_state.ExplicitIfElseStateMachineV1(other_context) + with pytest.raises(ValueError, match="policy identity"): + other_machine.validate_memory( + machine.reset(camp=0, episode_id="episode-1") + ) + + +def test_pair_provider_recomputes_strict_one_hot_on_same_state_memory_and_support( + monkeypatch, +): + config = _explicit_config() + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=config + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + observation = {"camp": 0} + support = build_trusted_action_support(observation) + + def fake_select(version, observation, memory, config): + assert memory.decision_step == 1 + assert observation["camp"] == 0 + command = { + "player": 0, + "round": 0, + "operation_type": "init", + "operation_parameters": { + "artifacts": ["InfernoFlame"], + "creatures": ["Priest", "Archer", "Swordsman"], + }, + } + return policy_state.SelectedCommandV1(command, "turn_start") + + monkeypatch.setattr(policy_state, "_select_legacy_command", fake_select) + decision = machine.evaluate_pair(observation, memory, support) + assert decision.old_action_id == decision.new_action_id + assert set(decision.old_distribution) == set(support.action_ids) + assert set(decision.new_distribution) == set(support.action_ids) + assert sum(decision.old_distribution.values()) == 1.0 + assert sum(decision.new_distribution.values()) == 1.0 + assert set(decision.old_distribution.values()) <= {0.0, 1.0} + assert set(decision.new_distribution.values()) <= {0.0, 1.0} + assert decision.memory_before_sha256 == memory.sha256 + assert decision.m_after.decision_step == 2 + assert decision.m_after.previous_transition_sha256 == decision.transition_sha256 + + +def test_provider_rejects_incomplete_or_forged_support_and_uploaded_distribution( + monkeypatch, +): + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + observation = _observation() + support = build_trusted_action_support(observation) + + command = { + "player": 0, + "round": 0, + "operation_type": "init", + "operation_parameters": { + "artifacts": ["InfernoFlame"], + "creatures": ["Priest", "Archer", "Swordsman"], + }, + } + monkeypatch.setattr( + policy_state, + "_select_legacy_command", + lambda *args: policy_state.SelectedCommandV1(command, "turn_start"), + ) + forged = copy.deepcopy(support) + object.__setattr__(forged, "schema_version", "forged") + with pytest.raises(ValueError, match="support"): + machine.evaluate_pair(observation, memory, forged) + assert "distribution" not in machine.evaluate_pair.__annotations__ + + +def test_transition_tip_rejects_replayed_or_skipped_memory(monkeypatch): + _install_init_selector(monkeypatch) + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + observation = {"camp": 0} + support = build_trusted_action_support(observation) + + first = machine.evaluate_pair(observation, memory, support) + with pytest.raises(ValueError, match="current transition tip"): + machine.evaluate_pair(observation, memory, support) + + skipped = machine.reset(camp=0, episode_id="episode-2") + object.__setattr__(skipped, "decision_step", 3) + with pytest.raises(ValueError, match="issued"): + machine.evaluate_pair(observation, skipped, support) + + second = machine.evaluate_pair(observation, first.m_after, support) + assert second.decision_step == 2 + assert second.episode_id == "episode-1" + assert second.memory_before_sha256 == first.m_after.sha256 + + +def test_pair_decision_is_issuer_bound_and_snapshot_protected(monkeypatch): + _install_init_selector(monkeypatch) + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + observation = {"camp": 0} + support = build_trusted_action_support(observation) + decision = machine.evaluate_pair(observation, memory, support) + + machine.validate_decision( + decision, + episode_id="episode-1", + decision_step=1, + observation=observation, + complete_action_support=support, + ) + json.dumps(decision.to_dict(), allow_nan=False) + + copied = copy.copy(decision) + with pytest.raises(ValueError, match="issued decision"): + machine.validate_decision( + copied, + episode_id="episode-1", + decision_step=1, + observation=observation, + complete_action_support=support, + ) + + object.__setattr__(decision, "old_action_id", "forged") + with pytest.raises(ValueError, match="issued decision"): + machine.validate_decision( + decision, + episode_id="episode-1", + decision_step=1, + observation=observation, + complete_action_support=support, + ) + + with pytest.raises(TypeError, match="issued"): + policy_state.PolicyPairDecisionV1() + + +def test_pair_decision_binds_context_step_and_distinct_provider_choices(monkeypatch): + config = _explicit_config() + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=config + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + observation = {"camp": 0} + support = build_trusted_action_support(observation) + seen = [] + + def different_select(provider_identity, observation, supplied_memory, config): + seen.append((provider_identity, supplied_memory)) + creatures = ["Priest", "Archer", "Swordsman"] + if provider_identity == context.new_provider.sha256: + creatures = ["Archer", "Priest", "Swordsman"] + return policy_state.SelectedCommandV1( + _init_command(creatures=creatures), "turn_start", "init" + ) + + monkeypatch.setattr(policy_state, "_select_legacy_command", different_select) + decision = machine.evaluate_pair(observation, memory, support) + assert decision.old_action_id != decision.new_action_id + assert seen == [ + (context.old_provider.sha256, memory), + (context.new_provider.sha256, memory), + ] + with pytest.raises(ValueError, match="episode"): + machine.validate_decision( + decision, + episode_id="another-episode", + decision_step=1, + observation=observation, + complete_action_support=support, + ) + with pytest.raises(ValueError, match="step"): + machine.validate_decision( + decision, + episode_id="episode-1", + decision_step=2, + observation=observation, + complete_action_support=support, + ) + + +def test_invalid_state_transition_fields_fail_closed(monkeypatch): + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + memory = machine.reset(camp=0, episode_id="episode-1") + observation = {"camp": 0} + support = build_trusted_action_support(observation) + monkeypatch.setattr( + policy_state, + "_select_legacy_command", + lambda *args: policy_state.SelectedCommandV1( + _init_command(), + "move", + "move.move", + memory_updates={"current_unit_cursor": -1}, + ), + ) + with pytest.raises(ValueError, match="current_unit_cursor"): + machine.evaluate_pair(observation, memory, support) + + +def test_source_identity_detects_replacement(tmp_path): + source_root = tmp_path / "policy" + source_root.mkdir() + for name in policy_state.LEGACY_SOURCE_FILES: + (source_root / name).write_text( + "{}\n" if name == "Data.json" else f"# {name}\n", + encoding="utf-8", + newline="\n", + ) + source = policy_state.LegacyPolicySourceV1.open(source_root, version="v0") + source.revalidate() + (source_root / "main.py").write_text( + "# replaced\n", encoding="utf-8", newline="\n" + ) + with pytest.raises(policy_state.PolicySourceError, match="identity mismatch"): + source.revalidate() + + +def test_source_loader_is_read_only_and_never_writes_bytecode(tmp_path): + source_root = tmp_path / "policy" + source_root.mkdir() + for name in policy_state.LEGACY_SOURCE_FILES: + payload = ( + "{}\n" + if name == "Data.json" + else "class IfElseAI:\n pass\n" + if name == "main.py" + else f"# {name}\n" + ) + (source_root / name).write_text( + payload, encoding="utf-8", newline="\n" + ) + source = policy_state.LegacyPolicySourceV1.open(source_root, version="v0") + policy_state._LegacyIfElseRuntime(source) + assert not (source_root / "__pycache__").exists() + + +def test_all_source_files_require_strict_utf8_without_bom(tmp_path): + source_root = tmp_path / "policy" + source_root.mkdir() + for name in policy_state.LEGACY_SOURCE_FILES: + (source_root / name).write_bytes( + b"{}\n" if name == "Data.json" else f"# {name}\n".encode("utf-8") + ) + (source_root / "Data.json").write_bytes(b"\xef\xbb\xbf{}\n") + with pytest.raises(policy_state.PolicySourceError, match="BOM"): + policy_state.LegacyPolicySourceV1.open(source_root, version="v0") + + +def test_trace_replacement_after_preflight_is_rejected_even_with_same_bytes( + tmp_path, monkeypatch +): + observation = {"camp": 0} + encoded = json.dumps(observation, separators=(",", ":"), sort_keys=True) + framed = f"{len(encoded.encode('utf-8')):06d}{encoded}" + records = [ + { + "kind": "judge_frame", + "frame": {"listen": [0], "player": [0], "content": [framed]}, + }, + {"kind": "ai_operation", "player": 0, "operation": _init_command()}, + ] + trace = tmp_path / "trace.jsonl" + trace.write_text( + "".join(json.dumps(record, separators=(",", ":")) + "\n" for record in records), + encoding="utf-8", + newline="\n", + ) + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + _install_init_selector(monkeypatch) + original_evaluate = machine.evaluate_pair + + def replace_after_evaluate(*args, **kwargs): + decision = original_evaluate(*args, **kwargs) + replacement = tmp_path / "replacement.jsonl" + replacement.write_bytes(trace.read_bytes()) + os.replace(replacement, trace) + return decision + + monkeypatch.setattr(machine, "evaluate_pair", replace_after_evaluate) + with pytest.raises(ValueError, match="changed during"): + policy_state.audit_sequential_trace( + trace, + machine=machine, + evaluated_camp=0, + episode_id="trace-replacement", + ) + + +def test_sequential_audit_serializes_every_memory_before_and_pair_decision( + tmp_path, monkeypatch +): + observation = {"camp": 0} + encoded = json.dumps(observation, separators=(",", ":"), sort_keys=True) + framed = f"{len(encoded.encode('utf-8')):06d}{encoded}" + records = [ + { + "kind": "judge_frame", + "frame": {"listen": [0], "player": [0], "content": [framed]}, + }, + {"kind": "ai_operation", "player": 0, "operation": _init_command()}, + ] + trace = tmp_path / "trace.jsonl" + trace.write_text( + "".join(json.dumps(record, separators=(",", ":")) + "\n" for record in records), + encoding="utf-8", + newline="\n", + ) + context = policy_state.PolicyComparisonIdentityV1.for_test( + old_version="v0", new_version="v1", config=_explicit_config() + ) + machine = policy_state.ExplicitIfElseStateMachineV1(context) + _install_init_selector(monkeypatch) + + audit = policy_state.audit_sequential_trace( + trace, + machine=machine, + evaluated_camp=0, + episode_id="serialized-memory", + ) + assert audit.complete is True + payload = audit.to_dict() + assert len(payload["decision_records"]) == 1 + record = payload["decision_records"][0] + assert record["decision_step"] == 1 + assert record["m_before"] == audit.memory_before[0].to_dict() + assert record["pair_decision"] == audit.decisions[0].to_dict() + assert record["pair_decision"]["memory_before_sha256"] == audit.memory_before[0].sha256 + assert record["recorded_action_id"] == record["pair_decision"]["new_action_id"] diff --git a/tests/miracle/test_information_gain_formal_contract.py b/tests/miracle/test_information_gain_formal_contract.py new file mode 100644 index 0000000..e05ef65 --- /dev/null +++ b/tests/miracle/test_information_gain_formal_contract.py @@ -0,0 +1,278 @@ +import math +from types import SimpleNamespace + +import pytest + +from agentbench_frame.eval.measurement import ActionCandidate, ActionSupport +from agentbench_frame.games.miracle import decision_kl_v1 as decision_kl + + +MAIN_EPSILON = 0.01 + + +def _observation(): + return { + "round": 1, + "camp": 0, + "map": {"units": [], "barracks": [-1] * 4, "miracles": [30, 30]}, + "players": [[[], 20, 20, [], []], [[], 0, 0, [], []]], + } + + +def _identity(support): + return { + "schema_version": support.schema_version, + "support_id": support.support_id, + "action_ids": list(support.action_ids), + } + + +def _evidence(old, new, *, step=1): + observation = _observation() + support = decision_kl.build_trusted_action_support(observation) + first, second = support.action_ids + aliases = {"a": first, "b": second} + return decision_kl.DecisionKLEvidence( + decision_step=step, + state_before=observation, + support_identity=_identity(support), + old_distribution={aliases[key]: value for key, value in old.items()}, + new_distribution={aliases[key]: value for key, value in new.items()}, + ) + + +def _smoothed(values, epsilon=MAIN_EPSILON): + uniform = 1.0 / len(values) + return [(1.0 - epsilon) * value + epsilon * uniform for value in values] + + +def _new_old_kl(new, old, epsilon=MAIN_EPSILON): + new_smoothed = _smoothed(new, epsilon) + old_smoothed = _smoothed(old, epsilon) + return math.fsum( + new_probability * math.log(new_probability / old_probability) + for new_probability, old_probability in zip( + new_smoothed, old_smoothed, strict=True + ) + ) + + +def test_miracle_local_kl_uses_new_old_direction_and_fixed_symmetric_smoothing(): + old = {"a": 0.8, "b": 0.2} + new = {"a": 0.55, "b": 0.45} + summary = decision_kl.compute_trajectory_kl([_evidence(old, new)]) + + expected = _new_old_kl([0.55, 0.45], [0.8, 0.2]) + reverse = _new_old_kl([0.8, 0.2], [0.55, 0.45]) + assert summary.status == "complete" + assert summary.trajectory_kl == pytest.approx(expected) + assert summary.trajectory_kl != pytest.approx(reverse) + assert summary.direction == "new||old" + assert summary.epsilon == MAIN_EPSILON + assert summary.smoothing == "symmetric_epsilon_uniform_full_support" + assert summary.log_base == "e" + + +def test_deterministic_hl_unchanged_is_zero_and_changed_is_finite_positive(): + unchanged = decision_kl.compute_trajectory_kl( + [_evidence({"a": 1.0, "b": 0.0}, {"a": 1.0, "b": 0.0})] + ) + changed = decision_kl.compute_trajectory_kl( + [_evidence({"a": 1.0, "b": 0.0}, {"a": 0.0, "b": 1.0})] + ) + + assert unchanged.status == "complete" + assert unchanged.trajectory_kl == pytest.approx(0.0) + assert changed.status == "complete" + assert changed.trajectory_kl == pytest.approx( + _new_old_kl([0.0, 1.0], [1.0, 0.0]) + ) + assert math.isfinite(changed.trajectory_kl) + assert changed.trajectory_kl > 0.0 + + +def test_episode_primary_information_gain_is_trace_mean_and_sum_is_distinct(): + evidence = [ + _evidence({"a": 0.8, "b": 0.2}, {"a": 0.6, "b": 0.4}, step=1), + _evidence({"a": 0.3, "b": 0.7}, {"a": 0.5, "b": 0.5}, step=2), + ] + summary = decision_kl.compute_trajectory_kl(evidence) + + assert summary.information_gain == pytest.approx(math.fsum(summary.trace) / 2) + assert summary.sum_local_kl == pytest.approx(math.fsum(summary.trace)) + assert summary.information_gain_unit == "nats / decision" + assert summary.sum_local_kl_unit == "nats / episode" + assert summary.aggregation == "arithmetic_mean" + + +def test_formal_runtime_config_has_no_noncanonical_epsilon_entry(): + from agentbench_frame.eval.trajectory_kl import TrajectoryKLConfig + + config = TrajectoryKLConfig.for_policy_information_gain( + "old", "new", metadata={"sensitivity_epsilon": 0.05} + ) + assert config.epsilon == MAIN_EPSILON + assert config.metadata["sensitivity_epsilon"] == 0.05 + with pytest.raises(TypeError): + TrajectoryKLConfig.for_policy_information_gain( + "old", "new", epsilon=0.05 + ) + + +def test_generic_runtime_epsilon_is_separate_from_formal_information_gain(): + from agentbench_frame.eval.information_gain import ( + FORMAL_POLICY_INFORMATION_GAIN_PROFILE, + GENERIC_TRAJECTORY_KL_PROFILE, + ) + from agentbench_frame.eval.trajectory_kl import TrajectoryKLConfig + + generic = TrajectoryKLConfig("old", "new", 0.05) + assert generic.measurement_profile == GENERIC_TRAJECTORY_KL_PROFILE + assert not generic.is_formal_policy_information_gain + + formal = TrajectoryKLConfig.for_policy_information_gain("old", "new") + assert formal.epsilon == MAIN_EPSILON + assert formal.measurement_profile == FORMAL_POLICY_INFORMATION_GAIN_PROFILE + assert formal.is_formal_policy_information_gain + with pytest.raises(TypeError): + TrajectoryKLConfig.for_policy_information_gain( + "old", "new", epsilon=0.05 + ) + + +def test_sensitivity_is_reproducible_without_changing_main_identity(): + from agentbench_frame.eval import information_gain + + sensitivity = information_gain.policy_kl_sensitivity( + [0.0, 1.0], + [1.0, 0.0], + ) + repeated = information_gain.policy_kl_sensitivity( + [0.0, 1.0], + [1.0, 0.0], + ) + + assert sensitivity == repeated + assert tuple(sensitivity) == (0.001, 0.01, 0.05) + assert sensitivity[MAIN_EPSILON] == pytest.approx( + information_gain.policy_kl( + [0.0, 1.0], [1.0, 0.0], epsilon=MAIN_EPSILON + ) + ) + assert information_gain.MAIN_POLICY_KL_EPSILON == MAIN_EPSILON + + +def test_generic_probability_boundary_rejects_bool_and_string_coercion(): + from agentbench_frame.eval.information_gain import ( + policy_kl, + validate_policy_distribution, + ) + + support = ActionSupport( + [ActionCandidate("a", "a"), ActionCandidate("b", "b")], + "test-support-v1", + ) + with pytest.raises(TypeError, match="probabil"): + validate_policy_distribution({"a": True, "b": 0.0}, support) + with pytest.raises(TypeError, match="probabil"): + policy_kl(["1", "0"], [1.0, 0.0], epsilon=MAIN_EPSILON) + + +def test_episode_trace_rejects_incomplete_policy_support_instead_of_zero_filling(): + from agentbench_frame.eval.information_gain import episode_policy_kl_trace + + with pytest.raises(ValueError, match="exactly match legal actions"): + episode_policy_kl_trace( + lambda _context: {"a": 1.0}, + lambda _context: {"a": 0.5, "b": 0.5}, + contexts=["state"], + legal_actions=lambda _context: ["a", "b"], + epsilon=MAIN_EPSILON, + ) + + +def test_support_size_one_and_action_id_reordering_are_well_defined(): + from agentbench_frame.eval.information_gain import epsilon_regularize, policy_kl + + assert policy_kl([1.0], [1.0], epsilon=MAIN_EPSILON) == pytest.approx(0.0) + assert epsilon_regularize([1.0, 0.0, 0.0], MAIN_EPSILON) == pytest.approx( + [1.0 - MAIN_EPSILON + MAIN_EPSILON / 3, MAIN_EPSILON / 3, MAIN_EPSILON / 3] + ) + summary = decision_kl.compute_trajectory_kl( + [_evidence({"b": 0.2, "a": 0.8}, {"b": 0.4, "a": 0.6})] + ) + assert summary.status == "complete" + assert summary.trajectory_kl == pytest.approx( + _new_old_kl([0.6, 0.4], [0.8, 0.2]) + ) + + +def test_iteration_validator_recomputes_local_kl_instead_of_trusting_upload(): + from agentbench_frame.games.miracle import iteration_protocol as protocol + + forged_local = 0.5 + payload = { + "episode": 1, + "version_before": "old", + "version_after": "new", + "epsilon": MAIN_EPSILON, + "measurement_profile": "24_miracle_policy_information_gain_v2", + "status": "complete", + "measurement_status": "complete", + "direction": "new||old", + "log_base": "e", + "rollout_source": "new_policy", + "estimand": "epsilon_regularized_local_kl_sum_under_new_policy_occupancy", + "decision_steps": 1, + "trace": [forged_local], + "trajectory_kl_episode": forged_local, + "mean_local_policy_kl": forged_local, + "errors": [], + "decisions": [ + { + "decision_step": 1, + "context_ref": "context", + "action_schema_version": "actions-v1", + "support_id": "support", + "legal_action_ids": ["a", "b"], + "selected_action_id": "a", + "new_distribution": {"a": 1.0, "b": 0.0}, + "old_distribution": {"a": 1.0, "b": 0.0}, + "new_probabilities": [1.0, 0.0], + "old_probabilities": [1.0, 0.0], + "local_policy_kl": forged_local, + "errors": [], + } + ], + } + plan = SimpleNamespace( + baseline_strategy_version="old", candidate_strategy_version="new" + ) + + with pytest.raises(protocol.IterationPreflightError, match="local policy KL"): + protocol._validate_trajectory_kl(payload, plan) + + payload.update( + status="incomplete", + measurement_status="incomplete", + errors=["later decision evidence missing"], + trajectory_kl_episode=None, + mean_local_policy_kl=None, + information_gain=None, + local_policy_kl_sum=None, + ) + with pytest.raises(protocol.IterationPreflightError, match="local policy KL"): + protocol._validate_trajectory_kl(payload, plan) + + +def test_occupancy_shift_remains_separate_from_episode_information_gain(): + from agentbench_frame.eval.information_gain import occupancy_shift + + summary = decision_kl.compute_trajectory_kl( + [_evidence({"a": 0.8, "b": 0.2}, {"a": 0.6, "b": 0.4})] + ) + shift = occupancy_shift(["new-state"], ["old-state"]) + + assert shift > 0.0 + assert "occupancy" not in summary.to_dict() + assert summary.information_gain == pytest.approx(summary.trace[0]) diff --git a/tests/miracle/test_iteration_acceptance.py b/tests/miracle/test_iteration_acceptance.py index 4407efa..5345492 100644 --- a/tests/miracle/test_iteration_acceptance.py +++ b/tests/miracle/test_iteration_acceptance.py @@ -9,6 +9,16 @@ from test_iteration_lifecycle import _candidate_strategy, _make_control_bundle +@pytest.fixture(autouse=True) +def _bind_test_only_internal_memory_evidence(monkeypatch): + protocol = _protocol() + monkeypatch.setattr( + protocol, + "CURRENT_INTERNAL_MEMORY_EVIDENCE", + protocol.INTERNAL_MEMORY_EVIDENCE_BOUND, + ) + + def _protocol(): from agentbench_frame.games.miracle import iteration_protocol @@ -82,22 +92,29 @@ def fake_runner(current): def _kl_result(protocol, *, episode=1, complete=True): - local = 0.02 if complete else None + local = 0.0 if complete else None return { "episode": episode, "version_before": "strategy-v0", "version_after": "strategy-v1", "epsilon": protocol.TRAJECTORY_KL_EPSILON, + "measurement_profile": "24_miracle_policy_information_gain_v2", "status": "complete" if complete else "incomplete", "measurement_status": "complete" if complete else "incomplete", "direction": protocol.KL_DIRECTION, "log_base": "e", "rollout_source": protocol.KL_ROLLOUT_SOURCE, "estimand": "epsilon_regularized_local_kl_sum_under_new_policy_occupancy", + "information_gain_estimand": "epsilon_regularized_mean_local_policy_kl_under_new_policy_occupancy", + "aggregation": "arithmetic_mean", + "information_gain_unit": "nats / decision", + "local_policy_kl_sum_unit": "nats / episode", "decision_steps": 1, "trace": [local], "trajectory_kl_episode": local, "mean_local_policy_kl": local, + "information_gain": local, + "local_policy_kl_sum": local, "errors": [] if complete else ["fake missing old distribution"], "metadata": {"fake_only": True}, "decisions": [ @@ -119,6 +136,64 @@ def _kl_result(protocol, *, episode=1, complete=True): } +def test_iteration_rejects_forged_measurement_profile(tmp_path, monkeypatch): + protocol = _protocol() + monkeypatch.setattr( + protocol, + "CURRENT_INTERNAL_MEMORY_EVIDENCE", + "state_and_internal_memory_bound_to_policy_identity", + raising=False, + ) + plan = type( + "Plan", + (), + { + "baseline_strategy_version": "strategy-v0", + "candidate_strategy_version": "strategy-v1", + }, + )() + payload = _kl_result(protocol) + payload["measurement_profile"] = "generic_trajectory_kl" + + with pytest.raises(protocol.IterationPreflightError, match="profile"): + protocol._validate_trajectory_kl(payload, plan) + + +def test_not_collected_internal_memory_makes_formal_ig_incomplete( + tmp_path, monkeypatch +): + protocol = _protocol() + monkeypatch.setattr( + protocol, + "CURRENT_INTERNAL_MEMORY_EVIDENCE", + "state_and_internal_memory_bound_to_policy_identity", + raising=False, + ) + plan = type( + "Plan", + (), + { + "baseline_strategy_version": "strategy-v0", + "candidate_strategy_version": "strategy-v1", + }, + )() + payload = _kl_result(protocol) + payload["measurement_profile"] = ( + "24_miracle_policy_information_gain_v2" + ) + monkeypatch.setattr( + protocol, + "CURRENT_INTERNAL_MEMORY_EVIDENCE", + "not_collected", + raising=False, + ) + + validated = protocol._validate_trajectory_kl(payload, plan) + assert not validated.complete + assert validated.information_gain is None + assert validated.local_policy_kl_sum is None + + def _evaluation_bundle(tmp_path, monkeypatch, *, candidate_score=0.75, kl_complete=True): protocol = _protocol() bundle, store, baseline, learning, candidate, runner_calls = _store_candidate( @@ -165,7 +240,7 @@ def _evaluation_bundle(tmp_path, monkeypatch, *, candidate_score=0.75, kl_comple candidate_score=candidate_score, terminal_status="complete", trajectory_kl=_kl_result(protocol, complete=kl_complete), - information_gain=0.1, + information_gain=(0.0 if kl_complete else None), failure_reason=None if kl_complete else "KL incomplete", ) evidence_manifest = protocol.EvaluationEvidenceManifest.create( @@ -463,6 +538,17 @@ def test_evaluation_evidence_never_enters_learning_payload_or_experience( ).training_evidence_sha256 +def test_case_information_gain_cannot_override_ordered_trajectory_evidence( + tmp_path, monkeypatch +): + protocol = _protocol() + data = _evaluation_bundle(tmp_path, monkeypatch) + forged = replace(data["evidence"], information_gain=0.5, sha256="") + + with pytest.raises(protocol.IterationPreflightError, match="authoritative ordered KL trace"): + forged.validate(data["plan"]) + + def test_score_improvement_with_incomplete_kl_derives_incomplete(tmp_path, monkeypatch): protocol = _protocol() data = _evaluation_bundle( @@ -492,6 +578,7 @@ def test_fake_complete_evidence_derives_fake_only_completed(tmp_path, monkeypatc assert summary["raw_score"] == 0.5 assert summary["evo_score"] == 0.75 assert summary["gain"] == 0.25 + assert summary["information_gain"] == 0.0 assert summary["research_manifest_sha256"] == protocol.research_manifest_sha256() assert summary["iteration_acceptance_sha256"] == data["acceptance"].sha256 assert summary["iteration_protocol_version"] == protocol.ITERATION_PROTOCOL_VERSION diff --git a/tests/miracle/test_research_protocol.py b/tests/miracle/test_research_protocol.py index 59d25c8..c422814 100644 --- a/tests/miracle/test_research_protocol.py +++ b/tests/miracle/test_research_protocol.py @@ -130,17 +130,25 @@ def test_each_logic_seed_realizes_the_frozen_map_and_day_draws(): def test_manifest_freezes_kl_and_explicitly_excludes_decision_change_rate(): manifest = research_protocol_manifest() assert research_protocol_module.MANIFEST_SCHEMA_VERSION == ( - "24-miracle-research-manifest-v1" + "24-miracle-research-manifest-v2" ) assert manifest["manifest_schema_version"] == ( research_protocol_module.MANIFEST_SCHEMA_VERSION ) assert research_protocol_module.BENCHMARK_VERSION == "24m-frozen-v1" - assert manifest["protocol_version"] == "24-miracle-research-v1" + assert manifest["protocol_version"] == "24-miracle-research-v2" assert manifest["benchmark_version"] == research_protocol_module.BENCHMARK_VERSION assert manifest["frozen_test"]["case_count"] == 72 assert manifest["trajectory_kl"]["epsilon"] == 0.01 + assert manifest["trajectory_kl"]["measurement_profile"] == ( + "24_miracle_policy_information_gain_v2" + ) assert manifest["trajectory_kl"]["direction"] == "new||old" + assert manifest["trajectory_kl"]["smoothing"] == "symmetric_epsilon_uniform_full_support" + assert manifest["trajectory_kl"]["primary_episode_information_gain"] == "arithmetic_mean" + assert manifest["trajectory_kl"]["primary_unit"] == "nats / decision" + assert manifest["trajectory_kl"]["optional_sum_unit"] == "nats / episode" + assert manifest["trajectory_kl"]["current_internal_memory_evidence"] == "not_collected" assert manifest["trajectory_kl"]["decision_change_rate"] == "not_collected" assert manifest["optimization_class"]["criterion"] == ( "no_backpropagation_or_gradient_updates" @@ -159,7 +167,7 @@ def test_research_manifest_canonical_bytes_and_hash_are_stable(): assert research_protocol_module.research_manifest_sha256( research_protocol_manifest() ) == expected - assert expected == "0eaa88a77ff215381f5058024cc0028b2cffe195094c2a4d2b86795e189ad9d0" + assert expected == "7557015c0979c5acfbb1c553fc18614e3ea7246f6d2627478a94baa262ce61c5" @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) @@ -214,7 +222,7 @@ def test_strict_deterministic_hl_adapters_produce_complete_finite_kl(): active, reference, lambda observation: support, - TrajectoryKLConfig("old", "new", TRAJECTORY_KL_EPSILON), + TrajectoryKLConfig.for_policy_information_gain("old", "new"), results.append, ) measured.reset() diff --git a/tests/test_local_report_research.py b/tests/test_local_report_research.py index 98c46be..4d9bea0 100644 --- a/tests/test_local_report_research.py +++ b/tests/test_local_report_research.py @@ -5,6 +5,75 @@ class LocalResearchReportTests(unittest.TestCase): + def test_report_does_not_promote_legacy_or_forged_rich_events_to_formal_ig(self): + from agentbench_frame.eval.information_gain import policy_kl + from agentbench_frame.report.builder import ReportBuilder + + local = policy_kl([0.75, 0.25], [0.5, 0.5], epsilon=0.01) + forged = { + "event_type": "policy_kl_trace", + "episode": 2, + "version_before": "v1", + "version_after": "v2", + "measurement_profile": "24_miracle_policy_information_gain_v2", + "measurement_status": "complete", + "epsilon": 0.01, + "direction": "new||old", + "log_base": "e", + "rollout_source": "new_policy", + "estimand": "epsilon_regularized_local_kl_sum_under_new_policy_occupancy", + "information_gain_estimand": "epsilon_regularized_mean_local_policy_kl_under_new_policy_occupancy", + "aggregation": "arithmetic_mean", + "information_gain_unit": "nats / decision", + "local_policy_kl_sum_unit": "nats / episode", + "decision_steps": 1, + "trace": [local + 0.25], + "trajectory_kl_episode": local + 0.25, + "mean_local_policy_kl": local + 0.25, + "information_gain": local + 0.25, + "local_policy_kl_sum": local + 0.25, + "errors": [], + "metadata": {}, + "decisions": [{ + "decision_step": 1, + "context_ref": "context", + "action_schema_version": "actions-v1", + "support_id": "support", + "legal_action_ids": ["a", "b"], + "selected_action_id": "a", + "new_distribution": {"a": 0.75, "b": 0.25}, + "old_distribution": {"a": 0.5, "b": 0.5}, + "new_probabilities": [0.75, 0.25], + "old_probabilities": [0.5, 0.5], + "local_policy_kl": local + 0.25, + "errors": [], + }], + } + valid = json.loads(json.dumps(forged)) + valid["episode"] = 3 + valid["trace"] = [local] + valid["trajectory_kl_episode"] = local + valid["mean_local_policy_kl"] = local + valid["information_gain"] = local + valid["local_policy_kl_sum"] = local + valid["decisions"][0]["local_policy_kl"] = local + history = ReportBuilder._derive_research( + {}, + [ + {"event_type": "policy_kl_trace", "episode": 1, "trace": [0.2]}, + forged, + valid, + ], + "missing-events.jsonl", + )["ig_history"] + + self.assertEqual(history[0]["status"], "legacy_unverified") + self.assertIsNone(history[0]["information_gain"]) + self.assertEqual(history[1]["status"], "incomplete") + self.assertIsNone(history[1]["information_gain"]) + self.assertEqual(history[2]["status"], "complete") + self.assertAlmostEqual(history[2]["information_gain"], local) + def test_local_report_reads_first_hand_events_and_renders_research_fields(self): from agentbench_frame.report.builder import ReportBuilder @@ -115,35 +184,35 @@ def test_local_report_reads_first_hand_events_and_renders_research_fields(self): quality = builder.runs[0]["research"]["quality"] html = (output / "index.html").read_text(encoding="utf-8") - self.assertAlmostEqual(ig_history[0]["trajectory_kl_episode"], 0.4) - self.assertAlmostEqual(ig_history[0]["mean_local_policy_kl"], 0.2) - self.assertEqual(ig_history[0]["status"], "complete") + self.assertIsNone(ig_history[0]["trajectory_kl_episode"]) + self.assertIsNone(ig_history[0]["mean_local_policy_kl"]) + self.assertIsNone(ig_history[0]["information_gain"]) + self.assertIsNone(ig_history[0]["local_policy_kl_sum"]) + self.assertEqual(ig_history[0]["status"], "legacy_unverified") self.assertEqual(ig_history[0]["estimand"], "legacy_unspecified") self.assertEqual(len(ig_history), 11) self.assertEqual(quality["malformed_lines"], 1) self.assertIsNone(ig_history[1]["trajectory_kl_episode"]) self.assertIsNone(ig_history[1]["mean_local_policy_kl"]) - self.assertEqual(ig_history[1]["status"], "incomplete") - self.assertAlmostEqual(ig_history[2]["trajectory_kl_episode"], 0.6) - self.assertAlmostEqual(ig_history[2]["mean_local_policy_kl"], 0.3) + self.assertEqual(ig_history[1]["status"], "legacy_unverified") + self.assertIsNone(ig_history[2]["trajectory_kl_episode"]) + self.assertIsNone(ig_history[2]["mean_local_policy_kl"]) + self.assertIsNone(ig_history[2]["information_gain"]) self.assertIsNone(ig_history[3]["trajectory_kl_episode"]) - self.assertEqual(ig_history[3]["status"], "quarantined") + self.assertEqual(ig_history[3]["status"], "legacy_unverified") self.assertIsNone(ig_history[4]["trajectory_kl_episode"]) - self.assertEqual(ig_history[4]["status"], "incomplete") + self.assertEqual(ig_history[4]["status"], "legacy_unverified") for point in ig_history[5:]: self.assertIsNone(point["trajectory_kl_episode"]) - self.assertEqual(point["status"], "incomplete") - self.assertEqual(len(ig_chart["segments"]), 2) + self.assertEqual(point["status"], "legacy_unverified") + self.assertEqual(len(ig_chart["segments"]), 0) self.assertIn("Information gain", html) - self.assertIn("Trajectory KL", html) - self.assertIn("Mean local policy KL", html) + self.assertIn("Policy IG", html) + self.assertIn("Local KL sum", html) self.assertIn("nats / episode", html) self.assertIn("nats / decision", html) - self.assertIn('aria-label="Trajectory KL by episode"', html) - self.assertIn('data-segment-count="2"', html) - self.assertIn("0.40", html) - self.assertIn("0.20", html) - self.assertIn("0.60", html) + self.assertIn('aria-label="Policy information gain by episode"', html) + self.assertIn('data-segment-count="0"', html) self.assertIn("missing", html) self.assertNotIn("999.00", html) self.assertIn("AUC / act", html) diff --git a/tests/test_tracking_contracts.py b/tests/test_tracking_contracts.py index ef90829..f238985 100644 --- a/tests/test_tracking_contracts.py +++ b/tests/test_tracking_contracts.py @@ -157,6 +157,10 @@ def test_run_persists_raw_policy_trace_and_occupancy_observations(self): occupancy = next(record for record in records if record["event_type"] == "occupancy") self.assertEqual(trace["trace"], [0.1, 0.2]) self.assertEqual(trace["context_refs"], ["s0", "s1"]) + self.assertIsNone(trace["information_gain"]) + self.assertAlmostEqual(trace["local_policy_kl_sum"], 0.3) + self.assertEqual(trace["information_gain_status"], "unverified") + self.assertEqual(trace["measurement_profile"], "legacy_policy_kl_trace") self.assertEqual( trace["estimand"], "epsilon_regularized_local_kl_sum_under_unspecified_occupancy", @@ -164,13 +168,112 @@ def test_run_persists_raw_policy_trace_and_occupancy_observations(self): self.assertEqual(trace["rollout_source"], "unspecified") self.assertEqual(occupancy["state_ids"], ["s0", "s1", "s1"]) + def test_legacy_trace_never_claims_formal_information_gain(self): + from agentbench_frame.tracking.run import Run + + with tempfile.TemporaryDirectory() as tmp: + run = Run.start("game", "agent", data_dir=tmp) + run.log_policy_kl_trace( + episode=1, + version_before="v1", + version_after="v2", + trace=[0.1, 0.2], + epsilon=None, + ) + run.log_policy_kl_trace( + episode=2, + version_before="v1", + version_after="v2", + trace=[0.1, 0.2], + epsilon=0.01, + ) + run.finish() + events = [ + json.loads(line) + for line in Path(run.run_dir, "events.jsonl").read_text().splitlines() + if json.loads(line)["event_type"] == "policy_kl_trace" + ] + + self.assertEqual(len(events), 2) + for event in events: + self.assertEqual(event["measurement_profile"], "legacy_policy_kl_trace") + self.assertIsNone(event["information_gain"]) + self.assertIsNone(event["information_gain_unit"]) + self.assertEqual(event["information_gain_status"], "unverified") + + def test_rich_logger_rejects_forged_uploaded_summary_and_missing_profile(self): + from agentbench_frame.eval.information_gain import policy_kl + from agentbench_frame.tracking.run import Run + + local = policy_kl([0.75, 0.25], [0.5, 0.5], epsilon=0.01) + payload = { + "episode": 1, + "version_before": "v1", + "version_after": "v2", + "epsilon": 0.01, + "status": "complete", + "measurement_status": "complete", + "direction": "new||old", + "log_base": "e", + "rollout_source": "new_policy", + "estimand": "epsilon_regularized_local_kl_sum_under_new_policy_occupancy", + "information_gain_estimand": "epsilon_regularized_mean_local_policy_kl_under_new_policy_occupancy", + "aggregation": "arithmetic_mean", + "information_gain_unit": "nats / decision", + "local_policy_kl_sum_unit": "nats / episode", + "decision_steps": 1, + "trace": [local], + "trajectory_kl_episode": local, + "mean_local_policy_kl": local + 0.25, + "information_gain": local + 0.25, + "local_policy_kl_sum": local, + "errors": [], + "metadata": {}, + "decisions": [{ + "decision_step": 1, + "context_ref": "context", + "action_schema_version": "actions-v1", + "support_id": "support", + "legal_action_ids": ["a", "b"], + "selected_action_id": "a", + "new_distribution": {"a": 0.75, "b": 0.25}, + "old_distribution": {"a": 0.5, "b": 0.5}, + "new_probabilities": [0.75, 0.25], + "old_probabilities": [0.5, 0.5], + "local_policy_kl": local, + "errors": [], + }], + } + + with tempfile.TemporaryDirectory() as tmp: + run = Run.start("game", "agent", data_dir=tmp) + try: + with self.assertRaisesRegex(ValueError, "profile"): + run.log_trajectory_kl_result(payload) + finally: + run.finish() + payload["measurement_profile"] = ( + "24_miracle_policy_information_gain_v2" + ) + with tempfile.TemporaryDirectory() as tmp: + run = Run.start("game", "agent", data_dir=tmp) + try: + with self.assertRaisesRegex(ValueError, "mean|summary|aggregate"): + run.log_trajectory_kl_result(payload) + finally: + run.finish() + + def test_run_persists_complete_first_hand_trajectory_kl_result(self): + from agentbench_frame.eval.information_gain import policy_kl from agentbench_frame.eval.trajectory_kl import ( TrajectoryKLDecisionRecord, TrajectoryKLEpisodeResult, ) from agentbench_frame.tracking.run import Run + local_1 = policy_kl([0.75, 0.25], [0.5, 0.5], epsilon=0.01) + local_2 = policy_kl([0.4, 0.6], [0.7, 0.3], epsilon=0.01) decisions = ( TrajectoryKLDecisionRecord( decision_step=1, @@ -183,7 +286,7 @@ def test_run_persists_complete_first_hand_trajectory_kl_result(self): old_distribution={"a": 0.5, "b": 0.5}, new_probabilities=(0.75, 0.25), old_probabilities=(0.5, 0.5), - local_policy_kl=0.1, + local_policy_kl=local_1, ), TrajectoryKLDecisionRecord( decision_step=2, @@ -196,7 +299,7 @@ def test_run_persists_complete_first_hand_trajectory_kl_result(self): old_distribution={"b": 0.7, "c": 0.3}, new_probabilities=(0.4, 0.6), old_probabilities=(0.7, 0.3), - local_policy_kl=0.2, + local_policy_kl=local_2, ), ) result = TrajectoryKLEpisodeResult( @@ -206,11 +309,12 @@ def test_run_persists_complete_first_hand_trajectory_kl_result(self): epsilon=0.01, status="complete", decisions=decisions, - trace=(0.1, 0.2), - trajectory_kl_episode=0.3, - mean_local_policy_kl=0.15, + trace=(local_1, local_2), + trajectory_kl_episode=local_1 + local_2, + mean_local_policy_kl=(local_1 + local_2) / 2, errors=(), metadata={"seed": 11, "opponent_name": "fixed-opponent"}, + measurement_profile="24_miracle_policy_information_gain_v2", ) with tempfile.TemporaryDirectory() as tmp: @@ -234,10 +338,14 @@ def test_run_persists_complete_first_hand_trajectory_kl_result(self): event["estimand"], "epsilon_regularized_local_kl_sum_under_new_policy_occupancy", ) - self.assertAlmostEqual(event["trajectory_kl_episode"], 0.3) - self.assertAlmostEqual(event["mean_local_policy_kl"], 0.15) + self.assertAlmostEqual(event["trajectory_kl_episode"], local_1 + local_2) + self.assertAlmostEqual(event["mean_local_policy_kl"], (local_1 + local_2) / 2) + self.assertAlmostEqual(event["information_gain"], (local_1 + local_2) / 2) + self.assertAlmostEqual(event["local_policy_kl_sum"], local_1 + local_2) + self.assertEqual(event["information_gain_unit"], "nats / decision") + self.assertEqual(event["local_policy_kl_sum_unit"], "nats / episode") self.assertEqual(event["decision_steps"], 2) - self.assertEqual(event["trace"], [0.1, 0.2]) + self.assertEqual(event["trace"], [local_1, local_2]) self.assertEqual(event["metadata"]["seed"], 11) self.assertEqual( event["decisions"][0]["legal_action_ids"], @@ -288,6 +396,7 @@ def test_run_persists_incomplete_trajectory_kl_without_partial_scalar(self): mean_local_policy_kl=None, errors=("decision 1: reference unavailable",), metadata={"seed": 12}, + measurement_profile="24_miracle_policy_information_gain_v2", ) with tempfile.TemporaryDirectory() as tmp: @@ -304,6 +413,8 @@ def test_run_persists_incomplete_trajectory_kl_without_partial_scalar(self): self.assertEqual(event["trace"], [None]) self.assertIsNone(event["trajectory_kl_episode"]) self.assertIsNone(event["mean_local_policy_kl"]) + self.assertIsNone(event["information_gain"]) + self.assertIsNone(event["local_policy_kl_sum"]) self.assertEqual(event["decisions"][0]["new_probabilities"], [1.0, 0.0]) self.assertIsNone(event["decisions"][0]["old_probabilities"]) @@ -327,19 +438,20 @@ def test_run_never_writes_complete_infinite_trajectory_kl_scalar(self): old_probabilities=(1.0,), local_policy_kl=1e308, ) - rich = TrajectoryKLEpisodeResult( - episode=2, - version_before="v1", - version_after="v2", - epsilon=0.01, - status="complete", - decisions=(decision, decision), - trace=(1e308, 1e308), - trajectory_kl_episode=float("inf"), - mean_local_policy_kl=float("inf"), - errors=(), - metadata={}, - ) + with self.assertRaisesRegex(ValueError, "decision steps|local policy KL|aggregate"): + TrajectoryKLEpisodeResult( + episode=2, + version_before="v1", + version_after="v2", + epsilon=0.01, + status="complete", + decisions=(decision, decision), + trace=(1e308, 1e308), + trajectory_kl_episode=float("inf"), + mean_local_policy_kl=float("inf"), + errors=(), + metadata={}, + ) with tempfile.TemporaryDirectory() as tmp: run = Run.start("game", "agent", data_dir=tmp) @@ -349,7 +461,6 @@ def test_run_never_writes_complete_infinite_trajectory_kl_scalar(self): version_after="v1", trace=[1e308, 1e308], ) - run.log_trajectory_kl_result(rich) run.finish() events = [ json.loads(line) @@ -360,14 +471,14 @@ def test_run_never_writes_complete_infinite_trajectory_kl_scalar(self): event for event in events if event["event_type"] == "policy_kl_trace" ] - self.assertEqual(len(traces), 2) + self.assertEqual(len(traces), 1) self.assertEqual(traces[0]["estimand"], "legacy_unspecified") self.assertEqual(traces[0]["rollout_source"], "unspecified") - for event in traces: - self.assertEqual(event["measurement_status"], "incomplete") - self.assertIsNone(event["trajectory_kl_episode"]) - self.assertIsNone(event["mean_local_policy_kl"]) - self.assertTrue(event["errors"]) + event = traces[0] + self.assertEqual(event["measurement_status"], "incomplete") + self.assertIsNone(event["trajectory_kl_episode"]) + self.assertIsNone(event["mean_local_policy_kl"]) + self.assertTrue(event["errors"]) def test_run_attaches_budget_coordinates_to_act_evaluation(self): from agentbench_frame.tracking.run import Run diff --git a/tests/test_trajectory_kl_runtime.py b/tests/test_trajectory_kl_runtime.py index 0fb53d2..388cb41 100644 --- a/tests/test_trajectory_kl_runtime.py +++ b/tests/test_trajectory_kl_runtime.py @@ -1,3 +1,4 @@ +import copy import unittest @@ -132,6 +133,162 @@ def _support(_observation): class TrajectoryKLRuntimeTests(unittest.TestCase): + def test_decision_record_freezes_source_distributions_and_metadata(self): + from agentbench_frame.eval.trajectory_kl import ( + TrajectoryKLDecisionRecord, + ) + + new_distribution = {"a": 0.75, "b": 0.25} + old_distribution = {"a": 0.5, "b": 0.5} + record = TrajectoryKLDecisionRecord( + decision_step=1, + context_ref="context", + action_schema_version="actions-v1", + support_id="support", + legal_action_ids=("a", "b"), + selected_action_id="a", + new_distribution=new_distribution, + old_distribution=old_distribution, + new_probabilities=(0.75, 0.25), + old_probabilities=(0.5, 0.5), + local_policy_kl=0.1, + ) + + new_distribution["a"] = 0.0 + old_distribution["a"] = 1.0 + self.assertEqual(record.to_dict()["new_distribution"]["a"], 0.75) + self.assertEqual(record.to_dict()["old_distribution"]["a"], 0.5) + with self.assertRaises(TypeError): + record.new_distribution["a"] = 0.0 + + def test_episode_result_rejects_forged_summary_derived_from_records(self): + from agentbench_frame.eval.information_gain import policy_kl + from agentbench_frame.eval.trajectory_kl import ( + TrajectoryKLDecisionRecord, + TrajectoryKLEpisodeResult, + ) + + local = policy_kl([0.75, 0.25], [0.5, 0.5], epsilon=0.01) + record = TrajectoryKLDecisionRecord( + decision_step=1, + context_ref="context", + action_schema_version="actions-v1", + support_id="support", + legal_action_ids=("a", "b"), + selected_action_id="a", + new_distribution={"a": 0.75, "b": 0.25}, + old_distribution={"a": 0.5, "b": 0.5}, + new_probabilities=(0.75, 0.25), + old_probabilities=(0.5, 0.5), + local_policy_kl=local, + ) + with self.assertRaisesRegex(ValueError, "mean|summary|aggregate"): + TrajectoryKLEpisodeResult( + episode=1, + version_before="v1", + version_after="v2", + epsilon=0.01, + status="complete", + decisions=(record,), + trace=(local,), + trajectory_kl_episode=local, + mean_local_policy_kl=local + 0.25, + errors=(), + metadata={}, + ) + + def test_episode_result_rejects_bool_aggregate_aliases(self): + from agentbench_frame.eval.trajectory_kl import ( + TrajectoryKLDecisionRecord, + TrajectoryKLEpisodeResult, + ) + + record = TrajectoryKLDecisionRecord( + decision_step=1, + context_ref="context", + action_schema_version="actions-v1", + support_id="support", + legal_action_ids=("a", "b"), + selected_action_id="a", + new_distribution={"a": 0.5, "b": 0.5}, + old_distribution={"a": 0.5, "b": 0.5}, + new_probabilities=(0.5, 0.5), + old_probabilities=(0.5, 0.5), + local_policy_kl=0.0, + ) + for field in ("trajectory_kl_episode", "mean_local_policy_kl"): + values = { + "trajectory_kl_episode": 0.0, + "mean_local_policy_kl": 0.0, + } + values[field] = False + with self.subTest(field=field), self.assertRaisesRegex( + ValueError, "aggregate|mean" + ): + TrajectoryKLEpisodeResult( + episode=1, + version_before="v1", + version_after="v2", + epsilon=0.01, + status="complete", + decisions=(record,), + trace=(0.0,), + errors=(), + metadata={}, + **values, + ) + + def test_payload_rejects_bool_numeric_aliases(self): + from agentbench_frame.eval.trajectory_kl import ( + TrajectoryKLDecisionRecord, + TrajectoryKLEpisodeResult, + trajectory_kl_result_from_payload, + ) + + record = TrajectoryKLDecisionRecord( + decision_step=1, + context_ref="context", + action_schema_version="actions-v1", + support_id="support", + legal_action_ids=("a", "b"), + selected_action_id="a", + new_distribution={"a": 0.5, "b": 0.5}, + old_distribution={"a": 0.5, "b": 0.5}, + new_probabilities=(0.5, 0.5), + old_probabilities=(0.5, 0.5), + local_policy_kl=0.0, + ) + result = TrajectoryKLEpisodeResult( + episode=1, + version_before="v1", + version_after="v2", + epsilon=0.01, + status="complete", + decisions=(record,), + trace=(0.0,), + trajectory_kl_episode=0.0, + mean_local_policy_kl=0.0, + errors=(), + metadata={}, + measurement_profile="24_miracle_policy_information_gain_v2", + ) + payload = result.to_dict() + + replacements = { + "decision_steps": True, + "trajectory_kl_episode": False, + "mean_local_policy_kl": False, + "information_gain": False, + "local_policy_kl_sum": False, + } + for field, forged in replacements.items(): + candidate = copy.deepcopy(payload) + candidate[field] = forged + with self.subTest(field=field), self.assertRaises( + (TypeError, ValueError) + ): + trajectory_kl_result_from_payload(candidate) + def test_match_measures_only_new_policy_decisions_on_the_actual_rollout(self): from agentbench_frame.arena.match import Match from agentbench_frame.eval.information_gain import policy_kl @@ -147,10 +304,9 @@ def test_match_measures_only_new_policy_decisions_on_the_actual_rollout(self): active_policy=active, reference_policy=reference, support_provider=_support, - config=TrajectoryKLConfig( + config=TrajectoryKLConfig.for_policy_information_gain( version_before="v1", version_after="v2", - epsilon=0.1, ), on_episode_complete=completed.append, ) @@ -185,13 +341,17 @@ def test_match_measures_only_new_policy_decisions_on_the_actual_rollout(self): episode = completed[0] expected = ( - policy_kl([0.75, 0.25], [0.5, 0.5], epsilon=0.1) - + policy_kl([0.5, 0.5], [0.5, 0.5], epsilon=0.1) + policy_kl([0.75, 0.25], [0.5, 0.5], epsilon=0.01) + + policy_kl([0.5, 0.5], [0.5, 0.5], epsilon=0.01) ) self.assertEqual(episode.status, "complete") self.assertEqual(episode.decision_steps, 2) self.assertAlmostEqual(episode.trajectory_kl_episode, expected) self.assertAlmostEqual(episode.mean_local_policy_kl, expected / 2) + self.assertAlmostEqual(episode.information_gain, expected / 2) + self.assertAlmostEqual(episode.local_policy_kl_sum, expected) + self.assertEqual(episode.information_gain_unit, "nats / decision") + self.assertEqual(episode.local_policy_kl_sum_unit, "nats / episode") self.assertEqual(episode.metadata["seed"], 17) self.assertEqual(episode.metadata["player_id"], 0) self.assertEqual(episode.metadata["opponent_name"], "opponent") @@ -216,10 +376,9 @@ def test_reference_failure_marks_episode_incomplete_without_replacing_active_act active_policy=_ActivePolicy(), reference_policy=_ReferencePolicy(fail=True), support_provider=_support, - config=TrajectoryKLConfig( + config=TrajectoryKLConfig.for_policy_information_gain( version_before="v1", version_after="v2", - epsilon=0.1, ), on_episode_complete=completed.append, ) @@ -255,7 +414,7 @@ def test_new_and_reference_sessions_receive_isolated_transition_snapshots(self): active_policy=active, reference_policy=reference, support_provider=_support, - config=TrajectoryKLConfig("v1", "v2", 0.1), + config=TrajectoryKLConfig.for_policy_information_gain("v1", "v2"), ) Match( @@ -289,7 +448,7 @@ def step(self, action): active_policy=_ActivePolicy(), reference_policy=_ReferencePolicy(), support_provider=_support, - config=TrajectoryKLConfig("v1", "v2", 0.1), + config=TrajectoryKLConfig.for_policy_information_gain("v1", "v2"), on_episode_complete=completed.append, ) @@ -320,7 +479,7 @@ def fail_persistence(_result): active_policy=_ActivePolicy(), reference_policy=_ReferencePolicy(), support_provider=_support, - config=TrajectoryKLConfig("v1", "v2", 0.1), + config=TrajectoryKLConfig.for_policy_information_gain("v1", "v2"), on_episode_complete=fail_persistence, ) @@ -343,7 +502,7 @@ def test_reset_aborts_unfinished_episode_before_clearing_decisions(self): active_policy=_ActivePolicy(), reference_policy=_ReferencePolicy(), support_provider=_support, - config=TrajectoryKLConfig("v1", "v2", 0.1), + config=TrajectoryKLConfig.for_policy_information_gain("v1", "v2"), on_episode_complete=completed.append, ) measured.reset() @@ -354,15 +513,18 @@ def test_reset_aborts_unfinished_episode_before_clearing_decisions(self): self.assertEqual(completed[0].status, "incomplete") self.assertIn("reset before terminal transition", completed[0].errors[-1]) - def test_config_requires_fixed_non_degenerate_epsilon_and_version_ids(self): + def test_generic_config_accepts_research_epsilon_but_rejects_invalid_values(self): from agentbench_frame.eval.trajectory_kl import TrajectoryKLConfig - for epsilon in (0.0, 1.0, -0.1, 1.1): + for epsilon in (0.0, 0.001, 0.05, 1.0): + config = TrajectoryKLConfig("v1", "v2", epsilon) + self.assertFalse(config.is_formal_policy_information_gain) + for epsilon in (-0.1, 1.1, True): with self.subTest(epsilon=epsilon): - with self.assertRaises(ValueError): + with self.assertRaises((TypeError, ValueError)): TrajectoryKLConfig("v1", "v2", epsilon) with self.assertRaises(ValueError): - TrajectoryKLConfig("", "v2", 0.1) + TrajectoryKLConfig("", "v2", 0.01) if __name__ == "__main__":