Retire the UK schema-3 gate path and keep household weights typed in-build (#654)

changelog.d/654-uk-rowwise-frame-entry.changed.md

@@ -0,0 +1 @@
+Narrow the UK rowwise clone entry to Frame or H5 inputs and refuse retired duck-typed in-memory dataset carriers.

changelog.d/654-uk-schema3-terminal-gates.removed.md

@@ -0,0 +1 @@
+Retire the legacy UK schema-3 terminal gate report path after verifying schema-4 battery parity with a saved differential receipt.

changelog.d/654-uk-typed-household-weights.changed.md

@@ -0,0 +1 @@
+Store UK national household weights as Frame typed weights during the build, materializing `household_weight` only at H5/export boundaries.

changelog.d/654-uk-weights-audit-absence-blocks.changed.md

@@ -0,0 +1 @@
+Block the build in every posture when the UK fit-weight audit evidence is absent: `uk_weights_audit` declares the new `evidence_absent_blocks` manifest flag and the battery honors it, porting the retired schema-3 path's strictness — an absent audit is not a passing audit.

packages/microcosm-build/src/microcosm/build/country_spec.py

@@ -108,7 +108,16 @@
 #: extension) would run the gate on defaults while the declared intent
 #: vanished from ``policy_sha256`` — an unattested threshold.
 _GATE_ENTRY_KEYS = frozenset(
-    {"id", "gate", "phase", "criticality", "parameters", "not_applicable", "notes"}
+    {
+        "id",
+        "gate",
+        "phase",
+        "criticality",
+        "parameters",
+        "not_applicable",
+        "evidence_absent_blocks",
+        "notes",
+    }
 )
 
 #: Build phases a gate selection may bind to — the shared vocabulary that
@@ -392,6 +401,13 @@ class GateSelectionSpec:
             it appears in every report as ``not_applicable`` and never
             evaluates. Mutually exclusive with ``parameters`` — an excused
             gate with tuned thresholds is a contradiction.
+        evidence_absent_blocks: When true, an ``evidence_absent`` outcome on
+            this entry blocks the build in every posture, not only under the
+            release-candidate posture. For entries whose declared intent is
+            that absence is never excusable (e.g. "an absent audit is not a
+            passing audit") — the outcome stays honestly ``evidence_absent``
+            in the report; only the enforcement changes. Meaningless on an
+            excused entry, so mutually exclusive with ``not_applicable``.
         notes: Free-text rationale.
     """
 
@@ -401,6 +417,7 @@ class GateSelectionSpec:
     criticality: str
     parameters: Mapping[str, Any] = field(default_factory=dict)
     not_applicable: str | None = None
+    evidence_absent_blocks: bool = False
     notes: str = ""
 
     def __post_init__(self) -> None:
@@ -409,6 +426,11 @@ def __post_init__(self) -> None:
                 "GateSelectionSpec parameters must be a mapping, got "
                 f"{type(self.parameters).__name__}."
             )
+        if not isinstance(self.evidence_absent_blocks, bool):
+            raise TypeError(
+                "GateSelectionSpec evidence_absent_blocks must be a bool, got "
+                f"{type(self.evidence_absent_blocks).__name__}."
+            )
         object.__setattr__(
             self,
             "parameters",
@@ -478,13 +500,26 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> GateSelectionSpec:
                     "mutually exclusive — an excused gate with tuned "
                     "thresholds is a contradiction."
                 )
+        evidence_absent_blocks = raw.get("evidence_absent_blocks", False)
+        if not isinstance(evidence_absent_blocks, bool):
+            raise ValueError(
+                f"gate {gate_id!r}: evidence_absent_blocks must be a JSON "
+                f"boolean, got {evidence_absent_blocks!r}."
+            )
+        if evidence_absent_blocks and not_applicable is not None:
+            raise ValueError(
+                f"gate {gate_id!r}: evidence_absent_blocks and not_applicable "
+                "are mutually exclusive — an excused entry never evaluates, "
+                "so demanding its absence block is a contradiction."
+            )
         return cls(
             id=gate_id,
             gate=gate,
             phase=phase,
             criticality=criticality,
             parameters=dict(parameters),
             not_applicable=not_applicable,
+            evidence_absent_blocks=evidence_absent_blocks,
             notes=str(raw.get("notes", "")),
         )
 

packages/microcosm-build/src/microcosm/build/gate_battery.py

@@ -190,6 +190,14 @@ def _gates_manifest_payload(gates: GatesManifest) -> dict[str, object]:
                 "criticality": entry.criticality,
                 "parameters": _json_safe(entry.parameters),
                 "not_applicable": entry.not_applicable,
+                # Present iff armed: a true flag must ride the policy hash,
+                # while the false default stays out so unflagged entries
+                # (and the US manifest) keep their serialized form.
+                **(
+                    {"evidence_absent_blocks": True}
+                    if entry.evidence_absent_blocks
+                    else {}
+                ),
                 "notes": entry.notes,
             }
             for entry in gates.gates
@@ -487,7 +495,11 @@ def blocking_outcomes(self, *, release_candidate: bool) -> tuple[GateOutcome, ..
         build without, say, an incumbent parity snapshot gets an honest
         non-shippable report instead of a crash, while a release build
         cannot excuse missing evidence — a missing frozen reference is not
-        a passing gate. Diagnostic entries never block.
+        a passing gate. An entry declaring ``evidence_absent_blocks`` opts
+        out of that dev-posture leniency: its absence blocks every posture
+        (the legacy UK weights-audit strictness, ported during the #654
+        schema-3 retirement — "an absent audit is not a passing audit").
+        Diagnostic entries never block.
         """
 
         blocking = []
@@ -496,7 +508,9 @@ def blocking_outcomes(self, *, release_candidate: bool) -> tuple[GateOutcome, ..
                 continue
             if outcome.status is GateStatus.FAILED:
                 blocking.append(outcome)
-            elif outcome.status is GateStatus.EVIDENCE_ABSENT and release_candidate:
+            elif outcome.status is GateStatus.EVIDENCE_ABSENT and (
+                release_candidate or outcome.entry.evidence_absent_blocks
+            ):
                 blocking.append(outcome)
         return tuple(blocking)
 
@@ -916,6 +930,14 @@ def _policy_sha256(self) -> str:
                     "criticality": entry.criticality,
                     "parameters": _json_safe(dict(entry.parameters)),
                     "not_applicable": entry.not_applicable,
+                    # Enforcement policy rides the policy hash when armed;
+                    # the false default stays out so unflagged entries keep
+                    # their digest.
+                    **(
+                        {"evidence_absent_blocks": True}
+                        if entry.evidence_absent_blocks
+                        else {}
+                    ),
                 }
                 for entry in sorted(self._gates.gates, key=lambda e: e.id)
             ]

packages/microcosm-build/src/microcosm/build/uk/gates.json

@@ -98,7 +98,8 @@
       "phase": "terminal",
       "criticality": "release_blocking",
       "parameters": {},
-      "notes": "Every fit-produced weight column carries a completed audit record. Armed by the SPI income stage; an absent audit is not a passing audit."
+      "evidence_absent_blocks": true,
+      "notes": "Every fit-produced weight column carries a completed audit record. Armed by the SPI income stage; an absent audit is not a passing audit — and blocks every posture, not only release candidates: the legacy schema-3 path's strictness, ported during the #654 retirement (microcosm#691 review)."
     },
     {
       "id": "uk_export_surface",

packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py

@@ -377,20 +377,14 @@
 from microcosm.build.uk_runtime.terminal_gates import (
     UK_DEFAULT_ZERO_WEIGHT_STRATA,
     UK_MAX_TARGET_ABS_RELATIVE_ERROR,
-    UK_MAX_TO_MEDIAN_WEIGHT_RATIO,
-    UK_MIN_ESS_FRACTION,
-    UK_TERMINAL_GATE_SCHEMA_VERSION,
-    UKReleaseParityEvidence,
     UKZeroWeightStratumDeclaration,
     uk_degenerate_release_surface_gate,
     uk_export_surface_gate,
     uk_target_fit_gate,
     uk_target_surface_gate,
-    uk_terminal_gate_report,
     uk_weight_ess_gate,
     uk_weight_ratio_gate,
     uk_zero_weight_strata_gate,
-    write_uk_terminal_gate_report,
 )
 from microcosm.build.uk_runtime.weighted_integrity import (
     UKInputMassParityPolicy,
@@ -719,13 +713,9 @@
     "write_hmrc_replay_report",
     "UK_DEFAULT_ZERO_WEIGHT_STRATA",
     "UK_MAX_TARGET_ABS_RELATIVE_ERROR",
-    "UK_MAX_TO_MEDIAN_WEIGHT_RATIO",
-    "UK_MIN_ESS_FRACTION",
-    "UK_TERMINAL_GATE_SCHEMA_VERSION",
     "UKInputMassParityPolicy",
     "UKInputMassReference",
     "UKQRFTailConcentrationPolicy",
-    "UKReleaseParityEvidence",
     "UKZeroWeightStratumDeclaration",
     "load_uk_input_mass_reference",
     "load_uk_reviewed_exclusion_register",
@@ -737,9 +727,7 @@
     "uk_qrf_tail_concentration_gate",
     "uk_target_fit_gate",
     "uk_target_surface_gate",
-    "uk_terminal_gate_report",
     "uk_weight_ess_gate",
     "uk_weight_ratio_gate",
     "uk_zero_weight_strata_gate",
-    "write_uk_terminal_gate_report",
 ]

packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_calibration.py

@@ -146,8 +146,12 @@ def materialize_uk_cgt_calibration_frame(
             f"UK CGT person {UK_CGT_SOURCE_COLUMN!r} values must be finite."
         )
 
+    household_weight_by_id = pd.Series(
+        national_frame.weights_for("household").values,
+        index=household["household_id"].to_numpy(),
+    )
     mapped_mass = person["person_household_id"].map(
-        household.set_index("household_id")["household_weight"]
+        household_weight_by_id
     )
     if mapped_mass.isna().any() or not mapped_mass.gt(0.0).all():
         raise ValueError(
@@ -188,7 +192,7 @@ def materialize_uk_cgt_calibration_frame(
         EntitySchema(group_entities=("household",)),
         {
             "household": Weights(
-                household["household_weight"].to_numpy(dtype=float),
+                national_frame.weights_for("household").values,
                 uk_household_weight_kind(national_frame),
             )
         },

packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py

@@ -427,9 +427,11 @@ def impute_uk_capital_gains(
     if "capital_gains" not in person.columns:
         raise ValueError("Person table has no capital_gains column to redraw.")
 
-    weights_by_household = frame.table("household").set_index("household_id")[
-        "household_weight"
-    ]
+    household = frame.table("household")
+    weights_by_household = pd.Series(
+        frame.weights_for("household").values,
+        index=household["household_id"],
+    )
     missing_households = set(person["person_household_id"]) - set(
         weights_by_household.index
     )
@@ -521,6 +523,7 @@ def impute_uk_capital_gains(
         household=frame.table("household"),
         time_period=time_period,
         weight_kind=uk_household_weight_kind(frame),
+        household_weights=frame.weights_for("household").values,
         mass_log=frame.mass_log,
     )
     validate_uk_national_frame(result_frame)
@@ -541,9 +544,11 @@ def summarize_uk_cgt_imputation(
     calibration adjudication's question.
     """
     person = after.table("person").reset_index(drop=True)
-    weights_by_household = after.table("household").set_index("household_id")[
-        "household_weight"
-    ]
+    household = after.table("household")
+    weights_by_household = pd.Series(
+        after.weights_for("household").values,
+        index=household["household_id"],
+    )
     weight = (
         person["person_household_id"].map(weights_by_household).to_numpy(dtype=float)
     )

packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py

@@ -31,6 +31,7 @@
     effective_sample_size,
 )
 from microcosm.calibrate.solve import CalibrationResult
+from microcosm.frame import Frame
 
 __all__ = [
     "UK_DIAGNOSTICS_SCHEMA_VERSION",
@@ -402,7 +403,7 @@ def _target_pass_rates(
 
 def uk_calibration_diagnostics_payload(
     result: CalibrationResult,
-    household: pd.DataFrame,
+    frame: Frame,
     *,
     target_geography_levels: Mapping[str, object],
     target_registry: TargetRegistry,
@@ -417,14 +418,11 @@ def uk_calibration_diagnostics_payload(
     """
 
     registry = _require_uk_target_registry(target_registry)
-    if not isinstance(household, pd.DataFrame):
-        raise TypeError("UK diagnostic household data must be a pandas DataFrame.")
-    if "household_weight" not in household:
-        raise ValueError(
-            "UK diagnostic household data must contain 'household_weight'."
-        )
+    if not isinstance(frame, Frame):
+        raise TypeError("UK diagnostic data must be a Frame.")
+    household = frame.table("household")
     result_weights = _as_weights(result.weights)
-    shipped_weights = _as_weights(household["household_weight"].to_numpy())
+    shipped_weights = _as_weights(frame.weights_for("household").values)
     if shipped_weights.shape != result_weights.shape or not np.array_equal(
         shipped_weights,
         result_weights,
@@ -465,7 +463,7 @@ def uk_calibration_diagnostics_payload(
 def write_uk_calibration_diagnostics(
     result: CalibrationResult,
     path: Path | str,
-    household: pd.DataFrame,
+    frame: Frame,
     *,
     target_geography_levels: Mapping[str, object],
     target_registry: TargetRegistry,
@@ -478,7 +476,7 @@ def write_uk_calibration_diagnostics(
     encoded = json.dumps(
         uk_calibration_diagnostics_payload(
             result,
-            household,
+            frame,
             target_geography_levels=target_geography_levels,
             target_registry=target_registry,
             stratum_columns=stratum_columns,

packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py

@@ -434,6 +434,7 @@ def retain_uk_frs_hmrc_leaves(
         household=frame.table("household"),
         time_period=time_period,
         weight_kind=uk_household_weight_kind(frame),
+        household_weights=frame.weights_for("household").values,
         mass_log=frame.mass_log,
     )
     validate_uk_national_frame(result_frame)

packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_calibration.py

@@ -228,7 +228,10 @@ def materialize_uk_hmrc_calibration_frame(
     ):
         raise RuntimeError("HMRC TI must equal derived TEI + TII exactly.")
 
-    positive_household_mass = household.set_index("household_id")["household_weight"]
+    positive_household_mass = pd.Series(
+        frame.weights_for("household").values,
+        index=household["household_id"].to_numpy(),
+    )
     mapped_mass = person["person_household_id"].map(positive_household_mass)
     if mapped_mass.isna().any() or not mapped_mass.gt(0.0).all():
         raise ValueError(
@@ -294,7 +297,7 @@ def materialize_uk_hmrc_calibration_frame(
         EntitySchema(group_entities=("household",)),
         {
             "household": Weights(
-                household["household_weight"].to_numpy(dtype=float),
+                frame.weights_for("household").values,
                 uk_household_weight_kind(frame),
             )
         },
@@ -392,10 +395,8 @@ def _validate_materialization_inputs(
         raise ValueError(
             "HMRC target materialization requires rebuilt importance weights."
         )
-    weights = pd.to_numeric(
-        frame.table("household")["household_weight"], errors="coerce"
-    )
-    if weights.isna().any() or not weights.gt(0.0).all():
+    weights = frame.weights_for("household").values
+    if not (weights > 0.0).all():
         raise ValueError(
             "HMRC target materialization requires every household prior weight "
             "to be strictly positive."

packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py

@@ -67,7 +67,7 @@
     replace_uk_spi_support_tables,
     support_channel_column,
 )
-from microcosm.frame import Frame, WeightKind
+from microcosm.frame import Frame, WeightKind, engine_tables
 
 __all__ = [
     "CERTIFIED_UK_CANDIDATE_FILENAME",
@@ -543,10 +543,11 @@ def restore_uk_hmrc_income_family(
         build_period=time_period,
     )
 
+    tables = engine_tables(frame, weighted_entities=("household",))
     support = replace_uk_spi_support_tables(
         person=frame.table("person"),
         benunit=frame.table("benunit"),
-        household=frame.table("household"),
+        household=tables["household"],
         seed=seed,
         source_year=int(time_period),
         spi_prior_mass_share=spi_prior_mass_share,
@@ -729,9 +730,11 @@ def _distributional_mass_shares(frame: Frame) -> dict[str, float]:
     )
     if not spi_people.any():
         raise RuntimeError("Rebuilt HMRC family contains no SPI support people.")
-    household_weights = frame.table("household").set_index("household_id")[
-        "household_weight"
-    ]
+    household = frame.table("household")
+    household_weights = pd.Series(
+        frame.weights_for("household").values,
+        index=household["household_id"].to_numpy(),
+    )
     mapped = pd.to_numeric(
         person["person_household_id"].map(household_weights),
         errors="coerce",