diff --git a/changelog.d/723-uk-frs-2024-25-retarget.changed.md b/changelog.d/723-uk-frs-2024-25-retarget.changed.md new file mode 100644 index 00000000..c7b06615 --- /dev/null +++ b/changelog.d/723-uk-frs-2024-25-retarget.changed.md @@ -0,0 +1 @@ +Retarget the UK build to FRS 2024-25 (#723): a new frozen release object (`uk/frs_release.json` — survey/base 2024, calibration year 2025, SN 9563, DOI, UKDS zip sha, HF acquisition pins) drives lockstep asserts over the re-pinned raw-tab manifest (all 21 `frs_table` artifacts at sha+size for the 2024-25 tabs, SPI-convention `tax_year_start`/`ukds_study_number`/`doi` keys), the build period moves to `"2024"`, the take-up contract's `build_year` follows (tax-free-childcare and TV-evasion rates flip to their 2024 date keys), and the HMRC 2023-24 published surface is re-mapped as a signed nearest-available-vintage declaration (`period_mapping: "latest_published_tax_year"`, values and vintage labels unchanged — no uprating). The parity bar re-freezes on `enhanced_frs_2024_25.h5` @ the v1.56.14 tag: the eFRS parity reference (145 columns, entity counts (16,288 raw + 10,000 SPI) × 2 + 270 CGT band donors = 52,846 — now test-pinned), the release-input coverage manifest, and the input-mass reference registry identity all move to the 2024-25 pins; gate thresholds do not move (#686 arms the re-measured baselines). Six stages' SN 9252 (FRS 2022-23) source citations were a standing prose defect, fixed and signed. diff --git a/changelog.d/723-uk-ingest-scale-ladder.added.md b/changelog.d/723-uk-ingest-scale-ladder.added.md new file mode 100644 index 00000000..2045631b --- /dev/null +++ b/changelog.d/723-uk-ingest-scale-ladder.added.md @@ -0,0 +1 @@ +The UK ingest spine driver joins the #627 scale ladder (#723): `--sample-fraction` (0.01/0.10/1.0) subsamples households right after the raw `frs_spine` stage through the generic frame-sampling helpers, records a sampling receipt and rung token in the build sidecar, refuses `--checkpoint-dir` on sampled rungs, and adopts the named-edge abort receipt. Sampled rungs get an explicit receipt posture on three full-scale fences — raw-tab person coverage restricts to survivors, the SPI synthetic stack scales by the fraction, and the ultra-sparse effective-mass restoration check records instead of raising — while f100 keeps every fence strict. diff --git a/changelog.d/723-uk-was-bridge-donor-pin.fixed.md b/changelog.d/723-uk-was-bridge-donor-pin.fixed.md new file mode 100644 index 00000000..11fb9c06 --- /dev/null +++ b/changelog.d/723-uk-was-bridge-donor-pin.fixed.md @@ -0,0 +1 @@ +The WAS round-8 bridge-donor artifact in `lcfs_consumption` declared a different `locator` than `was_wealth`'s pin for the same tab, so the full-roster spine driver refused every licensed run at the pin-coherence check — a break invisible to secrets-free PR CI (#723). The locator is aligned to the caller-supplied convention and a hermetic regression test now collects `_artifact_pins()` over the complete manifest. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 31175588..1c7d9d1d 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -5309,6 +5309,15 @@ }, "runtime_sha256_required": { "type": "boolean" + }, + "tax_year_start": { + "type": "integer" + }, + "ukds_study_number": { + "type": "integer" + }, + "doi": { + "type": "string" } } }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json index 914b021d..9e237c09 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json @@ -35,8 +35,8 @@ "3_3_2021-22", "3_4_2020-21" ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "basis": "Individuals with a CGT liability only; gains after losses and attributed gains but before the annual exempt amount; taxable income after reliefs and the Personal Allowance; trusts excluded; counts publish in thousands and amounts in GBP millions.", "runtime_sha256_required": true }, @@ -93,7 +93,7 @@ "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", "shortfall_policy": "proportional scale-down when the population holds less gainer mass than published taxpayers", - "minimum_allocation_people": 1.0, + "minimum_allocation_people": 1, "weights": "household_weight mapped to persons; no person splits across bands" }, { @@ -116,7 +116,7 @@ "kind": "record_mass_conservation_receipt", "entity": "household", "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "declared_factor": 1.0, + "declared_factor": 1, "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 0ab84156..9bc16e85 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -52,6 +52,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "frs_release.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "gates.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json index 5fd03c09..36d94b40 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json @@ -33,13 +33,18 @@ "household_id" ], "unknown_export_columns_excluded": [ + "bus_fare_spending", "clone_index", "constituency_code_oa", + "employment_sector", "esa_health_condition_proxy", "esa_support_group_proxy", "free_school_breakfasts", "has_fuel_consumption", "higher_earner_tie_break", + "household_is_capital_gains_clone", + "household_is_cgt_band_donor", + "household_is_spi_synthetic", "la_code_oa", "legacy_jobseeker_proxy", "lsoa_code", @@ -47,7 +52,10 @@ "oa_code", "region_code_oa", "salary_sacrifice_asked", - "salary_sacrifice_reported" + "salary_sacrifice_reported", + "sic_industry_division", + "source_household_id", + "source_year" ], "version": "2.89.0", "zero_share_input_columns_excluded": [ @@ -61,19 +69,19 @@ "export_columns": 11, "input_columns": 10, "populated_input_columns": 10, - "records": 61898 + "records": 61223 }, "household": { - "export_columns": 60, + "export_columns": 66, "input_columns": 51, "populated_input_columns": 51, - "records": 53508 + "records": 52846 }, "person": { - "export_columns": 97, + "export_columns": 99, "input_columns": 87, "populated_input_columns": 84, - "records": 115710 + "records": 113617 } }, "input_entities": { @@ -226,160 +234,160 @@ "nonzero_shares": { "a_and_e_visits": 1.0, "aa_category": 1.0, - "access_fund": 0.000207, + "access_fund": 0.000422, "accommodation_type": 1.0, "admitted_patient_visits": 1.0, - "adult_ema": 0.000259, - "afcs_reported": 0.006101, - "age": 0.989128, + "adult_ema": 0.000194, + "afcs_reported": 0.001884, + "age": 0.990829, "age_started_or_accepted_current_education_or_training": 1.0, - "alcohol_and_tobacco_consumption": 0.573372, + "alcohol_and_tobacco_consumption": 0.562975, "attends_private_school_random_draw": 1.0, "brma": 1.0, - "bsp_reported": 0.015349, - "bus_subsidy_spending": 0.226321, - "capital_gains": 0.231216, - "carers_allowance_reported": 0.019912, - "charitable_investment_gifts": 0.000173, - "child_benefit_opts_out": 0.229377, - "child_benefit_reported": 0.094495, - "child_ema": 0.000691, - "child_tax_credit_reported": 0.022401, - "childcare_expenses": 0.036609, - "clothing_and_footwear_consumption": 0.475817, - "communication_consumption": 0.79846, - "corporate_wealth": 0.810683, - "council_tax": 0.883905, + "bsp_reported": 0.004929, + "bus_subsidy_spending": 0.314347, + "capital_gains": 0.23375, + "carers_allowance_reported": 0.010535, + "charitable_investment_gifts": 0.00029, + "child_benefit_opts_out": 0.228084, + "child_benefit_reported": 0.08849, + "child_ema": 0.000907, + "child_tax_credit_reported": 0.002588, + "childcare_expenses": 0.037098, + "clothing_and_footwear_consumption": 0.500908, + "communication_consumption": 0.795084, + "corporate_wealth": 0.822465, + "council_tax": 0.819116, "council_tax_band": 1.0, - "council_tax_benefit_reported": 0.062829, + "council_tax_benefit_reported": 0.058468, "current_education": 1.0, - "dfe_education_spending": 0.000411, - "diesel_spending": 0.213688, - "dividend_income": 0.130706, + "dfe_education_spending": 0.000265, + "diesel_spending": 0.190346, + "dividend_income": 0.088455, "dla_m_category": 1.0, "dla_sc_category": 1.0, - "domestic_energy_consumption": 0.973163, - "domestic_rates": 0.110675, - "education_consumption": 0.130298, - "education_grants": 0.006032, - "electricity_consumption": 0.930141, - "employee_pension_contributions": 0.235079, - "employer_pension_contributions": 0.291194, - "employment_income": 0.481929, + "domestic_energy_consumption": 0.954093, + "domestic_rates": 0.106933, + "education_consumption": 0.125818, + "education_grants": 0.005483, + "electricity_consumption": 0.861352, + "employee_pension_contributions": 0.273489, + "employer_pension_contributions": 0.314865, + "employment_income": 0.492928, "employment_status": 1.0, - "esa_contrib_reported": 0.014796, - "esa_income_reported": 0.016438, - "external_child_payments": 0.011438, - "food_and_non_alcoholic_beverages_consumption": 0.992113, - "free_school_fruit_veg": 0.004719, - "free_school_meals": 0.034016, + "esa_contrib_reported": 0.008053, + "esa_income_reported": 0.009558, + "external_child_payments": 0.008875, + "food_and_non_alcoholic_beverages_consumption": 0.997086, + "free_school_fruit_veg": 0.004524, + "free_school_meals": 0.03422, "full_rate_vat_expenditure_rate": 1.0, - "gas_consumption": 0.969201, + "gas_consumption": 0.952314, "gender": 1.0, - "gift_aid": 0.037525, - "gross_financial_wealth": 0.977499, - "health_consumption": 0.484376, - "healthy_start_vouchers": 0.003094, + "gift_aid": 0.020904, + "gross_financial_wealth": 0.987189, + "health_consumption": 0.512849, + "healthy_start_vouchers": 0.002236, "highest_education": 1.0, - "hours_worked": 0.431717, - "household_furnishings_consumption": 0.829184, - "household_owns_tv": 0.951073, + "hours_worked": 0.428193, + "household_furnishings_consumption": 0.812625, + "household_owns_tv": 0.949022, "household_weight": 1.0, - "housing_benefit_reported": 0.030144, - "housing_service_charges": 0.105405, - "housing_water_and_electricity_consumption": 0.999813, - "iidb_reported": 0.016334, - "income_support_reported": 0.006274, - "is_before_universal_credit_qualifying_young_person_terminal_date": 0.000847, - "is_benunit_head": 0.534941, - "is_disabled_for_benefits": 0.064714, - "is_enhanced_disabled_for_benefits": 0.014657, - "is_household_head": 0.462432, - "is_in_non_advanced_education": 0.163581, - "is_married": 0.262916, - "is_parent": 0.204269, - "is_severely_disabled_for_benefits": 0.039098, - "jsa_contrib_reported": 0.00038, - "jsa_income_reported": 0.004408, - "lump_sum_income": 0.002783, - "main_residence_value": 0.590491, - "main_residential_property_purchased_is_first_home": 0.384541, - "maintenance_expenses": 0.008106, - "maintenance_income": 0.01184, + "housing_benefit_reported": 0.021423, + "housing_service_charges": 0.106971, + "housing_water_and_electricity_consumption": 0.999924, + "iidb_reported": 0.005589, + "income_support_reported": 0.000757, + "is_before_universal_credit_qualifying_young_person_terminal_date": 0.000898, + "is_benunit_head": 0.538854, + "is_disabled_for_benefits": 0.08908, + "is_enhanced_disabled_for_benefits": 0.045794, + "is_household_head": 0.465124, + "is_in_non_advanced_education": 0.161833, + "is_married": 0.256587, + "is_parent": 0.20335, + "is_severely_disabled_for_benefits": 0.056532, + "jsa_contrib_reported": 0.000378, + "jsa_income_reported": 0.000334, + "lump_sum_income": 0.003732, + "main_residence_value": 0.676078, + "main_residential_property_purchased_is_first_home": 0.380653, + "maintenance_expenses": 0.007437, + "maintenance_income": 0.01094, "marital_status": 1.0, - "maximum_extended_childcare_hours_usage": 0.998611, - "miscellaneous_consumption": 0.913471, - "miscellaneous_income": 0.012808, - "mortgage_capital_repayment": 0.252448, - "mortgage_interest_repayment": 0.250542, - "net_financial_wealth": 0.999551, + "maximum_extended_childcare_hours_usage": 0.99853, + "miscellaneous_consumption": 0.897532, + "miscellaneous_income": 0.013, + "mortgage_capital_repayment": 0.248098, + "mortgage_interest_repayment": 0.246206, + "net_financial_wealth": 0.998789, "nhs_a_and_e_spending": 1.0, "nhs_admitted_patient_spending": 1.0, "nhs_outpatient_spending": 1.0, - "non_residential_property_value": 0.056365, - "num_vehicles": 0.666368, - "other_residential_property_value": 0.125514, + "non_residential_property_value": 0.010786, + "num_vehicles": 0.760171, + "other_residential_property_value": 0.074991, "outpatient_visits": 1.0, - "owned_land": 0.045003, - "pension_contributions_via_salary_sacrifice": 0.077003, - "pension_credit_reported": 0.01305, - "personal_pension_contributions": 0.049417, - "petrol_spending": 0.507625, + "owned_land": 0.014306, + "pension_contributions_via_salary_sacrifice": 0.083139, + "pension_credit_reported": 0.016142, + "personal_pension_contributions": 0.035796, + "petrol_spending": 0.444177, "pip_dl_category": 1.0, "pip_m_category": 1.0, - "private_pension_income": 0.220569, - "private_transfer_income": 0.008366, - "property_income": 0.051508, - "property_purchased": 0.038985, - "property_wealth": 0.634073, - "rail_subsidy_spending": 0.044442, - "rail_usage": 0.044442, - "receives_benefits_in_own_right": 0.061239, - "recreation_consumption": 0.970173, + "private_pension_income": 0.231391, + "private_transfer_income": 0.009919, + "property_income": 0.039413, + "property_purchased": 0.038584, + "property_wealth": 0.708152, + "rail_subsidy_spending": 0.126443, + "rail_usage": 0.126443, + "receives_benefits_in_own_right": 0.081361, + "recreation_consumption": 0.973924, "region": 1.0, - "rent": 0.312252, - "restaurants_and_hotels_consumption": 0.574867, - "savings": 0.635344, - "savings_interest_income": 0.343566, - "sda_reported": 0.000104, - "self_employment_income": 0.061533, - "ssmg_reported": 0.000415, - "state_pension_reported": 0.23571, - "statutory_maternity_pay": 0.002506, - "statutory_sick_pay": 0.002057, - "structural_insurance_payments": 0.615983, - "student_loan_balance": 0.047256, + "rent": 0.311452, + "restaurants_and_hotels_consumption": 0.630549, + "savings": 0.662075, + "savings_interest_income": 0.424963, + "sda_reported": 8.8e-05, + "self_employment_income": 0.054939, + "ssmg_reported": 0.000405, + "state_pension_reported": 0.245694, + "statutory_maternity_pay": 0.002139, + "statutory_sick_pay": 0.001866, + "structural_insurance_payments": 0.616054, + "student_loan_balance": 0.01983, "student_loan_plan": 1.0, - "student_loan_repayments": 0.047584, - "student_loans": 0.011633, - "tax_free_savings_income": 0.157705, + "student_loan_repayments": 0.049042, + "student_loans": 0.012965, + "tax_free_savings_income": 0.189664, "tenure_type": 1.0, - "transport_consumption": 0.780145, - "universal_credit_reported": 0.042537, - "water_and_sewerage_charges": 0.852209, - "winter_fuel_allowance_reported": 0.23908, - "working_tax_credit_reported": 0.014761, - "would_claim_child_benefit": 0.893341, - "would_claim_extended_childcare": 0.808459, - "would_claim_marriage_allowance": 0.500147, - "would_claim_pc": 0.702091, - "would_claim_scp": 0.860254, - "would_claim_targeted_childcare": 0.594623, - "would_claim_tfc": 0.581667, - "would_claim_uc": 0.550163, - "would_claim_universal_childcare": 0.560923, - "would_evade_tv_licence_fee": 0.105666 + "transport_consumption": 0.86226, + "universal_credit_reported": 0.057359, + "water_and_sewerage_charges": 0.776937, + "winter_fuel_allowance_reported": 0.018598, + "working_tax_credit_reported": 0.000493, + "would_claim_child_benefit": 0.892573, + "would_claim_extended_childcare": 0.813893, + "would_claim_marriage_allowance": 0.498587, + "would_claim_pc": 0.70088, + "would_claim_scp": 0.858287, + "would_claim_targeted_childcare": 0.593404, + "would_claim_tfc": 0.879931, + "would_claim_uc": 0.550594, + "would_claim_universal_childcare": 0.560933, + "would_evade_tv_licence_fee": 0.127011 }, "schema_version": 3, "source": { - "filename": "enhanced_frs_2023_24.h5", - "period": "2023", + "filename": "enhanced_frs_2024_25.h5", + "period": "2024", "repo_id": "policyengine/policyengine-uk-data-private", "repo_type": "model", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "size_bytes": 125434652, - "url": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/655dd07e4bb9c777b00dac044949611f1feb824f/enhanced_frs_2023_24.h5", - "vintage": "2023_24" + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "size_bytes": 126579434, + "url": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/a2039519d3b92aecc06c66dfd175cb46ac24cada/enhanced_frs_2024_25.h5", + "vintage": "2024_25" } } diff --git a/packages/microcosm-build/src/microcosm/build/uk/frs_release.json b/packages/microcosm-build/src/microcosm/build/uk/frs_release.json new file mode 100644 index 00000000..b9832513 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/frs_release.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "country": "uk", + "policy": "The UK FRS release pin freezes the raw Family Resources Survey vintage that seeds the UK build; runtime loaders and source manifests must agree on its survey, base, calibration, UKDS, DOI, acquisition, and hash identity before licensed tabs are consumed.", + "name": "frs_2024_25", + "survey_year": 2024, + "base_year": 2024, + "calibration_year": 2025, + "time_period": "2024", + "vintage": "2024_25", + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1", + "ukds_tab_zip": { + "filename": "9563tab_05DD0069587DBD25E5719D355CE05FC0827D5EDD58C24ECE9AB85ACD954A9AEB_V1.zip", + "sha256": "05dd0069587dbd25e5719d355ce05fc0827d5edd58c24ece9ab85acd954a9aeb" + }, + "acquisition": { + "huggingface_repo": "policyengine/policyengine-uk-data-private", + "huggingface_revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "zip_filename": "frs_2024_25.zip", + "zip_sha256": "05dd0069587dbd25e5719d355ce05fc0827d5edd58c24ece9ab85acd954a9aeb", + "zip_size_bytes": 46637202 + }, + "notes": "Tape years labeled by tax year (2024_25) per the SPI-stage convention. Donor vintages (SPI 2022-23, WAS r8, LCFS 2023-24, ETB) unchanged. Adopted from the old E11 scope (microcosm#723)." +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index ce9119aa..5b33c56c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -313,13 +313,13 @@ "reference_registry": { "efrs-post-calibration": { "identity": { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24" + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25" }, - "totals_sha256": "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e", - "scope_note": "Post-calibration eFRS production incumbent; structurally lacks the SPI clone channel, so SPI-channel-exclusive columns are comparable only through per-reference reviewed exclusions." + "totals_sha256": "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659", + "scope_note": "Channel-blind post-calibration enhanced-FRS production incumbent, pinned to the 2024-25 line; its artifact carries the SPI-synthetic rows structurally but no admin-restored mass in the SPI-channel-exclusive columns, so those columns are comparable only through per-reference reviewed exclusions." } }, "relative_tolerance": 4.521811483823806, diff --git a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json index 298b6b8c..995d4824 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json @@ -7,15 +7,15 @@ "reason": "SPI-channel-exclusive column on a channel-blind reference: the efrs-post-calibration incumbent structurally lacks the SPI clone channel, so its reference mass is survey-side scraps while the staged candidate's mass is the admin-captured SPI channel functioning as designed (microcosm#630 case 2). Compared normally against any future channel-aware reference.", "approved_by": "juaristi22", "adjudication": "microcosm#630", - "approved_on": "2026-08-17", - "expires_on": "2027-02-17" + "approved_on": "2026-08-20", + "expires_on": "2027-02-20" }, "owned_land": { "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the E5 stability receipt (data/ukds/acceptance/e5/owned_land_stability_receipt.json) measures a 37.7 percent national and 2.41x London swing between adjacent seeds, the same realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases). Register parity at this grain is not meaningful until the whole-spine comparison; the one-month expiry enforces the end-of-workstream revisit registered on microcosm#145 (winsorised donor or separate land imputation are the candidate remedies).", "approved_by": "juaristi22", "adjudication": "microcosm#714", - "approved_on": "2026-08-19", - "expires_on": "2026-09-19" + "approved_on": "2026-08-20", + "expires_on": "2026-09-20" } } } diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 72f881b3..7c166b3c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -478,7 +478,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -497,7 +497,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -518,10 +518,10 @@ ], "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "16b68f1d222116278421b210d3d9de0d06edcdf25d68f5a29dbf0f52ca63384a", + "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", "source_vintages": { "hmrc_surface": "2023-24", - "mapped_build_period": "2023" + "mapped_build_period": "2024" }, "stage": "hmrc_cgt_gains", "status": "required_at_build" @@ -531,6 +531,8 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "calibration_permitted": false, + "canonical_source_manifest": "source_stages.json", + "canonical_source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -603,7 +605,8 @@ "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", "source_vintages": { "hmrc_surface": "2023-24", - "mapped_build_period": "2023", + "mapped_build_period": "2024", + "period_mapping": "latest_published_tax_year", "spi_donor": "2022-23" }, "spi_prior_national_household_mass_share": 0.5, @@ -639,7 +642,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -659,7 +662,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -690,7 +693,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" @@ -701,12 +704,12 @@ }, "reference": { "derived_from": "efrs_parity_reference.json", - "filename": "enhanced_frs_2023_24.h5", - "period": "2023", + "filename": "enhanced_frs_2024_25.h5", + "period": "2024", "populated_input_columns": 145, - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24" + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25" }, "restoration_evidence": { "derived_from": "efrs_parity_known_gaps.json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 20a473d7..9e052b4c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -5,8 +5,8 @@ "stages": [ { "stage": "frs_spine", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2; local licensed 2023_24 tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "household", "artifacts": [ { @@ -14,154 +14,196 @@ "table": "accounts", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "accounts.tab", - "sha256": "c5e31932bfd06087f835d2c83c0984c85a93409bf5ef85b699cb0958abcba1ea", - "size_bytes": 1807921, - "runtime_sha256_required": true + "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", + "size_bytes": 1812923, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "benefits", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "benefits.tab", - "sha256": "ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b", - "size_bytes": 4460292, - "runtime_sha256_required": true + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "benunit", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "benunit.tab", - "sha256": "88946815eace8561516d5cbb442c27e319c1e90abc381fb2338f0126e3b9e05b", - "size_bytes": 21213867, - "runtime_sha256_required": true + "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", + "size_bytes": 13986782, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "child", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "child.tab", - "sha256": "b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e", - "size_bytes": 2913156, - "runtime_sha256_required": true + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "size_bytes": 2753961, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "chldcare", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "chldcare.tab", - "sha256": "566e0ebca1d5e2f3e424e556c91f4cb583d17dadfdfa59feb3841eda7e5976a3", - "size_bytes": 273837, - "runtime_sha256_required": true + "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", + "size_bytes": 275878, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "extchild", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "extchild.tab", - "sha256": "8d358d7ee66ee4a7ceab87b4f24fbbf21ac86dc038dc7831e51fb271f96a57ec", - "size_bytes": 18677, - "runtime_sha256_required": true + "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", + "size_bytes": 15150, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "househol", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "househol.tab", - "sha256": "5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30", - "size_bytes": 12387117, - "runtime_sha256_required": true + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "job", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "job.tab", - "sha256": "88b77ffe06865f029f713bb1d55ff12bdea8a1234de5bc293e72458fe64f3a74", - "size_bytes": 10934873, - "runtime_sha256_required": true + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "size_bytes": 10518760, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "maint", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "maint.tab", - "sha256": "f2dc924eb5a51b0c357791693d15b431327dc39c6421011efb313d88bf839695", - "size_bytes": 15440, - "runtime_sha256_required": true + "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", + "size_bytes": 13993, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "mortgage", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "mortgage.tab", - "sha256": "ce36b477d67837c469608a0d68f7ef269ac04758974235f1157d2f6b92cdbfdc", - "size_bytes": 631783, - "runtime_sha256_required": true + "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", + "size_bytes": 600552, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "oddjob", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "oddjob.tab", - "sha256": "b4ba3dd3151f73a01422983c60514a3e38458ddfa4fb33ae4ed0326873406305", - "size_bytes": 5165, - "runtime_sha256_required": true + "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", + "size_bytes": 5339, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "penprov", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "penprov.tab", - "sha256": "ee001461c40306ec24b38b2881e1774121114266a2ee449d606cd0a811c37731", - "size_bytes": 522313, - "runtime_sha256_required": true + "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "size_bytes": 513614, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "pension", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "pension.tab", - "sha256": "150d6fad1fce81254fb7aea1526fbb00b63d4027d6e2ac4c26bb90aea3127eb7", - "size_bytes": 1225838, - "runtime_sha256_required": true + "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", + "size_bytes": 1232411, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -306,12 +348,12 @@ "housing_service_charges", "external_child_payments" ], - "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage." + "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity." }, { "stage": "frs_employment", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "person", "artifacts": [ { @@ -319,11 +361,14 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -352,8 +397,8 @@ }, { "stage": "frs_council_tax", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "household", "artifacts": [ { @@ -361,11 +406,14 @@ "table": "househol", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "househol.tab", - "sha256": "5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30", - "size_bytes": 12387117, - "runtime_sha256_required": true + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -399,8 +447,8 @@ }, { "stage": "frs_disability", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DWP parameters.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", "grain": "person", "artifacts": [], "operations": [ @@ -427,8 +475,8 @@ }, { "stage": "frs_education", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "person", "artifacts": [ { @@ -436,22 +484,28 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "child", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "child.tab", - "sha256": "b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e", - "size_bytes": 2913156, - "runtime_sha256_required": true + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "size_bytes": 2753961, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -493,12 +547,12 @@ "child_ema", "age_started_or_accepted_current_education_or_training" ], - "notes": "Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2023-24 data dictionary (UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Code 87 is undocumented in the dictionary and falls to the default. EMA uses the shared weeks-in-year constant." + "notes": "Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2024-25 data dictionary (UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Codes 1-87 are empirically present at 2024-25; code 87 remains undocumented in the SN 9563 dictionary and falls to the default. EMA uses the shared weeks-in-year constant." }, { "stage": "frs_legacy_proxies", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DWP parameters.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", "grain": "person", "artifacts": [ { @@ -506,11 +560,14 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -543,8 +600,8 @@ }, { "stage": "frs_education_grant_split", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DfE grant parameters.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DfE grant parameters.", "grain": "person", "artifacts": [], "operations": [ @@ -571,12 +628,12 @@ "nonnegative_outputs": [ "disabled_students_allowance_eligible_expenses" ], - "notes": "Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read." + "notes": "Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted." }, { "stage": "frs_take_up", - "survey": "Family Resources Survey 2023-24", - "source": "Family Resources Survey 2023-24 reported receipt anchors plus sourced UK take-up contract rates.", + "survey": "Family Resources Survey 2024-25", + "source": "Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates.", "grain": "benunit", "artifacts": [], "operations": [ @@ -675,7 +732,7 @@ }, { "stage": "frs_person_draws", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Sourced UK take-up contract rates and identity-keyed deterministic draws.", "grain": "person", "artifacts": [], @@ -721,7 +778,7 @@ }, { "stage": "frs_household_draws", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", "grain": "household", "artifacts": [], @@ -765,7 +822,7 @@ }, { "stage": "frs_brma", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor.", "grain": "household", "artifacts": [ @@ -1048,7 +1105,7 @@ "kind": "private_microdata", "format": "tab", "vintage": "2018_20", - "locator": "was_round_8_hhold_eul_may_2025_230525.tab", + "locator": "caller-supplied local input", "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "size_bytes": 39073613, "runtime_sha256_required": true, @@ -1457,8 +1514,8 @@ }, { "stage": "frs_hmrc_spine_leaves", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied local input", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", "grain": "person", "artifacts": [ { @@ -1466,30 +1523,36 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "benefits", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "benefits.tab", - "sha256": "ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b", - "size_bytes": 4460292, - "runtime_sha256_required": true + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ { "kind": "retain_adjudicated_frs_hmrc_leaves", "population": "uk_frs_raw_spine", - "source_vintage": "2023-24", - "mapped_build_period": 2023, + "source_vintage": "2024-25", + "mapped_build_period": 2024, "annualization": "weekly raw FRS amounts * (365.25 / 7)", "status": "adjudicated_partial_replay", "retained_full_constituents": { @@ -1589,7 +1652,7 @@ }, { "stage": "spi_support_channel", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", "grain": "household", "artifacts": [], @@ -1695,8 +1758,8 @@ "Table_3_6", "Table_3_7" ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "runtime_sha256_required": true } ], @@ -1902,8 +1965,8 @@ "require_all_predictors": true, "require_all_outputs": true, "initialize_frs_channel_columns": { - "gift_aid": 0.0, - "charitable_investment_gifts": 0.0 + "gift_aid": 0, + "charitable_investment_gifts": 0 }, "n_estimators": 100, "seed": 42 @@ -2000,8 +2063,8 @@ { "kind": "materialize_hmrc_income_bands_fail_closed", "artifact_role": "published_fact_surface", - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "column_index_base": 0, "data_row_start_index": 5, "stop_label": "All ranges", @@ -2258,7 +2321,7 @@ "support_channel_column": "person_support_channel", "required_support_channel": "spi", "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 1e-06, + "minimum_nondefault_mass_share": 0.000001, "fail_below_floor": true } ], @@ -2337,8 +2400,8 @@ }, { "stage": "frs_hmrc_retained_leaves", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied local input", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", "grain": "person", "artifacts": [], "operations": [ @@ -2351,8 +2414,8 @@ { "kind": "retain_adjudicated_frs_hmrc_leaves", "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2023-24", - "mapped_build_period": 2023, + "source_vintage": "2024-25", + "mapped_build_period": 2024, "annualization": "weekly raw FRS amounts * (365.25 / 7)", "status": "adjudicated_partial_replay", "retained_full_constituents": { @@ -2474,8 +2537,8 @@ "Table_3_6", "Table_3_7" ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "runtime_sha256_required": true } ], @@ -2778,8 +2841,8 @@ { "kind": "materialize_hmrc_income_bands_fail_closed", "artifact_role": "published_fact_surface", - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "column_index_base": 0, "data_row_start_index": 5, "stop_label": "All ranges", @@ -3036,7 +3099,7 @@ "support_channel_column": "person_support_channel", "required_support_channel": "spi", "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 1e-06, + "minimum_nondefault_mass_share": 0.000001, "fail_below_floor": true } ], diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 99f44cc1..e4194750 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -13,142 +13,181 @@ sources: stage_manifest: version: 1 country: uk - policy: The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact - must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay - keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure. + policy: The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure. stages: - stage: frs_spine - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2; - local licensed 2023_24 tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: household artifacts: - role: frs_table table: accounts kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: accounts.tab - sha256: c5e31932bfd06087f835d2c83c0984c85a93409bf5ef85b699cb0958abcba1ea - size_bytes: 1807921 + sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f + size_bytes: 1812923 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: benefits kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: benefits.tab - sha256: ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b - size_bytes: 4460292 + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + size_bytes: 2362329 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: benunit kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: benunit.tab - sha256: 88946815eace8561516d5cbb442c27e319c1e90abc381fb2338f0126e3b9e05b - size_bytes: 21213867 + sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a + size_bytes: 13986782 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: child kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: child.tab - sha256: b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e - size_bytes: 2913156 + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + size_bytes: 2753961 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: chldcare kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: chldcare.tab - sha256: 566e0ebca1d5e2f3e424e556c91f4cb583d17dadfdfa59feb3841eda7e5976a3 - size_bytes: 273837 + sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 + size_bytes: 275878 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: extchild kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: extchild.tab - sha256: 8d358d7ee66ee4a7ceab87b4f24fbbf21ac86dc038dc7831e51fb271f96a57ec - size_bytes: 18677 + sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 + size_bytes: 15150 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: househol kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: househol.tab - sha256: 5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30 - size_bytes: 12387117 + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + size_bytes: 12108606 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: job kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: job.tab - sha256: 88b77ffe06865f029f713bb1d55ff12bdea8a1234de5bc293e72458fe64f3a74 - size_bytes: 10934873 + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + size_bytes: 10518760 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: maint kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: maint.tab - sha256: f2dc924eb5a51b0c357791693d15b431327dc39c6421011efb313d88bf839695 - size_bytes: 15440 + sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d + size_bytes: 13993 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: mortgage kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: mortgage.tab - sha256: ce36b477d67837c469608a0d68f7ef269ac04758974235f1157d2f6b92cdbfdc - size_bytes: 631783 + sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 + size_bytes: 600552 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: oddjob kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: oddjob.tab - sha256: b4ba3dd3151f73a01422983c60514a3e38458ddfa4fb33ae4ed0326873406305 - size_bytes: 5165 + sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead + size_bytes: 5339 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: penprov kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: penprov.tab - sha256: ee001461c40306ec24b38b2881e1774121114266a2ee449d606cd0a811c37731 - size_bytes: 522313 + sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + size_bytes: 513614 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: pension kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: pension.tab - sha256: 150d6fad1fce81254fb7aea1526fbb00b63d4027d6e2ac4c26bb90aea3127eb7 - size_bytes: 1225838 + sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 + size_bytes: 1232411 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -157,9 +196,7 @@ stages: numeric_errors: coerce runtime_sha256_required: true - kind: replace_sentinels - sentinel_policy: 'raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled - to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays - unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate.' + sentinel_policy: 'raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate.' - kind: assemble_group_entities household_id: sernum benunit_id: sernum * 100 + benunit @@ -278,23 +315,24 @@ stages: - structural_insurance_payments - housing_service_charges - external_child_payments - notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields - remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally - absent from this stage. + notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. - stage: frs_employment - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -310,22 +348,24 @@ stages: - sic_industry_division nonnegative_outputs: - sic_industry_division - notes: Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, - so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent. + notes: Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent. - stage: frs_council_tax - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: household artifacts: - role: frs_table table: househol kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: househol.tab - sha256: 5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30 - size_bytes: 12387117 + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + size_bytes: 12108606 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -345,12 +385,10 @@ stages: - council_tax nonnegative_outputs: - council_tax - notes: Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted - before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped. + notes: Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped. - stage: frs_disability - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus - policyengine-uk DWP parameters. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters. grain: person artifacts: [] operations: @@ -367,31 +405,36 @@ stages: - is_disabled_for_benefits - is_enhanced_disabled_for_benefits - is_severely_disabled_for_benefits - notes: Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped - at E10; export allowlists stay fail-closed. + notes: Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped at E10; export allowlists stay fail-closed. - stage: frs_education - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: child kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: child.tab - sha256: b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e - size_bytes: 2913156 + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + size_bytes: 2753961 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -403,8 +446,7 @@ stages: scope: current/highest education, QYP inputs, EMA cell-mean degenerate fills, and benefits-in-own-right flag - kind: impute_cell_means cells: - - 'single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt - for children)' + - 'single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt for children)' donor_filter: participants with non-negative reported amounts missing: participants with sentinel negative reported amounts value: donor-mean fill, floored at zero, annualized with 365.25 / 7 @@ -422,26 +464,24 @@ stages: - adult_ema - child_ema - age_started_or_accepted_current_education_or_training - notes: 'Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries - the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2023-24 data dictionary (UK Data - Service SN 9367, DOI 10.5255/UKDA-SN-9367-2, adult table), corroborated against the raw aggregates and adopted as a signed - difference (PR #703); an upstream defect report records the incumbent inversion. Code 87 is undocumented in the dictionary - and falls to the default. EMA uses the shared weeks-in-year constant.' + notes: 'Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2024-25 data dictionary (UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Codes 1-87 are empirically present at 2024-25; code 87 remains undocumented in the SN 9563 dictionary and falls to the default. EMA uses the shared weeks-in-year constant.' - stage: frs_legacy_proxies - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus - policyengine-uk DWP parameters. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters. grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -459,12 +499,10 @@ stages: - legacy_jobseeker_proxy - esa_health_condition_proxy - esa_support_group_proxy - notes: The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine - hours scale; state_pension_age is consumed but not persisted. + notes: The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine hours scale; state_pension_age is consumed but not persisted. - stage: frs_education_grant_split - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus - policyengine-uk DfE grant parameters. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DfE grant parameters. grain: person artifacts: [] operations: @@ -482,11 +520,10 @@ stages: - education_grants nonnegative_outputs: - disabled_students_allowance_eligible_expenses - notes: Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned - zero vector rather than an engine shape-sizing read. + notes: Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted. - stage: frs_take_up - survey: Family Resources Survey 2023-24 - source: Family Resources Survey 2023-24 reported receipt anchors plus sourced UK take-up contract rates. + survey: Family Resources Survey 2024-25 + source: Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates. grain: benunit artifacts: [] operations: @@ -557,10 +594,9 @@ stages: - maximum_extended_childcare_hours_usage nonnegative_outputs: - maximum_extended_childcare_hours_usage - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. - Reported positive receipt anchors are transient and consumed only. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. - stage: frs_person_draws - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Sourced UK take-up contract rates and identity-keyed deterministic draws. grain: person artifacts: [] @@ -588,10 +624,9 @@ stages: - attends_private_school_random_draw nonnegative_outputs: - attends_private_school_random_draw - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally - not produced. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. - stage: frs_household_draws - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Sourced UK stochastic contract rates and identity-keyed deterministic draws. grain: household artifacts: [] @@ -621,10 +656,9 @@ stages: - would_evade_tv_licence_fee - main_residential_property_purchased_is_first_home - property_purchased - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent - draw, matching the incumbent reference share. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share. - stage: frs_brma - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor. grain: household artifacts: @@ -649,12 +683,10 @@ stages: seed: 0 outputs: - brma - notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt - brma:household_pick. + notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick. - stage: was_wealth survey: Wealth and Assets Survey round 8 - source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local - licensed 2006-22 household tab. + source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. grain: household base_candidate: filename: populace_uk_2023.h5 @@ -790,18 +822,8 @@ stages: - cash_isa - stocks_and_shares_isa - student_loan_balance - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces - the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds - stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales - for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. - Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the - incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy - encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully - like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes - (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: - 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality - revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s - -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero + for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' - stage: regional_property_uprating survey: Public regional property reference source: MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices. @@ -826,13 +848,10 @@ stages: rewrites: - main_residence_value - property_wealth - notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. - Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean - follows the incumbent behavior. + notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. - stage: lcfs_consumption survey: Living Costs and Food Survey 2023-24 - source: UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, - Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor. + source: UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor. grain: household base_candidate: filename: populace_uk_2023.h5 @@ -862,7 +881,7 @@ stages: kind: private_microdata format: tab vintage: '2018_20' - locator: was_round_8_hhold_eul_may_2025_230525.tab + locator: caller-supplied local input sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 size_bytes: 39073613 runtime_sha256_required: true @@ -1025,11 +1044,7 @@ stages: - electricity_consumption - gas_consumption - has_fuel_consumption - notes: Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed - seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity - at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating - is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build - year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh. + notes: Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh. - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. @@ -1082,14 +1097,10 @@ stages: outputs: - full_rate_vat_expenditure_rate nonnegative_outputs: [] - notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather - than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: - totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the - support gate is the guard - the net_financial_wealth precedent from was_wealth.' + notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth.' - stage: etb_services survey: Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table - source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost - table. + source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table. grain: household+person base_candidate: filename: populace_uk_2023.h5 @@ -1183,40 +1194,41 @@ stages: - nhs_a_and_e_spending - nhs_admitted_patient_spending - nhs_outpatient_spending - notes: Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS - visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves - to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides - with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this - vintage. + notes: Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this vintage. - stage: frs_hmrc_spine_leaves - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied - local input + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: benefits kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: benefits.tab - sha256: ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b - size_bytes: 4460292 + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + size_bytes: 2362329 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: retain_adjudicated_frs_hmrc_leaves population: uk_frs_raw_spine - source_vintage: 2023-24 - mapped_build_period: 2023 + source_vintage: 2024-25 + mapped_build_period: 2024 annualization: weekly raw FRS amounts * (365.25 / 7) status: adjudicated_partial_replay retained_full_constituents: @@ -1291,10 +1303,9 @@ stages: - ossben_identifiable_subset - srp_regular_code5 - employer_pension_contributions - notes: Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, - then ports the incumbent employer-pension-contributions estimate. + notes: Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then ports the incumbent employer-pension-contributions estimate. - stage: spi_support_channel - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning. grain: household artifacts: [] @@ -1344,8 +1355,7 @@ stages: - household_source_id - source_household_id - source_year - notes: Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, - and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation. + notes: Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation. - stage: hmrc_spi_income_spine survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -1380,8 +1390,8 @@ stages: sheets: - Table_3_6 - Table_3_7 - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year runtime_sha256_required: true operations: - kind: verify_pinned_hmrc_source_pair @@ -1511,15 +1521,11 @@ stages: - hmrc_spi_other_income - hmrc_spi_state_pension_income joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine - gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; - it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM - + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels + savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input + employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure + hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates - and TI equals TEI + TII exactly + assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly source_ti_identity_fields: - TI - TEI @@ -1528,8 +1534,7 @@ stages: documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf composite_indicator: AGERANGE == -1 formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, - PROFITS - CAPALL - LOSSBF) + TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS TI: TEI + TII maximum_absolute_difference_gbp: @@ -1541,9 +1546,7 @@ stages: TEI: 180 TII: 10 TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income - fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain - exact. + rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. ti_identity_absolute_tolerance_gbp: 5 stochastic_aggregates_forbidden: - hmrc_spi_employed_income @@ -1571,9 +1574,7 @@ stages: - private_pension_income - property_income reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only - stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified - Microcosm UK base candidate has no other_investment_income column.' + other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' categorical_predictors: - gender - region @@ -1610,10 +1611,8 @@ stages: - esa_contrib_reported - esa_income_reported reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not - a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training - source can be materialized for this stage. + incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. + maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. postprocess: gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income refresh_disability_categories: @@ -1639,8 +1638,8 @@ stages: rows: base_support_channel - kind: materialize_hmrc_income_bands_fail_closed artifact_role: published_fact_surface - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year column_index_base: 0 data_row_start_index: 5 stop_label: All ranges @@ -1723,9 +1722,7 @@ stages: directional_pass: 0 directional_fail: 0 excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize - full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound - is valid. + classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. reviewed_fences: - fence_id: frs_epb_source_absent constituents: @@ -1737,12 +1734,9 @@ stages: - JOB.FUELAMT - JOB.VCHAMT - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce - complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this - is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an - imputation or proxy. + finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. + mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. + rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. dependent_fence_ids: [] - fence_id: frs_exps_source_absent constituents: @@ -1753,12 +1747,9 @@ stages: - JOB.UMILEAMT/JOB.UMOTAMT - JOB.DEDUC1-DEDUC9 - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense - amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the - true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity - with a negative sign. + finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. + mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. + rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. dependent_fence_ids: [] - fence_id: frs_taxterm_source_absent constituents: @@ -1767,8 +1758,7 @@ stages: - ADULT.REDAMT - ADULT and JOB taxable-termination split search finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable - mass is unknown. + mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. dependent_fence_ids: [] - fence_id: frs_mothinc_source_absent @@ -1779,8 +1769,7 @@ stages: - ADULT.ALLPAY2 - ADULT.ROYYR2-ROYYR4 - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI - miscellaneous employment income would be a proxy. + finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. dependent_fence_ids: [] @@ -1793,10 +1782,8 @@ stages: - ACCOUNTS - ASSETS - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be - split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate - FRS effective person mass. + finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. + mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. dependent_fence_ids: [] - fence_id: frs_ossben_identifiable_subset @@ -1808,10 +1795,8 @@ stages: - BENEFITS.BENEFIT - BENEFITS.VAR2 - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax - treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; - it is not full OSSBEN support. + finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. + mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. dependent_fence_ids: [] - fence_id: frs_srp_regular_code5_subset @@ -1821,12 +1806,9 @@ stages: raw_sources_searched: - BENEFITS.BENAMT where BENEFIT == 5 - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination - of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; - it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published - state-pension measure. + finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. + mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. + rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. dependent_fence_ids: [] - fence_id: full_frs_tei_band_unavailable constituents: @@ -1838,12 +1820,9 @@ stages: - OSSBEN - SRP raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income - band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable - like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into - or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' + finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. + mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. + rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' dependent_fence_ids: - frs_epb_source_absent - frs_exps_source_absent @@ -1936,13 +1915,10 @@ stages: - hmrc_spi_pay - hmrc_spi_unemployment_benefit_income - hmrc_spi_incapacity_benefit_income - notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains - FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine - path. + notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path. - stage: frs_hmrc_retained_leaves - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied - local input + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input grain: person artifacts: [] operations: @@ -1952,8 +1928,8 @@ stages: fail_on_mismatch: true - kind: retain_adjudicated_frs_hmrc_leaves population: certified_microcosm_uk_candidate_base_channel - source_vintage: 2023-24 - mapped_build_period: 2023 + source_vintage: 2024-25 + mapped_build_period: 2024 annualization: weekly raw FRS amounts * (365.25 / 7) status: adjudicated_partial_replay retained_full_constituents: @@ -2016,9 +1992,7 @@ stages: - hmrc_spi_incapacity_benefit_income - ossben_identifiable_subset - srp_regular_code5 - notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and - INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these - leaves.' + notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves.' - stage: hmrc_spi_income survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -2053,8 +2027,8 @@ stages: sheets: - Table_3_6 - Table_3_7 - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year runtime_sha256_required: true operations: - kind: verify_pinned_hmrc_source_pair @@ -2075,8 +2049,7 @@ stages: output_weight_kind: importance preserve_total_household_mass: true require_mass_change_record: true - mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; - total national mass is conserved. + mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved. fail_on_live_existing_spi_mass: true - kind: strict_read_private_table artifact_role: qrf_donor @@ -2197,15 +2170,11 @@ stages: - hmrc_spi_other_income - hmrc_spi_state_pension_income joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine - gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; - it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM - + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels + savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input + employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure + hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates - and TI equals TEI + TII exactly + assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly source_ti_identity_fields: - TI - TEI @@ -2214,8 +2183,7 @@ stages: documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf composite_indicator: AGERANGE == -1 formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, - PROFITS - CAPALL - LOSSBF) + TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS TI: TEI + TII maximum_absolute_difference_gbp: @@ -2227,9 +2195,7 @@ stages: TEI: 180 TII: 10 TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income - fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain - exact. + rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. ti_identity_absolute_tolerance_gbp: 5 stochastic_aggregates_forbidden: - hmrc_spi_employed_income @@ -2252,9 +2218,7 @@ stages: - private_pension_income - property_income reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only - stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified - Microcosm UK base candidate has no other_investment_income column.' + other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' categorical_predictors: - gender - region @@ -2291,10 +2255,8 @@ stages: - esa_contrib_reported - esa_income_reported reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not - a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training - source can be materialized for this stage. + incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. + maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. postprocess: gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income refresh_disability_categories: @@ -2313,8 +2275,8 @@ stages: require_all_outputs: false - kind: materialize_hmrc_income_bands_fail_closed artifact_role: published_fact_surface - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year column_index_base: 0 data_row_start_index: 5 stop_label: All ranges @@ -2397,9 +2359,7 @@ stages: directional_pass: 0 directional_fail: 0 excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize - full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound - is valid. + classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. reviewed_fences: - fence_id: frs_epb_source_absent constituents: @@ -2411,12 +2371,9 @@ stages: - JOB.FUELAMT - JOB.VCHAMT - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce - complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this - is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an - imputation or proxy. + finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. + mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. + rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. dependent_fence_ids: [] - fence_id: frs_exps_source_absent constituents: @@ -2427,12 +2384,9 @@ stages: - JOB.UMILEAMT/JOB.UMOTAMT - JOB.DEDUC1-DEDUC9 - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense - amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the - true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity - with a negative sign. + finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. + mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. + rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. dependent_fence_ids: [] - fence_id: frs_taxterm_source_absent constituents: @@ -2441,8 +2395,7 @@ stages: - ADULT.REDAMT - ADULT and JOB taxable-termination split search finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable - mass is unknown. + mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. dependent_fence_ids: [] - fence_id: frs_mothinc_source_absent @@ -2453,8 +2406,7 @@ stages: - ADULT.ALLPAY2 - ADULT.ROYYR2-ROYYR4 - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI - miscellaneous employment income would be a proxy. + finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. dependent_fence_ids: [] @@ -2467,10 +2419,8 @@ stages: - ACCOUNTS - ASSETS - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be - split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate - FRS effective person mass. + finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. + mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. dependent_fence_ids: [] - fence_id: frs_ossben_identifiable_subset @@ -2482,10 +2432,8 @@ stages: - BENEFITS.BENEFIT - BENEFITS.VAR2 - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax - treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; - it is not full OSSBEN support. + finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. + mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. dependent_fence_ids: [] - fence_id: frs_srp_regular_code5_subset @@ -2495,12 +2443,9 @@ stages: raw_sources_searched: - BENEFITS.BENAMT where BENEFIT == 5 - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination - of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; - it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published - state-pension measure. + finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. + mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. + rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. dependent_fence_ids: [] - fence_id: full_frs_tei_band_unavailable constituents: @@ -2512,12 +2457,9 @@ stages: - OSSBEN - SRP raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income - band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable - like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into - or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' + finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. + mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. + rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' dependent_fence_ids: - frs_epb_source_absent - frs_exps_source_absent @@ -2569,15 +2511,5 @@ stages: - hmrc_spi_total_earned_income - hmrc_spi_total_investment_income - hmrc_spi_assessable_income - notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned - by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed - income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. - PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent - UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, - private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, - but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful - full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, - MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, - none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence - record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt - positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' + notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, + and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' diff --git a/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json b/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json index 85cad6b3..72c6f4d6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json +++ b/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json @@ -1,8 +1,8 @@ { "version": 1, "country": "uk", - "build_year": 2023, - "policy": "UK stochastic assignment contract for the FRS 2023-24 source spine. Date-keyed values use the incumbent parameter loader semantics: choose the latest value with year less than or equal to the build year. Reported-receipt anchors use an unweighted full-population target denominator, target = int(rate * n_units), and accept anchor overshoot. Random streams are intentionally identity-keyed by seed, output variable name, and entity id; this is a signed difference from the incumbent UK pipeline sequential RNG stream. Incumbent provenance: ebf733c.", + "build_year": 2024, + "policy": "UK stochastic assignment contract for the FRS 2024-25 source spine. Date-keyed values use the incumbent parameter loader semantics: choose the latest value with year less than or equal to the build year. Reported-receipt anchors use an unweighted full-population target denominator, target = int(rate * n_units), and accept anchor overshoot. Random streams are intentionally identity-keyed by seed, output variable name, and entity id; this is a signed difference from the incumbent UK pipeline sequential RNG stream. Incumbent provenance: ebf733c.", "incumbent_provenance": { "commit": "ebf733c", "description": "incumbent UK pipeline stochastic layer", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py new file mode 100644 index 00000000..c8472abe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py @@ -0,0 +1,112 @@ +"""Frozen UK FRS raw-vintage release loader.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from functools import lru_cache +from importlib.resources import files +from typing import Any + +__all__ = ["UKFRSRelease", "load_uk_frs_release"] + +_LOWER_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_VINTAGE = re.compile(r"^\d{4}_\d{2}$") + + +@dataclass(frozen=True) +class UKFRSRelease: + """Checked-in identity for the UK raw FRS release used by the build.""" + + version: int + country: str + policy: str + name: str + survey_year: int + base_year: int + calibration_year: int + time_period: str + vintage: str + ukds_study_number: int + doi: str + ukds_tab_zip: Mapping[str, Any] + acquisition: Mapping[str, Any] + notes: str + + +def _release_path(): + return files("microcosm.build.uk").joinpath("frs_release.json") + + +@lru_cache(maxsize=1) +def load_uk_frs_release() -> UKFRSRelease: + """Load and validate ``build/uk/frs_release.json``.""" + + raw = json.loads(_release_path().read_text(encoding="utf-8")) + if not isinstance(raw, Mapping): + raise ValueError("UK FRS release must contain a JSON object.") + release = UKFRSRelease( + version=_int(raw, "version"), + country=_str(raw, "country"), + policy=_str(raw, "policy"), + name=_str(raw, "name"), + survey_year=_int(raw, "survey_year"), + base_year=_int(raw, "base_year"), + calibration_year=_int(raw, "calibration_year"), + time_period=_str(raw, "time_period"), + vintage=_str(raw, "vintage"), + ukds_study_number=_int(raw, "ukds_study_number"), + doi=_str(raw, "doi"), + ukds_tab_zip=_mapping(raw, "ukds_tab_zip"), + acquisition=_mapping(raw, "acquisition"), + notes=_str(raw, "notes"), + ) + _validate(release) + return release + + +def _int(raw: Mapping[str, Any], key: str) -> int: + value = raw.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"UK FRS release requires integer {key}.") + return value + + +def _str(raw: Mapping[str, Any], key: str) -> str: + value = raw.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"UK FRS release requires non-empty string {key}.") + return value + + +def _mapping(raw: Mapping[str, Any], key: str) -> Mapping[str, Any]: + value = raw.get(key) + if not isinstance(value, Mapping): + raise ValueError(f"UK FRS release requires object {key}.") + return dict(value) + + +def _validate(release: UKFRSRelease) -> None: + if release.country != "uk": + raise ValueError("UK FRS release country must be 'uk'.") + if not (release.survey_year <= release.base_year < release.calibration_year): + raise ValueError( + "UK FRS release years must satisfy survey_year <= base_year < " + "calibration_year." + ) + if release.time_period != str(release.base_year): + raise ValueError("UK FRS release time_period must equal base_year.") + if not _VINTAGE.fullmatch(release.vintage): + raise ValueError("UK FRS release vintage must match YYYY_YY.") + if not release.vintage.startswith(str(release.survey_year)): + raise ValueError("UK FRS release vintage must start with survey_year.") + _validate_sha256(release.ukds_tab_zip, "ukds_tab_zip.sha256", "sha256") + _validate_sha256(release.acquisition, "acquisition.zip_sha256", "zip_sha256") + + +def _validate_sha256(raw: Mapping[str, Any], label: str, key: str) -> None: + value = raw.get(key) + if not isinstance(value, str) or not _LOWER_SHA256.fullmatch(value): + raise ValueError(f"UK FRS release {label} must be 64 lowercase hex chars.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 1ac77a43..323feea4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -55,7 +55,7 @@ ) WEEKS_IN_YEAR = 365.25 / 7 -TIME_PERIOD = "2023" +TIME_PERIOD = "2024" # FRS GVTREGNO uses skip-3 coding: code 3 (the retired Merseyside code) is # absent from the domain, so the real codes are [1, 2, 4..13] with diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py index 75f4f21c..93785b56 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py @@ -69,7 +69,7 @@ HMRC_CGT_JOINT_ODS_SIZE_BYTES = 11_996 HMRC_CGT_SOURCE_LABEL = "HMRC capital gains table 3" HMRC_CGT_SOURCE_VINTAGE = "2023-24" -HMRC_CGT_BUILD_PERIOD = "2023" +HMRC_CGT_BUILD_PERIOD = "2024" #: Worksheet per tax year, newest first. The publication carries four years, #: so a stage can fit each year rather than aging one forward. @@ -325,7 +325,9 @@ def materialize_hmrc_capital_gains_joint_distribution( Args: path: Local copy of the published ODS. tax_year: Which published year to read, as ``"2023-24"``. - build_period: The build period these facts are declared against. + build_period: The build period these facts are declared against. The + latest published tax year 2023-24 is replayed against build period + 2024 (signed, microcosm#723). verify_fingerprint: Whether to require the pinned hash and size. Only a caller reading a different vintage should turn this off, and it then owns checking what it read. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py index c3b6236e..f6e9f580 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py @@ -59,7 +59,7 @@ HMRC_SPI_SOURCE_VINTAGE = "2023-24" HMRC_SPI_SOURCE_TAX_YEAR = HMRC_SPI_SOURCE_VINTAGE HMRC_SPI_SOURCE_TAX_YEAR_START = 2023 -HMRC_SPI_BUILD_PERIOD = "2023" +HMRC_SPI_BUILD_PERIOD = "2024" HMRC_SPI_ASSESSABLE_INCOME_COLUMN = "hmrc_spi_assessable_income" HMRC_SPI_INCOME_BAND_LOWER_BOUNDS = ( @@ -357,16 +357,16 @@ def materialize_hmrc_spi_income_band_targets( ) -> HMRCIncomeTargetSet: """Parse the complete 2023-24 HMRC SPI income-band target surface. - The source tax year is mapped by its tax-year start, so 2023-24 is valid - only for the Microcosm build period ``"2023"``. Any missing, duplicate, or - malformed source record raises; this function never narrows the family. + The latest published tax year 2023-24 is replayed against build period + ``"2024"`` (signed, microcosm#723). Any missing, duplicate, or malformed + source record raises; this function never narrows the family. """ period = str(build_period) if period != HMRC_SPI_BUILD_PERIOD: raise ValueError( - f"HMRC SPI source tax year {HMRC_SPI_SOURCE_TAX_YEAR} starts in " - f"{HMRC_SPI_SOURCE_TAX_YEAR_START} and maps to build period " + f"HMRC SPI source tax year {HMRC_SPI_SOURCE_TAX_YEAR} is the " + "latest published tax year 2023-24 replayed against build period " f"{HMRC_SPI_BUILD_PERIOD!r}; got {period!r}." ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py index d8d715b9..b4f78b30 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import json from collections.abc import Mapping, Sequence from importlib.resources import files @@ -69,6 +70,7 @@ ] UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE = "hmrc_income_source_stages.json" +UK_CANONICAL_SOURCE_STAGES_RESOURCE = "source_stages.json" HMRC_DISTRIBUTIONAL_INPUTS = ( "gift_aid", "charitable_investment_gifts", @@ -90,9 +92,10 @@ _STAGE2_REVIEWED_ABSENT_PREDICTORS = { "other_investment_income": ( "This remains a stage-1 SPI draw and an official HMRC fact component, " - "but it is not an FRS-only stage-2 predictor: policyengine-uk-data " - "frs_only.py defines exactly six income predictors and the certified " - "Microcosm UK base candidate has no other_investment_income column." + "but it is not an FRS-only stage-2 predictor: the incumbent UK data " + "build's frs_only.py defines exactly six income predictors and the " + "certified Microcosm UK base candidate has no other_investment_income " + "column." ) } @@ -239,7 +242,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "published_fact_surface.period_mapping", surface.get("period_mapping"), - "tax_year_start", + "latest_published_tax_year", ) _expect( failures, @@ -300,7 +303,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "frs_leaves.source_vintage", frs_leaves.get("source_vintage"), - "2023-24", + "2024-25", ) _expect( failures, @@ -708,7 +711,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "materialize.period_mapping", materialize.get("period_mapping"), - "tax_year_start", + "latest_published_tax_year", ) _expect( failures, @@ -905,7 +908,12 @@ def assert_uk_hmrc_income_source_contract_current( "stage.outputs", tuple(stage.get("outputs", ())), ( - *HMRC_SPI_INCOME_COMPONENTS, + *( + "hmrc_spi_state_pension_income" + if component == "state_pension" + else component + for component in HMRC_SPI_INCOME_COMPONENTS + ), *HMRC_DISTRIBUTIONAL_INPUTS, *SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, ), @@ -971,11 +979,63 @@ def uk_hmrc_weighted_qrf_output_columns( def _load_payload(resource: Any | None) -> Mapping[str, Any]: - target = ( - files("microcosm.build.uk").joinpath(UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE) - if resource is None - else resource - ) + if resource is None: + frozen_payload = json.loads( + files("microcosm.build.uk") + .joinpath(UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE) + .read_text(encoding="utf-8") + ) + if not isinstance(frozen_payload, Mapping): + raise ValueError("UK HMRC source manifest root must be a JSON object.") + frozen_stages = frozen_payload.get("stages") + if not isinstance(frozen_stages, Sequence) or isinstance( + frozen_stages, (str, bytes) + ): + raise ValueError("UK HMRC source manifest stages must be a list.") + if len(frozen_stages) != 1 or not isinstance(frozen_stages[0], Mapping): + raise ValueError( + "UK HMRC source manifest must contain exactly one source stage." + ) + payload = json.loads( + files("microcosm.build.uk") + .joinpath(UK_CANONICAL_SOURCE_STAGES_RESOURCE) + .read_text(encoding="utf-8") + ) + if not isinstance(payload, Mapping): + raise ValueError("UK source manifest root must be a JSON object.") + stages = payload.get("stages") + if not isinstance(stages, Sequence) or isinstance(stages, (str, bytes)): + raise ValueError("UK source manifest stages must be a list.") + retained = [ + stage + for stage in stages + if isinstance(stage, Mapping) + and stage.get("stage") == "frs_hmrc_retained_leaves" + ] + hmrc = [ + stage + for stage in stages + if isinstance(stage, Mapping) and stage.get("stage") == "hmrc_spi_income" + ] + if len(retained) != 1 or len(hmrc) != 1: + raise ValueError( + "UK source manifest must contain exactly one " + "frs_hmrc_retained_leaves stage and one hmrc_spi_income stage." + ) + stage = copy.deepcopy(dict(hmrc[0])) + stage["base_candidate"] = copy.deepcopy( + dict(frozen_stages[0].get("base_candidate", {})) + ) + stage["operations"] = [ + *copy.deepcopy(list(retained[0].get("operations", ()))), + *copy.deepcopy(list(hmrc[0].get("operations", ()))), + ] + return { + "country": payload.get("country"), + "version": payload.get("version"), + "stages": [stage], + } + target = resource if hasattr(target, "read_text"): raw = target.read_text(encoding="utf-8") else: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 57b79833..6acb9a3b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -250,12 +250,22 @@ def evidence(self) -> dict[str, object]: class UKFRSHMRCSpineLeavesStageTransform: frs_raw_dir: Path stage: SourceStageSpec + # A #627 scale-ladder receipt build: the spine was subsampled after + # frs_spine, so raw-tab person coverage legitimately exceeds the frame. + sampled_rung: bool = False # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKFRSHMRCSpineLeavesResult | None = field(default=None, init=False) - def __init__(self, frs_raw_dir: str | Path, *, stage: SourceStageSpec) -> None: + def __init__( + self, + frs_raw_dir: str | Path, + *, + stage: SourceStageSpec, + sampled_rung: bool = False, + ) -> None: object.__setattr__(self, "frs_raw_dir", Path(frs_raw_dir)) object.__setattr__(self, "stage", stage) + object.__setattr__(self, "sampled_rung", bool(sampled_rung)) object.__setattr__(self, "last_result", None) def __call__(self, frame: Frame) -> Frame: @@ -278,11 +288,18 @@ def __call__(self, frame: Frame) -> Frame: source_leaves = _materialize_source_leaves(adult, benefits) person = frame.table("person").copy() missing_ids = sorted(set(source_leaves.index) - set(person["person_id"])) - if missing_ids: + if missing_ids and not self.sampled_rung: raise ValueError( "Raw FRS retained leaves contain person identity value(s) absent " f"from the raw spine: {missing_ids[:5]}." ) + if missing_ids: + # A rung sample deliberately drops most source people; restrict + # the raw surface to the survivors (the full-scale fence above + # stays strict — mirrors frs_hmrc_leaves' sampled_rung posture). + source_leaves = source_leaves.loc[ + source_leaves.index.isin(person["person_id"].to_numpy()) + ] aligned = source_leaves.reindex(person["person_id"], fill_value=0.0) for column in FRS_HMRC_RETAINED_LEAF_COLUMNS: person[column] = aligned[column].to_numpy(dtype=float) @@ -339,6 +356,10 @@ def checkpoint_metadata(self) -> dict[str, object]: class UKSPISupportChannelStageTransform: stage: SourceStageSpec seed: int = 42 + # A #627 scale-ladder receipt build: scale the declared synthetic stack + # by the survey-side sample fraction so the prior-mass pairing holds at + # every rung (f100 keeps the declared count exactly). + sample_fraction: float = 1.0 # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKSPISupportResult | None = field(default=None, init=False) @@ -349,6 +370,8 @@ def __call__(self, frame: Frame) -> Frame: self.stage, seed=self.seed, ) + if self.sample_fraction != 1.0 and count is not None: + count = max(1, int(round(count * self.sample_fraction))) result = build_uk_spi_support_channel( tables["person"], tables["benunit"], @@ -416,6 +439,10 @@ class UKSPIIncomeSpineStageTransform: seed: int = 42 qrf_estimators: int = 100 donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE + # A #627 scale-ladder receipt build: a sampled stack cannot carry mass in + # ultra-sparse restored columns, so the effective-mass fence records the + # gap instead of raising (f100 keeps the strict fence). + sampled_rung: bool = False # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKSPIIncomeSpineResult | None = field(default=None, init=False) @@ -428,6 +455,7 @@ def __init__( seed: int = 42, qrf_estimators: int = 100, donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE, + sampled_rung: bool = False, ) -> None: object.__setattr__(self, "spi_tab_path", Path(spi_tab_path)) object.__setattr__(self, "hmrc_ods_path", Path(hmrc_ods_path)) @@ -435,6 +463,7 @@ def __init__( object.__setattr__(self, "seed", seed) object.__setattr__(self, "qrf_estimators", qrf_estimators) object.__setattr__(self, "donor_sample_size", donor_sample_size) + object.__setattr__(self, "sampled_rung", bool(sampled_rung)) object.__setattr__(self, "last_result", None) @property @@ -492,11 +521,16 @@ def __call__(self, frame: Frame) -> Frame: for name, share in distributional_mass_shares.items() if share < DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE } - if insufficient: + if insufficient and not self.sampled_rung: raise RuntimeError( "Rebuilt SPI spine channel did not restore required " f"effective-mass coverage: {insufficient}." ) + # On a sampled rung the scaled stack cannot carry mass in + # ultra-sparse restored columns (e.g. charitable_investment_gifts at + # ~0.02 percent density); the gap is receipted via the replay + # report's distributional_mass_shares — the full-scale fence above + # stays strict. report = _build_spine_replay_report( source_targets=source_targets, support=support, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 8759a77f..cefeeb86 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -70,7 +70,10 @@ ] UK_CANDIDATE_DATASET_NAME = "populace_uk_2023" -UK_REFERENCE_DATASET_NAME = "enhanced_frs_2023_24_recalibrated" +# The label names the pinned reference artifact exactly: the 2024-25 line's +# published enhanced_frs_2024_25.h5 (no separate "recalibrated" variant +# exists at this vintage; the June report strings keep their own label). +UK_REFERENCE_DATASET_NAME = "enhanced_frs_2024_25" UK_MAX_TARGET_ABS_RELATIVE_ERROR = 0.25 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py index b17a03c2..e82962d6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py @@ -160,21 +160,12 @@ UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE = "degenerate_reviewed_exclusions.json" # Canonical sha256 of {"reference": {"identity": ..., "totals": ...}} for -# the 131-column weighted input surface emitted from the pinned enhanced-FRS -# artifact (sha 584ae33d...) by build_uk_efrs_parity_reference.py. The totals -# remain uncommitted under the UKDS EUL; this reviewed digest lets the gate and -# publication contract bind them without disclosing them (PR #610 review). -# -# Re-frozen keys-only under the #611 A4 dedup: totals keys went from -# "entity.column" to flat frame column names, every value byte-identical to -# the #610 freeze (whose saved totals reproduce the prior digest 11b22dd4… -# exactly), 131 columns before and after, zero cross-entity collisions — -# also enforced at mint time by the migrated tool's collision guard. The -# identity is unchanged. Reviewed by María Juaristi (2026-08-12); verified -# by loading the re-keyed totals through load_uk_input_mass_reference -# unpatched. +# the weighted input surface emitted from the pinned enhanced-FRS artifact by +# build_uk_efrs_parity_reference.py. The totals remain uncommitted under the +# UKDS EUL; this reviewed digest lets the gate and publication contract bind +# them without disclosing them. UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e" + "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" ) @@ -216,7 +207,9 @@ def __post_init__(self) -> None: ) for field_name in ("sha256", "totals_sha256"): value = getattr(self, field_name) - if len(value) != 64 or any(character not in _SHA256_HEX for character in value): + if len(value) != 64 or any( + character not in _SHA256_HEX for character in value + ): raise ValueError( f"UK input-mass reference descriptor {field_name} must be " "a lowercase sha256." @@ -245,14 +238,16 @@ def spec_payload(self) -> dict[str, object]: _UK_INPUT_MASS_REFERENCE_DESCRIPTOR = UKInputMassReferenceDescriptor( name="efrs-post-calibration", - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256="584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - vintage="2023_24", + filename="enhanced_frs_2024_25.h5", + revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", + sha256="97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + vintage="2024_25", totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note=( - "Post-calibration eFRS production incumbent; structurally lacks the " - "SPI clone channel, so SPI-channel-exclusive columns are comparable " + "Channel-blind post-calibration enhanced-FRS production incumbent, " + "pinned to the 2024-25 line; its artifact carries the SPI-synthetic " + "rows structurally but no admin-restored mass in the " + "SPI-channel-exclusive columns, so those columns are comparable " "only through per-reference reviewed exclusions." ), ) @@ -439,8 +434,7 @@ def coerce_input_mass_reference_registry( identity = entry["identity"] if not isinstance(identity, Mapping): raise TypeError( - f"{label} reference_registry entry {name!r} identity must be " - "an object." + f"{label} reference_registry entry {name!r} identity must be an object." ) expected_identity_fields = {"filename", "revision", "sha256", "vintage"} if set(identity) != expected_identity_fields: @@ -542,9 +536,9 @@ def _premature_exclusion_failure( ) -def _read_register_payload(source: str | Path | None, *, resource: str) -> tuple[ - Mapping[str, object], str -]: +def _read_register_payload( + source: str | Path | None, *, resource: str +) -> tuple[Mapping[str, object], str]: if source is None: raw = files("microcosm.build.uk").joinpath(resource).read_text("utf-8") label = resource diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 19ff0df4..a7998c23 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -281,6 +281,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "degenerate_reviewed_exclusions.json", "efrs_parity_known_gaps.json", "efrs_parity_reference.json", + "frs_release.json", "gates.json", "brma_rent_counts.json", "hmrc_income_release_gate_report.json", @@ -346,6 +347,7 @@ def test_uk_package_loads(self) -> None: "degenerate_reviewed_exclusions.json", "efrs_parity_known_gaps.json", "efrs_parity_reference.json", + "frs_release.json", "gates.json", "brma_rent_counts.json", "hmrc_income_release_gate_report.json", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 3661721b..617eff55 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e65bbdfba881d64618713df765cf0ccd21c119b05a5b11ba7fbcf336065654dd", + "e12a2cb87c0e096af0173bd51fbede4b7df5e7c118ffe07ef616bbb30640e4ea", { "benunit.benunit_id", "household.household_id", @@ -118,8 +118,7 @@ def test_country_kernel_contract_ids_are_closed_in_the_compiler_registry() -> No assert country_contract_ids <= F0_KERNEL_REGISTRY.ids assert all(F0_KERNEL_REGISTRY.contains(value) for value in country_contract_ids) assert not any( - F0_KERNEL_REGISTRY.has_implementation(value) - for value in country_contract_ids + F0_KERNEL_REGISTRY.has_implementation(value) for value in country_contract_ids ) diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index 1b9a86a1..d0d6f281 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -121,7 +121,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: ) compliant = SimpleNamespace( household_weight_kind=WeightKind.IMPORTANCE, - time_period="2023", + time_period="2024", mass_log=( MassChangeRecord( entity="household", @@ -152,7 +152,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: missing_receipt = SimpleNamespace( household_weight_kind=WeightKind.IMPORTANCE, - time_period="2023", + time_period="2024", mass_log=compliant.mass_log[:1], ) _, failures = _family_build_state_diagnostics(missing_receipt, manifest) @@ -160,7 +160,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: wrong_kind = SimpleNamespace( household_weight_kind=WeightKind.DESIGN, - time_period="2023", + time_period="2024", mass_log=compliant.mass_log, ) _, failures = _family_build_state_diagnostics(wrong_kind, manifest) diff --git a/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py b/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py index 4f599627..a4fe3a3c 100644 --- a/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py +++ b/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py @@ -75,7 +75,7 @@ def synthetic_efrs(tmp_path, monkeypatch) -> Path: "council_tax": np.array([1_000.0, 2_000.0, 3_000.0]), } ) - path = tmp_path / "enhanced_frs_2023_24.h5" + path = tmp_path / "enhanced_frs_2024_25.h5" with pd.HDFStore(path) as store: store.put("person", person, format="table", data_columns=True) store.put("benunit", benunit, format="table", data_columns=True) diff --git a/packages/microcosm-build/tests/test_uk_frs_release.py b/packages/microcosm-build/tests/test_uk_frs_release.py new file mode 100644 index 00000000..48cfda58 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_frs_release.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import copy +import json +from importlib.resources import files + +import pytest + +from microcosm.build.source_manifest import SourceManifest +from microcosm.build.uk_runtime import frs_spine +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract + + +def _source_manifest() -> SourceManifest: + raw = json.loads( + files("microcosm.build.uk").joinpath("source_stages.json").read_text() + ) + return SourceManifest.from_mapping(raw) + + +def _reload_with(monkeypatch, mutated: dict) -> None: + load_uk_frs_release.cache_clear() + payload = json.dumps(mutated) + + class _FakePath: + def read_text(self, *args, **kwargs): + return payload + + monkeypatch.setattr( + "microcosm.build.uk_runtime.frs_release._release_path", + lambda: _FakePath(), + ) + + +def test_uk_frs_release_loads_and_controls_runtime_period() -> None: + release = load_uk_frs_release() + + assert release.country == "uk" + assert release.name == "frs_2024_25" + assert release.survey_year == 2024 + assert release.base_year == 2024 + assert release.calibration_year == 2025 + assert frs_spine.TIME_PERIOD == release.time_period == str(release.base_year) + assert release.vintage == "2024_25" + assert release.ukds_study_number == 9563 + assert release.doi == "10.5255/UKDA-SN-9563-1" + + +def test_release_lockstep_with_source_manifest_and_take_up_contract() -> None: + release = load_uk_frs_release() + manifest = _source_manifest() + + for stage in manifest.stages: + for artifact in stage.artifacts: + if artifact.get("role") != "frs_table": + continue + assert artifact["vintage"] == release.vintage + assert artifact["ukds_study_number"] == release.ukds_study_number + assert artifact["doi"] == release.doi + assert artifact["tax_year_start"] == release.base_year + assert load_uk_take_up_contract().build_year == release.base_year + assert "SN 9563" in manifest.stage_map()["frs_spine"].source + + +def test_release_validation_refuses_bad_years_and_hashes(monkeypatch) -> None: + raw = json.loads( + files("microcosm.build.uk").joinpath("frs_release.json").read_text() + ) + + mutated = copy.deepcopy(raw) + mutated["time_period"] = "2023" + _reload_with(monkeypatch, mutated) + with pytest.raises(ValueError, match="time_period"): + load_uk_frs_release() + + mutated = copy.deepcopy(raw) + mutated["ukds_tab_zip"]["sha256"] = "ABC" + _reload_with(monkeypatch, mutated) + with pytest.raises(ValueError, match="lowercase hex"): + load_uk_frs_release() + + mutated = copy.deepcopy(raw) + mutated["vintage"] = "2023_24" + _reload_with(monkeypatch, mutated) + with pytest.raises(ValueError, match="survey_year"): + load_uk_frs_release() diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index e01ed24a..82241439 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -323,7 +323,7 @@ def _write_fixture( "table": table, "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "size_bytes": path.stat().st_size, @@ -761,13 +761,26 @@ def test_manifest_stage_and_runtime_agree_on_artifacts_and_operations() -> None: assert set(stage.outputs) == set(UKFRSSpineStageTransform.output_columns()) +def test_full_roster_artifact_pins_are_coherent() -> None: + # Regression armor for the #723 licensed-run finding: the WAS round-8 tab + # was declared by two stages (was_wealth, lcfs_consumption) with different + # locator strings, so `_artifact_pins()` refused the full-roster driver + # run — a break only a licensed build could see. Collecting pins over the + # complete manifest keeps the same-artifact-same-pin invariant CI-visible. + tool = _load_tool() + spec = load_country_spec("uk") + stages = list(spec.sources.stage_map().values()) + pins = tool._artifact_pins(stages) + assert pins, "full-roster pin collection must not be empty" + + def test_builds_structural_frame_from_pinned_tabs(tmp_path: Path) -> None: stage = _write_fixture(tmp_path) frame = build_uk_frs_spine_frame(tmp_path, stage=stage) validate_uk_national_frame(frame) - assert uk_time_period(frame) == "2023" + assert uk_time_period(frame) == "2024" assert frame.weights_for("household").kind is WeightKind.DESIGN np.testing.assert_array_equal( frame.weights_for("household").values, @@ -1058,7 +1071,7 @@ def _fake_value(column: str, rows: int): if column.endswith("_support_channel"): return ["frs"] * rows if column == "source_household_key": - return [f"2023:{index + 1}" for index in range(rows)] + return [f"2024:{index + 1}" for index in range(rows)] if column == "household_is_spi_synthetic": return [False] * rows return np.arange(1, rows + 1, dtype=float) @@ -1163,6 +1176,8 @@ def test_driver_writes_spine_h5_sidecars_and_logbook( "stage1": 42, "stage2": 43, } + assert sidecar["source_vintages"] == {"frs": "2024_25"} + assert sidecar["sampling"] is None replay_bytes = output.with_suffix(".hmrc_replay.json").read_bytes() assert json.loads(replay_bytes) == {"report_kind": "fake_spine_replay"} # The synthetic spec declares no non-table pinned artifacts, so the pin @@ -1352,7 +1367,7 @@ def test_driver_refuses_misnamed_hmrc_ods(tmp_path: Path) -> None: tool._validate_args(args) -def test_driver_has_no_sample_fraction_path(tmp_path: Path) -> None: +def test_driver_rejects_non_rung_sample_fraction(tmp_path: Path) -> None: tool = _load_tool() with pytest.raises(SystemExit): @@ -1372,6 +1387,144 @@ def test_driver_has_no_sample_fraction_path(tmp_path: Path) -> None: ) +def test_driver_derives_rung_tokens_from_sample_fraction() -> None: + tool = _load_tool() + + assert tool.UK_SAMPLE_RUNG_TOKENS[tool._rung_sample_fraction("0.01")] == "f001" + assert tool.UK_SAMPLE_RUNG_TOKENS[tool._rung_sample_fraction("0.10")] == "f010" + assert tool.UK_SAMPLE_RUNG_TOKENS[tool._rung_sample_fraction("1.0")] == "f100" + + +def test_driver_refuses_checkpoint_dir_on_sampled_rung(tmp_path: Path) -> None: + tool = _load_tool() + + with pytest.raises(SystemExit): + tool._parse_args( + [ + "--frs-raw-dir", + str(tmp_path), + "--spine-h5", + str(tmp_path / "spine.h5"), + "--spi-tab", + str(tmp_path / "put2223uk.tab"), + "--hmrc-ods", + str(tmp_path / "hmrc.ods"), + "--sample-fraction", + "0.10", + "--checkpoint-dir", + str(tmp_path / "checkpoints"), + ] + ) + + +def test_driver_records_sampled_spine_sidecar( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pytest.importorskip("tables") + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + output = tmp_path / "spine.h5" + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + monkeypatch.setattr( + tool, + "sample_frame_households", + lambda frame, **kwargs: ( + frame, + {"eligible_households": len(frame.table("household"))}, + ), + ) + monkeypatch.setattr( + tool, + "normalize_sampled_household_mass", + lambda frame, **kwargs: (frame, 1.0), + ) + + assert ( + tool.main( + [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--sample-fraction", + "0.10", + "--sample-seed", + "999", + ] + ) + == 0 + ) + + sidecar = json.loads(output.with_suffix(".build.json").read_text()) + assert sidecar["sampling"] == { + "fraction": 0.1, + "seed": 999, + "rung_token": "f010", + "pre_household_count": 2, + "post_household_count": 2, + "normalization_factor": 1.0, + "receipt": {"eligible_households": 2}, + } + rows = load_spool_rows(tmp_path / "logbook-spool") + assert rows[0].rung == "f010" + + +def test_driver_sampled_named_edge_aborts_with_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + output = tmp_path / "spine.h5" + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + + def _raise_named_edge(*args, **kwargs): + raise ValueError(tool._RUNG_NAMED_EDGE_SIGNATURE) + + monkeypatch.setattr(tool, "sample_frame_households", _raise_named_edge) + + assert ( + tool.main( + [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--sample-fraction", + "0.10", + ] + ) + == tool._RUNG_ABORT_EXIT_CODE + ) + + receipt = json.loads(output.with_suffix(".rung_abort.json").read_text()) + assert receipt["named_edge"] == "spine_split_singleton_class" + assert receipt["sampling"]["rung_token"] == "f010" + assert not output.exists() + rows = load_spool_rows(tmp_path / "logbook-spool") + assert rows[0].disposition == "discarded" + assert rows[0].rung == "f010" + + def test_refuses_missing_tab(tmp_path: Path) -> None: stage = _write_fixture(tmp_path) (tmp_path / "adult.tab").unlink() diff --git a/packages/microcosm-build/tests/test_uk_hmrc_income.py b/packages/microcosm-build/tests/test_uk_hmrc_income.py index aaa151d3..eeeaa680 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_income.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_income.py @@ -189,10 +189,10 @@ def should_not_parse(_path: Path): monkeypatch.setattr(hmrc_income, "_read_ods_tables", should_not_parse) - with pytest.raises(ValueError, match="maps to build period '2023'"): + with pytest.raises(ValueError, match="replayed against build period '2024'"): materialize_hmrc_spi_income_band_targets( ods_path, - build_period="2024", + build_period="2023", ) assert called is False diff --git a/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py b/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py index c659a642..92a4009a 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py @@ -19,6 +19,14 @@ / "uk" / "hmrc_income_source_stages.json" ) +_CANONICAL_SOURCE_STAGES_PATH = ( + Path(__file__).resolve().parents[1] + / "src" + / "microcosm" + / "build" + / "uk" + / "source_stages.json" +) _COLLATED_ODS_URL = ( "https://assets.publishing.service.gov.uk/media/" "69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods" @@ -105,6 +113,29 @@ def _manifest() -> dict[str, object]: return json.loads(_MANIFEST_PATH.read_text(encoding="utf-8")) +def _runtime_manifest() -> dict[str, object]: + payload = json.loads(_CANONICAL_SOURCE_STAGES_PATH.read_text(encoding="utf-8")) + retained = [ + stage + for stage in payload["stages"] + if stage["stage"] == "frs_hmrc_retained_leaves" + ] + hmrc = [stage for stage in payload["stages"] if stage["stage"] == "hmrc_spi_income"] + assert len(retained) == 1 + assert len(hmrc) == 1 + stage = dict(hmrc[0]) + stage["base_candidate"] = dict(_manifest()["stages"][0]["base_candidate"]) + stage["operations"] = [ + *retained[0]["operations"], + *hmrc[0]["operations"], + ] + return { + "country": payload["country"], + "version": payload["version"], + "stages": [stage], + } + + def _stage(payload: dict[str, object]) -> dict[str, object]: stages = payload["stages"] assert isinstance(stages, list) @@ -638,7 +669,7 @@ def test_runtime_source_contract_rejects_manifest_drift( replacement, match, ) -> None: - payload = _manifest() + payload = _runtime_manifest() cursor = payload for segment in path[:-1]: cursor = cursor[segment] @@ -658,7 +689,7 @@ def test_runtime_source_contract_rejects_duplicate_keys( collection, key, ) -> None: - payload = _manifest() + payload = _runtime_manifest() values = payload["stages"][0][collection] values.append(dict(values[0])) tampered = tmp_path / f"duplicate_{key}.json" diff --git a/packages/microcosm-build/tests/test_uk_hmrc_replay.py b/packages/microcosm-build/tests/test_uk_hmrc_replay.py index 04569c84..45e9869a 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_replay.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_replay.py @@ -98,7 +98,7 @@ def _evidence() -> dict[str, dict[str, object]]: "hmrc_surface": {"sha256": "d" * 64, "size_bytes": 166_693}, }, "build_evidence": { - "period": "2023", + "period": HMRC_SPI_BUILD_PERIOD, "seed": 42, "spi_prior_mass_share": 0.5, }, @@ -255,7 +255,7 @@ def test_future_exact_and_directional_fact_invariants() -> None: component="employment_income", measure="amount", unit="GBP", - period="2023", + period=HMRC_SPI_BUILD_PERIOD, total_income_lower_bound=12_570, total_income_upper_bound=15_000, published_value=100.0, diff --git a/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py b/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py index 081a0748..cf2c149c 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py @@ -121,6 +121,22 @@ def test_real_replay_binds_sources_identity_and_positive_mass() -> None: } assert sources["hmrc_surface"]["sha256"] == HMRC_SPI_COLLATED_ODS_SHA256 assert sources["hmrc_surface"]["mapped_build_period"] == "2023" + # June-freeze partition, made self-describing (adversarial-review + # disposition, microcosm#723): this report is evidence for the + # grandfathered June release and binds to the FROZEN manifest's period + # mapping - it deliberately does NOT follow the live build period, which + # moved to "2024" with the #723 signed re-map. It retires with the frozen + # manifest after #686 (#687's disposition), never regenerates against a + # different vintage. + frozen_stage = _resource("hmrc_income_source_stages.json")["stages"][0] + frozen_surface = next( + artifact + for artifact in frozen_stage["artifacts"] + if artifact.get("role") == "published_fact_surface" + ) + assert sources["hmrc_surface"]["mapped_build_period"] == str( + frozen_surface["mapped_build_period"] + ) assert qrf["fits"] == { "uk_frs_only_spi_fill": {"weight_kind": "importance"}, "uk_spi_2022_23_income": {"weight_kind": "design"}, diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index 6cc3fe26..e8edf9ea 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -251,7 +251,7 @@ def flags(true_count: int) -> list[bool]: "child_benefit_opts_out": flags(23), "would_claim_pc": flags(70), "would_claim_uc": flags(55), - "would_claim_tfc": flags(59), + "would_claim_tfc": flags(88), "would_claim_extended_childcare": flags(81), "would_claim_universal_childcare": flags(56), "would_claim_targeted_childcare": flags(60), @@ -279,7 +279,7 @@ def flags(true_count: int) -> list[bool]: index % 4 >= 2 for index in range(n) ], "household_owns_tv": flags(95), - "would_evade_tv_licence_fee": flags(11), + "would_evade_tv_licence_fee": flags(13), "main_residential_property_purchased_is_first_home": flags(38), "property_purchased": flags(4), "brma": [ diff --git a/packages/microcosm-build/tests/test_uk_parity_reference.py b/packages/microcosm-build/tests/test_uk_parity_reference.py index bc5c9337..d5d9c3af 100644 --- a/packages/microcosm-build/tests/test_uk_parity_reference.py +++ b/packages/microcosm-build/tests/test_uk_parity_reference.py @@ -88,15 +88,31 @@ def test_source_provenance_is_complete_and_immutable(self) -> None: source = load_efrs_parity_reference().source assert source.repo_id == "policyengine/policyengine-uk-data-private" assert source.repo_type == "model" - assert source.filename == "enhanced_frs_2023_24.h5" - assert source.revision == "655dd07e4bb9c777b00dac044949611f1feb824f" + assert source.filename == "enhanced_frs_2024_25.h5" + assert source.revision == "a2039519d3b92aecc06c66dfd175cb46ac24cada" assert source.sha256 == ( - "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" ) - assert source.size_bytes == 125_434_652 + assert source.size_bytes == 126_579_434 assert source.url.endswith(f"/{source.revision}/{source.filename}") - assert source.period == "2023" - assert source.vintage == "2023_24" + assert source.period == "2024" + assert source.vintage == "2024_25" + + def test_entity_record_counts_match_reviewed_2024_25_surface(self) -> None: + raw = json.loads(_resource(EFRS_PARITY_REFERENCE_RESOURCE).read_text()) + stats = raw["entity_stats"] + + # 52846 households = (16288 raw FRS + 10000 SPI) * 2 + 270 CGT band + # donors (30 * 9 bands; 181 FRS-side + 89 SPI-side). Unlike 2023-24, + # no raw household is dropped at 2024-25. + assert {entity: values["records"] for entity, values in stats.items()} == { + "person": 113_617, + "benunit": 61_223, + "household": 52_846, + } + assert set(stats) == {"person", "benunit", "household"} + for values in stats.values(): + assert {"records", "export_columns"} <= set(values) def test_recorded_sha256_is_lowercase_hex(self) -> None: sha = load_efrs_parity_reference().source.sha256 @@ -146,9 +162,7 @@ def test_uk_loader_migration_inputs_are_covered(self) -> None: def test_formula_owned_persisted_overrides_are_hard_covered(self) -> None: raw = json.loads(_resource(EFRS_PARITY_REFERENCE_RESOURCE).read_text()) - included = set( - raw["engine"]["formula_owned_persisted_overrides_included"] - ) + included = set(raw["engine"]["formula_owned_persisted_overrides_included"]) assert included == _FORMULA_OWNED_PERSISTED_OVERRIDES assert included <= set(raw["nonzero_shares"]) assert "formula_owned_export_columns_excluded" not in raw["engine"] diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py index 5c5f90a1..c9e2d390 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py @@ -126,6 +126,55 @@ def test_cached_candidate_regeneration_matches_committed_evidence() -> None: assert regenerated == committed +def test_hmrc_family_period_fields_come_from_the_bytes_their_hash_names() -> None: + # Adversarial-review finding (2026-08-20): the family block renders the + # #723 re-mapped period fields from the CANONICAL manifest while the + # frozen mirror keeps its June bytes; each field set must bind to the + # sha256 of the file it actually came from. + import hashlib + + manifest = _resource("release_input_coverage_manifest.json") + family = manifest["family_coverage"]["hmrc_spi_income"] + + frozen_bytes = ( + files(_UK_PACKAGE).joinpath("hmrc_income_source_stages.json").read_bytes() + ) + canonical_bytes = files(_UK_PACKAGE).joinpath("source_stages.json").read_bytes() + assert family["source_manifest_sha256"] == hashlib.sha256(frozen_bytes).hexdigest() + assert ( + family["canonical_source_manifest_sha256"] + == hashlib.sha256(canonical_bytes).hexdigest() + ) + + frozen_stage = json.loads(frozen_bytes)["stages"][0] + frozen_surface = next( + artifact + for artifact in frozen_stage["artifacts"] + if artifact.get("role") == "published_fact_surface" + ) + canonical_stage = next( + stage + for stage in json.loads(canonical_bytes)["stages"] + if stage.get("stage") == "hmrc_spi_income" + ) + canonical_surface = next( + artifact + for artifact in canonical_stage["artifacts"] + if artifact.get("role") == "published_fact_surface" + ) + # The re-mapped fields equal the canonical declaration; the frozen mirror + # still declares the June mapping (its bytes are pinned elsewhere). + assert family["source_vintages"]["mapped_build_period"] == str( + canonical_surface["mapped_build_period"] + ) + assert ( + family["source_vintages"]["period_mapping"] + == canonical_surface["period_mapping"] + ) + assert str(frozen_surface["mapped_build_period"]) == "2023" + assert frozen_surface["period_mapping"] == "tax_year_start" + + def test_promoted_manifest_requires_the_full_reference_surface() -> None: reference = _resource("efrs_parity_reference.json") manifest = _resource("release_input_coverage_manifest.json") @@ -168,9 +217,12 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No assert family["source_vintages"] == { "spi_donor": "2022-23", "hmrc_surface": "2023-24", - "mapped_build_period": "2023", + "mapped_build_period": "2024", + "period_mapping": "latest_published_tax_year", } assert family["spi_prior_national_household_mass_share"] == 0.5 + assert family["canonical_source_manifest"] == "source_stages.json" + assert len(family["canonical_source_manifest_sha256"]) == 64 assert family["required_mass_change_reason"] == ( "Allocate 50% of certified UK national household prior mass to the " "rebuilt 2022-23 SPI support channel; total national mass is conserved." diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index a151d80f..8eef5ec3 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -145,6 +145,13 @@ def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( expected_operations[6]["reviewed_absent_predictors"][ "other_investment_income" ] = _rephrase_stage2_predictor_note(predictor_note) + # FRS retained leaves now come from the FRS 2024-25 spine while the + # frozen HMRC fact surface stays byte-pinned. + expected_operations[1]["source_vintage"] = "2024-25" + expected_operations[1]["mapped_build_period"] = 2024 + # Signed period re-map (#723) for materialized HMRC SPI facts. + expected_operations[7]["mapped_build_period"] = 2024 + expected_operations[7]["period_mapping"] = "latest_published_tax_year" assert stage1["operations"] + stage2["operations"] == expected_operations _assert_no_forbidden_dependency( @@ -155,6 +162,11 @@ def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( expected_artifacts = copy.deepcopy(frozen_stage["artifacts"]) expected_artifacts[0]["reviewed_source"] = _expected_reviewed_source() + # Signed period re-map (#723): the ODS source surface remains the + # frozen 2023-24 file, but the canonical manifest declares that it is + # replayed against build period 2024. + expected_artifacts[1]["mapped_build_period"] = 2024 + expected_artifacts[1]["period_mapping"] = "latest_published_tax_year" # Declared output-name correction (licensed-data acceptance finding): # the frozen original listed the SPI concept "state_pension", but the # stage writes the auxiliary column SPI_HMRC_STATE_PENSION_INCOME_COLUMN @@ -611,9 +623,7 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: ) assert ( tuple( - lcfs_ops["materialize_rules_engine_predictors"].parameters[ - "predictors" - ] + lcfs_ops["materialize_rules_engine_predictors"].parameters["predictors"] ) == UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS ) @@ -638,14 +648,9 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: tuple(stages["etb_services"].operations[1].parameters["predictors"]) == UK_ETB_SERVICES_ENGINE_VARIABLES ) - assert ( - set( - stages["etb_services"] - .operations[1] - .parameters["derived_predictors"] - ) - == set(UK_ETB_SERVICES_EDUCATION_COUNTS) - ) + assert set( + stages["etb_services"].operations[1].parameters["derived_predictors"] + ) == set(UK_ETB_SERVICES_EDUCATION_COUNTS) assert ( tuple(stages["etb_services"].operations[2].parameters["targets"]) == UK_ETB_SERVICES_OUTPUT_COLUMNS[:3] diff --git a/packages/microcosm-build/tests/test_uk_spi_spine.py b/packages/microcosm-build/tests/test_uk_spi_spine.py index 7543f26c..fe8e681c 100644 --- a/packages/microcosm-build/tests/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/test_uk_spi_spine.py @@ -181,6 +181,28 @@ def test_spine_leaves_fail_closed_on_unknown_raw_person(tmp_path: Path) -> None: UKFRSHMRCSpineLeavesStageTransform(tmp_path, stage=stage)(_base_frame()) +def test_spine_leaves_sampled_rung_restricts_to_surviving_people( + tmp_path: Path, +) -> None: + # A #627 rung subsamples households after frs_spine, so raw-tab person + # coverage legitimately exceeds the frame; the sampled posture restricts + # the raw surface to the survivors while f100 keeps the strict fence. + stage = _leaves_stage( + tmp_path, + extra_adult_rows=({"sernum": 9, "person": 1, "inearns": 1.0},), + ) + + result = UKFRSHMRCSpineLeavesStageTransform( + tmp_path, stage=stage, sampled_rung=True + )(_base_frame()) + person = result.table("person") + + assert person["person_id"].tolist() == [2001, 1001] + assert person["hmrc_spi_pay"].tolist() == pytest.approx( + [5.0 * 365.25 / 7.0, 4.0 * 365.25 / 7.0] + ) + + def _support_stage() -> SourceStageSpec: return SourceStageSpec.from_mapping( { diff --git a/packages/microcosm-build/tests/test_uk_take_up_contract.py b/packages/microcosm-build/tests/test_uk_take_up_contract.py index 99056791..d9965b83 100644 --- a/packages/microcosm-build/tests/test_uk_take_up_contract.py +++ b/packages/microcosm-build/tests/test_uk_take_up_contract.py @@ -24,7 +24,7 @@ def test_uk_contract_loads_and_selects_build_year_rates() -> None: contract = load_uk_take_up_contract() assert contract.country == "uk" - assert contract.build_year == 2023 + assert contract.build_year == 2024 assert [entry.key for entry in contract.programs] == [ "child_benefit", "child_benefit_opts_out_rate", @@ -44,7 +44,7 @@ def test_uk_contract_loads_and_selects_build_year_rates() -> None: 0.7, 0.55, 0.5, - 0.586, + 0.88, 0.812, 0.563, 0.597, @@ -53,7 +53,7 @@ def test_uk_contract_loads_and_selects_build_year_rates() -> None: ] assert [contract.rate(entry.key) for entry in contract.stochastic] == [ 0.95, - 0.1058, + 0.1252, 0.384, 0.0385, ] diff --git a/packages/microcosm-build/tests/test_uk_terminal_gates.py b/packages/microcosm-build/tests/test_uk_terminal_gates.py index b8fe3468..2a67eace 100644 --- a/packages/microcosm-build/tests/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/test_uk_terminal_gates.py @@ -284,20 +284,20 @@ def test_ported_june_parity_gates_reject_empty_evidence() -> None: def _input_mass_reference(totals=None) -> UKInputMassReference: return UKInputMassReference( totals=({"employment_income": 10.0} if totals is None else totals), - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256=("584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d"), - vintage="2023_24", + filename="enhanced_frs_2024_25.h5", + revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", + sha256=("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + vintage="2024_25", ) def _input_mass_descriptor() -> UKInputMassReferenceDescriptor: return UKInputMassReferenceDescriptor( name="efrs-post-calibration", - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256="584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - vintage="2023_24", + filename="enhanced_frs_2024_25.h5", + revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", + sha256="97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + vintage="2024_25", totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note="Seeded scoped-reference note.", ) @@ -332,9 +332,7 @@ def test_zeroed_input_column_fails_by_name() -> None: assert gate.name == "input_mass_parity" assert "employment_income" in gate.failures[0] assert "mass is zero" in gate.failures[0] - assert gate.details["reference_identity"]["filename"] == ( - "enhanced_frs_2023_24.h5" - ) + assert gate.details["reference_identity"]["filename"] == ("enhanced_frs_2024_25.h5") def test_999_permille_mass_loss_fails_by_name() -> None: diff --git a/packages/microcosm-build/tests/test_uk_weighted_integrity.py b/packages/microcosm-build/tests/test_uk_weighted_integrity.py index 31b8fab6..c73eaaf2 100644 --- a/packages/microcosm-build/tests/test_uk_weighted_integrity.py +++ b/packages/microcosm-build/tests/test_uk_weighted_integrity.py @@ -91,10 +91,10 @@ def _frame( def _reference(totals, **overrides) -> UKInputMassReference: fields = { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": ("584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d"), - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": ("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + "vintage": "2024_25", } fields.update(overrides) return UKInputMassReference(totals=totals, **fields) @@ -103,10 +103,10 @@ def _reference(totals, **overrides) -> UKInputMassReference: def _descriptor(**overrides) -> UKInputMassReferenceDescriptor: fields = { "name": "efrs-post-calibration", - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25", "totals_sha256": UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, "scope_note": "Seeded scoped-reference note.", } @@ -272,10 +272,10 @@ def test_input_mass_reference_identity_is_recorded() -> None: ) assert gate.details["reference_identity"] == { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": ("584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d"), - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": ("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + "vintage": "2024_25", } @@ -1001,12 +1001,12 @@ def test_input_mass_reference_round_trips_the_measurement_schema(tmp_path) -> No { "schema_version": 1, "identity": { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", "sha256": ( - "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" ), - "vintage": "2023_24", + "vintage": "2024_25", }, "totals": {"employment_income": 10.5}, } @@ -1023,7 +1023,7 @@ def test_input_mass_reference_round_trips_the_measurement_schema(tmp_path) -> No ): reference = load_uk_input_mass_reference(path) - assert reference.filename == "enhanced_frs_2023_24.h5" + assert reference.filename == "enhanced_frs_2024_25.h5" assert dict(reference.totals) == {"employment_income": 10.5} with pytest.raises(ValueError, match="schema_version"): path.write_text(json.dumps({"schema_version": 9})) diff --git a/packages/microcosm-build/tests/test_us_plan.py b/packages/microcosm-build/tests/test_us_plan.py index d6157d8c..50c8b379 100644 --- a/packages/microcosm-build/tests/test_us_plan.py +++ b/packages/microcosm-build/tests/test_us_plan.py @@ -983,6 +983,7 @@ def test_no_incumbent_data_package_references_in_live_tree(self) -> None: # name only the immutable, sha-verified enhanced-FRS reference and # never import or execute the retired data package. "packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json", + "packages/microcosm-build/src/microcosm/build/uk/frs_release.json", "packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json", # The UK national contract's registry-parity accounting names the # retired data package by necessity: 651 rows at pinned ref ebf733c diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index eb91cc38..3e5190c2 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -187,10 +187,10 @@ # uk/gates.json reference_registry["efrs-post-calibration"].identity. _UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" _UK_INPUT_MASS_REFERENCE_IDENTITY = { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25", } # Independent publication pin for the canonical # {"reference": {"identity": ..., "totals": ...}} evidence emitted from the @@ -198,7 +198,7 @@ # under the UKDS EUL; keep this in lockstep with # UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 in the build shard. _UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e" + "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" ) _UK_TERMINAL_GATE_DETAIL_FIELDS = { "uk_release_input_coverage": frozenset( @@ -344,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "47d41dc3643818e049034b4333392aeb352bed5f76fc4556c1da5ba2fe8fb86d" + "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "687bb4ae4b085ac9abf2e77a7a0c8bcf11204eb9b00fdac1fd7232e16f81161f" + "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e54e2f5880f2380854d9764dcd5482aaa51e37ca0915a3ffaa0c9897626ba656" + "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -430,7 +430,7 @@ # canonical hash; this pins the wrapped digest so the entry's evidence line # still binds the enhanced-FRS incumbent totals. _UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "df74556909990345bc9032f6f5db7f817d9c273f75522a85ab6e332dd8dc7355" + "806f46de90a0bf08c70c977ab63dad1ed644088c89e40df1869ce07b97f63c0c" ) # The degenerate binding's evidence payload digests the resolved exclusion # records; for a release that must be the committed register, so its digest diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 09068cea..8daf0078 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -64,19 +64,20 @@ "qrf_tail_concentration": ("qrf_tail_concentration",), } UK_INPUT_MASS_REFERENCE_IDENTITY = { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25", } UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e" + "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" ) UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" UK_INPUT_MASS_REFERENCE_SCOPE_NOTE = ( - "Post-calibration eFRS production incumbent; structurally lacks the SPI " - "clone channel, so SPI-channel-exclusive columns are comparable only " - "through per-reference reviewed exclusions." + "Channel-blind post-calibration enhanced-FRS production incumbent, pinned " + "to the 2024-25 line; structurally lacks the SPI clone channel, so " + "SPI-channel-exclusive columns are comparable only through per-reference " + "reviewed exclusions." ) UK_INPUT_MASS_REVIEWED_EXCLUSIONS = { "charitable_investment_gifts": { @@ -90,8 +91,8 @@ ), "approved_by": "juaristi22", "adjudication": "microcosm#630", - "approved_on": "2026-08-17", - "expires_on": "2027-02-17", + "approved_on": "2026-08-20", + "expires_on": "2027-02-20", }, "owned_land": { "reason": ( @@ -111,8 +112,8 @@ ), "approved_by": "juaristi22", "adjudication": "microcosm#714", - "approved_on": "2026-08-19", - "expires_on": "2026-09-19", + "approved_on": "2026-08-20", + "expires_on": "2026-09-20", }, } GIT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" @@ -144,19 +145,19 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "47d41dc3643818e049034b4333392aeb352bed5f76fc4556c1da5ba2fe8fb86d" + "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "687bb4ae4b085ac9abf2e77a7a0c8bcf11204eb9b00fdac1fd7232e16f81161f" + "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e54e2f5880f2380854d9764dcd5482aaa51e37ca0915a3ffaa0c9897626ba656" + "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" ) UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "df74556909990345bc9032f6f5db7f817d9c273f75522a85ab6e332dd8dc7355" + "806f46de90a0bf08c70c977ab63dad1ed644088c89e40df1869ce07b97f63c0c" ) #: Spec entry id -> (neutral gate name, phase, legacy detail-schema name). UK_GATE_BATTERY_ENTRIES = { diff --git a/tools/build_uk_efrs_parity_reference.py b/tools/build_uk_efrs_parity_reference.py index e8785bed..26dc7bfe 100644 --- a/tools/build_uk_efrs_parity_reference.py +++ b/tools/build_uk_efrs_parity_reference.py @@ -17,7 +17,7 @@ Run with a licensed local artifact (the source SHA is always verified): python tools/build_uk_efrs_parity_reference.py \ - --input-h5 /path/to/enhanced_frs_2023_24.h5 + --input-h5 /path/to/enhanced_frs_2024_25.h5 Without ``--input-h5``, the tool first resolves the exact immutable revision from the Hugging Face cache and then downloads it if necessary. It accepts the @@ -47,12 +47,12 @@ # adjudication inputs. The licensed data lives in a private HF *model* repo. SOURCE_REPO_ID = "policyengine/policyengine-uk-data-private" SOURCE_REPO_TYPE = "model" -SOURCE_FILENAME = "enhanced_frs_2023_24.h5" -SOURCE_REVISION = "655dd07e4bb9c777b00dac044949611f1feb824f" -SOURCE_SHA256 = "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" -SOURCE_SIZE_BYTES = 125_434_652 -SOURCE_VINTAGE = "2023_24" -SOURCE_PERIOD = "2023" +SOURCE_FILENAME = "enhanced_frs_2024_25.h5" +SOURCE_REVISION = "a2039519d3b92aecc06c66dfd175cb46ac24cada" +SOURCE_SHA256 = "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" +SOURCE_SIZE_BYTES = 126_579_434 +SOURCE_VINTAGE = "2024_25" +SOURCE_PERIOD = "2024" SOURCE_URL = ( f"https://huggingface.co/{SOURCE_REPO_ID}/resolve/" f"{SOURCE_REVISION}/{SOURCE_FILENAME}" @@ -85,7 +85,7 @@ def _parse_args() -> argparse.Namespace: "--input-h5", type=Path, help=( - "Licensed enhanced_frs_2023_24.h5. If omitted, resolve the exact " + "Licensed enhanced_frs_2024_25.h5. If omitted, resolve the exact " "pinned revision from the HF cache/download API." ), ) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index e3a0f57a..5b4822a9 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -12,6 +12,10 @@ from pathlib import Path from microcosm.build.country_spec import country_stage_plan, load_country_spec +from microcosm.build.frame_sampling import ( + normalize_sampled_household_mass, + sample_frame_households, +) from microcosm.build.logbook import canonical_json_bytes from microcosm.build.logbook_adoption import ( AttemptState, @@ -46,6 +50,7 @@ UKFRSLegacyProxiesStageTransform, ) from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.frs_spine import ( UKFRSSpineStageTransform, uk_frs_spine_seed_frame, @@ -57,6 +62,10 @@ ) from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind +from microcosm.build.uk_runtime.national_sampling import ( + UK_SAMPLE_RUNG_TOKENS, + UK_SAMPLE_SEED_DEFAULT, +) from microcosm.build.uk_runtime.regional_uprating import ( UKRegionalPropertyUpratingStageTransform, ) @@ -70,8 +79,9 @@ from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine _PIPELINE = "uk-frs-spine" -_RUNG = "f100" _REPOSITORY = Path(__file__).resolve().parents[1] +_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" +_RUNG_ABORT_EXIT_CODE = 3 _STAGE_NAMES = ( "frs_spine", "frs_employment", @@ -95,6 +105,22 @@ ) +def _rung_sample_fraction(value: str) -> float: + """CLI rung policy (#624) over the permissive library validator.""" + + try: + fraction = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError( + f"sample fraction must be a number; got {value!r}." + ) from error + if fraction not in UK_SAMPLE_RUNG_TOKENS: + raise argparse.ArgumentTypeError( + "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." + ) + return fraction + + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -107,7 +133,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--frs-raw-dir", type=Path, required=True, - help="Directory containing the 14 licensed FRS 2023-24 tab files.", + help="Directory containing the 14 licensed FRS 2024-25 tab files.", ) parser.add_argument( "--spine-h5", @@ -132,6 +158,22 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Optional directory for a copy of the completed spine checkpoint.", ) + parser.add_argument( + "--sample-fraction", + type=_rung_sample_fraction, + default=1.0, + help=( + "Scale-ladder rung (#624): 0.01 smoke, 0.10 dev, or 1.0 full. " + "Below 1.0 the raw FRS spine is sampled immediately after ingest, " + "renormalized to full household mass, and treated as a receipt." + ), + ) + parser.add_argument( + "--sample-seed", + type=int, + default=UK_SAMPLE_SEED_DEFAULT, + help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", + ) parser.add_argument( "--was-tab", type=Path, @@ -162,7 +204,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=sha256_argument, help="Optional current Logbook chain head.", ) - return parser.parse_args(argv) + args = parser.parse_args(argv) + if args.sample_seed < 0: + parser.error("sample seed must be a non-negative integer.") + if args.sample_fraction != 1.0 and args.checkpoint_dir is not None: + parser.error( + "sampled spine rungs refuse --checkpoint-dir; rung artifacts are " + "receipts, never releases." + ) + return args def _validate_args(args: argparse.Namespace) -> None: @@ -370,6 +420,8 @@ def _build_sidecar( input_artifact_pins: dict[str, dict[str, object]], hmrc_replay: dict[str, object], stochastic_contract_sha256: str, + frs_vintage: str, + sampling: dict[str, object] | None, ) -> dict[str, object]: household_weight = frame.weights_for("household") return { @@ -401,6 +453,8 @@ def _build_sidecar( for stage in stages }, "declared_seeds": _declared_seeds(stages), + "source_vintages": {"frs": frs_vintage}, + "sampling": sampling, "stochastic_contract_sha256": stochastic_contract_sha256, "rules_engine": _rules_engine_provenance(), } @@ -434,6 +488,7 @@ def _record_attempt( code_pin: str, disposition: str, predecessor: str | None, + rung: str, spool_dir: Path, ) -> Path: return record_terminal_attempt( @@ -441,7 +496,7 @@ def _record_attempt( started_at=started_at, started_ts=started_ts, pipeline=_PIPELINE, - rung=_RUNG, + rung=rung, seed=None, code_pin=code_pin, disposition=disposition, @@ -450,8 +505,92 @@ def _record_attempt( ) +def _sample_spine_frame( + frame, + *, + fraction: float, + seed: int, +) -> tuple[object, dict[str, object] | None]: + if fraction == 1.0: + return frame, None + household_weight = frame.weights_for("household") + pre_households = int(len(frame.table("household"))) + sampled, receipt = sample_frame_households( + frame, + fraction=fraction, + seed=seed, + source_name="UK FRS spine", + ) + normalized, factor = normalize_sampled_household_mass( + sampled, + target_mass=float(household_weight.total), + source_name="UK FRS spine", + ) + return normalized, { + "fraction": float(fraction), + "seed": int(seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + "pre_household_count": pre_households, + "post_household_count": int(len(normalized.table("household"))), + "normalization_factor": float(factor), + "receipt": dict(receipt), + } + + +def _run_plan_with_spine_sampling( + plan, + *, + sample_fraction: float, + sample_seed: int, +) -> tuple[object, tuple[object, ...], dict[str, object] | None]: + if not plan.stages or plan.stages[0].name != "frs_spine": + frame, records = plan.run(uk_frs_spine_seed_frame()) + return frame, records, None + + from microcosm.build.plan import StagePlan + + spine_frame, spine_records = StagePlan(plan.stages[:1]).run( + uk_frs_spine_seed_frame() + ) + spine_frame, sampling = _sample_spine_frame( + spine_frame, + fraction=sample_fraction, + seed=sample_seed, + ) + if len(plan.stages) == 1: + return spine_frame, spine_records, sampling + frame, tail_records = StagePlan(plan.stages[1:]).run(spine_frame) + return frame, (*spine_records, *tail_records), sampling + + +def _rung_abort_receipt( + args: argparse.Namespace, + *, + error: BaseException, +) -> dict[str, object]: + return { + "schema_version": 1, + "artifact_kind": "uk_frs_spine_rung_abort_receipt", + "build_kind": "uk_frs_spine", + "sampling": { + "sample_fraction": float(args.sample_fraction), + "sample_seed": int(args.sample_seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + }, + "named_edge": "spine_split_singleton_class", + "stage": "frs_spine", + "error": str(error), + "disposition": "aborted_with_receipt", + "remedy": ( + "Re-roll --sample-seed; accepted dev-scale statistical edge. " + "The computation is never altered to avoid it." + ), + } + + def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) + rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] started_at = time.perf_counter() started_ts = datetime.now(UTC) predecessor = resolve_predecessor(args.logbook_prev_row_digest) @@ -481,6 +620,7 @@ def main(argv: list[str] | None = None) -> int: args.spine_h5, args.spine_h5.with_suffix(".build.json"), args.spine_h5.with_suffix(".hmrc_replay.json"), + args.spine_h5.with_suffix(".rung_abort.json"), ] if args.emit_nonzero_shares is not None: stale_outputs.append(args.emit_nonzero_shares) @@ -543,10 +683,12 @@ def main(argv: list[str] | None = None) -> int: append_phase(state, "inputs_pinned") engine = _rules_engine() stochastic_contract = load_uk_take_up_contract() + frs_release = load_uk_frs_release() hmrc_spine_transform = UKSPIIncomeSpineStageTransform( args.spi_tab, args.hmrc_ods, stage=stages_by_name["hmrc_spi_income_spine"], + sampled_rung=args.sample_fraction != 1.0, ) implementations = { "frs_spine": UKFRSSpineStageTransform( @@ -631,9 +773,11 @@ def main(argv: list[str] | None = None) -> int: implementations["frs_hmrc_spine_leaves"] = UKFRSHMRCSpineLeavesStageTransform( args.frs_raw_dir, stage=stages_by_name["frs_hmrc_spine_leaves"], + sampled_rung=args.sample_fraction != 1.0, ) implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( stage=stages_by_name["spi_support_channel"], + sample_fraction=args.sample_fraction, ) implementations["hmrc_spi_income_spine"] = hmrc_spine_transform plan = country_stage_plan( @@ -641,7 +785,11 @@ def main(argv: list[str] | None = None) -> int: implementations, stage_names=stage_names, ) - frame, records = plan.run(uk_frs_spine_seed_frame()) + frame, records, sampling = _run_plan_with_spine_sampling( + plan, + sample_fraction=args.sample_fraction, + sample_seed=args.sample_seed, + ) append_phase(state, "spine_built") output = write_uk_national_frame(frame, args.spine_h5) append_phase(state, "spine_written") @@ -673,6 +821,8 @@ def main(argv: list[str] | None = None) -> int: input_artifact_pins=input_artifact_pins, hmrc_replay=replay_binding, stochastic_contract_sha256=stochastic_contract.resource_sha256, + frs_vintage=frs_release.vintage, + sampling=sampling, ) atomic_write_json(sidecar_path, sidecar) append_phase(state, "build_sidecar_written") @@ -712,12 +862,43 @@ def main(argv: list[str] | None = None) -> int: code_pin=code_pin, disposition="iterating", predecessor=predecessor, + rung=rung, spool_dir=spool_dir, ) print(f"Wrote FRS spine H5: {output}", file=sys.stderr) print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) return 0 except Exception as error: + if ( + isinstance(error, ValueError) + and args.sample_fraction != 1.0 + and _RUNG_NAMED_EDGE_SIGNATURE in str(error) + ): + rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") + receipt = _rung_abort_receipt(args, error=error) + atomic_write_json(rung_abort_path, receipt) + state.gate_verdicts = { + "uk_frs_spine_rung_abort": { + "verdict": "aborted", + "receipt": ( + f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" + "#/named_edge" + ), + } + } + append_phase(state, "rung_aborted") + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="discarded", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + print(json.dumps(receipt, indent=2, sort_keys=True)) + return _RUNG_ABORT_EXIT_CODE try: receipt_path = write_error_receipt( error_receipt_path(args.spine_h5.parent, build_id=state.build_id), @@ -736,6 +917,7 @@ def main(argv: list[str] | None = None) -> int: code_pin=code_pin, disposition="failed", predecessor=predecessor, + rung=rung, spool_dir=spool_dir, ) except Exception: diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index ffd67928..31d19595 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -41,6 +41,7 @@ from microcosm.build.uk_runtime.frs_hmrc_leaves import ( UKFRSHMRCRetainedLeavesStageTransform, ) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.hmrc_replay import write_hmrc_replay_report from microcosm.build.uk_runtime.hmrc_restoration import ( UKHMRCIncomeStageTransform, @@ -159,7 +160,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, required=True, help=( - "Raw FRS 2023-24 directory containing adult.tab and benefits.tab " + "Raw FRS 2024-25 directory containing adult.tab and benefits.tab " "for source-faithful retained HMRC leaves." ), ) @@ -1133,6 +1134,7 @@ def _aggregate_build_record( source_vintages = dict(family_evidence.get("source_vintages", {})) if ledger_artifact_provenance is not None: source_vintages["ledger_facts"] = ledger_artifact_provenance + source_vintages["frs"] = load_uk_frs_release().vintage return { "schema_version": 3, "build_kind": "uk_national_staging_dataset", diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 557eb7bd..f0b17562 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -994,6 +994,20 @@ def _hmrc_family_coverage_contract( stage = stages[0] if not isinstance(stage, dict) or stage.get("stage") != "hmrc_spi_income": raise ValueError(f"{HMRC_SOURCE_STAGES_PATH}: expected hmrc_spi_income stage.") + canonical_payload = _load(SOURCE_STAGES_PATH) + canonical_stages = canonical_payload.get("stages") + if not isinstance(canonical_stages, list): + raise ValueError(f"{SOURCE_STAGES_PATH}: expected source stages list.") + canonical_matches = [ + candidate + for candidate in canonical_stages + if isinstance(candidate, dict) and candidate.get("stage") == "hmrc_spi_income" + ] + if len(canonical_matches) != 1: + raise ValueError( + f"{SOURCE_STAGES_PATH}: expected exactly one hmrc_spi_income stage." + ) + canonical_stage = canonical_matches[0] base_candidate = stage.get("base_candidate") if not isinstance(base_candidate, dict): raise ValueError( @@ -1011,6 +1025,11 @@ def _hmrc_family_coverage_contract( for artifact in stage.get("artifacts", []) if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) } + canonical_artifacts = { + artifact["role"]: artifact + for artifact in canonical_stage.get("artifacts", []) + if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) + } operations = { operation["kind"]: operation for operation in stage.get("operations", []) @@ -1018,6 +1037,7 @@ def _hmrc_family_coverage_contract( } required_artifacts = {"qrf_donor", "published_fact_surface"} missing_artifacts = sorted(required_artifacts - set(artifacts)) + missing_canonical_artifacts = sorted(required_artifacts - set(canonical_artifacts)) required_operations = { "retain_adjudicated_frs_hmrc_leaves", "verify_pinned_hmrc_source_pair", @@ -1026,10 +1046,11 @@ def _hmrc_family_coverage_contract( "gate_distributional_effective_mass", } missing_operations = sorted(required_operations - set(operations)) - if missing_artifacts or missing_operations: + if missing_artifacts or missing_canonical_artifacts or missing_operations: raise ValueError( f"{HMRC_SOURCE_STAGES_PATH}: incomplete HMRC family contract; " f"missing_artifacts={missing_artifacts}, " + f"missing_canonical_artifacts={missing_canonical_artifacts}, " f"missing_operations={missing_operations}." ) classification = operations["classify_hmrc_income_facts_with_reviewed_fences"] @@ -1076,13 +1097,23 @@ def _hmrc_family_coverage_contract( "stage": "hmrc_spi_income", "source_manifest": HMRC_SOURCE_STAGES_PATH.name, "source_manifest_sha256": _sha256(HMRC_SOURCE_STAGES_PATH), + # The two re-mapped period fields below come from the CANONICAL + # manifest (the #723 signed re-map lives there; the frozen mirror + # keeps its June bytes), so the bytes they derive from are pinned + # separately - evidence fields and their hash must name the same + # source (adversarial-review finding, 2026-08-20). + "canonical_source_manifest": SOURCE_STAGES_PATH.name, + "canonical_source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), "base_candidate_sha256": str(base_candidate["sha256"]), "base_candidate_tier": base_candidate_tier, "source_vintages": { "spi_donor": str(artifacts["qrf_donor"]["vintage"]), "hmrc_surface": str(artifacts["published_fact_surface"]["vintage"]), "mapped_build_period": str( - artifacts["published_fact_surface"]["mapped_build_period"] + canonical_artifacts["published_fact_surface"]["mapped_build_period"] + ), + "period_mapping": str( + canonical_artifacts["published_fact_surface"]["period_mapping"] ), }, "spi_prior_national_household_mass_share": float( diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index c816571e..68b428ad 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -348,6 +348,7 @@ def e6_identity_receipt( allocate_nhs_by_age_gender, load_etb_services_anchors, ) + rail_fare_index = float( load_etb_services_anchors()["rail_fare_index_2023"]["value"] ) @@ -360,9 +361,7 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: if {"electricity_consumption", "gas_consumption"} <= set(household.columns): household_out["domestic_energy_consumption"] = household[ "electricity_consumption" - ].to_numpy(dtype=float) + household["gas_consumption"].to_numpy( - dtype=float - ) + ].to_numpy(dtype=float) + household["gas_consumption"].to_numpy(dtype=float) if "rail_subsidy_spending" in household.columns: household_out["rail_usage"] = ( household["rail_subsidy_spending"].to_numpy(dtype=float) @@ -424,9 +423,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: for op in channel["operations"] if op["kind"] == "allocate_zero_weight_prior_mass" ) - household["household_weight"] = household[ - "household_weight" - ].to_numpy(dtype=float) / (1.0 - float(share)) + household["household_weight"] = household["household_weight"].to_numpy( + dtype=float + ) / (1.0 - float(share)) original = recompute(person, benunit, household) rng = np.random.default_rng(permutation_seed) permuted = recompute( @@ -510,9 +509,14 @@ def main() -> int: from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine + # E4 draws ran on the pre-stack FRS spine and its take-up targets are + # unweighted int(rate * n_units): recompute on the survey channel + # only, or every target moves with the synthetic rows (the post-#717 + # scoping rule the E5 receipt already applies). + frame = _frs_only_frame(frame) engine = PolicyEngineUKEngine() lha_category = engine.materialize( - frame, ("LHA_category",), uk_time_period(frame) + _engine_safe_frame(frame), ("LHA_category",), uk_time_period(frame) )["LHA_category"] receipt = e4_identity_receipt( frame, @@ -549,6 +553,75 @@ def main() -> int: return 0 if ok else 1 +def _frs_only_frame(frame): + """Scope a post-#717 artifact to the survey channel (FRS rows only).""" + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + + household = frame.table("household") + if "household_is_spi_synthetic" not in household.columns: + return frame + keep = ~household["household_is_spi_synthetic"].astype(bool) + weights = frame.weights_for("household").values[keep.to_numpy()] + household = household.loc[keep].reset_index(drop=True) + ids = set(household["household_id"].tolist()) + person = ( + frame.table("person") + .loc[lambda t: t["person_household_id"].isin(ids)] + .reset_index(drop=True) + ) + benunit_ids = set(person["person_benunit_id"].tolist()) + benunit = ( + frame.table("benunit") + .loc[lambda t: t["benunit_id"].isin(benunit_ids)] + .reset_index(drop=True) + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=weights, + mass_log=frame.mass_log, + ) + + +def _engine_safe_frame(frame): + """Fill by-design NaN on channel-only auxiliary columns for engine reads. + + The #717 SPI channel leaves hmrc_spi_* auxiliaries (e.g. + other_investment_income) NaN on FRS rows by design; instruments fill 0, + matching stage-time semantics, because the engine adapter rejects NaN. + LHA_category derivation does not read these columns. + """ + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + + tables = {} + for entity in ("person", "benunit", "household"): + table = frame.table(entity).copy() + for column in table.columns: + if table[column].dtype.kind == "f" and table[column].isna().any(): + table[column] = table[column].fillna(0.0) + tables[entity] = table + return uk_national_frame( + person=tables["person"], + benunit=tables["benunit"], + household=tables["household"], + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + + def _reverse_rows(frame): from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind,