diff --git a/.gitignore b/.gitignore index 1e629c60..9a741bc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ *.egg-info **/__pycache__ **/.DS_STORE +**/.DS_Store +**/__MACOSX +**/._* **/*.h5 **/*.csv **/*.zip diff --git a/changelog.d/scottish-ct-water-charges.md b/changelog.d/scottish-ct-water-charges.md new file mode 100644 index 00000000..1223fb62 --- /dev/null +++ b/changelog.d/scottish-ct-water-charges.md @@ -0,0 +1 @@ +Net Scottish Water and sewerage charges out of FRS-reported council tax (CTANNUAL includes them in Scotland), fixing a ~£500-per-household double count that overstated Scottish council tax by roughly 25% relative to Scottish Government receipts; the charges remain captured in water_and_sewerage_charges. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index c717d97e..df4cb3a3 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -817,6 +817,28 @@ def determine_education_level(fted_val, typeed2_val, age_val): # Impute Council Tax + # In Scotland, council tax bills are collected together with Scottish + # Water and sewerage charges, and the FRS CTANNUAL variable includes + # them. Net them off (they are weekly variables; CTANNUAL is annual) so + # council_tax is tax only: the water charges are already captured + # separately in water_and_sewerage_charges, so leaving them in both + # double-counts them and overstates Scottish council tax by roughly + # £500 per household (~25% of the Scottish total). + SCOTLAND_GVTREGNO = 12 + scottish_water_annual = pd.Series( + np.where( + household.gvtregno == SCOTLAND_GVTREGNO, + ( + np.maximum(household.csewamt.fillna(0), 0) + + np.maximum(household.cwatamtd.fillna(0), 0) + ) + * (365.25 / 7), + 0, + ), + index=household.index, + ) + ctannual_tax_only = np.maximum(household.ctannual - scottish_water_annual, 0) + # Only ~25% of household report Council Tax bills - use # these to build a model to impute missing values CT_valid = household.ctannual > 0 @@ -826,7 +848,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): region = household.gvtregno[CT_valid] band = household.ctband[CT_valid] single_person = (household.adulth == 1)[CT_valid] - ctannual = household.ctannual[CT_valid] + ctannual = ctannual_tax_only[CT_valid] # Build the table ct_mean = ctannual.groupby([region, band, single_person], dropna=False).mean() @@ -851,7 +873,7 @@ def determine_education_level(fted_val, typeed2_val, age_val): # uses -1 for missing values (household.ctannual < 0) | household.ctannual.isna(), np.maximum(ct_imputed, 0).values, - household.ctannual, + ctannual_tax_only, ) ) pe_household["council_tax"] = council_tax.fillna(0) diff --git a/policyengine_uk_data/storage/spi_2022_23/__MACOSX/._README.md b/policyengine_uk_data/storage/spi_2022_23/__MACOSX/._README.md deleted file mode 100644 index 3fdac2e3..00000000 Binary files a/policyengine_uk_data/storage/spi_2022_23/__MACOSX/._README.md and /dev/null differ