diff --git a/FETCH-MANIFEST-EUROSTAT.json b/FETCH-MANIFEST-EUROSTAT.json new file mode 100644 index 0000000..df260f5 --- /dev/null +++ b/FETCH-MANIFEST-EUROSTAT.json @@ -0,0 +1,30 @@ +{ + "schema_version": "ledger.fetch_manifest.v1", + "network_policy": "Fetch only in an authorized network-enabled handoff environment.", + "fetches": [ + { + "dataset_id": "gov_10a_taxag", + "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/gov_10a_taxag?format=JSON&lang=en&freq=A&unit=MIO_EUR§or=S13&na_item=D2&na_item=D5&na_item=D51&na_item=D61&geo=BE&geo=DE&geo=FR&time=2023&time=2024", + "destination": "db/data/eurostat/gov_10a_taxag/gov_10a_taxag.json", + "sha256": "5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a" + }, + { + "dataset_id": "spr_exp_func", + "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/spr_exp_func?format=JSON&lang=en&freq=A&spdeps=SPR&spfunc=TOTAL&spfunc=SICK&spfunc=DIS&spfunc=OLD&spfunc=SRV&spfunc=FAM&spfunc=UNE&spfunc=HOU&spfunc=EXCL&unit=MIO_EUR&geo=BE&geo=DE&geo=FR&time=2023", + "destination": "db/data/eurostat/spr_exp_func/spr_exp_func.json", + "sha256": "24ad3c0115efef790ff8cb2ff8fb269d4ceb7de70ac3fad673ebd1340e4f650f" + }, + { + "dataset_id": "ilc_li02", + "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_li02?format=JSON&lang=en&freq=A&statinfo=MED_EI&unit=PC&rskpovth=B_60&sex=T&age=TOTAL&geo=BE&geo=DE&geo=FR&time=2024", + "destination": "db/data/eurostat/ilc_li02/ilc_li02.json", + "sha256": "789535c86cac56c4c5b29559ef037eeb2f0b06f882a8393bfe7ad4d0c09e0052" + }, + { + "dataset_id": "ilc_di01", + "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_di01?format=JSON&lang=en&freq=A&quant_inc=D1&quant_inc=D2&quant_inc=D3&quant_inc=D4&quant_inc=D5&quant_inc=D6&quant_inc=D7&quant_inc=D8&quant_inc=D9&statinfo=TC&statinfo=SHARE&unit=EUR&geo=BE&geo=DE&geo=FR&time=2024", + "destination": "db/data/eurostat/ilc_di01/ilc_di01.json", + "sha256": "1c3bb1277f98f257d00725ded6b6d8b1678834329eb6b66954b8f42c19a0ad17" + } + ] +} \ No newline at end of file diff --git a/chronicle/core.py b/chronicle/core.py index 9bf90f4..744b255 100644 --- a/chronicle/core.py +++ b/chronicle/core.py @@ -60,12 +60,14 @@ "government", "dwelling", "institutional_sector", + "social_protection_scheme", "firm", } ALLOWED_AGGREGATIONS = { "sum", "mean", "median", + "quantile", "rate", "ratio", "share", diff --git a/chronicle/source_package.py b/chronicle/source_package.py index 589fd47..815f7d1 100644 --- a/chronicle/source_package.py +++ b/chronicle/source_package.py @@ -52,6 +52,7 @@ source_rows_from_json_table, source_rows_from_statxplore_table, source_rows_from_ees_permalink_table_html, + source_rows_from_json_stat_2, source_rows_from_kff_state_indicator_gdocs_html, source_rows_from_ons_timeseries_json, source_rows_from_delimited_text, @@ -250,6 +251,10 @@ "hmrc-vat-firm-sector-targets-2024-25": Path( "hmrc/vat_firm_sector_targets_2024_25" ), + "eurostat-gov-10a-taxag": Path("eurostat/gov_10a_taxag"), + "eurostat-spr-exp-func": Path("eurostat/spr_exp_func"), + "eurostat-ilc-li02": Path("eurostat/ilc_li02"), + "eurostat-ilc-di01": Path("eurostat/ilc_di01"), "kff-marketplace-effectuated-enrollment": Path( "kff/marketplace_effectuated_enrollment" ), @@ -392,6 +397,12 @@ def build_source_rows(self, year: int) -> list[SourceRow]: artifact, sheet_name=self._sheet_name(filename, year=year), ) + if self.parser == "json_stat_2_full_rows": + return source_rows_from_json_stat_2( + content, + artifact, + sheet_name=self._sheet_name(filename, year=year), + ) if self.parser == "census_acs_s0101_age_json_rows": return source_rows_from_census_acs_s0101_age_json( content, @@ -577,6 +588,26 @@ def build_source_cells( for row in self.selected_rows ), ) + if self.parser == "json_stat_2_full_rows": + rows = ( + source_rows + if source_rows is not None + else source_rows_from_json_stat_2( + content, + artifact, + sheet_name=self._sheet_name(filename, year=year), + ) + ) + return source_cells_from_source_rows( + rows, + selected_rows=tuple( + { + key: str(_render_value(value, year=year)) + for key, value in row.items() + } + for row in self.selected_rows + ), + ) if self.parser == "census_acs_s0101_age_json_rows": rows = ( source_rows diff --git a/chronicle/sources/rows.py b/chronicle/sources/rows.py index 79419cf..ba2fe30 100644 --- a/chronicle/sources/rows.py +++ b/chronicle/sources/rows.py @@ -11,6 +11,7 @@ from dataclasses import asdict, dataclass from datetime import date, datetime from io import BytesIO, StringIO +from itertools import product from pathlib import Path from typing import Any @@ -338,6 +339,217 @@ def source_rows_from_json_table( raise ValueError("JSON table must be an array of arrays or objects.") +def source_rows_from_json_stat_2( + content: bytes, + artifact: SourceArtifactMetadata, + *, + sheet_name: str, +) -> list[SourceRow]: + """Flatten a JSON-stat 2.0 dataset cube into deterministic source rows. + + JSON-stat stores observations in row-major order, with the last dimension + changing fastest. The returned rows preserve that order, retain every cube + position (including null observations), and expose dimension codes before + the observation value so declarative packages can select exact series. + """ + data = json.loads(content.decode("utf-8")) + if not isinstance(data, dict): + raise ValueError("JSON-stat 2.0 payload must be an object.") + if data.get("class") != "dataset": + raise ValueError("JSON-stat 2.0 payload class must be 'dataset'.") + if data.get("version") != "2.0": + raise ValueError("JSON-stat payload version must be '2.0'.") + + dimension_ids = data.get("id") + sizes = data.get("size") + dimensions = data.get("dimension") + if not isinstance(dimension_ids, list) or not dimension_ids: + raise ValueError("JSON-stat dataset id must be a non-empty array.") + if not all(isinstance(item, str) and item for item in dimension_ids): + raise ValueError("JSON-stat dataset dimension IDs must be non-empty strings.") + if len(set(dimension_ids)) != len(dimension_ids): + raise ValueError("JSON-stat dataset dimension IDs must be unique.") + if not isinstance(sizes, list) or len(sizes) != len(dimension_ids): + raise ValueError("JSON-stat dataset size must align with id.") + if not all(type(size) is int and size > 0 for size in sizes): + raise ValueError("JSON-stat dataset sizes must be positive integers.") + if not isinstance(dimensions, dict): + raise ValueError("JSON-stat dataset dimension must be an object.") + + codes_by_dimension: list[list[str]] = [] + labels_by_dimension: list[dict[str, str]] = [] + for dimension_id, size in zip(dimension_ids, sizes, strict=True): + codes, labels = _json_stat_dimension_categories( + dimensions, + dimension_id=dimension_id, + expected_size=size, + ) + codes_by_dimension.append(codes) + labels_by_dimension.append(labels) + + observation_count = 1 + for size in sizes: + observation_count *= size + values = _json_stat_observations( + data.get("value"), + field="value", + observation_count=observation_count, + required=True, + ) + statuses = _json_stat_observations( + data.get("status"), + field="status", + observation_count=observation_count, + required=False, + ) + + rows: list[SourceRow] = [] + coordinates = product(*(range(size) for size in sizes)) + for source_index, positions in enumerate(coordinates): + dimension_codes = { + dimension_id: codes_by_dimension[index][position] + for index, (dimension_id, position) in enumerate( + zip(dimension_ids, positions, strict=True) + ) + } + row_values: dict[str, Scalar] = dict(dimension_codes) + row_values["value"] = _json_stat_scalar(values[source_index], field="value") + row_values["status"] = _json_stat_scalar( + statuses[source_index], + field="status", + ) + row_values["source_index"] = source_index + for index, dimension_id in enumerate(dimension_ids): + code = dimension_codes[dimension_id] + row_values[f"{dimension_id}_label"] = labels_by_dimension[index].get( + code, + code, + ) + rows.append( + SourceRow( + artifact=artifact, + sheet_name=sheet_name, + row_number=source_index + 1, + values=row_values, + ) + ) + return rows + + +def _json_stat_dimension_categories( + dimensions: dict[str, Any], + *, + dimension_id: str, + expected_size: int, +) -> tuple[list[str], dict[str, str]]: + dimension = dimensions.get(dimension_id) + if not isinstance(dimension, dict): + raise ValueError(f"JSON-stat dimension {dimension_id!r} is missing.") + category = dimension.get("category") + if not isinstance(category, dict): + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category must be an object." + ) + category_index = category.get("index") + if isinstance(category_index, list): + if not all(isinstance(code, str) and code for code in category_index): + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category index values " + "must be non-empty strings." + ) + codes = list(category_index) + elif isinstance(category_index, dict): + positions: dict[int, str] = {} + for code, position in category_index.items(): + if not isinstance(code, str) or not code: + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category codes " + "must be non-empty strings." + ) + if type(position) is not int or position < 0: + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category positions " + "must be non-negative integers." + ) + if position in positions: + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category positions " + "must be unique." + ) + positions[position] = code + if set(positions) != set(range(expected_size)): + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category positions must " + f"cover 0 through {expected_size - 1}." + ) + codes = [positions[position] for position in range(expected_size)] + else: + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category index must be an " + "array or object." + ) + + if len(codes) != expected_size or len(set(codes)) != len(codes): + raise ValueError( + f"JSON-stat dimension {dimension_id!r} categories must contain " + f"{expected_size} unique codes." + ) + + raw_labels = category.get("label", {}) + if not isinstance(raw_labels, dict): + raise ValueError( + f"JSON-stat dimension {dimension_id!r} category labels must be an object." + ) + labels = { + code: str(raw_labels[code]) + for code in codes + if raw_labels.get(code) is not None + } + return codes, labels + + +def _json_stat_observations( + observations: Any, + *, + field: str, + observation_count: int, + required: bool, +) -> list[Any]: + if observations is None: + if required: + raise ValueError(f"JSON-stat dataset {field} is required.") + return [None] * observation_count + if isinstance(observations, list): + if len(observations) != observation_count: + raise ValueError( + f"JSON-stat dataset {field} array must contain " + f"{observation_count} entries." + ) + return list(observations) + if isinstance(observations, dict): + dense = [None] * observation_count + for raw_index, value in observations.items(): + try: + index = int(raw_index) + except (TypeError, ValueError) as exc: + raise ValueError( + f"JSON-stat dataset {field} keys must be integer indexes." + ) from exc + if str(index) != str(raw_index) or not 0 <= index < observation_count: + raise ValueError( + f"JSON-stat dataset {field} index {raw_index!r} is out of range." + ) + dense[index] = value + return dense + raise ValueError(f"JSON-stat dataset {field} must be an array or object.") + + +def _json_stat_scalar(value: Any, *, field: str) -> Scalar: + if value is None or isinstance(value, bool | int | float | str): + return _json_scalar(value) + raise ValueError(f"JSON-stat dataset {field} entries must be scalar values.") + + S0101_TOTAL_AGE_COLUMNS: tuple[tuple[str, str], ...] = ( ("S0101_C01_002E", "Aged 0-4"), ("S0101_C01_003E", "Aged 5-9"), diff --git a/chronicle/sources/specs.py b/chronicle/sources/specs.py index 1798b0f..c0d75ca 100644 --- a/chronicle/sources/specs.py +++ b/chronicle/sources/specs.py @@ -5,6 +5,7 @@ import hashlib import json from dataclasses import asdict, dataclass, field +from decimal import Decimal from chronicle.core import ( DEFAULT_ASSERTION, @@ -711,6 +712,14 @@ def _scale_value(value: Scalar, scale: int | float) -> int | float | str: raise ValueError(f"Cannot scale nonnumeric source value {value!r}") if isinstance(value, int | float): scaled = value * scale + # Integrality is decided in decimal, not binary: publisher lexemes + # like 16448.06 scaled by 1_000_000 are exactly 16448060000, where + # binary multiplication alone emits 16448060000.000002. Non-integral + # products keep the binary result unchanged, so no existing + # non-integral fact value shifts. + exact = Decimal(str(value)) * Decimal(str(scale)) + if exact == exact.to_integral_value(): + return int(exact) if isinstance(scaled, float) and scaled.is_integer(): return int(scaled) return scaled diff --git a/db/data/eurostat/gov_10a_taxag/gov_10a_taxag.json b/db/data/eurostat/gov_10a_taxag/gov_10a_taxag.json new file mode 100644 index 0000000..3e1a4d3 --- /dev/null +++ b/db/data/eurostat/gov_10a_taxag/gov_10a_taxag.json @@ -0,0 +1 @@ +{"version":"2.0","class":"dataset","label":"Main national accounts tax aggregates","source":"ESTAT","updated":"2026-07-21T11:00:00+0200","value":{"0":72825.5,"1":73248.5,"2":428710.0,"3":443779.0,"4":446580.0,"5":453677.0,"6":96585.0,"7":104740.8,"8":533331.0,"9":552833.0,"10":357078.0,"11":366049.0,"12":94031.3,"13":101965.9,"14":517508.0,"15":536831.0,"16":351268.0,"17":359026.0,"18":90443.5,"19":94416.8,"20":710828.0,"21":756551.0,"22":461666.0,"23":482281.0},"status":{"2":"p","3":"p","8":"p","9":"p","14":"p","15":"p","20":"p","21":"p"},"id":["freq","unit","sector","na_item","geo","time"],"size":[1,1,1,4,3,2],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"unit":{"label":"Unit of measure","category":{"index":{"MIO_EUR":0},"label":{"MIO_EUR":"Million euro"}}},"sector":{"label":"Sector","category":{"index":{"S13":0},"label":{"S13":"General government"}}},"na_item":{"label":"National accounts indicator (ESA 2010)","category":{"index":{"D2":0,"D5":1,"D51":2,"D61":3},"label":{"D2":"Taxes on production and imports","D5":"Current taxes on income, wealth, etc.","D51":"Taxes on income","D61":"Net social contributions"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"BE":0,"DE":1,"FR":2},"label":{"BE":"Belgium","DE":"Germany","FR":"France"}}},"time":{"label":"Time","category":{"index":{"2023":0,"2024":1},"label":{"2023":"2023","2024":"2024"}}}},"extension":{"lang":"EN","id":"GOV_10A_TAXAG","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"GOV_10A_TAXAG","agencyId":"ESTAT","version":"68.0"},"annotation":[{"type":"CREATED","date":"2015-02-18T14:28:42+0100"},{"type":"DISSEMINATION_DOI_XML","title":"10.2908/GOV_10A_TAXAG2023-01-19"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-07-21T11:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-07-21T11:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-07-21T11:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/gov_10a_taxag_esms.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/gov_10a_taxag_esms.sdmx.zip"},{"type":"OBS_COUNT","title":"1470597"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2025"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"1995"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat"},{"type":"UPDATE_DATA","date":"2026-07-21T11:00:00+0200"},{"type":"UPDATE_STRUCTURE","date":"2026-04-22T11:00:00+0200"}],"status":{"label":{"p":"provisional"}},"positions-with-no-data":{"freq":[],"unit":[],"sector":[],"na_item":[],"geo":[],"time":[]}}} \ No newline at end of file diff --git a/db/data/eurostat/gov_10a_taxag/manifest.yaml b/db/data/eurostat/gov_10a_taxag/manifest.yaml new file mode 100644 index 0000000..2d66f89 --- /dev/null +++ b/db/data/eurostat/gov_10a_taxag/manifest.yaml @@ -0,0 +1,23 @@ +source_id: eurostat-gov-10a-taxag +source_name: Eurostat gov_10a_taxag +publisher: Eurostat +source_page: https://ec.europa.eu/eurostat/databrowser/view/gov_10a_taxag/default/table?lang=en +files: + 2024: + filename: gov_10a_taxag.json + source_url: https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/gov_10a_taxag?format=JSON&lang=en&freq=A&unit=MIO_EUR§or=S13&na_item=D2&na_item=D5&na_item=D51&na_item=D61&geo=BE&geo=DE&geo=FR&time=2023&time=2024 + source_table: Eurostat gov_10a_taxag Main national accounts tax aggregates for Belgium, Germany, and France + sha256: 5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a + size_bytes: 3346 + storage: + r2: + provider: r2 + bucket: ledger-raw + key: raw/eurostat/eurostat-gov-10a-taxag/2024/5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a/gov_10a_taxag.json + uri: r2://ledger-raw/raw/eurostat/eurostat-gov-10a-taxag/2024/5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a/gov_10a_taxag.json + source_urls: + - https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/gov_10a_taxag?format=JSON&lang=en&freq=A&unit=MIO_EUR§or=S13&na_item=D2&na_item=D5&na_item=D51&na_item=D61&geo=BE&geo=DE&geo=FR&time=2023&time=2024 + notes: >- + Real Eurostat dissemination API bytes for the filtered request in + source_url; hash-pinned and content-addressed. Different bytes at + the mutable API endpoint are a new vintage, never a pin update. diff --git a/db/data/eurostat/ilc_di01/ilc_di01.json b/db/data/eurostat/ilc_di01/ilc_di01.json new file mode 100644 index 0000000..6f16d3b --- /dev/null +++ b/db/data/eurostat/ilc_di01/ilc_di01.json @@ -0,0 +1 @@ +{"version":"2.0","class":"dataset","label":"Distribution of income by quantiles","source":"ESTAT","updated":"2026-07-27T23:00:00+0200","value":{"3":4.1,"4":3.4,"5":3.1,"0":17661,"1":14325,"2":13290,"9":5.8,"10":5.1,"11":5.1,"6":20540,"7":18190,"8":16800,"15":6.7,"16":6.2,"17":6.3,"12":23766,"13":21500,"14":19824,"21":7.7,"22":7.2,"23":7.2,"18":27161,"19":24538,"20":22620,"27":8.7,"28":8.2,"29":8.2,"24":30446,"25":27619,"26":25557,"33":9.7,"34":9.2,"35":9.3,"30":33915,"31":31246,"32":28586,"39":10.8,"40":10.5,"41":10.3,"36":37538,"37":35571,"38":32278,"45":12.0,"46":12.0,"47":11.9,"42":41766,"43":41419,"44":37590,"51":13.7,"52":14.4,"53":14.3,"48":49815,"49":51787,"50":47759},"id":["freq","quant_inc","statinfo","unit","geo","time"],"size":[1,9,2,1,3,1],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"quant_inc":{"label":"Income quantile","category":{"index":{"D1":0,"D2":1,"D3":2,"D4":3,"D5":4,"D6":5,"D7":6,"D8":7,"D9":8},"label":{"D1":"First decile","D2":"Second decile","D3":"Third decile","D4":"Fourth decile","D5":"Fifth decile","D6":"Sixth decile","D7":"Seventh decile","D8":"Eighth decile","D9":"Ninth decile"}}},"statinfo":{"label":"Statistical information","category":{"index":{"TC":0,"SHARE":1},"label":{"TC":"Top cut-off point","SHARE":"Share of national equivalised income"}}},"unit":{"label":"Unit of measure","category":{"index":{"EUR":0},"label":{"EUR":"Euro"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"BE":0,"DE":1,"FR":2},"label":{"BE":"Belgium","DE":"Germany","FR":"France"}}},"time":{"label":"Time","category":{"index":{"2024":0},"label":{"2024":"2024"}}}},"extension":{"lang":"EN","id":"ILC_DI01","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"ILC_DI01","agencyId":"ESTAT","version":"39.0"},"annotation":[{"type":"CREATED","date":"2011-04-14T12:07:26+0200"},{"type":"DISSEMINATION_DOI_XML","title":"10.2908/ILC_DI012023-01-19"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"info-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_ILC_20251009.pdf","text":"Information note"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"info-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_ILC_20251009.pdf"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"info-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_ILC_20251009.pdf"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-07-27T12:34:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-07-27T12:34:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-07-06T23:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/ilc_sieusilc.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/ilc_sieusilc.sdmx.zip"},{"type":"OBS_COUNT","title":"139244"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2025"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"1995"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat"},{"type":"UPDATE_DATA","date":"2026-07-06T23:00:00+0200"},{"type":"UPDATE_STRUCTURE","date":"2026-07-27T23:00:00+0200"}],"positions-with-no-data":{"freq":[],"quant_inc":[],"statinfo":[],"unit":[],"geo":[],"time":[]}}} \ No newline at end of file diff --git a/db/data/eurostat/ilc_di01/manifest.yaml b/db/data/eurostat/ilc_di01/manifest.yaml new file mode 100644 index 0000000..b64da50 --- /dev/null +++ b/db/data/eurostat/ilc_di01/manifest.yaml @@ -0,0 +1,24 @@ +source_id: eurostat-ilc-di01 +source_name: Eurostat ilc_di01 +publisher: Eurostat +source_page: https://ec.europa.eu/eurostat/databrowser/view/ilc_di01/default/table?lang=en +files: + 2024: + filename: ilc_di01.json + source_url: https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_di01?format=JSON&lang=en&freq=A&quant_inc=D1&quant_inc=D2&quant_inc=D3&quant_inc=D4&quant_inc=D5&quant_inc=D6&quant_inc=D7&quant_inc=D8&quant_inc=D9&statinfo=TC&statinfo=SHARE&unit=EUR&geo=BE&geo=DE&geo=FR&time=2024 + source_table: Eurostat ilc_di01 Distribution of income by quantiles for Belgium, Germany, and France + sha256: 1c3bb1277f98f257d00725ded6b6d8b1678834329eb6b66954b8f42c19a0ad17 + size_bytes: 4091 + storage: + r2: + provider: r2 + bucket: ledger-raw + key: raw/eurostat/eurostat-ilc-di01/2024/1c3bb1277f98f257d00725ded6b6d8b1678834329eb6b66954b8f42c19a0ad17/ilc_di01.json + uri: r2://ledger-raw/raw/eurostat/eurostat-ilc-di01/2024/1c3bb1277f98f257d00725ded6b6d8b1678834329eb6b66954b8f42c19a0ad17/ilc_di01.json + source_urls: + - https://ec.europa.eu/eurostat/databrowser/view/ilc_di01/default/table?lang=en + - https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_di01?format=JSON&lang=en&freq=A&quant_inc=D1&quant_inc=D2&quant_inc=D3&quant_inc=D4&quant_inc=D5&quant_inc=D6&quant_inc=D7&quant_inc=D8&quant_inc=D9&statinfo=TC&statinfo=SHARE&unit=EUR&geo=BE&geo=DE&geo=FR&time=2024 + notes: >- + Real Eurostat dissemination API bytes for the filtered request in + source_url; hash-pinned and content-addressed. Different bytes at + the mutable API endpoint are a new vintage, never a pin update. diff --git a/db/data/eurostat/ilc_li02/ilc_li02.json b/db/data/eurostat/ilc_li02/ilc_li02.json new file mode 100644 index 0000000..c5e78ad --- /dev/null +++ b/db/data/eurostat/ilc_li02/ilc_li02.json @@ -0,0 +1 @@ +{"version":"2.0","class":"dataset","label":"At-risk-of-poverty rate by poverty threshold, age and sex - EU-SILC and ECHP surveys","source":"ESTAT","updated":"2026-06-10T23:00:00+0200","value":{"0":11.4,"1":15.5,"2":15.9},"id":["freq","statinfo","unit","rskpovth","sex","age","geo","time"],"size":[1,1,1,1,1,1,3,1],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"statinfo":{"label":"Statistical information","category":{"index":{"MED_EI":0},"label":{"MED_EI":"Median equivalised income"}}},"unit":{"label":"Unit of measure","category":{"index":{"PC":0},"label":{"PC":"Percentage"}}},"rskpovth":{"label":"Risk of poverty threshold","category":{"index":{"B_60":0},"label":{"B_60":"Below 60%"}}},"sex":{"label":"Sex","category":{"index":{"T":0},"label":{"T":"Total"}}},"age":{"label":"Age class","category":{"index":{"TOTAL":0},"label":{"TOTAL":"Total"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"BE":0,"DE":1,"FR":2},"label":{"BE":"Belgium","DE":"Germany","FR":"France"}}},"time":{"label":"Time","category":{"index":{"2024":0},"label":{"2024":"2024"}}}},"extension":{"lang":"EN","id":"ILC_LI02","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"ILC_LI02","agencyId":"ESTAT","version":"76.1"},"annotation":[{"type":"CREATED","date":"2026-06-10T12:26:53+0200"},{"type":"DISSEMINATION_DOI_XML","title":"10.2908/ILC_LI022023-01-19"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"info-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_ILC_20251009.pdf","text":"Information note"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"info-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_ILC_20251009.pdf"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"info-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_ILC_20251009.pdf"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-06-10T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-06-10T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-06-10T23:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/ilc_sieusilc.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/ilc_sieusilc.sdmx.zip"},{"type":"OBS_COUNT","title":"1053428"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2025"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"1995"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat"},{"type":"UPDATE_DATA","date":"2026-06-10T23:00:00+0200"},{"type":"UPDATE_STRUCTURE","date":"2026-06-10T23:00:00+0200"}],"positions-with-no-data":{"freq":[],"statinfo":[],"unit":[],"rskpovth":[],"sex":[],"age":[],"geo":[],"time":[]}}} \ No newline at end of file diff --git a/db/data/eurostat/ilc_li02/manifest.yaml b/db/data/eurostat/ilc_li02/manifest.yaml new file mode 100644 index 0000000..1ab769b --- /dev/null +++ b/db/data/eurostat/ilc_li02/manifest.yaml @@ -0,0 +1,23 @@ +source_id: eurostat-ilc-li02 +source_name: Eurostat ilc_li02 +publisher: Eurostat +source_page: https://ec.europa.eu/eurostat/databrowser/view/ilc_li02/default/table?lang=en +files: + 2024: + filename: ilc_li02.json + source_url: https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_li02?format=JSON&lang=en&freq=A&statinfo=MED_EI&unit=PC&rskpovth=B_60&sex=T&age=TOTAL&geo=BE&geo=DE&geo=FR&time=2024 + source_table: Eurostat ilc_li02 At-risk-of-poverty rate by poverty threshold, age and sex - EU-SILC and ECHP surveys for Belgium, Germany, and France + sha256: 789535c86cac56c4c5b29559ef037eeb2f0b06f882a8393bfe7ad4d0c09e0052 + size_bytes: 3573 + storage: + r2: + provider: r2 + bucket: ledger-raw + key: raw/eurostat/eurostat-ilc-li02/2024/789535c86cac56c4c5b29559ef037eeb2f0b06f882a8393bfe7ad4d0c09e0052/ilc_li02.json + uri: r2://ledger-raw/raw/eurostat/eurostat-ilc-li02/2024/789535c86cac56c4c5b29559ef037eeb2f0b06f882a8393bfe7ad4d0c09e0052/ilc_li02.json + source_urls: + - https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_li02?format=JSON&lang=en&freq=A&statinfo=MED_EI&unit=PC&rskpovth=B_60&sex=T&age=TOTAL&geo=BE&geo=DE&geo=FR&time=2024 + notes: >- + Real Eurostat dissemination API bytes for the filtered request in + source_url; hash-pinned and content-addressed. Different bytes at + the mutable API endpoint are a new vintage, never a pin update. diff --git a/db/data/eurostat/spr_exp_func/manifest.yaml b/db/data/eurostat/spr_exp_func/manifest.yaml new file mode 100644 index 0000000..2e99995 --- /dev/null +++ b/db/data/eurostat/spr_exp_func/manifest.yaml @@ -0,0 +1,24 @@ +source_id: eurostat-spr-exp-func +source_name: Eurostat spr_exp_func ESSPROS social protection expenditure by function +publisher: Eurostat +source_page: https://ec.europa.eu/eurostat/databrowser/view/spr_exp_func/default/table?lang=en +files: + 2023: + filename: spr_exp_func.json + source_url: https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/spr_exp_func?format=JSON&lang=en&freq=A&spdeps=SPR&spfunc=TOTAL&spfunc=SICK&spfunc=DIS&spfunc=OLD&spfunc=SRV&spfunc=FAM&spfunc=UNE&spfunc=HOU&spfunc=EXCL&unit=MIO_EUR&geo=BE&geo=DE&geo=FR&time=2023 + source_table: Eurostat spr_exp_func Expenditure on social benefits by function for Belgium, Germany, and France + sha256: 24ad3c0115efef790ff8cb2ff8fb269d4ceb7de70ac3fad673ebd1340e4f650f + size_bytes: 4130 + storage: + r2: + provider: r2 + bucket: ledger-raw + key: raw/eurostat/eurostat-spr-exp-func/2023/24ad3c0115efef790ff8cb2ff8fb269d4ceb7de70ac3fad673ebd1340e4f650f/spr_exp_func.json + uri: r2://ledger-raw/raw/eurostat/eurostat-spr-exp-func/2023/24ad3c0115efef790ff8cb2ff8fb269d4ceb7de70ac3fad673ebd1340e4f650f/spr_exp_func.json + source_urls: + - https://ec.europa.eu/eurostat/databrowser/view/spr_exp_func/default/table?lang=en + - https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/spr_exp_func?format=JSON&lang=en&freq=A&spdeps=SPR&spfunc=TOTAL&spfunc=SICK&spfunc=DIS&spfunc=OLD&spfunc=SRV&spfunc=FAM&spfunc=UNE&spfunc=HOU&spfunc=EXCL&unit=MIO_EUR&geo=BE&geo=DE&geo=FR&time=2023 + notes: >- + Real Eurostat dissemination API bytes for the filtered request in + source_url; hash-pinned and content-addressed. Different bytes at + the mutable API endpoint are a new vintage, never a pin update. diff --git a/db/data/eurostat/spr_exp_func/spr_exp_func.json b/db/data/eurostat/spr_exp_func/spr_exp_func.json new file mode 100644 index 0000000..565c97d --- /dev/null +++ b/db/data/eurostat/spr_exp_func/spr_exp_func.json @@ -0,0 +1 @@ +{"version":"2.0","class":"dataset","label":"Expenditure on social benefits by function","source":"ESTAT","updated":"2026-08-07T23:00:00+0200","value":{"6":16448.06,"7":85802.38,"8":58066.1,"24":5395.12,"25":7811.95,"26":35179.03,"15":12232.62,"16":146525.87,"17":62858.38,"21":1262.27,"22":25522.36,"23":15751.83,"9":69583.94,"10":436747.13,"11":357486.85,"3":46448.89,"4":402730.52,"5":267651.26,"12":8914.51,"13":66036.89,"14":43230.73,"0":165806.45,"1":1210659.67,"2":889478.92,"18":5521.04,"19":39482.57,"20":49254.74},"status":{"7":"p","8":"p","25":"p","26":"p","16":"p","17":"p","22":"p","23":"p","10":"p","11":"p","4":"p","5":"p","13":"p","14":"p","1":"p","2":"p","19":"p","20":"p"},"id":["freq","spdeps","spfunc","unit","geo","time"],"size":[1,1,9,1,3,1],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"spdeps":{"label":"Main expenditure type","category":{"index":{"SPR":0},"label":{"SPR":"Social protection benefits"}}},"spfunc":{"label":"Social protection function","category":{"index":{"TOTAL":0,"SICK":1,"DIS":2,"OLD":3,"SRV":4,"FAM":5,"UNE":6,"HOU":7,"EXCL":8},"label":{"TOTAL":"Total","SICK":"Sickness/health care","DIS":"Disability","OLD":"Old age","SRV":"Survivors","FAM":"Family/children","UNE":"Unemployment","HOU":"Housing","EXCL":"Social exclusion, n.e.c."}}},"unit":{"label":"Unit of measure","category":{"index":{"MIO_EUR":0},"label":{"MIO_EUR":"Million euro"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"BE":0,"DE":1,"FR":2},"label":{"BE":"Belgium","DE":"Germany","FR":"France"}}},"time":{"label":"Time","category":{"index":{"2023":0},"label":{"2023":"2023"}}}},"extension":{"lang":"EN","id":"SPR_EXP_FUNC","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"SPR_EXP_FUNC","agencyId":"ESTAT","version":"23.0"},"annotation":[{"type":"CREATED","date":"2024-11-07T14:26:55+0100"},{"type":"DISSEMINATION_DOI_XML","title":"10.2908/SPR_EXP_FUNC2025-03-31"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_SPR_20241119.pdf","text":"Information note"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_SPR_20241119.pdf"},{"type":"DISSEMINATION_EXPLANATORY_LINK","title":"text-category","href":"https://ec.europa.eu/eurostat/databrowser-backend/api/public/explanatory-notes/get/Info_note_SPR_20241119.pdf"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-08-07T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-08-07T23:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-08-07T23:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/spr_esms.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/spr_esms.sdmx.zip"},{"type":"OBS_COUNT","title":"163448"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2024"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"1990"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat"},{"type":"UPDATE_DATA","date":"2026-08-07T23:00:00+0200"},{"type":"UPDATE_STRUCTURE","date":"2026-05-22T23:00:00+0200"}],"status":{"label":{"p":"provisional"}},"positions-with-no-data":{"freq":[],"spdeps":[],"spfunc":[],"unit":[],"geo":[],"time":[]}}} \ No newline at end of file diff --git a/packages/eurostat/gov_10a_taxag/source_package.yaml b/packages/eurostat/gov_10a_taxag/source_package.yaml new file mode 100644 index 0000000..3d45961 --- /dev/null +++ b/packages/eurostat/gov_10a_taxag/source_package.yaml @@ -0,0 +1,567 @@ +schema_version: ledger.source_package.v1 +package_id: eurostat-gov-10a-taxag +label: Eurostat gov_10a_taxag tax revenue for Belgium, Germany, and France +artifact: + source_name: eurostat + source_table: Eurostat gov_10a_taxag Main national accounts tax aggregates for Belgium, Germany, and France + resource_package: db + resource_directory: data/eurostat/gov_10a_taxag + manifest: manifest.yaml + vintage: eurostat_gov_10a_taxag_2024 + extracted_at: '2026-07-22' + extraction_method: >- + Eurostat dissemination API v1.0 JSON-stat 2.0 full-cube row parse of + the filtered request recorded in the manifest source_url + parser: json_stat_2_full_rows + artifact_year: 2024 + sheet_name: gov_10a_taxag + selected_rows: + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D2, geo: BE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D2, geo: DE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D2, geo: FR, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D5, geo: BE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D5, geo: DE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D5, geo: FR, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D51, geo: BE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D51, geo: DE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D51, geo: FR, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D61, geo: BE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D61, geo: DE, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D61, geo: FR, time: '2023'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D2, geo: BE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D2, geo: DE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D2, geo: FR, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D5, geo: BE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D5, geo: DE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D5, geo: FR, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D51, geo: BE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D51, geo: DE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D51, geo: FR, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D61, geo: BE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D61, geo: DE, time: '2024'} + - {freq: A, unit: MIO_EUR, sector: S13, na_item: D61, geo: FR, time: '2024'} +record_sets: + - record_set_id: eurostat.gov_10a_taxag.cy2023.tax_revenue.country + provenance_class: administrative + record_set_spec_id: eurostat.gov_10a_taxag.tax_revenue.country.v1 + source_record_id_prefix: eurostat.gov_10a_taxag.cy2023.tax_revenue.country + sheet_name: gov_10a_taxag + period_type: calendar_year + period: 2023 + geography_id: BE + geography_level: country + geography_name: Belgium + geography_vintage: current + entity: government + entity_role: general_government + domain: taxes_and_social_contributions + groupby_dimension: na_item + shared_filters: + freq: A + unit: MIO_EUR + sector: S13 + time: '2023' + shared_constraints: + - {variable: freq, operator: '==', value: A, label: Eurostat frequency} + - {variable: unit, operator: '==', value: MIO_EUR, label: Eurostat source unit} + - {variable: sector, operator: '==', value: S13, label: Eurostat institutional sector} + - {variable: time, operator: '==', value: '2023', label: Eurostat time} + rows: + - value_id: d2_be + label: Belgium taxes on production and imports + ordinal: 0 + row_number: 2 + expected_row_header_column: D + expected_row_header: D2 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D2, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D2, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D2, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d2_de + label: Germany taxes on production and imports + ordinal: 1 + row_number: 3 + expected_row_header_column: D + expected_row_header: D2 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D2, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D2, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D2, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d2_fr + label: France taxes on production and imports + ordinal: 2 + row_number: 4 + expected_row_header_column: D + expected_row_header: D2 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D2, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D2, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D2, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + - value_id: d5_be + label: Belgium current taxes on income and wealth + ordinal: 3 + row_number: 5 + expected_row_header_column: D + expected_row_header: D5 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D5, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D5, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D5, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d5_de + label: Germany current taxes on income and wealth + ordinal: 4 + row_number: 6 + expected_row_header_column: D + expected_row_header: D5 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D5, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D5, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D5, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d5_fr + label: France current taxes on income and wealth + ordinal: 5 + row_number: 7 + expected_row_header_column: D + expected_row_header: D5 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D5, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D5, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D5, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + - value_id: d51_be + label: Belgium taxes on income + ordinal: 6 + row_number: 8 + expected_row_header_column: D + expected_row_header: D51 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D51, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D51, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D51, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d51_de + label: Germany taxes on income + ordinal: 7 + row_number: 9 + expected_row_header_column: D + expected_row_header: D51 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D51, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D51, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D51, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d51_fr + label: France taxes on income + ordinal: 8 + row_number: 10 + expected_row_header_column: D + expected_row_header: D51 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D51, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D51, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D51, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + - value_id: d61_be + label: Belgium net social contributions + ordinal: 9 + row_number: 11 + expected_row_header_column: D + expected_row_header: D61 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D61, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D61, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D61, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d61_de + label: Germany net social contributions + ordinal: 10 + row_number: 12 + expected_row_header_column: D + expected_row_header: D61 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D61, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D61, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D61, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d61_fr + label: France net social contributions + ordinal: 11 + row_number: 13 + expected_row_header_column: D + expected_row_header: D61 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D61, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D61, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D61, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + measures: + - measure_id: revenue + label: Tax or net social contribution revenue + ordinal: 0 + column: G + source_column_id: value + expected_column_header_row: 1 + expected_column_header: value + concept: eurostat.gov_10a_taxag.revenue + source_concept: eurostat.gov_10a_taxag.value + concept_relation: source_label + concept_authority: eurostat + concept_evidence_url: https://ec.europa.eu/eurostat/databrowser/view/gov_10a_taxag/default/table?lang=en + concept_evidence_notes: >- + Source na_item codes identify the published tax and net + social contribution categories. + unit: eur + aggregation: sum + value_scale: 1000000 + expected_cell_type: number + - record_set_id: eurostat.gov_10a_taxag.cy2024.tax_revenue.country + provenance_class: administrative + record_set_spec_id: eurostat.gov_10a_taxag.tax_revenue.country.v1 + source_record_id_prefix: eurostat.gov_10a_taxag.cy2024.tax_revenue.country + sheet_name: gov_10a_taxag + period_type: calendar_year + period: 2024 + geography_id: BE + geography_level: country + geography_name: Belgium + geography_vintage: current + entity: government + entity_role: general_government + domain: taxes_and_social_contributions + groupby_dimension: na_item + shared_filters: + freq: A + unit: MIO_EUR + sector: S13 + time: '2024' + shared_constraints: + - {variable: freq, operator: '==', value: A, label: Eurostat frequency} + - {variable: unit, operator: '==', value: MIO_EUR, label: Eurostat source unit} + - {variable: sector, operator: '==', value: S13, label: Eurostat institutional sector} + - {variable: time, operator: '==', value: '2024', label: Eurostat time} + rows: + - value_id: d2_be + label: Belgium taxes on production and imports + ordinal: 0 + row_number: 14 + expected_row_header_column: D + expected_row_header: D2 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D2, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D2, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D2, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d2_de + label: Germany taxes on production and imports + ordinal: 1 + row_number: 15 + expected_row_header_column: D + expected_row_header: D2 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D2, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D2, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D2, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d2_fr + label: France taxes on production and imports + ordinal: 2 + row_number: 16 + expected_row_header_column: D + expected_row_header: D2 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D2, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D2, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D2, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + - value_id: d5_be + label: Belgium current taxes on income and wealth + ordinal: 3 + row_number: 17 + expected_row_header_column: D + expected_row_header: D5 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D5, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D5, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D5, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d5_de + label: Germany current taxes on income and wealth + ordinal: 4 + row_number: 18 + expected_row_header_column: D + expected_row_header: D5 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D5, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D5, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D5, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d5_fr + label: France current taxes on income and wealth + ordinal: 5 + row_number: 19 + expected_row_header_column: D + expected_row_header: D5 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D5, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D5, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D5, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + - value_id: d51_be + label: Belgium taxes on income + ordinal: 6 + row_number: 20 + expected_row_header_column: D + expected_row_header: D51 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D51, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D51, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D51, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d51_de + label: Germany taxes on income + ordinal: 7 + row_number: 21 + expected_row_header_column: D + expected_row_header: D51 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D51, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D51, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D51, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d51_fr + label: France taxes on income + ordinal: 8 + row_number: 22 + expected_row_header_column: D + expected_row_header: D51 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D51, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D51, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D51, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + - value_id: d61_be + label: Belgium net social contributions + ordinal: 9 + row_number: 23 + expected_row_header_column: D + expected_row_header: D61 + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D61, label: Eurostat na_item dimension} + - {column: E, expected_value: BE, label: Eurostat geo dimension} + filters: {na_item: D61, geo: BE} + constraints: + - {variable: na_item, operator: '==', value: D61, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: BE, label: Eurostat geography} + - value_id: d61_de + label: Germany net social contributions + ordinal: 10 + row_number: 24 + expected_row_header_column: D + expected_row_header: D61 + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D61, label: Eurostat na_item dimension} + - {column: E, expected_value: DE, label: Eurostat geo dimension} + filters: {na_item: D61, geo: DE} + constraints: + - {variable: na_item, operator: '==', value: D61, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: DE, label: Eurostat geography} + - value_id: d61_fr + label: France net social contributions + ordinal: 11 + row_number: 25 + expected_row_header_column: D + expected_row_header: D61 + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MIO_EUR, label: Eurostat unit dimension} + - {column: C, expected_value: S13, label: Eurostat sector dimension} + - {column: D, expected_value: D61, label: Eurostat na_item dimension} + - {column: E, expected_value: FR, label: Eurostat geo dimension} + filters: {na_item: D61, geo: FR} + constraints: + - {variable: na_item, operator: '==', value: D61, label: Eurostat national accounts item} + - {variable: geo, operator: '==', value: FR, label: Eurostat geography} + measures: + - measure_id: revenue + label: Tax or net social contribution revenue + ordinal: 0 + column: G + source_column_id: value + expected_column_header_row: 1 + expected_column_header: value + concept: eurostat.gov_10a_taxag.revenue + source_concept: eurostat.gov_10a_taxag.value + concept_relation: source_label + concept_authority: eurostat + concept_evidence_url: https://ec.europa.eu/eurostat/databrowser/view/gov_10a_taxag/default/table?lang=en + concept_evidence_notes: >- + Source na_item codes identify the published tax and net + social contribution categories. + unit: eur + aggregation: sum + value_scale: 1000000 + expected_cell_type: number diff --git a/packages/eurostat/ilc_di01/source_package.yaml b/packages/eurostat/ilc_di01/source_package.yaml new file mode 100644 index 0000000..a507a12 --- /dev/null +++ b/packages/eurostat/ilc_di01/source_package.yaml @@ -0,0 +1,488 @@ +schema_version: ledger.source_package.v1 +package_id: eurostat-ilc-di01 +label: Eurostat EU-SILC income distribution by decile for BE, DE, and FR +artifact: + source_name: eurostat + source_table: Eurostat ilc_di01 Distribution of income by quantiles for Belgium, Germany, and France + resource_package: db + resource_directory: data/eurostat/ilc_di01 + manifest: manifest.yaml + vintage: eurostat_ilc_di01_2024 + extracted_at: "2026-08-09" + extraction_method: >- + Eurostat dissemination API v1.0 JSON-stat 2.0 full-cube row parse of + the filtered request recorded in the manifest source_url + parser: json_stat_2_full_rows + artifact_year: 2024 + sheet_name: ilc_di01 + # statinfo determines measure semantics in the live cube. SHARE is a + # percentage share even though its raw unit/currency context is EUR; TC is a + # top cut-off point measured in EUR. Chronicle preserves both source fields. + selected_rows: + - {freq: A, quant_inc: D1, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D2, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D3, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D4, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D5, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D6, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D7, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D8, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D9, statinfo: SHARE, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D1, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D2, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D3, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D4, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D5, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D6, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D7, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D8, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D9, statinfo: SHARE, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D1, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D2, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D3, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D4, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D5, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D6, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D7, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D8, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D9, statinfo: SHARE, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D1, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D2, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D3, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D4, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D5, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D6, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D7, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D8, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D9, statinfo: TC, unit: EUR, geo: BE, time: "2024"} + - {freq: A, quant_inc: D1, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D2, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D3, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D4, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D5, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D6, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D7, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D8, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D9, statinfo: TC, unit: EUR, geo: DE, time: "2024"} + - {freq: A, quant_inc: D1, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D2, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D3, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D4, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D5, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D6, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D7, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D8, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + - {freq: A, quant_inc: D9, statinfo: TC, unit: EUR, geo: FR, time: "2024"} + +# EVALUATION-ONLY by class: Ledger ingests everything; the Populace gate decides use. +record_sets: + - &share_be_record_set + record_set_id: eurostat.ilc_di01.cy2024.income_share.be + provenance_class: survey_aggregate + survey_instrument: EU-SILC + record_set_spec_id: eurostat.ilc_di01.income_share.by_decile.v1 + source_record_id_prefix: eurostat.ilc_di01.cy2024.income_share.be + sheet_name: ilc_di01 + period_type: calendar_year + period: 2024 + period_coverage: &period_coverage + basis: survey_reference + source_period_label: "2024" + start_date: "2023-01-01" + end_date: "2023-12-31" + notes: >- + The fact period preserves Eurostat TIME_PERIOD 2024 (survey year). + For BE, DE, and FR the EU-SILC income reference period is the + previous calendar year, 2023; start_date and end_date record that + income reference span. + geography_id: BE + geography_level: country + geography_name: Belgium + geography_vintage: current + entity: person + entity_role: eu_silc_equivalised_disposable_income_recipient + domain: income_distribution + groupby_dimension: quant_inc + shared_filters: + freq: A + statinfo: SHARE + unit: EUR + geo: BE + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency code} + - {variable: statinfo, operator: "==", value: SHARE, label: Share of national equivalised income} + - {variable: unit, operator: "==", value: EUR, label: Eurostat unit code} + - {variable: geo, operator: "==", value: BE, label: Eurostat geography code} + - {variable: time, operator: "==", value: "2024", label: Eurostat time code} + rows: + - &share_d1 + value_id: d1 + label: First income decile share + ordinal: 0 + row_number: 2 + expected_row_header_column: B + expected_row_header: D1 + guard_cells: &share_be_guards + - {column: A, expected_value: A, label: Eurostat frequency code} + - {column: C, expected_value: SHARE, label: Eurostat statistical information} + - {column: D, expected_value: EUR, label: Eurostat unit code} + - {column: E, expected_value: BE, label: Eurostat geography code} + filters: {quant_inc: D1} + constraints: + - {variable: quant_inc, operator: "==", value: D1, label: Eurostat income quantile code} + - &share_d2 + value_id: d2 + label: Second income decile share + ordinal: 1 + row_number: 3 + expected_row_header_column: B + expected_row_header: D2 + guard_cells: *share_be_guards + filters: {quant_inc: D2} + constraints: + - {variable: quant_inc, operator: "==", value: D2, label: Eurostat income quantile code} + - &share_d3 + value_id: d3 + label: Third income decile share + ordinal: 2 + row_number: 4 + expected_row_header_column: B + expected_row_header: D3 + guard_cells: *share_be_guards + filters: {quant_inc: D3} + constraints: + - {variable: quant_inc, operator: "==", value: D3, label: Eurostat income quantile code} + - &share_d4 + value_id: d4 + label: Fourth income decile share + ordinal: 3 + row_number: 5 + expected_row_header_column: B + expected_row_header: D4 + guard_cells: *share_be_guards + filters: {quant_inc: D4} + constraints: + - {variable: quant_inc, operator: "==", value: D4, label: Eurostat income quantile code} + - &share_d5 + value_id: d5 + label: Fifth income decile share + ordinal: 4 + row_number: 6 + expected_row_header_column: B + expected_row_header: D5 + guard_cells: *share_be_guards + filters: {quant_inc: D5} + constraints: + - {variable: quant_inc, operator: "==", value: D5, label: Eurostat income quantile code} + - &share_d6 + value_id: d6 + label: Sixth income decile share + ordinal: 5 + row_number: 7 + expected_row_header_column: B + expected_row_header: D6 + guard_cells: *share_be_guards + filters: {quant_inc: D6} + constraints: + - {variable: quant_inc, operator: "==", value: D6, label: Eurostat income quantile code} + - &share_d7 + value_id: d7 + label: Seventh income decile share + ordinal: 6 + row_number: 8 + expected_row_header_column: B + expected_row_header: D7 + guard_cells: *share_be_guards + filters: {quant_inc: D7} + constraints: + - {variable: quant_inc, operator: "==", value: D7, label: Eurostat income quantile code} + - &share_d8 + value_id: d8 + label: Eighth income decile share + ordinal: 7 + row_number: 9 + expected_row_header_column: B + expected_row_header: D8 + guard_cells: *share_be_guards + filters: {quant_inc: D8} + constraints: + - {variable: quant_inc, operator: "==", value: D8, label: Eurostat income quantile code} + - &share_d9 + value_id: d9 + label: Ninth income decile share + ordinal: 8 + row_number: 10 + expected_row_header_column: B + expected_row_header: D9 + guard_cells: *share_be_guards + filters: {quant_inc: D9} + constraints: + - {variable: quant_inc, operator: "==", value: D9, label: Eurostat income quantile code} + measures: &share_measure + - measure_id: income_share + label: Income share of decile + ordinal: 0 + column: G + source_column_id: value + expected_column_header_row: 1 + expected_column_header: value + concept: eurostat.ilc_di01.income_share + source_concept: eurostat.ilc_di01.SHARE + concept_relation: source_label + concept_authority: eurostat + concept_evidence_url: https://ec.europa.eu/eurostat/databrowser/view/ilc_di01/default/table?lang=en + concept_evidence_notes: >- + statinfo SHARE is "Share of national equivalised income" and is a + percentage share. The live cube carries raw unit EUR as currency + context, so statinfo—not unit—controls this Ledger measure mapping. + EU-SILC universe: private households and their current members; + persons in collective households and institutions are generally + excluded, and France is covered excluding Mayotte, per the + Eurostat ilc_sieusilc reference metadata. + unit: percent + aggregation: share + expected_cell_type: number + - <<: *share_be_record_set + record_set_id: eurostat.ilc_di01.cy2024.income_share.de + provenance_class: survey_aggregate + survey_instrument: EU-SILC + source_record_id_prefix: eurostat.ilc_di01.cy2024.income_share.de + geography_id: DE + geography_name: Germany + shared_filters: + freq: A + statinfo: SHARE + unit: EUR + geo: DE + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency code} + - {variable: statinfo, operator: "==", value: SHARE, label: Share of national equivalised income} + - {variable: unit, operator: "==", value: EUR, label: Eurostat unit code} + - {variable: geo, operator: "==", value: DE, label: Eurostat geography code} + - {variable: time, operator: "==", value: "2024", label: Eurostat time code} + rows: + - <<: *share_d1 + row_number: 11 + guard_cells: &share_de_guards + - {column: A, expected_value: A, label: Eurostat frequency code} + - {column: C, expected_value: SHARE, label: Eurostat statistical information} + - {column: D, expected_value: EUR, label: Eurostat unit code} + - {column: E, expected_value: DE, label: Eurostat geography code} + - {<<: *share_d2, row_number: 12, guard_cells: *share_de_guards} + - {<<: *share_d3, row_number: 13, guard_cells: *share_de_guards} + - {<<: *share_d4, row_number: 14, guard_cells: *share_de_guards} + - {<<: *share_d5, row_number: 15, guard_cells: *share_de_guards} + - {<<: *share_d6, row_number: 16, guard_cells: *share_de_guards} + - {<<: *share_d7, row_number: 17, guard_cells: *share_de_guards} + - {<<: *share_d8, row_number: 18, guard_cells: *share_de_guards} + - {<<: *share_d9, row_number: 19, guard_cells: *share_de_guards} + - <<: *share_be_record_set + record_set_id: eurostat.ilc_di01.cy2024.income_share.fr + provenance_class: survey_aggregate + survey_instrument: EU-SILC + source_record_id_prefix: eurostat.ilc_di01.cy2024.income_share.fr + geography_id: FR + geography_name: France + shared_filters: + freq: A + statinfo: SHARE + unit: EUR + geo: FR + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency code} + - {variable: statinfo, operator: "==", value: SHARE, label: Share of national equivalised income} + - {variable: unit, operator: "==", value: EUR, label: Eurostat unit code} + - {variable: geo, operator: "==", value: FR, label: Eurostat geography code} + - {variable: time, operator: "==", value: "2024", label: Eurostat time code} + rows: + - <<: *share_d1 + row_number: 20 + guard_cells: &share_fr_guards + - {column: A, expected_value: A, label: Eurostat frequency code} + - {column: C, expected_value: SHARE, label: Eurostat statistical information} + - {column: D, expected_value: EUR, label: Eurostat unit code} + - {column: E, expected_value: FR, label: Eurostat geography code} + - {<<: *share_d2, row_number: 21, guard_cells: *share_fr_guards} + - {<<: *share_d3, row_number: 22, guard_cells: *share_fr_guards} + - {<<: *share_d4, row_number: 23, guard_cells: *share_fr_guards} + - {<<: *share_d5, row_number: 24, guard_cells: *share_fr_guards} + - {<<: *share_d6, row_number: 25, guard_cells: *share_fr_guards} + - {<<: *share_d7, row_number: 26, guard_cells: *share_fr_guards} + - {<<: *share_d8, row_number: 27, guard_cells: *share_fr_guards} + - {<<: *share_d9, row_number: 28, guard_cells: *share_fr_guards} + - &cut_point_be_record_set + record_set_id: eurostat.ilc_di01.cy2024.income_cut_point.be + provenance_class: survey_aggregate + survey_instrument: EU-SILC + record_set_spec_id: eurostat.ilc_di01.income_cut_point.by_decile.v1 + source_record_id_prefix: eurostat.ilc_di01.cy2024.income_cut_point.be + sheet_name: ilc_di01 + period_type: calendar_year + period: 2024 + period_coverage: *period_coverage + geography_id: BE + geography_level: country + geography_name: Belgium + geography_vintage: current + entity: person + entity_role: eu_silc_equivalised_disposable_income_recipient + domain: income_distribution + groupby_dimension: quant_inc + shared_filters: + freq: A + statinfo: TC + unit: EUR + geo: BE + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency code} + - {variable: statinfo, operator: "==", value: TC, label: Top cut-off point} + - {variable: unit, operator: "==", value: EUR, label: Eurostat unit code} + - {variable: geo, operator: "==", value: BE, label: Eurostat geography code} + - {variable: time, operator: "==", value: "2024", label: Eurostat time code} + rows: + - &cut_d1 + <<: *share_d1 + label: First income decile cut-point + row_number: 29 + guard_cells: &cut_be_guards + - {column: A, expected_value: A, label: Eurostat frequency code} + - {column: C, expected_value: TC, label: Eurostat statistical information} + - {column: D, expected_value: EUR, label: Eurostat unit code} + - {column: E, expected_value: BE, label: Eurostat geography code} + - &cut_d2 + <<: *share_d2 + label: Second income decile cut-point + row_number: 30 + guard_cells: *cut_be_guards + - &cut_d3 + <<: *share_d3 + label: Third income decile cut-point + row_number: 31 + guard_cells: *cut_be_guards + - &cut_d4 + <<: *share_d4 + label: Fourth income decile cut-point + row_number: 32 + guard_cells: *cut_be_guards + - &cut_d5 + <<: *share_d5 + label: Fifth income decile cut-point + row_number: 33 + guard_cells: *cut_be_guards + - &cut_d6 + <<: *share_d6 + label: Sixth income decile cut-point + row_number: 34 + guard_cells: *cut_be_guards + - &cut_d7 + <<: *share_d7 + label: Seventh income decile cut-point + row_number: 35 + guard_cells: *cut_be_guards + - &cut_d8 + <<: *share_d8 + label: Eighth income decile cut-point + row_number: 36 + guard_cells: *cut_be_guards + - &cut_d9 + <<: *share_d9 + label: Ninth income decile cut-point + row_number: 37 + guard_cells: *cut_be_guards + measures: &cut_point_measure + - measure_id: income_cut_point + label: Income decile top cut-off point + ordinal: 0 + column: G + source_column_id: value + expected_column_header_row: 1 + expected_column_header: value + concept: eurostat.ilc_di01.income_cut_point + source_concept: eurostat.ilc_di01.TC + concept_relation: source_label + concept_authority: eurostat + concept_evidence_url: https://ec.europa.eu/eurostat/databrowser/view/ilc_di01/default/table?lang=en + concept_evidence_notes: >- + statinfo TC is "Top cut-off point". For D1-D9 with raw unit EUR, + Eurostat publishes the euro-denominated income decile cut-points. + EU-SILC universe: private households and their current members; + persons in collective households and institutions are generally + excluded, and France is covered excluding Mayotte, per the + Eurostat ilc_sieusilc reference metadata. + unit: eur + aggregation: quantile + expected_cell_type: number + - <<: *cut_point_be_record_set + record_set_id: eurostat.ilc_di01.cy2024.income_cut_point.de + provenance_class: survey_aggregate + survey_instrument: EU-SILC + source_record_id_prefix: eurostat.ilc_di01.cy2024.income_cut_point.de + geography_id: DE + geography_name: Germany + shared_filters: + freq: A + statinfo: TC + unit: EUR + geo: DE + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency code} + - {variable: statinfo, operator: "==", value: TC, label: Top cut-off point} + - {variable: unit, operator: "==", value: EUR, label: Eurostat unit code} + - {variable: geo, operator: "==", value: DE, label: Eurostat geography code} + - {variable: time, operator: "==", value: "2024", label: Eurostat time code} + rows: + - <<: *cut_d1 + row_number: 38 + guard_cells: &cut_de_guards + - {column: A, expected_value: A, label: Eurostat frequency code} + - {column: C, expected_value: TC, label: Eurostat statistical information} + - {column: D, expected_value: EUR, label: Eurostat unit code} + - {column: E, expected_value: DE, label: Eurostat geography code} + - {<<: *cut_d2, row_number: 39, guard_cells: *cut_de_guards} + - {<<: *cut_d3, row_number: 40, guard_cells: *cut_de_guards} + - {<<: *cut_d4, row_number: 41, guard_cells: *cut_de_guards} + - {<<: *cut_d5, row_number: 42, guard_cells: *cut_de_guards} + - {<<: *cut_d6, row_number: 43, guard_cells: *cut_de_guards} + - {<<: *cut_d7, row_number: 44, guard_cells: *cut_de_guards} + - {<<: *cut_d8, row_number: 45, guard_cells: *cut_de_guards} + - {<<: *cut_d9, row_number: 46, guard_cells: *cut_de_guards} + - <<: *cut_point_be_record_set + record_set_id: eurostat.ilc_di01.cy2024.income_cut_point.fr + provenance_class: survey_aggregate + survey_instrument: EU-SILC + source_record_id_prefix: eurostat.ilc_di01.cy2024.income_cut_point.fr + geography_id: FR + geography_name: France + shared_filters: + freq: A + statinfo: TC + unit: EUR + geo: FR + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency code} + - {variable: statinfo, operator: "==", value: TC, label: Top cut-off point} + - {variable: unit, operator: "==", value: EUR, label: Eurostat unit code} + - {variable: geo, operator: "==", value: FR, label: Eurostat geography code} + - {variable: time, operator: "==", value: "2024", label: Eurostat time code} + rows: + - <<: *cut_d1 + row_number: 47 + guard_cells: &cut_fr_guards + - {column: A, expected_value: A, label: Eurostat frequency code} + - {column: C, expected_value: TC, label: Eurostat statistical information} + - {column: D, expected_value: EUR, label: Eurostat unit code} + - {column: E, expected_value: FR, label: Eurostat geography code} + - {<<: *cut_d2, row_number: 48, guard_cells: *cut_fr_guards} + - {<<: *cut_d3, row_number: 49, guard_cells: *cut_fr_guards} + - {<<: *cut_d4, row_number: 50, guard_cells: *cut_fr_guards} + - {<<: *cut_d5, row_number: 51, guard_cells: *cut_fr_guards} + - {<<: *cut_d6, row_number: 52, guard_cells: *cut_fr_guards} + - {<<: *cut_d7, row_number: 53, guard_cells: *cut_fr_guards} + - {<<: *cut_d8, row_number: 54, guard_cells: *cut_fr_guards} + - {<<: *cut_d9, row_number: 55, guard_cells: *cut_fr_guards} diff --git a/packages/eurostat/ilc_li02/source_package.yaml b/packages/eurostat/ilc_li02/source_package.yaml new file mode 100644 index 0000000..e6dd6b8 --- /dev/null +++ b/packages/eurostat/ilc_li02/source_package.yaml @@ -0,0 +1,150 @@ +schema_version: ledger.source_package.v1 +package_id: eurostat-ilc-li02 +label: Eurostat ilc_li02 at-risk-of-poverty rate for Belgium, Germany, and France +artifact: + source_name: eurostat + source_table: Eurostat ilc_li02 At-risk-of-poverty rate by poverty threshold, age and sex - EU-SILC and ECHP surveys for Belgium, Germany, and France + resource_package: db + resource_directory: data/eurostat/ilc_li02 + manifest: manifest.yaml + vintage: eurostat_ilc_li02_2024 + extracted_at: "2026-08-09" + extraction_method: >- + Eurostat dissemination API v1.0 JSON-stat 2.0 full-cube row parse of + the filtered request recorded in the manifest source_url + parser: json_stat_2_full_rows + artifact_year: 2024 + sheet_name: ilc_li02 + # Live category labels: B_60 is "Below 60%" and selects the AROP side of + # the threshold; A_60 is "Above 60%" and is therefore rejected. MED_EI is + # "Median equivalised income", the statistic that anchors the 60% threshold. + selected_rows: + - {freq: A, statinfo: MED_EI, unit: PC, rskpovth: B_60, sex: T, age: TOTAL, geo: BE, time: "2024"} + - {freq: A, statinfo: MED_EI, unit: PC, rskpovth: B_60, sex: T, age: TOTAL, geo: DE, time: "2024"} + - {freq: A, statinfo: MED_EI, unit: PC, rskpovth: B_60, sex: T, age: TOTAL, geo: FR, time: "2024"} +# EVALUATION-ONLY by class: Ledger ingests everything; the Populace gate decides use. +record_sets: + - record_set_id: eurostat.ilc_li02.cy2024.at_risk_of_poverty_rate.country + provenance_class: survey_aggregate + survey_instrument: EU-SILC + record_set_spec_id: eurostat.ilc_li02.at_risk_of_poverty_rate.country.v1 + source_record_id_prefix: eurostat.ilc_li02.cy2024.at_risk_of_poverty_rate.country + sheet_name: ilc_li02 + period_type: calendar_year + period: 2024 + period_coverage: + basis: survey_reference + source_period_label: "2024" + start_date: "2023-01-01" + end_date: "2023-12-31" + notes: >- + The fact period preserves Eurostat TIME_PERIOD 2024 (survey year). + For BE, DE, and FR the EU-SILC income reference period is the + previous calendar year, 2023; start_date and end_date record that + income reference span. + geography_id: BE + geography_level: country + geography_name: Belgium + geography_vintage: current + entity: person + entity_role: private_household_resident + domain: income_and_living_conditions + groupby_dimension: geo + shared_filters: + freq: A + statinfo: MED_EI + unit: PC + rskpovth: B_60 + sex: T + age: TOTAL + time: "2024" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: Eurostat frequency} + - {variable: statinfo, operator: "==", value: MED_EI, label: Median equivalised income} + - {variable: unit, operator: "==", value: PC, label: Eurostat source unit} + - {variable: rskpovth, operator: "==", value: B_60, label: Below 60% threshold} + - {variable: sex, operator: "==", value: T, label: Eurostat sex total} + - {variable: age, operator: "==", value: TOTAL, label: Eurostat age total} + - {variable: time, operator: "==", value: "2024", label: Eurostat time} + rows: + - value_id: be + label: Belgium at-risk-of-poverty rate + ordinal: 0 + row_number: 2 + expected_row_header_column: G + expected_row_header: BE + geography_id: BE + geography_name: Belgium + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MED_EI, label: Eurostat statinfo dimension} + - {column: C, expected_value: PC, label: Eurostat unit dimension} + - {column: D, expected_value: B_60, label: Eurostat threshold dimension} + - {column: E, expected_value: T, label: Eurostat sex dimension} + - {column: F, expected_value: TOTAL, label: Eurostat age dimension} + - {column: G, expected_value: BE, label: Eurostat geo dimension} + filters: {geo: BE} + constraints: + - {variable: geo, operator: "==", value: BE, label: Eurostat geography} + - value_id: de + label: Germany at-risk-of-poverty rate + ordinal: 1 + row_number: 3 + expected_row_header_column: G + expected_row_header: DE + geography_id: DE + geography_name: Germany + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MED_EI, label: Eurostat statinfo dimension} + - {column: C, expected_value: PC, label: Eurostat unit dimension} + - {column: D, expected_value: B_60, label: Eurostat threshold dimension} + - {column: E, expected_value: T, label: Eurostat sex dimension} + - {column: F, expected_value: TOTAL, label: Eurostat age dimension} + - {column: G, expected_value: DE, label: Eurostat geo dimension} + filters: {geo: DE} + constraints: + - {variable: geo, operator: "==", value: DE, label: Eurostat geography} + - value_id: fr + label: France at-risk-of-poverty rate + ordinal: 2 + row_number: 4 + expected_row_header_column: G + expected_row_header: FR + geography_id: FR + geography_name: France + guard_cells: + - {column: A, expected_value: A, label: Eurostat freq dimension} + - {column: B, expected_value: MED_EI, label: Eurostat statinfo dimension} + - {column: C, expected_value: PC, label: Eurostat unit dimension} + - {column: D, expected_value: B_60, label: Eurostat threshold dimension} + - {column: E, expected_value: T, label: Eurostat sex dimension} + - {column: F, expected_value: TOTAL, label: Eurostat age dimension} + - {column: G, expected_value: FR, label: Eurostat geo dimension} + filters: {geo: FR} + constraints: + - {variable: geo, operator: "==", value: FR, label: Eurostat geography} + measures: + - measure_id: at_risk_of_poverty_rate + label: At-risk-of-poverty rate after social transfers + ordinal: 0 + column: I + source_column_id: value + expected_column_header_row: 1 + expected_column_header: value + concept: at_risk_of_poverty_rate_after_social_transfers + source_concept: eurostat.ilc_li02.MED_EI.B_60 + concept_relation: exact + concept_authority: eurostat + concept_evidence_url: https://ec.europa.eu/eurostat/databrowser/view/ilc_li02/default/table?lang=en + concept_evidence_notes: >- + ilc_li02 is the after-social-transfers AROP table. MED_EI (Median + equivalised income), B_60 (Below 60%), PC, total sex, and total age + select its headline rate. EU-SILC universe: private households and + their current members; persons in collective households and + institutions are generally excluded, and France is covered + excluding Mayotte, per the Eurostat ilc_sieusilc reference + metadata. + unit: percent + aggregation: rate + expected_cell_type: number diff --git a/packages/eurostat/spr_exp_func/source_package.yaml b/packages/eurostat/spr_exp_func/source_package.yaml new file mode 100644 index 0000000..ed1d328 --- /dev/null +++ b/packages/eurostat/spr_exp_func/source_package.yaml @@ -0,0 +1,271 @@ +schema_version: ledger.source_package.v1 +package_id: eurostat-spr-exp-func +label: Eurostat ESSPROS social protection expenditure by function pilot +artifact: + source_name: eurostat + source_table: Eurostat spr_exp_func Expenditure on social benefits by function for Belgium, Germany, and France + resource_package: db + resource_directory: data/eurostat/spr_exp_func + manifest: manifest.yaml + vintage: eurostat_spr_exp_func_2023 + extracted_at: "2026-08-09" + extraction_method: >- + Eurostat dissemination API v1.0 JSON-stat 2.0 full-cube row parse of + the filtered request recorded in the manifest source_url + parser: json_stat_2_full_rows + artifact_year: 2023 + sheet_name: spr_exp_func + # The live API identifies social protection benefits with spdeps=SPR and + # their function with spfunc. TOTAL is Eurostat's own published aggregate; + # Chronicle does not derive it from the eight component observations. + selected_rows: + - {freq: A, spdeps: SPR, spfunc: TOTAL, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: SICK, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: DIS, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: OLD, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: SRV, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: FAM, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: UNE, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: HOU, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: EXCL, unit: MIO_EUR, geo: BE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: TOTAL, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: SICK, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: DIS, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: OLD, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: SRV, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: FAM, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: UNE, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: HOU, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: EXCL, unit: MIO_EUR, geo: DE, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: TOTAL, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: SICK, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: DIS, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: OLD, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: SRV, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: FAM, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: UNE, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: HOU, unit: MIO_EUR, geo: FR, time: "2023"} + - {freq: A, spdeps: SPR, spfunc: EXCL, unit: MIO_EUR, geo: FR, time: "2023"} +record_sets: + - &belgium_record_set + record_set_id: eurostat.spr_exp_func.cy2023.social_protection_expenditure_by_function.be + provenance_class: administrative + record_set_spec_id: eurostat.spr_exp_func.social_protection_expenditure_by_function.country.v1 + source_record_id_prefix: eurostat.spr_exp_func.cy2023.social_protection_expenditure_by_function.be + sheet_name: spr_exp_func + period_type: calendar_year + period: 2023 + geography_id: BE + geography_level: country + geography_name: Belgium + geography_vintage: current + entity: social_protection_scheme + entity_role: social_protection_provider + domain: social_protection_expenditure + groupby_dimension: spfunc + shared_filters: + freq: A + spdeps: SPR + unit: MIO_EUR + geo: BE + time: "2023" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: annual frequency} + - {variable: spdeps, operator: "==", value: SPR, label: social protection benefits} + - {variable: unit, operator: "==", value: MIO_EUR, label: source unit} + - {variable: geo, operator: "==", value: BE, label: geography} + - {variable: time, operator: "==", value: "2023", label: source time} + rows: + - &total_row + value_id: total + label: Total social protection benefits + ordinal: 0 + row_number: 2 + expected_row_header_column: C + expected_row_header: TOTAL + filters: {spfunc: TOTAL} + constraints: + - {variable: spfunc, operator: "==", value: TOTAL, label: Total function} + guard_cells: &belgium_guards + - {column: A, expected_value: A, label: annual frequency} + - {column: B, expected_value: SPR, label: social protection benefits} + - {column: D, expected_value: MIO_EUR, label: source unit} + - {column: E, expected_value: BE, label: geography} + - &sickness_row + value_id: sickness_health_care + label: Sickness and health care + ordinal: 1 + row_number: 3 + expected_row_header_column: C + expected_row_header: SICK + filters: {spfunc: SICK} + constraints: + - {variable: spfunc, operator: "==", value: SICK, label: Sickness and health care function} + guard_cells: *belgium_guards + - &disability_row + value_id: disability + label: Disability + ordinal: 2 + row_number: 4 + expected_row_header_column: C + expected_row_header: DIS + filters: {spfunc: DIS} + constraints: + - {variable: spfunc, operator: "==", value: DIS, label: Disability function} + guard_cells: *belgium_guards + - &old_age_row + value_id: old_age + label: Old age + ordinal: 3 + row_number: 5 + expected_row_header_column: C + expected_row_header: OLD + filters: {spfunc: OLD} + constraints: + - {variable: spfunc, operator: "==", value: OLD, label: Old age function} + guard_cells: *belgium_guards + - &survivors_row + value_id: survivors + label: Survivors + ordinal: 4 + row_number: 6 + expected_row_header_column: C + expected_row_header: SRV + filters: {spfunc: SRV} + constraints: + - {variable: spfunc, operator: "==", value: SRV, label: Survivors function} + guard_cells: *belgium_guards + - &family_row + value_id: family_children + label: Family and children + ordinal: 5 + row_number: 7 + expected_row_header_column: C + expected_row_header: FAM + filters: {spfunc: FAM} + constraints: + - {variable: spfunc, operator: "==", value: FAM, label: Family and children function} + guard_cells: *belgium_guards + - &unemployment_row + value_id: unemployment + label: Unemployment + ordinal: 6 + row_number: 8 + expected_row_header_column: C + expected_row_header: UNE + filters: {spfunc: UNE} + constraints: + - {variable: spfunc, operator: "==", value: UNE, label: Unemployment function} + guard_cells: *belgium_guards + - &housing_row + value_id: housing + label: Housing + ordinal: 7 + row_number: 9 + expected_row_header_column: C + expected_row_header: HOU + filters: {spfunc: HOU} + constraints: + - {variable: spfunc, operator: "==", value: HOU, label: Housing function} + guard_cells: *belgium_guards + - &social_exclusion_row + value_id: social_exclusion_nec + label: Social exclusion not elsewhere classified + ordinal: 8 + row_number: 10 + expected_row_header_column: C + expected_row_header: EXCL + filters: {spfunc: EXCL} + constraints: + - {variable: spfunc, operator: "==", value: EXCL, label: Social exclusion not elsewhere classified function} + guard_cells: *belgium_guards + measures: &expenditure_measure + - measure_id: expenditure + label: Social protection expenditure + ordinal: 0 + column: G + source_column_id: value + expected_column_header_row: 1 + expected_column_header: value + concept: social_protection_expenditure + source_concept: eurostat.spr_exp_func.social_protection_expenditure_by_function + concept_relation: source_label + concept_authority: eurostat + concept_evidence_url: https://ec.europa.eu/eurostat/databrowser/view/spr_exp_func/default/table?lang=en + concept_evidence_notes: >- + spr_exp_func publishes social protection benefits for TOTAL and the + eight ESSPROS functions in spfunc. ESSPROS counts interventions by + public and private social protection schemes; the scheme, not + general government, is the ESSPROS statistical unit. + unit: eur + aggregation: sum + value_scale: 1000000 + expected_cell_type: number + - <<: *belgium_record_set + record_set_id: eurostat.spr_exp_func.cy2023.social_protection_expenditure_by_function.de + provenance_class: administrative + source_record_id_prefix: eurostat.spr_exp_func.cy2023.social_protection_expenditure_by_function.de + geography_id: DE + geography_name: Germany + shared_filters: + freq: A + spdeps: SPR + unit: MIO_EUR + geo: DE + time: "2023" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: annual frequency} + - {variable: spdeps, operator: "==", value: SPR, label: social protection benefits} + - {variable: unit, operator: "==", value: MIO_EUR, label: source unit} + - {variable: geo, operator: "==", value: DE, label: geography} + - {variable: time, operator: "==", value: "2023", label: source time} + rows: + - <<: *total_row + row_number: 11 + guard_cells: &germany_guards + - {column: A, expected_value: A, label: annual frequency} + - {column: B, expected_value: SPR, label: social protection benefits} + - {column: D, expected_value: MIO_EUR, label: source unit} + - {column: E, expected_value: DE, label: geography} + - {<<: *sickness_row, row_number: 12, guard_cells: *germany_guards} + - {<<: *disability_row, row_number: 13, guard_cells: *germany_guards} + - {<<: *old_age_row, row_number: 14, guard_cells: *germany_guards} + - {<<: *survivors_row, row_number: 15, guard_cells: *germany_guards} + - {<<: *family_row, row_number: 16, guard_cells: *germany_guards} + - {<<: *unemployment_row, row_number: 17, guard_cells: *germany_guards} + - {<<: *housing_row, row_number: 18, guard_cells: *germany_guards} + - {<<: *social_exclusion_row, row_number: 19, guard_cells: *germany_guards} + - <<: *belgium_record_set + record_set_id: eurostat.spr_exp_func.cy2023.social_protection_expenditure_by_function.fr + provenance_class: administrative + source_record_id_prefix: eurostat.spr_exp_func.cy2023.social_protection_expenditure_by_function.fr + geography_id: FR + geography_name: France + shared_filters: + freq: A + spdeps: SPR + unit: MIO_EUR + geo: FR + time: "2023" + shared_constraints: + - {variable: freq, operator: "==", value: A, label: annual frequency} + - {variable: spdeps, operator: "==", value: SPR, label: social protection benefits} + - {variable: unit, operator: "==", value: MIO_EUR, label: source unit} + - {variable: geo, operator: "==", value: FR, label: geography} + - {variable: time, operator: "==", value: "2023", label: source time} + rows: + - <<: *total_row + row_number: 20 + guard_cells: &france_guards + - {column: A, expected_value: A, label: annual frequency} + - {column: B, expected_value: SPR, label: social protection benefits} + - {column: D, expected_value: MIO_EUR, label: source unit} + - {column: E, expected_value: FR, label: geography} + - {<<: *sickness_row, row_number: 21, guard_cells: *france_guards} + - {<<: *disability_row, row_number: 22, guard_cells: *france_guards} + - {<<: *old_age_row, row_number: 23, guard_cells: *france_guards} + - {<<: *survivors_row, row_number: 24, guard_cells: *france_guards} + - {<<: *family_row, row_number: 25, guard_cells: *france_guards} + - {<<: *unemployment_row, row_number: 26, guard_cells: *france_guards} + - {<<: *housing_row, row_number: 27, guard_cells: *france_guards} + - {<<: *social_exclusion_row, row_number: 28, guard_cells: *france_guards} diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 243a90f..375678f 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -25,18 +25,18 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): assert summary["valid"] assert summary["counts"] == { "aggregate_duplicate_key_count": 0, - "entity_count": 9, + "entity_count": 10, "error_count": 0, - "fact_count": 155288, - "geography_count": 12534, + "fact_count": 155396, + "geography_count": 12536, "period_count": 147, "semantic_duplicate_key_count": 12, "skipped_source_count": 10, - "source_count": 40, - "source_package_count": 123, + "source_count": 41, + "source_package_count": 127, "warning_count": 1, } - assert len(rows) == 155288 + assert len(rows) == 155396 assert {row["provenance_class"] for row in rows} <= { "administrative", "census", @@ -54,7 +54,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ) assert rows[0]["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert rows[0]["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert source_packages["source_package_count"] == 123 + assert source_packages["source_package_count"] == 127 assert source_packages["skipped_source_count"] == 10 assert sorted(item["source"] for item in source_packages["skipped_sources"]) == [ "census-acs-s0101-congressional-district-age-2024", @@ -68,7 +68,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 155288 + assert coverage["fact_count"] == 155396 assert coverage["counts"]["by_source"] == { "bea": 445, "bfp_economic_outlook": 5, @@ -82,6 +82,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "cms_nhe": 3, "dft": 81, "dwp": 6327, + "eurostat": 108, "federal_reserve": 1, "hhs_acf_liheap": 2, "hhs_acf_tanf": 110, @@ -112,7 +113,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "welshgov": 198, } table_counts = coverage["counts"]["by_source_table"] - assert len(table_counts) == 118 + assert len(table_counts) == 122 assert table_counts["usda_snap:SNAP FY2025 Monthly State Participation"] == 636 assert table_counts["irs_soi:Congressional District Data 2022"] == 26880 assert table_counts["irs_soi:IRS SOI County Data 2022"] == 6286 @@ -234,6 +235,35 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ] == 18 ) + assert ( + table_counts[ + "eurostat:Eurostat gov_10a_taxag Main national accounts tax " + "aggregates for Belgium, Germany, and France" + ] + == 24 + ) + assert ( + table_counts[ + "eurostat:Eurostat spr_exp_func Expenditure on social benefits by " + "function for Belgium, Germany, and France" + ] + == 27 + ) + assert ( + table_counts[ + "eurostat:Eurostat ilc_li02 At-risk-of-poverty rate by poverty " + "threshold, age and sex - EU-SILC and ECHP surveys for Belgium, " + "Germany, and France" + ] + == 3 + ) + assert ( + table_counts[ + "eurostat:Eurostat ilc_di01 Distribution of income by quantiles " + "for Belgium, Germany, and France" + ] + == 54 + ) assert coverage["counts"]["by_period"] == { "academic_year:2013": 6, "academic_year:2014": 6, @@ -284,8 +314,8 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "calendar_year:2020": 83, "calendar_year:2021": 3990, "calendar_year:2022": 1911, - "calendar_year:2023": 6161, - "calendar_year:2024": 33722, + "calendar_year:2023": 6200, + "calendar_year:2024": 33791, "calendar_year:2025": 4438, "calendar_year:2026": 235, "calendar_year:2027": 214, @@ -383,7 +413,9 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "tax_year:2023": 48436, "tax_year:2024": 40, } - assert coverage["counts"]["by_geography"]["country:BE"] == 31 + assert coverage["counts"]["by_geography"]["country:BE"] == 67 + assert coverage["counts"]["by_geography"]["country:DE"] == 36 + assert coverage["counts"]["by_geography"]["country:FR"] == 36 assert coverage["counts"]["by_geography"]["nuts1:BE1"] == 6 assert coverage["counts"]["by_geography"]["nuts1:BE2"] == 17 assert coverage["counts"]["by_geography"]["nuts1:BE3"] == 6 @@ -395,16 +427,17 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ) assert coverage["counts"]["by_geography"]["country:K02000001"] == 4189 assert coverage["counts"]["by_geography"]["country:K03000001"] == 277 - assert len(coverage["counts"]["by_geography"]) == 12534 + assert len(coverage["counts"]["by_geography"]) == 12536 assert coverage["counts"]["by_entity"] == { "dwelling": 12708, "family": 107, "firm": 1439, - "government": 213, + "government": 237, "household": 40432, "institutional_sector": 103, "pension_plan": 2, - "person": 60215, + "person": 60272, + "social_protection_scheme": 27, "tax_unit": 40069, } assert not coverage["duplicates"]["aggregate_fact_keys"] @@ -548,6 +581,13 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): / "hmrc-vat-firm-sector-targets-2024-25" / "consumer_facts.jsonl" ).exists() + for source in ( + "eurostat-gov-10a-taxag", + "eurostat-spr-exp-func", + "eurostat-ilc-li02", + "eurostat-ilc-di01", + ): + assert (output_dir / "sources" / source / "consumer_facts.jsonl").exists() def test_build_bundle_cli_supports_explicit_sources(tmp_path, capsys): diff --git a/tests/test_chronicle_core.py b/tests/test_chronicle_core.py index b0b1ac5..8828ea4 100644 --- a/tests/test_chronicle_core.py +++ b/tests/test_chronicle_core.py @@ -63,6 +63,17 @@ def test_unknown_period_type_is_rejected(): assert any(issue.code == "malformed_period" for issue in validate_fact(fact)) +def test_quantile_aggregation_passes_validation(): + fact = _fact( + measure=Measure(concept="income_quantile_cut_point", unit="eur"), + aggregation=Aggregation(method="quantile"), + filters={"eurostat.quant_inc": "D1"}, + ) + + assert validate_fact(fact) == () + + + def test_stable_key_ignores_human_label(): fact = _fact() relabeled = _fact(label="A different display label") diff --git a/tests/test_chronicle_source_rows.py b/tests/test_chronicle_source_rows.py index 72c6416..90b0fa0 100644 --- a/tests/test_chronicle_source_rows.py +++ b/tests/test_chronicle_source_rows.py @@ -13,6 +13,7 @@ source_rows_from_census_b01001_female_age_json, source_rows_from_cdc_vsrr_live_births_json, source_rows_from_ees_permalink_table_html, + source_rows_from_json_stat_2, source_rows_from_json_table, ) @@ -168,6 +169,121 @@ def test_json_table_parser_reads_object_rows(): assert rows[0].values == {"state": "ALABAMA", "data_value": 4_932} +def test_json_stat_2_parser_flattens_dense_dimensions_and_sparse_observations(): + payload = { + "version": "2.0", + "class": "dataset", + "id": ["geo", "time"], + "size": [2, 2], + "dimension": { + "geo": { + "category": { + "index": {"BE": 0, "DE": 1}, + "label": {"BE": "Belgium", "DE": "Germany"}, + } + }, + "time": { + "category": { + "index": ["2023", "2024"], + "label": {"2023": "2023", "2024": "2024"}, + } + }, + }, + "value": {"0": 11.5, "1": 12.0, "3": 15.0}, + "status": {"1": "p"}, + } + + rows = source_rows_from_json_stat_2( + json.dumps(payload).encode(), + _artifact(), + sheet_name="api", + ) + + assert [row.row_number for row in rows] == [1, 2, 3, 4] + assert [(row.values["geo"], row.values["time"]) for row in rows] == [ + ("BE", "2023"), + ("BE", "2024"), + ("DE", "2023"), + ("DE", "2024"), + ] + assert [row.values["value"] for row in rows] == [11.5, 12.0, None, 15.0] + assert [row.values["status"] for row in rows] == [None, "p", None, None] + assert rows[0].values == { + "geo": "BE", + "time": "2023", + "value": 11.5, + "status": None, + "source_index": 0, + "geo_label": "Belgium", + "time_label": "2023", + } + + +def test_json_stat_2_parser_preserves_dense_values_and_statuses(): + payload = { + "version": "2.0", + "class": "dataset", + "id": ["geo", "time"], + "size": [1, 2], + "dimension": { + "geo": {"category": {"index": ["BE"]}}, + "time": {"category": {"index": ["2023", "2024"]}}, + }, + "value": [10, 11.5], + "status": ["p", None], + } + + rows = source_rows_from_json_stat_2( + json.dumps(payload).encode(), + _artifact(), + sheet_name="api", + ) + + assert [row.values["value"] for row in rows] == [10, 11.5] + assert [row.values["status"] for row in rows] == ["p", None] + assert [row.values["source_index"] for row in rows] == [0, 1] + + +def test_json_stat_2_parser_rejects_non_contiguous_category_positions(): + payload = { + "version": "2.0", + "class": "dataset", + "id": ["geo"], + "size": [2], + "dimension": { + "geo": {"category": {"index": {"BE": 0, "DE": 2}}}, + }, + "value": [1, 2], + } + + with pytest.raises(ValueError, match="positions must cover 0 through 1"): + source_rows_from_json_stat_2( + json.dumps(payload).encode(), + _artifact(), + sheet_name="api", + ) + + +def test_json_stat_2_parser_rejects_misaligned_dense_values(): + payload = { + "version": "2.0", + "class": "dataset", + "id": ["geo"], + "size": [2], + "dimension": { + "geo": {"category": {"index": ["BE", "DE"]}}, + }, + "value": [1], + } + + with pytest.raises(ValueError, match="value array must contain 2 entries"): + source_rows_from_json_stat_2( + json.dumps(payload).encode(), + _artifact(), + sheet_name="api", + ) + + def test_census_acs_s0101_age_parser_unpivots_age_columns(): rows = source_rows_from_census_acs_s0101_age_json( json.dumps( diff --git a/tests/test_etl_eurostat.py b/tests/test_etl_eurostat.py new file mode 100644 index 0000000..5618269 --- /dev/null +++ b/tests/test_etl_eurostat.py @@ -0,0 +1,374 @@ +"""Offline ETL coverage for the Eurostat BE/DE/FR pilot packages.""" + +from __future__ import annotations + +from collections import Counter +from functools import lru_cache +import hashlib +import json +import re +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import pytest +import yaml + +from chronicle.consumer_contract import validate_consumer_fact_contract +from chronicle.core import validate_facts +from chronicle.source_package import ( + SOURCE_PACKAGE_ALIASES, + load_source_package, + validate_source_package, +) +from chronicle.sources.cells import validate_source_cells +from chronicle.sources.rows import build_source_row_key, validate_source_rows +from chronicle.suite import build_source_suite + + +REPO_ROOT = Path(__file__).resolve().parents[1] +EVALUATION_ONLY_COMMENT = ( + "# EVALUATION-ONLY by class: Ledger ingests everything; " + "the Populace gate decides use." +) +EUROSTAT_PACKAGES = { + "eurostat-gov-10a-taxag": ("gov_10a_taxag", 2024, 24, 24), + "eurostat-spr-exp-func": ("spr_exp_func", 2023, 27, 27), + "eurostat-ilc-li02": ("ilc_li02", 2024, 3, 3), + "eurostat-ilc-di01": ("ilc_di01", 2024, 54, 54), +} +EUROSTAT_DATASET_IDS = {values[0] for values in EUROSTAT_PACKAGES.values()} +EXPECTED_QUERY_FILTERS = { + "gov_10a_taxag": { + "freq": ["A"], + "unit": ["MIO_EUR"], + "sector": ["S13"], + "na_item": ["D2", "D5", "D51", "D61"], + "geo": ["BE", "DE", "FR"], + "time": ["2023", "2024"], + }, + "spr_exp_func": { + "freq": ["A"], + "spdeps": ["SPR"], + "spfunc": [ + "TOTAL", + "SICK", + "DIS", + "OLD", + "SRV", + "FAM", + "UNE", + "HOU", + "EXCL", + ], + "unit": ["MIO_EUR"], + "geo": ["BE", "DE", "FR"], + "time": ["2023"], + }, + "ilc_li02": { + "freq": ["A"], + "statinfo": ["MED_EI"], + "unit": ["PC"], + "rskpovth": ["B_60"], + "sex": ["T"], + "age": ["TOTAL"], + "geo": ["BE", "DE", "FR"], + "time": ["2024"], + }, + "ilc_di01": { + "freq": ["A"], + "quant_inc": [f"D{index}" for index in range(1, 10)], + "statinfo": ["TC", "SHARE"], + "unit": ["EUR"], + "geo": ["BE", "DE", "FR"], + "time": ["2024"], + }, +} + + +@lru_cache +def _package_outputs(alias: str): + _dataset_id, year, _expected_count, _expected_row_count = EUROSTAT_PACKAGES[alias] + package = load_source_package(alias) + rows = package.build_source_rows(year) + cells = package.build_source_cells(year, source_rows=rows) + facts = package.build_facts(year, cells=cells, source_rows=rows) + return package, rows, cells, facts + + +def test_eurostat_aliases_and_packages_validate_end_to_end(): + assert set(EUROSTAT_PACKAGES) <= set(SOURCE_PACKAGE_ALIASES) + + for alias, ( + _dataset_id, + year, + expected_count, + expected_row_count, + ) in EUROSTAT_PACKAGES.items(): + package, rows, cells, facts = _package_outputs(alias) + package_report = validate_source_package(alias, year=year) + + assert package.artifact.parser == "json_stat_2_full_rows" + assert package_report.valid, package_report.to_dict() + assert len(rows) == expected_row_count + assert len(facts) == expected_count + assert validate_source_rows(rows).valid + assert validate_source_cells(cells).valid + assert validate_facts(facts).valid + contract_report = validate_consumer_fact_contract(facts) + assert contract_report.valid, contract_report.to_dict() + + +@pytest.mark.parametrize("alias", EUROSTAT_PACKAGES) +def test_eurostat_packages_pass_full_agent_acceptance(alias, tmp_path): + _dataset_id, year, expected_count, _expected_row_count = EUROSTAT_PACKAGES[alias] + + report = build_source_suite(alias, tmp_path / alias, year=year) + + assert report.valid, report.to_dict() + assert report.agent_acceptance.valid, report.agent_acceptance.to_dict() + assert report.agent_acceptance.counts["fact_count"] == expected_count + assert report.agent_acceptance.counts["row_semantic_error_count"] == 0 + + +def test_eurostat_facts_preserve_raw_cube_dimensions_and_scaling(): + unit_mapping = { + "MIO_EUR": ("eur", 1_000_000), + "PC": ("percent", 1), + "EUR": ("eur", 1), + } + + for alias in EUROSTAT_PACKAGES: + _package, rows, _cells, facts = _package_outputs(alias) + rows_by_key = {build_source_row_key(row): row for row in rows} + + for fact in facts: + assert len(fact.source_row_keys) == 1 + source_row = rows_by_key[fact.source_row_keys[0]] + raw_unit = str(source_row.values["unit"]) + if alias == "eurostat-ilc-di01": + expected_unit = { + "SHARE": "percent", + "TC": "eur", + }[str(source_row.values["statinfo"])] + scale = 1 + else: + expected_unit, scale = unit_mapping[raw_unit] + + assert source_row.values["geo"] == fact.geography.id + assert str(source_row.values["time"]) == str(fact.period.value) + assert fact.geography.level == "country" + assert fact.geography.vintage == "current" + assert fact.measure.unit == expected_unit + assert fact.value == pytest.approx(source_row.values["value"] * scale) + + +def test_eurostat_provenance_and_evaluation_only_boundary(): + administrative_aliases = { + "eurostat-gov-10a-taxag", + "eurostat-spr-exp-func", + } + survey_aliases = {"eurostat-ilc-li02", "eurostat-ilc-di01"} + + for alias in administrative_aliases: + facts = _package_outputs(alias)[3] + assert {fact.provenance_class for fact in facts} == {"administrative"} + assert {fact.survey_instrument for fact in facts} == {None} + + for alias in survey_aliases: + package, _rows, _cells, facts = _package_outputs(alias) + package_text = (package.package_path / "source_package.yaml").read_text() + assert EVALUATION_ONLY_COMMENT in package_text + assert {fact.provenance_class for fact in facts} == {"survey_aggregate"} + assert {fact.survey_instrument for fact in facts} == {"EU-SILC"} + + di01_facts = _package_outputs("eurostat-ilc-di01")[3] + assert Counter( + (fact.measure.unit, fact.aggregation.method) for fact in di01_facts + ) == { + ("percent", "share"): 27, + ("eur", "quantile"): 27, + } + + +def test_eurostat_production_values_match_publisher_bytes(): + # Decoded independently from the raw JSON-stat cube (row-major index over + # freq/unit/sector/na_item/geo/time): D2 taxes on production and imports, + # calendar year 2023, MIO_EUR scaled by 1,000,000. Exact equality, not + # approx: publisher lexemes must survive scaling bit-for-bit. + _package, rows, _cells, facts = _package_outputs("eurostat-gov-10a-taxag") + rows_by_key = {build_source_row_key(row): row for row in rows} + expected = {"BE": 72_825_500_000, "DE": 428_710_000_000, "FR": 446_580_000_000} + + d2_2023 = { + source_row.values["geo"]: fact + for fact in facts + for source_row in (rows_by_key[fact.source_row_keys[0]],) + if source_row.values["na_item"] == "D2" + and str(source_row.values["time"]) == "2023" + } + + assert set(d2_2023) == set(expected) + for geo, value in expected.items(): + assert d2_2023[geo].value == value + assert isinstance(d2_2023[geo].value, int) + assert d2_2023[geo].measure.unit == "eur" + assert d2_2023[geo].geography.id == geo + + # The decimal-scaling regression case: 16448.06 MIO_EUR must scale to + # exactly 16,448,060,000 (binary-float multiply emitted ...000.000002). + _package, spr_rows, _cells, spr_facts = _package_outputs("eurostat-spr-exp-func") + spr_rows_by_key = {build_source_row_key(row): row for row in spr_rows} + be_disability = [ + fact + for fact in spr_facts + for source_row in (spr_rows_by_key[fact.source_row_keys[0]],) + if source_row.values["geo"] == "BE" and source_row.values["spfunc"] == "DIS" + ] + assert len(be_disability) == 1 + assert be_disability[0].value == 16_448_060_000 + assert isinstance(be_disability[0].value, int) + + +def test_eurostat_production_surfaces_carry_no_fixture_identity(): + # Fixture identity must be absent everywhere it could reach a fact: + # package specs (vintage, sheet names, notes), db manifests, and the + # artifacts themselves. Case-insensitive, marker-word based. + forbidden = re.compile(r"fixture|sentinel|synthetic", re.IGNORECASE) + surfaces = sorted( + list((REPO_ROOT / "packages" / "eurostat").rglob("source_package.yaml")) + + list((REPO_ROOT / "db" / "data" / "eurostat").rglob("manifest.yaml")) + + list((REPO_ROOT / "db" / "data" / "eurostat").rglob("*.json")) + ) + assert len(surfaces) >= 12 + for surface in surfaces: + match = forbidden.search(surface.read_text()) + assert match is None, f"{surface}: {match.group(0)!r}" + + +def test_eurostat_artifacts_match_verified_live_dimension_shapes_and_labels(): + expected_shapes = { + "spr_exp_func": ( + ["freq", "spdeps", "spfunc", "unit", "geo", "time"], + [1, 1, 9, 1, 3, 1], + ), + "ilc_li02": ( + ["freq", "statinfo", "unit", "rskpovth", "sex", "age", "geo", "time"], + [1, 1, 1, 1, 1, 1, 3, 1], + ), + "ilc_di01": ( + ["freq", "quant_inc", "statinfo", "unit", "geo", "time"], + [1, 9, 2, 1, 3, 1], + ), + } + + fixtures = {} + for dataset_id, (expected_id, expected_size) in expected_shapes.items(): + package_dir = REPO_ROOT / "db" / "data" / "eurostat" / dataset_id + manifest = yaml.safe_load((package_dir / "manifest.yaml").read_text()) + filename = next(iter(manifest["files"].values()))["filename"] + fixture = json.loads((package_dir / filename).read_text()) + fixtures[dataset_id] = fixture + + assert fixture["id"] == expected_id + assert fixture["size"] == expected_size + + assert fixtures["spr_exp_func"]["dimension"]["spfunc"]["category"]["index"] == { + "TOTAL": 0, + "SICK": 1, + "DIS": 2, + "OLD": 3, + "SRV": 4, + "FAM": 5, + "UNE": 6, + "HOU": 7, + "EXCL": 8, + } + assert fixtures["ilc_li02"]["dimension"]["statinfo"]["category"]["label"] == { + "MED_EI": "Median equivalised income" + } + assert fixtures["ilc_li02"]["dimension"]["rskpovth"]["category"]["label"] == { + "B_60": "Below 60%" + } + assert fixtures["ilc_di01"]["dimension"]["statinfo"]["category"]["label"] == { + "TC": "Top cut-off point", + "SHARE": "Share of national equivalised income", + } + assert ( + "D10" not in fixtures["ilc_di01"]["dimension"]["quant_inc"]["category"]["index"] + ) + + li02_package_text = ( + REPO_ROOT / "packages" / "eurostat" / "ilc_li02" / "source_package.yaml" + ).read_text() + assert 'B_60 is "Below 60%"' in li02_package_text + assert 'A_60 is "Above 60%"' in li02_package_text + + +def test_eurostat_manifests_pin_real_publisher_artifacts(): + for ( + dataset_id, + _year, + _expected_count, + _expected_row_count, + ) in EUROSTAT_PACKAGES.values(): + package_dir = REPO_ROOT / "db" / "data" / "eurostat" / dataset_id + manifest = yaml.safe_load((package_dir / "manifest.yaml").read_text()) + file_spec = next(iter(manifest["files"].values())) + artifact_path = package_dir / file_spec["filename"] + artifact = json.loads(artifact_path.read_text()) + content = artifact_path.read_bytes() + markers = ( + " ".join( + str(artifact.get(field) or "") for field in ("label", "source", "note") + ) + + " " + + str(manifest.get("source_name", "")) + + " " + + str(file_spec.get("source_table", "")) + ) + + assert file_spec["filename"] == f"{dataset_id}.json" + assert file_spec["sha256"] == hashlib.sha256(content).hexdigest() + assert file_spec["size_bytes"] == len(content) + assert file_spec["storage"]["r2"]["uri"].startswith("r2://ledger-raw/") + assert file_spec["sha256"] in file_spec["storage"]["r2"]["uri"] + assert artifact["source"] == "ESTAT" + assert "TEST FIXTURE" not in markers + assert "sentinel" not in markers.lower() + + +def test_fetch_manifest_has_exact_filtered_eurostat_requests(): + fetch_manifest = json.loads( + (REPO_ROOT / "FETCH-MANIFEST-EUROSTAT.json").read_text() + ) + fetches = fetch_manifest["fetches"] + + assert fetch_manifest["schema_version"] == "ledger.fetch_manifest.v1" + assert {fetch["dataset_id"] for fetch in fetches} == EUROSTAT_DATASET_IDS + for fetch in fetches: + dataset_id = fetch["dataset_id"] + parsed = urlsplit(fetch["source_url"]) + query = parse_qs(parsed.query) + package_manifest = yaml.safe_load( + ( + REPO_ROOT / "db" / "data" / "eurostat" / dataset_id / "manifest.yaml" + ).read_text() + ) + artifact_spec = next(iter(package_manifest["files"].values())) + + assert parsed.scheme == "https" + assert parsed.netloc == "ec.europa.eu" + assert parsed.path.endswith(f"/data/{dataset_id}") + assert query.pop("format") == ["JSON"] + assert query.pop("lang") == ["en"] + assert query == EXPECTED_QUERY_FILTERS[dataset_id] + assert fetch["destination"] == ( + f"db/data/eurostat/{dataset_id}/{dataset_id}.json" + ) + sha = fetch["sha256"] + dest = REPO_ROOT / fetch["destination"] + assert re.fullmatch(r"[0-9a-f]{64}", sha) + assert dest.exists() + assert hashlib.sha256(dest.read_bytes()).hexdigest() == sha + assert artifact_spec["sha256"] == sha + assert artifact_spec["source_url"] == fetch["source_url"] diff --git a/tests/test_scale_value.py b/tests/test_scale_value.py new file mode 100644 index 0000000..3a8fcf1 --- /dev/null +++ b/tests/test_scale_value.py @@ -0,0 +1,157 @@ +"""Unit coverage for the shared measure value scaler. + +The scaler decides integrality in decimal so publisher lexemes that scale to +whole numbers come out as exact ints, while every non-integral product keeps +the binary result bit-for-bit (existing fact values must not shift). +""" + +import pytest + +from chronicle.sources.specs import _scale_value + + +def test_integral_decimal_product_is_exact_int(): + # The spr_exp_func regression: binary multiplication alone emits + # 16448060000.000002 for this publisher lexeme. + assert _scale_value(16448.06, 1_000_000) == 16_448_060_000 + assert isinstance(_scale_value(16448.06, 1_000_000), int) + + +def test_integral_float_product_is_int(): + assert _scale_value(2.5, 2) == 5 + assert isinstance(_scale_value(2.5, 2), int) + + +def test_int_inputs_stay_int(): + assert _scale_value(5, 1000) == 5000 + assert isinstance(_scale_value(5, 1000), int) + + +def test_non_integral_product_preserves_binary_result_bit_for_bit(): + # Non-integral results must be the pre-existing binary product. These + # pairs are chosen because binary multiplication and the rejected + # all-Decimal implementation (float(Decimal(str(v)) * Decimal(str(s)))) + # land on DIFFERENT adjacent doubles, so this test fails under a + # full-Decimal scaler; the expected reprs are the binary products. + cases = { + (247322.9728, 1000): "247322972.79999998", # all-Decimal: ...72.8 + (944577.8457, 1000): "944577845.6999999", # all-Decimal: ...45.7 + (288204.28227, 1000): "288204282.27000004", # all-Decimal: ...82.27 + (322810.9418, 1000): "322810941.79999995", # all-Decimal: ...41.8 + } + for (value, scale), expected_repr in cases.items(): + assert repr(value * scale) == expected_repr + assert repr(_scale_value(value, scale)) == expected_repr + + +def test_string_passthrough_only_at_scale_one(): + assert _scale_value("suppressed", 1) == "suppressed" + with pytest.raises(ValueError, match="Cannot scale"): + _scale_value("suppressed", 1000) + + +def test_bool_and_none_are_rejected(): + with pytest.raises(ValueError, match="Cannot scale"): + _scale_value(True, 1) + with pytest.raises(ValueError, match="Cannot scale"): + _scale_value(None, 1) + + +def test_scaler_corrections_pin_previously_dusty_package_values(): + # The complete set of existing facts whose serialized values change under + # decimal integrality (verified by a full five-package diff against the + # pre-change tree): four integral publisher values that binary + # multiplication had emitted with float dust (e.g. a recipient headcount + # of 1028438.0000000001). Everything else is bit-identical. + from chronicle.source_package import load_source_package + + expected = { + ( + "slc-student-support-england-2025", + 2025, + "slc.support_2025.table_3a.recipients.ay2017.grand_total.recipients", + ): 1_028_438, + ( + "slc-student-support-england-2025", + 2025, + "slc.support_2025.table_4c.recipients.ay2018" + ".adult_dependants_grant.recipients", + ): 16_336, + ( + "welshgov-council-tax-levels-2026-27", + 2026, + "welshgov.ct_levels_2026_27.budget.authority" + ".fy2026.w06000005.council_tax_income", + ): 131_972_547, + ( + "scotgov-scottish-budget-social-security-assistance-2026", + 2026, + "scotgov.budget_2026_27.table_5_08.fy2025" + ".carer_support_payment.amount", + ): 520_700_000, + } + + facts_by_alias = {} + for alias, year, record_id in expected: + if alias not in facts_by_alias: + package = load_source_package(alias) + rows = package.build_source_rows(year) + cells = package.build_source_cells(year, source_rows=rows) + facts = package.build_facts(year, cells=cells, source_rows=rows) + facts_by_alias[alias] = { + fact.source_record_id: fact for fact in facts + } + fact = facts_by_alias[alias][record_id] + assert fact.value == expected[(alias, year, record_id)] + assert isinstance(fact.value, int) + + +def test_non_integral_package_values_keep_their_binary_doubles(): + # Discriminating fact-level pins: for each of these, the rejected + # all-Decimal scaler emits a DIFFERENT adjacent double (shown in the + # comment), verified by building all five packages under both + # implementations and diffing every serialized value. The pinned reprs + # are the pre-existing binary products, unchanged from main. + from chronicle.source_package import load_source_package + + expected = { + ("obr-efo-expenditure-march-2026", 2026): { + # all-Decimal: 180697298614.88257 + "obr.efo_2026_03.expenditure.state_pension" + ".fy2030.state_pension.amount": "180697298614.8826", + # all-Decimal: 2938385000.0 + "obr.efo_2026_03.expenditure.council_tax_scotland" + ".fy2024.council_tax_scotland.amount": "2938385000.0000005", + }, + ("obr-efo-receipts-march-2026", 2026): { + # all-Decimal: 331437583074.4428 + "obr.efo_2026_03.receipts.income_tax" + ".fy2025.income_tax.amount": "331437583074.4429", + # all-Decimal: 115446805221.42703 + "obr.efo_2026_03.receipts.ni_employer" + ".fy2024.ni_employer.amount": "115446805221.42705", + }, + ("slc-student-support-england-2025", 2025): { + # all-Decimal: 1117591.0 — the upstream lexeme is genuinely + # non-integral, so the scaler must NOT round it to a clean count. + "slc.support_2025.table_3a.recipients" + ".ay2020.grand_total.recipients": "1117591.0000000002", + # all-Decimal: 168349636.59000003 + "slc.support_2025.table_4c.amount" + ".ay2023.parents_learning_allowance.amount_awarded": "168349636.59", + }, + ("welshgov-council-tax-levels-2026-27", 2026): { + # all-Decimal: 708199216.3799999 + "welshgov.ct_levels_2026_27.budget.authority" + ".fy2026.w06000016.budget_requirement": "708199216.38", + }, + } + + for (alias, year), pins in expected.items(): + package = load_source_package(alias) + rows = package.build_source_rows(year) + cells = package.build_source_cells(year, source_rows=rows) + facts = package.build_facts(year, cells=cells, source_rows=rows) + facts_by_id = {fact.source_record_id: fact for fact in facts} + for record_id, value_repr in pins.items(): + assert repr(facts_by_id[record_id].value) == value_repr