Eurostat pilot calibration packages: BE, DE, FR

FETCH-MANIFEST-EUROSTAT.json

@@ -0,0 +1,30 @@
+{
+ "schema_version": "ledger.fetch_manifest.v1",
+ "network_policy": "Fetch only in an authorized network-enabled handoff environment.",
+ "fetches": [
+  {
+   "dataset_id": "gov_10a_taxag",
+   "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/gov_10a_taxag?format=JSON&lang=en&freq=A&unit=MIO_EUR&sector=S13&na_item=D2&na_item=D5&na_item=D51&na_item=D61&geo=BE&geo=DE&geo=FR&time=2023&time=2024",
+   "destination": "db/data/eurostat/gov_10a_taxag/gov_10a_taxag.json",
+   "sha256": "5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a"
+  },
+  {
+   "dataset_id": "spr_exp_func",
+   "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/spr_exp_func?format=JSON&lang=en&freq=A&spdeps=SPR&spfunc=TOTAL&spfunc=SICK&spfunc=DIS&spfunc=OLD&spfunc=SRV&spfunc=FAM&spfunc=UNE&spfunc=HOU&spfunc=EXCL&unit=MIO_EUR&geo=BE&geo=DE&geo=FR&time=2023",
+   "destination": "db/data/eurostat/spr_exp_func/spr_exp_func.json",
+   "sha256": "24ad3c0115efef790ff8cb2ff8fb269d4ceb7de70ac3fad673ebd1340e4f650f"
+  },
+  {
+   "dataset_id": "ilc_li02",
+   "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_li02?format=JSON&lang=en&freq=A&statinfo=MED_EI&unit=PC&rskpovth=B_60&sex=T&age=TOTAL&geo=BE&geo=DE&geo=FR&time=2024",
+   "destination": "db/data/eurostat/ilc_li02/ilc_li02.json",
+   "sha256": "789535c86cac56c4c5b29559ef037eeb2f0b06f882a8393bfe7ad4d0c09e0052"
+  },
+  {
+   "dataset_id": "ilc_di01",
+   "source_url": "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/ilc_di01?format=JSON&lang=en&freq=A&quant_inc=D1&quant_inc=D2&quant_inc=D3&quant_inc=D4&quant_inc=D5&quant_inc=D6&quant_inc=D7&quant_inc=D8&quant_inc=D9&statinfo=TC&statinfo=SHARE&unit=EUR&geo=BE&geo=DE&geo=FR&time=2024",
+   "destination": "db/data/eurostat/ilc_di01/ilc_di01.json",
+   "sha256": "1c3bb1277f98f257d00725ded6b6d8b1678834329eb6b66954b8f42c19a0ad17"
+  }
+ ]
+}

chronicle/core.py

@@ -60,12 +60,14 @@
     "government",
     "dwelling",
     "institutional_sector",
+    "social_protection_scheme",
     "firm",
 }
 ALLOWED_AGGREGATIONS = {
     "sum",
     "mean",
     "median",
+    "quantile",
     "rate",
     "ratio",
     "share",

chronicle/source_package.py

@@ -52,6 +52,7 @@
     source_rows_from_json_table,
     source_rows_from_statxplore_table,
     source_rows_from_ees_permalink_table_html,
+    source_rows_from_json_stat_2,
     source_rows_from_kff_state_indicator_gdocs_html,
     source_rows_from_ons_timeseries_json,
     source_rows_from_delimited_text,
@@ -250,6 +251,10 @@
     "hmrc-vat-firm-sector-targets-2024-25": Path(
         "hmrc/vat_firm_sector_targets_2024_25"
     ),
+    "eurostat-gov-10a-taxag": Path("eurostat/gov_10a_taxag"),
+    "eurostat-spr-exp-func": Path("eurostat/spr_exp_func"),
+    "eurostat-ilc-li02": Path("eurostat/ilc_li02"),
+    "eurostat-ilc-di01": Path("eurostat/ilc_di01"),
     "kff-marketplace-effectuated-enrollment": Path(
         "kff/marketplace_effectuated_enrollment"
     ),
@@ -392,6 +397,12 @@ def build_source_rows(self, year: int) -> list[SourceRow]:
                 artifact,
                 sheet_name=self._sheet_name(filename, year=year),
             )
+        if self.parser == "json_stat_2_full_rows":
+            return source_rows_from_json_stat_2(
+                content,
+                artifact,
+                sheet_name=self._sheet_name(filename, year=year),
+            )
         if self.parser == "census_acs_s0101_age_json_rows":
             return source_rows_from_census_acs_s0101_age_json(
                 content,
@@ -577,6 +588,26 @@ def build_source_cells(
                     for row in self.selected_rows
                 ),
             )
+        if self.parser == "json_stat_2_full_rows":
+            rows = (
+                source_rows
+                if source_rows is not None
+                else source_rows_from_json_stat_2(
+                    content,
+                    artifact,
+                    sheet_name=self._sheet_name(filename, year=year),
+                )
+            )
+            return source_cells_from_source_rows(
+                rows,
+                selected_rows=tuple(
+                    {
+                        key: str(_render_value(value, year=year))
+                        for key, value in row.items()
+                    }
+                    for row in self.selected_rows
+                ),
+            )
         if self.parser == "census_acs_s0101_age_json_rows":
             rows = (
                 source_rows

chronicle/sources/rows.py

@@ -11,6 +11,7 @@
 from dataclasses import asdict, dataclass
 from datetime import date, datetime
 from io import BytesIO, StringIO
+from itertools import product
 from pathlib import Path
 from typing import Any
 
@@ -338,6 +339,217 @@ def source_rows_from_json_table(
     raise ValueError("JSON table must be an array of arrays or objects.")
 
 
+def source_rows_from_json_stat_2(
+    content: bytes,
+    artifact: SourceArtifactMetadata,
+    *,
+    sheet_name: str,
+) -> list[SourceRow]:
+    """Flatten a JSON-stat 2.0 dataset cube into deterministic source rows.
+
+    JSON-stat stores observations in row-major order, with the last dimension
+    changing fastest. The returned rows preserve that order, retain every cube
+    position (including null observations), and expose dimension codes before
+    the observation value so declarative packages can select exact series.
+    """
+    data = json.loads(content.decode("utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError("JSON-stat 2.0 payload must be an object.")
+    if data.get("class") != "dataset":
+        raise ValueError("JSON-stat 2.0 payload class must be 'dataset'.")
+    if data.get("version") != "2.0":
+        raise ValueError("JSON-stat payload version must be '2.0'.")
+
+    dimension_ids = data.get("id")
+    sizes = data.get("size")
+    dimensions = data.get("dimension")
+    if not isinstance(dimension_ids, list) or not dimension_ids:
+        raise ValueError("JSON-stat dataset id must be a non-empty array.")
+    if not all(isinstance(item, str) and item for item in dimension_ids):
+        raise ValueError("JSON-stat dataset dimension IDs must be non-empty strings.")
+    if len(set(dimension_ids)) != len(dimension_ids):
+        raise ValueError("JSON-stat dataset dimension IDs must be unique.")
+    if not isinstance(sizes, list) or len(sizes) != len(dimension_ids):
+        raise ValueError("JSON-stat dataset size must align with id.")
+    if not all(type(size) is int and size > 0 for size in sizes):
+        raise ValueError("JSON-stat dataset sizes must be positive integers.")
+    if not isinstance(dimensions, dict):
+        raise ValueError("JSON-stat dataset dimension must be an object.")
+
+    codes_by_dimension: list[list[str]] = []
+    labels_by_dimension: list[dict[str, str]] = []
+    for dimension_id, size in zip(dimension_ids, sizes, strict=True):
+        codes, labels = _json_stat_dimension_categories(
+            dimensions,
+            dimension_id=dimension_id,
+            expected_size=size,
+        )
+        codes_by_dimension.append(codes)
+        labels_by_dimension.append(labels)
+
+    observation_count = 1
+    for size in sizes:
+        observation_count *= size
+    values = _json_stat_observations(
+        data.get("value"),
+        field="value",
+        observation_count=observation_count,
+        required=True,
+    )
+    statuses = _json_stat_observations(
+        data.get("status"),
+        field="status",
+        observation_count=observation_count,
+        required=False,
+    )
+
+    rows: list[SourceRow] = []
+    coordinates = product(*(range(size) for size in sizes))
+    for source_index, positions in enumerate(coordinates):
+        dimension_codes = {
+            dimension_id: codes_by_dimension[index][position]
+            for index, (dimension_id, position) in enumerate(
+                zip(dimension_ids, positions, strict=True)
+            )
+        }
+        row_values: dict[str, Scalar] = dict(dimension_codes)
+        row_values["value"] = _json_stat_scalar(values[source_index], field="value")
+        row_values["status"] = _json_stat_scalar(
+            statuses[source_index],
+            field="status",
+        )
+        row_values["source_index"] = source_index
+        for index, dimension_id in enumerate(dimension_ids):
+            code = dimension_codes[dimension_id]
+            row_values[f"{dimension_id}_label"] = labels_by_dimension[index].get(
+                code,
+                code,
+            )
+        rows.append(
+            SourceRow(
+                artifact=artifact,
+                sheet_name=sheet_name,
+                row_number=source_index + 1,
+                values=row_values,
+            )
+        )
+    return rows
+
+
+def _json_stat_dimension_categories(
+    dimensions: dict[str, Any],
+    *,
+    dimension_id: str,
+    expected_size: int,
+) -> tuple[list[str], dict[str, str]]:
+    dimension = dimensions.get(dimension_id)
+    if not isinstance(dimension, dict):
+        raise ValueError(f"JSON-stat dimension {dimension_id!r} is missing.")
+    category = dimension.get("category")
+    if not isinstance(category, dict):
+        raise ValueError(
+            f"JSON-stat dimension {dimension_id!r} category must be an object."
+        )
+    category_index = category.get("index")
+    if isinstance(category_index, list):
+        if not all(isinstance(code, str) and code for code in category_index):
+            raise ValueError(
+                f"JSON-stat dimension {dimension_id!r} category index values "
+                "must be non-empty strings."
+            )
+        codes = list(category_index)
+    elif isinstance(category_index, dict):
+        positions: dict[int, str] = {}
+        for code, position in category_index.items():
+            if not isinstance(code, str) or not code:
+                raise ValueError(
+                    f"JSON-stat dimension {dimension_id!r} category codes "
+                    "must be non-empty strings."
+                )
+            if type(position) is not int or position < 0:
+                raise ValueError(
+                    f"JSON-stat dimension {dimension_id!r} category positions "
+                    "must be non-negative integers."
+                )
+            if position in positions:
+                raise ValueError(
+                    f"JSON-stat dimension {dimension_id!r} category positions "
+                    "must be unique."
+                )
+            positions[position] = code
+        if set(positions) != set(range(expected_size)):
+            raise ValueError(
+                f"JSON-stat dimension {dimension_id!r} category positions must "
+                f"cover 0 through {expected_size - 1}."
+            )
+        codes = [positions[position] for position in range(expected_size)]
+    else:
+        raise ValueError(
+            f"JSON-stat dimension {dimension_id!r} category index must be an "
+            "array or object."
+        )
+
+    if len(codes) != expected_size or len(set(codes)) != len(codes):
+        raise ValueError(
+            f"JSON-stat dimension {dimension_id!r} categories must contain "
+            f"{expected_size} unique codes."
+        )
+
+    raw_labels = category.get("label", {})
+    if not isinstance(raw_labels, dict):
+        raise ValueError(
+            f"JSON-stat dimension {dimension_id!r} category labels must be an object."
+        )
+    labels = {
+        code: str(raw_labels[code])
+        for code in codes
+        if raw_labels.get(code) is not None
+    }
+    return codes, labels
+
+
+def _json_stat_observations(
+    observations: Any,
+    *,
+    field: str,
+    observation_count: int,
+    required: bool,
+) -> list[Any]:
+    if observations is None:
+        if required:
+            raise ValueError(f"JSON-stat dataset {field} is required.")
+        return [None] * observation_count
+    if isinstance(observations, list):
+        if len(observations) != observation_count:
+            raise ValueError(
+                f"JSON-stat dataset {field} array must contain "
+                f"{observation_count} entries."
+            )
+        return list(observations)
+    if isinstance(observations, dict):
+        dense = [None] * observation_count
+        for raw_index, value in observations.items():
+            try:
+                index = int(raw_index)
+            except (TypeError, ValueError) as exc:
+                raise ValueError(
+                    f"JSON-stat dataset {field} keys must be integer indexes."
+                ) from exc
+            if str(index) != str(raw_index) or not 0 <= index < observation_count:
+                raise ValueError(
+                    f"JSON-stat dataset {field} index {raw_index!r} is out of range."
+                )
+            dense[index] = value
+        return dense
+    raise ValueError(f"JSON-stat dataset {field} must be an array or object.")
+
+
+def _json_stat_scalar(value: Any, *, field: str) -> Scalar:
+    if value is None or isinstance(value, bool | int | float | str):
+        return _json_scalar(value)
+    raise ValueError(f"JSON-stat dataset {field} entries must be scalar values.")
+
+
 S0101_TOTAL_AGE_COLUMNS: tuple[tuple[str, str], ...] = (
     ("S0101_C01_002E", "Aged 0-4"),
     ("S0101_C01_003E", "Aged 5-9"),

chronicle/sources/specs.py

@@ -5,6 +5,7 @@
 import hashlib
 import json
 from dataclasses import asdict, dataclass, field
+from decimal import Decimal
 
 from chronicle.core import (
     DEFAULT_ASSERTION,
@@ -711,6 +712,14 @@ def _scale_value(value: Scalar, scale: int | float) -> int | float | str:
         raise ValueError(f"Cannot scale nonnumeric source value {value!r}")
     if isinstance(value, int | float):
         scaled = value * scale
+        # Integrality is decided in decimal, not binary: publisher lexemes
+        # like 16448.06 scaled by 1_000_000 are exactly 16448060000, where
+        # binary multiplication alone emits 16448060000.000002. Non-integral
+        # products keep the binary result unchanged, so no existing
+        # non-integral fact value shifts.
+        exact = Decimal(str(value)) * Decimal(str(scale))
+        if exact == exact.to_integral_value():
+            return int(exact)
         if isinstance(scaled, float) and scaled.is_integer():
             return int(scaled)
         return scaled

db/data/eurostat/gov_10a_taxag/gov_10a_taxag.json

@@ -0,0 +1 @@
+{"version":"2.0","class":"dataset","label":"Main national accounts tax aggregates","source":"ESTAT","updated":"2026-07-21T11:00:00+0200","value":{"0":72825.5,"1":73248.5,"2":428710.0,"3":443779.0,"4":446580.0,"5":453677.0,"6":96585.0,"7":104740.8,"8":533331.0,"9":552833.0,"10":357078.0,"11":366049.0,"12":94031.3,"13":101965.9,"14":517508.0,"15":536831.0,"16":351268.0,"17":359026.0,"18":90443.5,"19":94416.8,"20":710828.0,"21":756551.0,"22":461666.0,"23":482281.0},"status":{"2":"p","3":"p","8":"p","9":"p","14":"p","15":"p","20":"p","21":"p"},"id":["freq","unit","sector","na_item","geo","time"],"size":[1,1,1,4,3,2],"dimension":{"freq":{"label":"Time frequency","category":{"index":{"A":0},"label":{"A":"Annual"}}},"unit":{"label":"Unit of measure","category":{"index":{"MIO_EUR":0},"label":{"MIO_EUR":"Million euro"}}},"sector":{"label":"Sector","category":{"index":{"S13":0},"label":{"S13":"General government"}}},"na_item":{"label":"National accounts indicator (ESA 2010)","category":{"index":{"D2":0,"D5":1,"D51":2,"D61":3},"label":{"D2":"Taxes on production and imports","D5":"Current taxes on income, wealth, etc.","D51":"Taxes on income","D61":"Net social contributions"}}},"geo":{"label":"Geopolitical entity (reporting)","category":{"index":{"BE":0,"DE":1,"FR":2},"label":{"BE":"Belgium","DE":"Germany","FR":"France"}}},"time":{"label":"Time","category":{"index":{"2023":0,"2024":1},"label":{"2023":"2023","2024":"2024"}}}},"extension":{"lang":"EN","id":"GOV_10A_TAXAG","agencyId":"ESTAT","version":"1.0","datastructure":{"id":"GOV_10A_TAXAG","agencyId":"ESTAT","version":"68.0"},"annotation":[{"type":"CREATED","date":"2015-02-18T14:28:42+0100"},{"type":"DISSEMINATION_DOI_XML","title":"<adms:identifier xmlns:adms=\"http://www.w3.org/ns/adms#\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core.html\" xmlns:dct=\"http://purl.org/dc/terms/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"><adms:Identifier rdf:about=\"https://doi.org/10.2908/GOV_10A_TAXAG\"><skos:notation rdf:datatype=\"http://purl.org/spar/datacite/doi\">10.2908/GOV_10A_TAXAG</skos:notation><dct:creator rdf:resource=\"http://publications.europa.eu/resource/authority/corporate-body/ESTAT\"/><dct:issued rdf:datatype=\"http://www.w3.org/2001/XMLSchema#date\">2023-01-19</dct:issued></adms:Identifier></adms:identifier>"},{"type":"DISSEMINATION_OBJECT_TYPE","title":"DATASET"},{"type":"DISSEMINATION_TIMESTAMP_DATA","date":"2026-07-21T11:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_GLOBAL","date":"2026-07-21T11:00:00+0200"},{"type":"DISSEMINATION_TIMESTAMP_PLANNED","date":"2026-07-21T11:00:00+0200"},{"type":"ESMS_HTML","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/cache/metadata/en/gov_10a_taxag_esms.htm"},{"type":"ESMS_SDMX","title":"Explanatory texts (metadata)","href":"https://ec.europa.eu/eurostat/api/dissemination/files?file=metadata/gov_10a_taxag_esms.sdmx.zip"},{"type":"OBS_COUNT","title":"1470597"},{"type":"OBS_PERIOD_OVERALL_LATEST","title":"2025"},{"type":"OBS_PERIOD_OVERALL_OLDEST","title":"1995"},{"type":"SOURCE_INSTITUTIONS","text":"Eurostat"},{"type":"UPDATE_DATA","date":"2026-07-21T11:00:00+0200"},{"type":"UPDATE_STRUCTURE","date":"2026-04-22T11:00:00+0200"}],"status":{"label":{"p":"provisional"}},"positions-with-no-data":{"freq":[],"unit":[],"sector":[],"na_item":[],"geo":[],"time":[]}}}

db/data/eurostat/gov_10a_taxag/manifest.yaml

@@ -0,0 +1,23 @@
+source_id: eurostat-gov-10a-taxag
+source_name: Eurostat gov_10a_taxag
+publisher: Eurostat
+source_page: https://ec.europa.eu/eurostat/databrowser/view/gov_10a_taxag/default/table?lang=en
+files:
+  2024:
+    filename: gov_10a_taxag.json
+    source_url: https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/gov_10a_taxag?format=JSON&lang=en&freq=A&unit=MIO_EUR&sector=S13&na_item=D2&na_item=D5&na_item=D51&na_item=D61&geo=BE&geo=DE&geo=FR&time=2023&time=2024
+    source_table: Eurostat gov_10a_taxag Main national accounts tax aggregates for Belgium, Germany, and France
+    sha256: 5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a
+    size_bytes: 3346
+    storage:
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/eurostat/eurostat-gov-10a-taxag/2024/5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a/gov_10a_taxag.json
+        uri: r2://ledger-raw/raw/eurostat/eurostat-gov-10a-taxag/2024/5b4e4b99f0778855e164481a52eb16306ef4ac99afef874a91f22315bd0c472a/gov_10a_taxag.json
+    source_urls:
+      - https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/gov_10a_taxag?format=JSON&lang=en&freq=A&unit=MIO_EUR&sector=S13&na_item=D2&na_item=D5&na_item=D51&na_item=D61&geo=BE&geo=DE&geo=FR&time=2023&time=2024
+    notes: >-
+      Real Eurostat dissemination API bytes for the filtered request in
+      source_url; hash-pinned and content-addressed. Different bytes at
+      the mutable API endpoint are a new vintage, never a pin update.