diff --git a/protea/core/operations/export_transitions_dataset.py b/protea/core/operations/export_transitions_dataset.py index 16481b0e..3b2de40b 100644 --- a/protea/core/operations/export_transitions_dataset.py +++ b/protea/core/operations/export_transitions_dataset.py @@ -461,7 +461,37 @@ def _streamed( ) -def _emit_state(st: WindowState, emit: EmitFn) -> WindowState: +# WHAT `discarded_off_axis` IS, BECAUSE A COMMENT INSIDE THIS FUNCTION SAID THE +# OPPOSITE UNTIL IT WAS MEASURED. +# +# It said the figure was expected to be zero while the axis is release 211's own +# corpus ontology, and that it was the first thing to read before trusting a +# bundle. Over the 65 published windows it is 689,055 rows, 1.54 per cent of the +# raw cards, and it is zero in neither. +# +# The reasoning behind the wrong comment was a real measurement applied to the +# wrong half: release 211 does drop nothing, because the pivot IS the ontology +# that release's corpus was loaded under. But this table reads BOTH ends of a +# window, and the end at t is an earlier release whose corpus named terms the +# 2022 ontology no longer has and cannot rename through an alias. So the drop +# tracks the window's distance from the axis's era: 0.02 per cent on 205 -> 211, +# 0.84 on 174 -> 202, 1.66 on 160 -> 177, 4.54 on 170 -> 171. +# +# TWO CONSEQUENCES, and the second is the one that leaves this file. +# +# It is not a signal to trust a bundle by. The signals that are: +# `remapped_across_aspects` and `discarded_without_evidence`, both zero, and +# both about a row the writer could have placed and chose not to. +# +# And the state at t is systematically THINNER for old windows than for recent +# ones, by up to four and a half per cent, because the axis is a 2022 ontology. +# That is a difference correlated with a window's age, which a model trained +# across mixed horizons can see. Which is why the event carries the split per +# moment AND both eras: a total cannot be crossed against "the end at t drops +# more than the end at t'", and a drop cannot be read without the age it tracks. +# The axis's own era is in the run's result as `cut_version`, once, because it +# does not vary per window. +def _emit_state(st: WindowState, facts: WindowFacts, emit: EmitFn) -> WindowState: emit( "transitions.query_state", f"{st.window[:8]}: {st.rows} cards, {st.collapse.gaf} collapsed by the GAF " @@ -480,15 +510,14 @@ def _emit_state(st: WindowState, emit: EmitFn) -> WindowState: # synonym map changed. "collapsed_by_gaf": st.collapse.gaf, "collapsed_by_remap": st.collapse.remap, - # Expected to be zero while the axis is release 211's own corpus - # ontology. It is the first thing to read before trusting a bundle. + # Not zero. See the block above this function. "discarded_off_axis": st.discards.off_axis, - # Split per moment, because the whole explanation of this figure is - # that the two ends sit at different distances from the axis's era. **{ f"off_axis_{k.replace(chr(39), '_prime')}": v for k, v in st.off_axis_by_moment.items() }, + "version_t": facts.version_t, + "version_t_prime": facts.version_t_prime, "remapped_across_aspects": st.discards.crossed_aspect, # The column is NULLABLE in the schema, so this branch is reachable # even though the live data does not take it. Dropped rather than @@ -682,7 +711,7 @@ def _per_window(self, session: Session, run: _Run, emit: EmitFn) -> _Computed: ) tabla, st = state_of(session, facts, [r.query for r in q.rows], run.frame) run.state.write(tabla) - hecho.state.append(_emit_state(st, emit)) + hecho.state.append(_emit_state(st, facts, emit)) if q is not None and "gains" in p.tables and axis is not None: poblacion = {r.query for r in q.rows} hecho.gains.append( diff --git a/tests/test_the_state_event_carries_the_split_and_the_eras.py b/tests/test_the_state_event_carries_the_split_and_the_eras.py new file mode 100644 index 00000000..d59ea95c --- /dev/null +++ b/tests/test_the_state_event_carries_the_split_and_the_eras.py @@ -0,0 +1,125 @@ +"""What ``transitions.query_state`` promises a reader of the training report. + +THE FIGURE THIS EVENT CARRIES WAS MISREAD, and the misreading was written down. +A comment beside ``discarded_off_axis`` said the count was expected to be zero +and was the first thing to read before trusting a bundle. Measured over the 65 +published windows it is 689.055 rows, 1,54 per cent of the raw cards, and it +tracks the window's distance from the axis's era: 0,02 per cent on ``205->211`` +and 4,54 on ``170->171``. Only the end AT THE CUT sits on the pivot's own era. + +So the event has to carry two things a total cannot carry, and these tests pin +both because the reader is in another repository and its silence is the failure +mode: a dropped field leaves a report that still renders, with a column of +blanks nobody notices. + +1. **The split per moment.** The explanation is about the two ends being at + different distances from the axis, and a total cannot be crossed against it. +2. **Both eras of the window.** A drop of 4,54 per cent means nothing until it + sits beside the age it tracks, and the age is not derivable from the window + id. + +And the window is the FULL id in both, because the loader joins by that string +as text: a prefix would join to nothing and nothing would fail. +""" + +from __future__ import annotations + +import uuid +from typing import Any + +from protea.core.operations import export_transitions_dataset as E +from protea.core.transitions_export._query_state import ( + Collapse, + Discards, + WindowFacts, + WindowState, +) + + +class _Capturado: + """An ``EmitFn`` that keeps what it was handed.""" + + def __init__(self) -> None: + self.eventos: list[tuple[str, str | None, dict[str, Any], str]] = [] + + def __call__(self, event: str, message: str | None, fields: dict[str, Any], level: Any) -> None: + self.eventos.append((event, message, fields, str(level))) + + @property + def campos(self) -> dict[str, Any]: + assert len(self.eventos) == 1, f"expected one event, got {len(self.eventos)}" + return self.eventos[0][2] + + +VENTANA = str(uuid.uuid4()) + + +def _state() -> WindowState: + """A window that drops at ``t`` and not at ``t'``, which is the live shape.""" + return WindowState( + window=VENTANA, + rows=1200, + by_moment={"t": 500, "t'": 700}, + evidence={"t": {"IDA": 500}, "t'": {"IDA": 700}}, + distinct_terms=43, + negated=4, + collapse=Collapse(gaf=300, remap=7), + discards=Discards(off_axis=55, crossed_aspect=0, no_evidence=0), + off_axis_by_moment={"t": 55, "t'": 0}, + ) + + +def _facts() -> WindowFacts: + return WindowFacts( + eset=None, + window=VENTANA, + data=None, + version_t="170", + version_t_prime="171", + horizon_days=61, + old_end_snapshot=None, + new_end_snapshot=None, + ) + + +def test_the_event_carries_the_off_axis_split_per_moment(): + """The total and both of its halves, so the explanation can be checked.""" + emit = _Capturado() + + E._emit_state(_state(), _facts(), emit) + + campos = emit.campos + assert campos["discarded_off_axis"] == 55 + assert campos["off_axis_t"] == 55 + assert campos["off_axis_t_prime"] == 0 + assert campos["off_axis_t"] + campos["off_axis_t_prime"] == campos["discarded_off_axis"] + + +def test_the_event_carries_both_eras_of_the_window(): + """The age the drop tracks, beside the drop. + + Without these two fields a report can print 4,54 per cent and cannot say + whether that is a window ten years from the axis or one beside it. + """ + emit = _Capturado() + + E._emit_state(_state(), _facts(), emit) + + campos = emit.campos + assert campos["version_t"] == "170" + assert campos["version_t_prime"] == "171" + + +def test_the_event_names_the_window_by_its_full_id(): + """The join key, as text, in the field and not only in the message. + + The message abbreviates to eight characters for a human reading a log. A + reader that joined on that would join to nothing, which is why the field is + separate from the message and is asserted here. + """ + emit = _Capturado() + + E._emit_state(_state(), _facts(), emit) + + assert emit.campos["window"] == VENTANA + assert len(emit.campos["window"]) == 36