Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 35 additions & 6 deletions protea/core/operations/export_transitions_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,37 @@ def _streamed(
)


def _emit_state(st: WindowState, emit: EmitFn) -> WindowState:
# WHAT `discarded_off_axis` IS, BECAUSE A COMMENT INSIDE THIS FUNCTION SAID THE
# OPPOSITE UNTIL IT WAS MEASURED.
#
# It said the figure was expected to be zero while the axis is release 211's own
# corpus ontology, and that it was the first thing to read before trusting a
# bundle. Over the 65 published windows it is 689,055 rows, 1.54 per cent of the
# raw cards, and it is zero in neither.
#
# The reasoning behind the wrong comment was a real measurement applied to the
# wrong half: release 211 does drop nothing, because the pivot IS the ontology
# that release's corpus was loaded under. But this table reads BOTH ends of a
# window, and the end at t is an earlier release whose corpus named terms the
# 2022 ontology no longer has and cannot rename through an alias. So the drop
# tracks the window's distance from the axis's era: 0.02 per cent on 205 -> 211,
# 0.84 on 174 -> 202, 1.66 on 160 -> 177, 4.54 on 170 -> 171.
#
# TWO CONSEQUENCES, and the second is the one that leaves this file.
#
# It is not a signal to trust a bundle by. The signals that are:
# `remapped_across_aspects` and `discarded_without_evidence`, both zero, and
# both about a row the writer could have placed and chose not to.
#
# And the state at t is systematically THINNER for old windows than for recent
# ones, by up to four and a half per cent, because the axis is a 2022 ontology.
# That is a difference correlated with a window's age, which a model trained
# across mixed horizons can see. Which is why the event carries the split per
# moment AND both eras: a total cannot be crossed against "the end at t drops
# more than the end at t'", and a drop cannot be read without the age it tracks.
# The axis's own era is in the run's result as `cut_version`, once, because it
# does not vary per window.
def _emit_state(st: WindowState, facts: WindowFacts, emit: EmitFn) -> WindowState:
emit(
"transitions.query_state",
f"{st.window[:8]}: {st.rows} cards, {st.collapse.gaf} collapsed by the GAF "
Expand All @@ -480,15 +510,14 @@ def _emit_state(st: WindowState, emit: EmitFn) -> WindowState:
# synonym map changed.
"collapsed_by_gaf": st.collapse.gaf,
"collapsed_by_remap": st.collapse.remap,
# Expected to be zero while the axis is release 211's own corpus
# ontology. It is the first thing to read before trusting a bundle.
# Not zero. See the block above this function.
"discarded_off_axis": st.discards.off_axis,
# Split per moment, because the whole explanation of this figure is
# that the two ends sit at different distances from the axis's era.
**{
f"off_axis_{k.replace(chr(39), '_prime')}": v
for k, v in st.off_axis_by_moment.items()
},
"version_t": facts.version_t,
"version_t_prime": facts.version_t_prime,
"remapped_across_aspects": st.discards.crossed_aspect,
# The column is NULLABLE in the schema, so this branch is reachable
# even though the live data does not take it. Dropped rather than
Expand Down Expand Up @@ -682,7 +711,7 @@ def _per_window(self, session: Session, run: _Run, emit: EmitFn) -> _Computed:
)
tabla, st = state_of(session, facts, [r.query for r in q.rows], run.frame)
run.state.write(tabla)
hecho.state.append(_emit_state(st, emit))
hecho.state.append(_emit_state(st, facts, emit))
if q is not None and "gains" in p.tables and axis is not None:
poblacion = {r.query for r in q.rows}
hecho.gains.append(
Expand Down
125 changes: 125 additions & 0 deletions tests/test_the_state_event_carries_the_split_and_the_eras.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
"""What ``transitions.query_state`` promises a reader of the training report.

THE FIGURE THIS EVENT CARRIES WAS MISREAD, and the misreading was written down.
A comment beside ``discarded_off_axis`` said the count was expected to be zero
and was the first thing to read before trusting a bundle. Measured over the 65
published windows it is 689.055 rows, 1,54 per cent of the raw cards, and it
tracks the window's distance from the axis's era: 0,02 per cent on ``205->211``
and 4,54 on ``170->171``. Only the end AT THE CUT sits on the pivot's own era.

So the event has to carry two things a total cannot carry, and these tests pin
both because the reader is in another repository and its silence is the failure
mode: a dropped field leaves a report that still renders, with a column of
blanks nobody notices.

1. **The split per moment.** The explanation is about the two ends being at
different distances from the axis, and a total cannot be crossed against it.
2. **Both eras of the window.** A drop of 4,54 per cent means nothing until it
sits beside the age it tracks, and the age is not derivable from the window
id.

And the window is the FULL id in both, because the loader joins by that string
as text: a prefix would join to nothing and nothing would fail.
"""

from __future__ import annotations

import uuid
from typing import Any

from protea.core.operations import export_transitions_dataset as E
from protea.core.transitions_export._query_state import (
Collapse,
Discards,
WindowFacts,
WindowState,
)


class _Capturado:
"""An ``EmitFn`` that keeps what it was handed."""

def __init__(self) -> None:
self.eventos: list[tuple[str, str | None, dict[str, Any], str]] = []

def __call__(self, event: str, message: str | None, fields: dict[str, Any], level: Any) -> None:
self.eventos.append((event, message, fields, str(level)))

@property
def campos(self) -> dict[str, Any]:
assert len(self.eventos) == 1, f"expected one event, got {len(self.eventos)}"
return self.eventos[0][2]


VENTANA = str(uuid.uuid4())


def _state() -> WindowState:
"""A window that drops at ``t`` and not at ``t'``, which is the live shape."""
return WindowState(
window=VENTANA,
rows=1200,
by_moment={"t": 500, "t'": 700},
evidence={"t": {"IDA": 500}, "t'": {"IDA": 700}},
distinct_terms=43,
negated=4,
collapse=Collapse(gaf=300, remap=7),
discards=Discards(off_axis=55, crossed_aspect=0, no_evidence=0),
off_axis_by_moment={"t": 55, "t'": 0},
)


def _facts() -> WindowFacts:
return WindowFacts(
eset=None,
window=VENTANA,
data=None,
version_t="170",
version_t_prime="171",
horizon_days=61,
old_end_snapshot=None,
new_end_snapshot=None,
)


def test_the_event_carries_the_off_axis_split_per_moment():
"""The total and both of its halves, so the explanation can be checked."""
emit = _Capturado()

E._emit_state(_state(), _facts(), emit)

campos = emit.campos
assert campos["discarded_off_axis"] == 55
assert campos["off_axis_t"] == 55
assert campos["off_axis_t_prime"] == 0
assert campos["off_axis_t"] + campos["off_axis_t_prime"] == campos["discarded_off_axis"]


def test_the_event_carries_both_eras_of_the_window():
"""The age the drop tracks, beside the drop.

Without these two fields a report can print 4,54 per cent and cannot say
whether that is a window ten years from the axis or one beside it.
"""
emit = _Capturado()

E._emit_state(_state(), _facts(), emit)

campos = emit.campos
assert campos["version_t"] == "170"
assert campos["version_t_prime"] == "171"


def test_the_event_names_the_window_by_its_full_id():
"""The join key, as text, in the field and not only in the message.

The message abbreviates to eight characters for a human reading a log. A
reader that joined on that would join to nothing, which is why the field is
separate from the message and is asserted here.
"""
emit = _Capturado()

E._emit_state(_state(), _facts(), emit)

assert emit.campos["window"] == VENTANA
assert len(emit.campos["window"]) == 36
Loading