|
| 1 | +"""``fd.learn`` on categorical columns. |
| 2 | +
|
| 3 | +Comparing two pandas Categoricals raises ``TypeError`` unless their categories |
| 4 | +are identical, and a messy/clean pair almost never shares categories -- the |
| 5 | +repair is what changes them. Learning must compare, align and learn such |
| 6 | +columns by value, exactly as it does for the object-dtype equivalent. The |
| 7 | +fixtures avoid version-specific pandas APIs so they run unchanged on pandas |
| 8 | +1.5 (Python 3.9 lane) and pandas 2.x. |
| 9 | +""" |
| 10 | + |
| 11 | +from __future__ import annotations |
| 12 | + |
| 13 | +import json |
| 14 | + |
| 15 | +import numpy as np |
| 16 | +import pandas as pd |
| 17 | +import pytest |
| 18 | + |
| 19 | +import freshdata as fd |
| 20 | +from freshdata.learning import LearningProfile, learn, load_profile, save_profile |
| 21 | +from freshdata.learning.align import align_pair |
| 22 | +from freshdata.learning.diff import compute_diff |
| 23 | + |
| 24 | +_RAW = [ |
| 25 | + "Deliverd", |
| 26 | + "SHIPPED", |
| 27 | + "shipped ", |
| 28 | + "delivered", |
| 29 | + "SHIPPED", |
| 30 | + "Deliverd", |
| 31 | + "shipped ", |
| 32 | + "SHIPPED", |
| 33 | +] |
| 34 | +_CLEAN = [ |
| 35 | + "delivered", |
| 36 | + "shipped", |
| 37 | + "shipped", |
| 38 | + "delivered", |
| 39 | + "shipped", |
| 40 | + "delivered", |
| 41 | + "shipped", |
| 42 | + "shipped", |
| 43 | +] |
| 44 | +_IDS = [f"A{i}" for i in range(len(_RAW))] |
| 45 | + |
| 46 | + |
| 47 | +def _frames(raw: list, clean: list) -> tuple[pd.DataFrame, pd.DataFrame]: |
| 48 | + ids = [f"A{i}" for i in range(len(raw))] |
| 49 | + return ( |
| 50 | + pd.DataFrame({"order_id": ids, "status": raw}), |
| 51 | + pd.DataFrame({"order_id": ids, "status": clean}), |
| 52 | + ) |
| 53 | + |
| 54 | + |
| 55 | +def _as_object(df: pd.DataFrame) -> pd.DataFrame: |
| 56 | + """The object-dtype equivalent of ``df`` (categoricals decoded).""" |
| 57 | + return df.assign( |
| 58 | + **{ |
| 59 | + str(c): df[c].astype(object) |
| 60 | + for c in df.columns |
| 61 | + if isinstance(df[c].dtype, pd.CategoricalDtype) |
| 62 | + } |
| 63 | + ) |
| 64 | + |
| 65 | + |
| 66 | +def _learned(profile: LearningProfile) -> str: |
| 67 | + """Everything a profile learned, as canonical JSON (timestamps excluded).""" |
| 68 | + audit = profile.audit().to_dict() |
| 69 | + payload = { |
| 70 | + "rules": [r.to_dict() for r in profile.rules], |
| 71 | + "value_maps": {c: m.to_dict() for c, m in sorted(profile.value_maps.items())}, |
| 72 | + "examples": profile.examples.to_dict() if profile.examples is not None else None, |
| 73 | + "value_patterns": profile.memory.value_patterns if profile.memory is not None else None, |
| 74 | + "notes": audit.get("notes"), |
| 75 | + "demotions": audit.get("demotions"), |
| 76 | + "holdout_metrics": audit.get("holdout_metrics"), |
| 77 | + } |
| 78 | + return json.dumps(payload, sort_keys=True, default=str) |
| 79 | + |
| 80 | + |
| 81 | +def _assert_parity(messy: pd.DataFrame, clean: pd.DataFrame, **kwargs) -> LearningProfile: |
| 82 | + profile = learn(messy, clean, **kwargs) |
| 83 | + reference = learn(_as_object(messy), _as_object(clean), **kwargs) |
| 84 | + assert _learned(profile) == _learned(reference) |
| 85 | + return profile |
| 86 | + |
| 87 | + |
| 88 | +def test_repro_categoricals_with_different_categories() -> None: |
| 89 | + messy = pd.DataFrame({"c": pd.Categorical([" a", "B ", "a"])}) |
| 90 | + clean = pd.DataFrame({"c": pd.Categorical(["a", "b", "a"])}) |
| 91 | + profile = fd.learn(messy, clean) |
| 92 | + assert isinstance(profile, LearningProfile) |
| 93 | + assert _learned(profile) == _learned(fd.learn(_as_object(messy), _as_object(clean))) |
| 94 | + |
| 95 | + |
| 96 | +def test_diff_compares_categoricals_by_value() -> None: |
| 97 | + messy, clean = _frames(pd.Categorical(_RAW), pd.Categorical(_CLEAN)) |
| 98 | + summary = compute_diff(align_pair(messy, clean, key="order_id")) |
| 99 | + reference = compute_diff(align_pair(_as_object(messy), _as_object(clean), key="order_id")) |
| 100 | + assert {col: [d.to_dict() for d in diffs] for col, diffs in summary.column_diffs.items()} == { |
| 101 | + col: [d.to_dict() for d in diffs] for col, diffs in reference.column_diffs.items() |
| 102 | + } |
| 103 | + pairs = {(d.raw_value, d.clean_value, d.support) for d in summary.column_diffs["status"]} |
| 104 | + assert pairs == { |
| 105 | + ("SHIPPED", "shipped", 3), |
| 106 | + ("Deliverd", "delivered", 2), |
| 107 | + ("shipped ", "shipped", 2), |
| 108 | + } |
| 109 | + # Only the values changed, never the dtype. |
| 110 | + assert summary.schema_diffs.dtype_changes == {} |
| 111 | + |
| 112 | + |
| 113 | +@pytest.mark.parametrize( |
| 114 | + ("messy_status", "clean_status"), |
| 115 | + [ |
| 116 | + pytest.param(pd.Categorical(_RAW), pd.Categorical(_CLEAN), id="both-different-categories"), |
| 117 | + pytest.param( |
| 118 | + pd.Categorical(_RAW, categories=sorted(set(_RAW) | set(_CLEAN))), |
| 119 | + pd.Categorical(_CLEAN, categories=sorted(set(_RAW) | set(_CLEAN))), |
| 120 | + id="both-same-categories", |
| 121 | + ), |
| 122 | + pytest.param(pd.Categorical(_RAW), list(_CLEAN), id="categorical-messy-object-clean"), |
| 123 | + pytest.param(list(_RAW), pd.Categorical(_CLEAN), id="object-messy-categorical-clean"), |
| 124 | + pytest.param( |
| 125 | + pd.Categorical(_RAW, ordered=True), |
| 126 | + pd.Categorical(_CLEAN, ordered=True), |
| 127 | + id="both-ordered", |
| 128 | + ), |
| 129 | + pytest.param( |
| 130 | + pd.Categorical(_RAW, ordered=True), |
| 131 | + pd.Categorical(_CLEAN, categories=["shipped", "delivered"], ordered=True), |
| 132 | + id="ordered-different-order", |
| 133 | + ), |
| 134 | + pytest.param( |
| 135 | + pd.Categorical(_RAW, ordered=True), pd.Categorical(_CLEAN), id="ordered-vs-unordered" |
| 136 | + ), |
| 137 | + ], |
| 138 | +) |
| 139 | +@pytest.mark.parametrize("key", [None, "order_id"]) |
| 140 | +def test_learning_matches_object_dtype(messy_status, clean_status, key) -> None: |
| 141 | + messy, clean = _frames(messy_status, clean_status) |
| 142 | + profile = _assert_parity(messy, clean, key=key, min_support=2) |
| 143 | + entries = {(e.raw_value, e.clean_value) for e in profile.value_maps["status"].entries} |
| 144 | + assert entries == { |
| 145 | + ("SHIPPED", "shipped"), |
| 146 | + ("shipped ", "shipped"), |
| 147 | + ("Deliverd", "delivered"), |
| 148 | + } |
| 149 | + |
| 150 | + |
| 151 | +def test_categorical_key_column() -> None: |
| 152 | + messy, clean = _frames(pd.Categorical(_RAW), pd.Categorical(_CLEAN)) |
| 153 | + messy["order_id"] = pd.Categorical(messy["order_id"]) |
| 154 | + clean = clean.iloc[::-1].reset_index(drop=True) |
| 155 | + clean["order_id"] = pd.Categorical(clean["order_id"], categories=list(reversed(_IDS))) |
| 156 | + _assert_parity(messy, clean, key="order_id", min_support=2) |
| 157 | + |
| 158 | + |
| 159 | +def test_missing_cells_in_categoricals() -> None: |
| 160 | + raw = ["x", "N/A", "y", np.nan, "x", "N/A", "y", np.nan, "x", "N/A"] |
| 161 | + clean = ["x", np.nan, "y", np.nan, "x", np.nan, "y", np.nan, "x", np.nan] |
| 162 | + messy_df, clean_df = _frames(pd.Categorical(raw), pd.Categorical(clean)) |
| 163 | + summary = compute_diff(align_pair(messy_df, clean_df, key="order_id")) |
| 164 | + # Both-missing cells are equal; only the sentinel cells differ. |
| 165 | + (diff,) = summary.column_diffs["status"] |
| 166 | + assert (diff.raw_value, diff.clean_value, diff.support, diff.kind) == ( |
| 167 | + "N/A", |
| 168 | + None, |
| 169 | + 3, |
| 170 | + "value_to_missing", |
| 171 | + ) |
| 172 | + profile = _assert_parity(messy_df, clean_df, key="order_id", min_support=2) |
| 173 | + assert any(r.rule == "sentinel" for r in profile.rules) |
| 174 | + |
| 175 | + |
| 176 | +def test_imputed_categorical_cells() -> None: |
| 177 | + raw = ["a", np.nan, "a", "b", np.nan, "a", "a", "b", np.nan, "a"] |
| 178 | + clean = ["a", "a", "a", "b", "a", "a", "a", "b", "a", "a"] |
| 179 | + messy_df, clean_df = _frames(pd.Categorical(raw), pd.Categorical(clean)) |
| 180 | + profile = _assert_parity(messy_df, clean_df, key="order_id", min_support=2) |
| 181 | + assert profile.value_maps == {} # imputation is never a literal map |
| 182 | + |
| 183 | + |
| 184 | +def test_sensitive_categorical_column_learns_like_object() -> None: |
| 185 | + raw = ["A@X.COM ", "b@@y.com", "c@z.com"] * 3 + ["d@@w.com"] |
| 186 | + clean = ["a@x.com", "b@y.com", "c@z.com"] * 3 + ["d@w.com"] |
| 187 | + messy = pd.DataFrame({"email": pd.Categorical(raw)}) |
| 188 | + clean_df = pd.DataFrame({"email": pd.Categorical(clean)}) |
| 189 | + _assert_parity(messy, clean_df, privacy="none", include_sensitive=True, min_support=2) |
| 190 | + masked = learn(messy, clean_df, min_support=2) |
| 191 | + reference = learn(_as_object(messy), _as_object(clean_df), min_support=2) |
| 192 | + assert [e.masked for e in masked.value_maps["email"].entries] == [ |
| 193 | + e.masked for e in reference.value_maps["email"].entries |
| 194 | + ] |
| 195 | + |
| 196 | + |
| 197 | +def test_categorical_profile_round_trips_and_replays_like_object(tmp_path) -> None: |
| 198 | + messy, clean = _frames(pd.Categorical(_RAW), pd.Categorical(_CLEAN)) |
| 199 | + profile = learn(messy, clean, key="order_id", min_support=2) |
| 200 | + reference = learn(_as_object(messy), _as_object(clean), key="order_id", min_support=2) |
| 201 | + |
| 202 | + path = tmp_path / "categorical.fdprofile" |
| 203 | + save_profile(profile, path) |
| 204 | + loaded = load_profile(path) |
| 205 | + assert loaded.profile_id == profile.profile_id == reference.profile_id |
| 206 | + assert _learned(loaded) == _learned(profile) |
| 207 | + |
| 208 | + batch = pd.DataFrame( |
| 209 | + {"order_id": ["B1", "B2", "B3"], "status": ["Deliverd", "SHIPPED", "shipped "]} |
| 210 | + ) |
| 211 | + categorical_batch = batch.assign(status=pd.Categorical(batch["status"])) |
| 212 | + replayed = fd.clean(categorical_batch, profile=loaded, semantic_mode="auto") |
| 213 | + expected = fd.clean(batch, profile=reference, semantic_mode="auto") |
| 214 | + assert replayed["status"].astype(object).tolist() == ["delivered", "shipped", "shipped"] |
| 215 | + assert replayed["status"].astype(object).tolist() == expected["status"].astype(object).tolist() |
0 commit comments