Skip to content

Commit dc8ac5e

Browse files
fix(learning): learn from categorical columns whose categories differ (#410)
fd.learn raised "TypeError: Categoricals can only be compared if 'categories' are the same" whenever a shared column was categorical on both sides with different categories (including the same categories in a different order, or differing orderedness). Root cause: the diff stage (learning/diff.py::_column_diffs) finds changed cells with an elementwise `messy.eq(clean)`. pandas only compares two Categoricals when their categories are identical, and a messy/clean pair almost never shares categories, because repairing the values is exactly what changes them. Every fd.learn path runs this comparison (positional and keyed alignment, privacy="mask"/"none", context=), so they all crashed before classification. Pairs that happened to share categories, or mixed categorical with object, compared fine and already learned the same profile as the object-dtype run. Fix: cells are compared by value, so the diff stage decodes a categorical side to its object values before comparing. That is exactly the object-dtype equivalent. Every later stage already handled categorical values, so after the diff the learned rules, value maps, examples, embedded memory and audit match the object-dtype run. The recorded dtype_changes still use the original dtypes. Non-categorical columns do not go through the new branch, so their output is unchanged.
1 parent 6967aa8 commit dc8ac5e

2 files changed

Lines changed: 231 additions & 0 deletions

File tree

‎src/freshdata/learning/diff.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,23 @@ def _diff_kind(raw_missing: bool, clean_missing: bool) -> str:
2424
return "value_change"
2525

2626

27+
def _by_value(series: pd.Series) -> pd.Series:
28+
"""``series`` with a categorical dtype replaced by its plain object values.
29+
30+
Comparing two Categorical Series raises ``TypeError`` unless both share
31+
identical categories (and ordering), which a messy/clean pair almost never
32+
does -- repairing the values is exactly what changes the categories. Cells
33+
are compared by value, so decode categoricals to the object values an
34+
object-dtype column would hold.
35+
"""
36+
if isinstance(series.dtype, pd.CategoricalDtype):
37+
return series.astype(object)
38+
return series
39+
40+
2741
def _column_diffs(messy: pd.Series, clean: pd.Series, column: str) -> list[ValueDiff]:
42+
messy = _by_value(messy)
43+
clean = _by_value(clean)
2844
raw_na = messy.isna()
2945
clean_na = clean.isna()
3046
# A cell differs when values are unequal, except both-missing which is equal.

‎tests/learning/test_categorical.py‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
"""``fd.learn`` on categorical columns.
2+
3+
Comparing two pandas Categoricals raises ``TypeError`` unless their categories
4+
are identical, and a messy/clean pair almost never shares categories -- the
5+
repair is what changes them. Learning must compare, align and learn such
6+
columns by value, exactly as it does for the object-dtype equivalent. The
7+
fixtures avoid version-specific pandas APIs so they run unchanged on pandas
8+
1.5 (Python 3.9 lane) and pandas 2.x.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
import json
14+
15+
import numpy as np
16+
import pandas as pd
17+
import pytest
18+
19+
import freshdata as fd
20+
from freshdata.learning import LearningProfile, learn, load_profile, save_profile
21+
from freshdata.learning.align import align_pair
22+
from freshdata.learning.diff import compute_diff
23+
24+
_RAW = [
25+
"Deliverd",
26+
"SHIPPED",
27+
"shipped ",
28+
"delivered",
29+
"SHIPPED",
30+
"Deliverd",
31+
"shipped ",
32+
"SHIPPED",
33+
]
34+
_CLEAN = [
35+
"delivered",
36+
"shipped",
37+
"shipped",
38+
"delivered",
39+
"shipped",
40+
"delivered",
41+
"shipped",
42+
"shipped",
43+
]
44+
_IDS = [f"A{i}" for i in range(len(_RAW))]
45+
46+
47+
def _frames(raw: list, clean: list) -> tuple[pd.DataFrame, pd.DataFrame]:
48+
ids = [f"A{i}" for i in range(len(raw))]
49+
return (
50+
pd.DataFrame({"order_id": ids, "status": raw}),
51+
pd.DataFrame({"order_id": ids, "status": clean}),
52+
)
53+
54+
55+
def _as_object(df: pd.DataFrame) -> pd.DataFrame:
56+
"""The object-dtype equivalent of ``df`` (categoricals decoded)."""
57+
return df.assign(
58+
**{
59+
str(c): df[c].astype(object)
60+
for c in df.columns
61+
if isinstance(df[c].dtype, pd.CategoricalDtype)
62+
}
63+
)
64+
65+
66+
def _learned(profile: LearningProfile) -> str:
67+
"""Everything a profile learned, as canonical JSON (timestamps excluded)."""
68+
audit = profile.audit().to_dict()
69+
payload = {
70+
"rules": [r.to_dict() for r in profile.rules],
71+
"value_maps": {c: m.to_dict() for c, m in sorted(profile.value_maps.items())},
72+
"examples": profile.examples.to_dict() if profile.examples is not None else None,
73+
"value_patterns": profile.memory.value_patterns if profile.memory is not None else None,
74+
"notes": audit.get("notes"),
75+
"demotions": audit.get("demotions"),
76+
"holdout_metrics": audit.get("holdout_metrics"),
77+
}
78+
return json.dumps(payload, sort_keys=True, default=str)
79+
80+
81+
def _assert_parity(messy: pd.DataFrame, clean: pd.DataFrame, **kwargs) -> LearningProfile:
82+
profile = learn(messy, clean, **kwargs)
83+
reference = learn(_as_object(messy), _as_object(clean), **kwargs)
84+
assert _learned(profile) == _learned(reference)
85+
return profile
86+
87+
88+
def test_repro_categoricals_with_different_categories() -> None:
89+
messy = pd.DataFrame({"c": pd.Categorical([" a", "B ", "a"])})
90+
clean = pd.DataFrame({"c": pd.Categorical(["a", "b", "a"])})
91+
profile = fd.learn(messy, clean)
92+
assert isinstance(profile, LearningProfile)
93+
assert _learned(profile) == _learned(fd.learn(_as_object(messy), _as_object(clean)))
94+
95+
96+
def test_diff_compares_categoricals_by_value() -> None:
97+
messy, clean = _frames(pd.Categorical(_RAW), pd.Categorical(_CLEAN))
98+
summary = compute_diff(align_pair(messy, clean, key="order_id"))
99+
reference = compute_diff(align_pair(_as_object(messy), _as_object(clean), key="order_id"))
100+
assert {col: [d.to_dict() for d in diffs] for col, diffs in summary.column_diffs.items()} == {
101+
col: [d.to_dict() for d in diffs] for col, diffs in reference.column_diffs.items()
102+
}
103+
pairs = {(d.raw_value, d.clean_value, d.support) for d in summary.column_diffs["status"]}
104+
assert pairs == {
105+
("SHIPPED", "shipped", 3),
106+
("Deliverd", "delivered", 2),
107+
("shipped ", "shipped", 2),
108+
}
109+
# Only the values changed, never the dtype.
110+
assert summary.schema_diffs.dtype_changes == {}
111+
112+
113+
@pytest.mark.parametrize(
114+
("messy_status", "clean_status"),
115+
[
116+
pytest.param(pd.Categorical(_RAW), pd.Categorical(_CLEAN), id="both-different-categories"),
117+
pytest.param(
118+
pd.Categorical(_RAW, categories=sorted(set(_RAW) | set(_CLEAN))),
119+
pd.Categorical(_CLEAN, categories=sorted(set(_RAW) | set(_CLEAN))),
120+
id="both-same-categories",
121+
),
122+
pytest.param(pd.Categorical(_RAW), list(_CLEAN), id="categorical-messy-object-clean"),
123+
pytest.param(list(_RAW), pd.Categorical(_CLEAN), id="object-messy-categorical-clean"),
124+
pytest.param(
125+
pd.Categorical(_RAW, ordered=True),
126+
pd.Categorical(_CLEAN, ordered=True),
127+
id="both-ordered",
128+
),
129+
pytest.param(
130+
pd.Categorical(_RAW, ordered=True),
131+
pd.Categorical(_CLEAN, categories=["shipped", "delivered"], ordered=True),
132+
id="ordered-different-order",
133+
),
134+
pytest.param(
135+
pd.Categorical(_RAW, ordered=True), pd.Categorical(_CLEAN), id="ordered-vs-unordered"
136+
),
137+
],
138+
)
139+
@pytest.mark.parametrize("key", [None, "order_id"])
140+
def test_learning_matches_object_dtype(messy_status, clean_status, key) -> None:
141+
messy, clean = _frames(messy_status, clean_status)
142+
profile = _assert_parity(messy, clean, key=key, min_support=2)
143+
entries = {(e.raw_value, e.clean_value) for e in profile.value_maps["status"].entries}
144+
assert entries == {
145+
("SHIPPED", "shipped"),
146+
("shipped ", "shipped"),
147+
("Deliverd", "delivered"),
148+
}
149+
150+
151+
def test_categorical_key_column() -> None:
152+
messy, clean = _frames(pd.Categorical(_RAW), pd.Categorical(_CLEAN))
153+
messy["order_id"] = pd.Categorical(messy["order_id"])
154+
clean = clean.iloc[::-1].reset_index(drop=True)
155+
clean["order_id"] = pd.Categorical(clean["order_id"], categories=list(reversed(_IDS)))
156+
_assert_parity(messy, clean, key="order_id", min_support=2)
157+
158+
159+
def test_missing_cells_in_categoricals() -> None:
160+
raw = ["x", "N/A", "y", np.nan, "x", "N/A", "y", np.nan, "x", "N/A"]
161+
clean = ["x", np.nan, "y", np.nan, "x", np.nan, "y", np.nan, "x", np.nan]
162+
messy_df, clean_df = _frames(pd.Categorical(raw), pd.Categorical(clean))
163+
summary = compute_diff(align_pair(messy_df, clean_df, key="order_id"))
164+
# Both-missing cells are equal; only the sentinel cells differ.
165+
(diff,) = summary.column_diffs["status"]
166+
assert (diff.raw_value, diff.clean_value, diff.support, diff.kind) == (
167+
"N/A",
168+
None,
169+
3,
170+
"value_to_missing",
171+
)
172+
profile = _assert_parity(messy_df, clean_df, key="order_id", min_support=2)
173+
assert any(r.rule == "sentinel" for r in profile.rules)
174+
175+
176+
def test_imputed_categorical_cells() -> None:
177+
raw = ["a", np.nan, "a", "b", np.nan, "a", "a", "b", np.nan, "a"]
178+
clean = ["a", "a", "a", "b", "a", "a", "a", "b", "a", "a"]
179+
messy_df, clean_df = _frames(pd.Categorical(raw), pd.Categorical(clean))
180+
profile = _assert_parity(messy_df, clean_df, key="order_id", min_support=2)
181+
assert profile.value_maps == {} # imputation is never a literal map
182+
183+
184+
def test_sensitive_categorical_column_learns_like_object() -> None:
185+
raw = ["A@X.COM ", "b@@y.com", "c@z.com"] * 3 + ["d@@w.com"]
186+
clean = ["a@x.com", "b@y.com", "c@z.com"] * 3 + ["d@w.com"]
187+
messy = pd.DataFrame({"email": pd.Categorical(raw)})
188+
clean_df = pd.DataFrame({"email": pd.Categorical(clean)})
189+
_assert_parity(messy, clean_df, privacy="none", include_sensitive=True, min_support=2)
190+
masked = learn(messy, clean_df, min_support=2)
191+
reference = learn(_as_object(messy), _as_object(clean_df), min_support=2)
192+
assert [e.masked for e in masked.value_maps["email"].entries] == [
193+
e.masked for e in reference.value_maps["email"].entries
194+
]
195+
196+
197+
def test_categorical_profile_round_trips_and_replays_like_object(tmp_path) -> None:
198+
messy, clean = _frames(pd.Categorical(_RAW), pd.Categorical(_CLEAN))
199+
profile = learn(messy, clean, key="order_id", min_support=2)
200+
reference = learn(_as_object(messy), _as_object(clean), key="order_id", min_support=2)
201+
202+
path = tmp_path / "categorical.fdprofile"
203+
save_profile(profile, path)
204+
loaded = load_profile(path)
205+
assert loaded.profile_id == profile.profile_id == reference.profile_id
206+
assert _learned(loaded) == _learned(profile)
207+
208+
batch = pd.DataFrame(
209+
{"order_id": ["B1", "B2", "B3"], "status": ["Deliverd", "SHIPPED", "shipped "]}
210+
)
211+
categorical_batch = batch.assign(status=pd.Categorical(batch["status"]))
212+
replayed = fd.clean(categorical_batch, profile=loaded, semantic_mode="auto")
213+
expected = fd.clean(batch, profile=reference, semantic_mode="auto")
214+
assert replayed["status"].astype(object).tolist() == ["delivered", "shipped", "shipped"]
215+
assert replayed["status"].astype(object).tolist() == expected["status"].astype(object).tolist()

0 commit comments

Comments
 (0)