|
| 1 | +"""Exotic Python values in an otherwise numeric or textual object column. |
| 2 | +
|
| 3 | +FD2-012: one ``complex`` value beside text crashed ``fd.clean`` on default |
| 4 | +settings. pandas' ``to_numeric`` returns a complex result as soon as one cell |
| 5 | +is complex, and it never writes a text, bytes or bool cell into that complex |
| 6 | +buffer -- those cells come back as whatever the last same-sized buffer freed |
| 7 | +by numpy held. The garbage counted as "parsed", the column passed the numeric |
| 8 | +threshold, and the real-number finalizer's ``% 1`` raised ``TypeError``. |
| 9 | +
|
| 10 | +The numeric targets here are real dtypes, so a complex value is not a number |
| 11 | +this step can hold. It is treated as any other unparseable straggler: it |
| 12 | +counts against ``numeric_threshold``; when the column still converts it is |
| 13 | +quarantined (set to missing, original kept in ``report.coerced_cells``); when |
| 14 | +the column does not convert it is left exactly as it was, and the |
| 15 | +type-contamination warning names it. |
| 16 | +""" |
| 17 | + |
| 18 | +import fractions |
| 19 | + |
| 20 | +import numpy as np |
| 21 | +import pandas as pd |
| 22 | +import pytest |
| 23 | + |
| 24 | +import freshdata as fd |
| 25 | +from freshdata.config import CleanConfig |
| 26 | +from freshdata.report import CleanReport |
| 27 | +from freshdata.steps.dtypes import ( |
| 28 | + _to_numeric_or_none, |
| 29 | + refine_numeric_after_semantic, |
| 30 | + suggest_conversion, |
| 31 | +) |
| 32 | + |
| 33 | +COMPLEX_VALUES = [ |
| 34 | + pytest.param(complex(1, 2), id="complex"), |
| 35 | + pytest.param(np.complex128(1 + 2j), id="np.complex128"), |
| 36 | + pytest.param(np.complex64(1 + 2j), id="np.complex64"), |
| 37 | +] |
| 38 | + |
| 39 | +NUMERIC_STRINGS = [str(i) for i in range(1, 21)] # 20 of 21 parse: >= 0.95 |
| 40 | + |
| 41 | + |
| 42 | +def _clean(values, **options): |
| 43 | + out, report = fd.clean(pd.DataFrame({"v": values}), return_report=True, |
| 44 | + verbose=False, drop_duplicates=False, **options) |
| 45 | + return out["v"], report |
| 46 | + |
| 47 | + |
| 48 | +def _poison_numpy_buffer_cache(n: int, value: complex) -> None: |
| 49 | + """Free an ``n``-cell complex128 buffer holding *value*. |
| 50 | +
|
| 51 | + numpy hands small freed buffers straight back to the next allocation of the |
| 52 | + same size, so a pandas call that reads an unwritten slot of its complex |
| 53 | + buffer sees *value*. This makes the pandas garbage deterministic instead of |
| 54 | + depending on whatever ran earlier in the process. |
| 55 | + """ |
| 56 | + buf = np.full(n, value, dtype=np.complex128) |
| 57 | + del buf |
| 58 | + |
| 59 | + |
| 60 | +# ── the reported crash ─────────────────────────────────────────────────────── |
| 61 | + |
| 62 | + |
| 63 | +@pytest.mark.parametrize("value", COMPLEX_VALUES) |
| 64 | +def test_complex_beside_text_is_left_intact(value): |
| 65 | + """The FD2-012 reproduction: the column is not numeric (one of three cells |
| 66 | + parses), so it is declined and returned exactly as given.""" |
| 67 | + column, report = _clean([value, "abc", "3"]) |
| 68 | + |
| 69 | + assert column.dtype == object |
| 70 | + assert column.tolist() == [value, "abc", "3"] |
| 71 | + assert column.iloc[0] == complex(1, 2) |
| 72 | + assert "v" not in report.coerced_cells |
| 73 | + |
| 74 | + |
| 75 | +@pytest.mark.parametrize("fill", [complex(7, 7), complex(np.nan, np.nan)]) |
| 76 | +def test_parse_beside_a_complex_value_does_not_read_pandas_garbage(fill): |
| 77 | + """The cause. Whatever the freed buffer held -- a plausible number or NaN -- |
| 78 | + the text cells are parsed as text: ``"abc"`` is missing, ``"3"`` is 3.0, |
| 79 | + and the complex cell is unparseable. Before the fix the result was |
| 80 | + complex128 and ``"abc"``/``"3"`` both came back as *fill*.""" |
| 81 | + s = pd.Series([complex(1, 2), "abc", "3"], dtype=object) |
| 82 | + _poison_numpy_buffer_cache(len(s), fill) |
| 83 | + parsed = _to_numeric_or_none(s) |
| 84 | + |
| 85 | + assert parsed is not None |
| 86 | + assert parsed.dtype == np.float64 |
| 87 | + assert parsed.isna().tolist() == [True, True, False] |
| 88 | + assert parsed.iloc[2] == 3.0 |
| 89 | + |
| 90 | + |
| 91 | +def test_result_does_not_depend_on_what_ran_earlier_in_the_process(): |
| 92 | + """The order dependence recorded in FD2-012: a NaN-filled freed buffer made |
| 93 | + the garbage fail the threshold (no crash), a 7+7j one made ``"abc"`` look |
| 94 | + parsed (crash). Both states must now give the same, intact frame.""" |
| 95 | + frame = pd.DataFrame({"v": [complex(1, 2), "abc", "3"]}) |
| 96 | + results = [] |
| 97 | + for fill in (complex(np.nan, np.nan), complex(7, 7)): |
| 98 | + _poison_numpy_buffer_cache(len(frame), fill) |
| 99 | + results.append(fd.clean(frame, verbose=False)["v"].tolist()) |
| 100 | + assert results == [[complex(1, 2), "abc", "3"]] * 2 |
| 101 | + |
| 102 | + |
| 103 | +# ── a complex straggler in a numeric column is quarantined, not lost ───────── |
| 104 | + |
| 105 | + |
| 106 | +@pytest.mark.parametrize("value", COMPLEX_VALUES) |
| 107 | +def test_complex_straggler_in_a_numeric_column_is_quarantined(value): |
| 108 | + """20 numeric strings and one complex value clear the 0.95 threshold, so the |
| 109 | + column converts. The complex value cannot live in Int64: it is set to |
| 110 | + missing, its original is in ``coerced_cells``, and a warning says so.""" |
| 111 | + column, report = _clean([value, *NUMERIC_STRINGS]) |
| 112 | + |
| 113 | + assert str(column.dtype) == "Int64" |
| 114 | + assert column.isna().tolist() == [True] + [False] * 20 |
| 115 | + assert column.iloc[1:].tolist() == list(range(1, 21)) |
| 116 | + assert report.coerced_cells["v"] == {0: value} |
| 117 | + assert report.coerced_rows["v"] == (0,) |
| 118 | + assert any("could not be parsed as Int64" in w and "(1+2j) (row 0)" in w |
| 119 | + for w in report.warnings) |
| 120 | + |
| 121 | + |
| 122 | +def test_complex_straggler_is_handled_exactly_like_other_unparseable_values(): |
| 123 | + """Alignment with the existing straggler path: a ``Fraction`` (which pandas |
| 124 | + cannot parse either) in the same position gets the same dtype, the same |
| 125 | + missing cell and the same ``coerced_cells`` entry.""" |
| 126 | + for values in ([None, *NUMERIC_STRINGS], [None, "1", "2", "3", "4"]): |
| 127 | + with_complex = _clean([complex(1, 2), *values[1:]]) |
| 128 | + with_fraction = _clean([fractions.Fraction(1, 3), *values[1:]]) |
| 129 | + assert str(with_complex[0].dtype) == str(with_fraction[0].dtype) |
| 130 | + assert with_complex[0].isna().tolist() == with_fraction[0].isna().tolist() |
| 131 | + assert (set(with_complex[1].coerced_cells.get("v", {})) |
| 132 | + == set(with_fraction[1].coerced_cells.get("v", {}))) |
| 133 | + |
| 134 | + |
| 135 | +def test_complex_below_threshold_leaves_the_column_and_is_named_in_the_warning(): |
| 136 | + """Four numbers and one complex value: 80% parse, below 0.95, so nothing is |
| 137 | + converted and the contamination warning points at the complex row.""" |
| 138 | + column, report = _clean([complex(1, 2), "1", "2", "3", "4"]) |
| 139 | + |
| 140 | + assert column.dtype == object |
| 141 | + assert column.tolist() == [complex(1, 2), "1", "2", "3", "4"] |
| 142 | + assert "v" not in report.coerced_cells |
| 143 | + assert any("looks numeric (80% of values parse)" in w and "(1+2j) (row 0)" in w |
| 144 | + and "values were NOT silently coerced" in w for w in report.warnings) |
| 145 | + |
| 146 | + |
| 147 | +def test_complex_beside_python_numbers_in_an_object_column_is_left_intact(): |
| 148 | + """No text at all: pandas parses this correctly (complex128), and the |
| 149 | + finalizer then crashed on ``% 1``. One of five is not real -> declined.""" |
| 150 | + values = pd.Series([complex(1, 2), 1, 2, 3, 4.5], dtype=object) |
| 151 | + column, report = _clean(values) |
| 152 | + |
| 153 | + assert column.dtype == object |
| 154 | + assert column.tolist() == [complex(1, 2), 1, 2, 3, 4.5] |
| 155 | + assert "v" not in report.coerced_cells |
| 156 | + |
| 157 | + |
| 158 | +def test_lower_threshold_quarantines_the_complex_value_beside_text(): |
| 159 | + """With ``numeric_threshold=0.3`` the FD2-012 column converts; the complex |
| 160 | + value and the word are both casualties, recorded with their originals.""" |
| 161 | + column, report = _clean([complex(1, 2), "abc", "3"], numeric_threshold=0.3) |
| 162 | + |
| 163 | + assert str(column.dtype) == "Int64" |
| 164 | + assert column.isna().tolist() == [True, True, False] |
| 165 | + assert column.iloc[2] == 3 |
| 166 | + assert report.coerced_cells["v"] == {0: complex(1, 2), 1: "abc"} |
| 167 | + |
| 168 | + |
| 169 | +def test_profile_preview_matches_cleaning(): |
| 170 | + """``suggest_conversion`` is shared with ``fd.profile``; it must report the |
| 171 | + same decision instead of crashing.""" |
| 172 | + config = CleanConfig() |
| 173 | + beside_text = pd.Series([complex(1, 2), "abc", "3"], dtype=object) |
| 174 | + assert suggest_conversion(beside_text, config) == ("none", None, 0) |
| 175 | + |
| 176 | + target, converted, n_coerced = suggest_conversion( |
| 177 | + pd.Series([complex(1, 2), *NUMERIC_STRINGS], dtype=object), config) |
| 178 | + assert (target, str(converted.dtype), n_coerced) == ("numeric", "Int64", 1) |
| 179 | + |
| 180 | + |
| 181 | +def test_post_semantic_retry_declines_a_complex_casualty(): |
| 182 | + """``refine_numeric_after_semantic`` only quarantines numeric-*shaped* text. |
| 183 | + A complex value is not, so the retry leaves the column untouched.""" |
| 184 | + df = pd.DataFrame({"v": [complex(1, 2), "1", "2", "3", "4", "5"]}) |
| 185 | + report = CleanReport() |
| 186 | + out = refine_numeric_after_semantic(df, ["v"], CleanConfig(), report) |
| 187 | + |
| 188 | + assert out["v"].tolist() == [complex(1, 2), "1", "2", "3", "4", "5"] |
| 189 | + assert "v" not in report.coerced_cells |
| 190 | + |
| 191 | + |
| 192 | +# ── controls: columns without a complex value convert exactly as before ────── |
| 193 | + |
| 194 | + |
| 195 | +@pytest.mark.parametrize( |
| 196 | + ("values", "dtype", "expected"), |
| 197 | + [ |
| 198 | + (["1", "2", "3"], "int64", [1, 2, 3]), |
| 199 | + (["1.5", "2", "-3"], "float64", [1.5, 2.0, -3.0]), |
| 200 | + (["1e3", "2", "-0.5"], "float64", [1000.0, 2.0, -0.5]), |
| 201 | + ([str(i) for i in range(19)] + ["$1,234.56"], "float64", |
| 202 | + [float(i) for i in range(19)] + [1234.56]), |
| 203 | + (["inf", "1", "2"], "float64", [float("inf"), 1.0, 2.0]), |
| 204 | + ], |
| 205 | +) |
| 206 | +def test_ordinary_numeric_text_converts_as_before(values, dtype, expected): |
| 207 | + column, report = _clean(values) |
| 208 | + assert str(column.dtype) == dtype |
| 209 | + assert column.tolist() == expected |
| 210 | + assert "v" not in report.coerced_cells |
| 211 | + |
| 212 | + |
| 213 | +def test_ordinary_text_straggler_is_still_quarantined_as_before(): |
| 214 | + column, report = _clean(["abc", *NUMERIC_STRINGS]) |
| 215 | + assert str(column.dtype) == "Int64" |
| 216 | + assert column.iloc[1:].tolist() == list(range(1, 21)) |
| 217 | + assert report.coerced_cells["v"] == {0: "abc"} |
| 218 | + |
| 219 | + |
| 220 | +def test_all_complex_object_column_is_still_declined_before_parsing(): |
| 221 | + """``infer_dtype`` reports ``"complex"`` and the column never reaches the |
| 222 | + numeric path -- unchanged by this fix.""" |
| 223 | + column, report = _clean(pd.Series([complex(1, 2), complex(3, 4)], dtype=object)) |
| 224 | + assert column.tolist() == [complex(1, 2), complex(3, 4)] |
| 225 | + assert "v" not in report.coerced_cells |
0 commit comments