Skip to content

Commit a5a34d0

Browse files
committed
fix(polars): align zero-column handling across dependency versions
Signed-off-by: WilliamK112 <164879897+WilliamK112@users.noreply.github.com>
1 parent 722183f commit a5a34d0

3 files changed

Lines changed: 11 additions & 4 deletions

File tree

‎src/freshdata/execution/backends/_polars.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,10 @@ def _stage_drop_duplicates(
472472
pl: Any,
473473
engine_config: EngineConfig | None = None,
474474
) -> Any:
475+
# With no columns, the pandas reference skips duplicate handling. Newer
476+
# Polars versions retain height here but unique() can collapse the rows.
477+
if not lf.collect_schema().names():
478+
return lf
475479
# Counts go through the streaming engine: a plain .collect() re-evaluates
476480
# the whole upstream plan in memory, defeating out-of-core execution.
477481
n_before = self._count(lf, engine_config, pl)

‎tests/test_execution/test_engine_parity_edge_cases.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,11 @@ def _all_empty() -> pd.DataFrame:
108108

109109

110110
@pytest.mark.parametrize("engine", ["polars", "duckdb"])
111-
def test_all_empty_columns_match_pandas(engine):
112-
ref_out, ref = fd.clean(_all_empty(), config=_config(), engine="pandas", return_report=True)
113-
out, report = fd.clean(_all_empty(), config=_config(), engine=engine, return_report=True)
111+
@pytest.mark.parametrize("drop_duplicates", [False, True])
112+
def test_all_empty_columns_match_pandas(engine, drop_duplicates):
113+
config = _config(drop_duplicates=drop_duplicates)
114+
ref_out, ref = fd.clean(_all_empty(), config=config, engine="pandas", return_report=True)
115+
out, report = fd.clean(_all_empty(), config=config, engine=engine, return_report=True)
114116
assert list(out.columns) == list(ref_out.columns) == []
115117
assert len(out) == len(ref_out) == 2
116118
assert (report.rows_after, report.cols_after) == (ref.rows_after, ref.cols_after) == (2, 0)

‎tests/test_execution/test_polars_engine.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ def test_zero_column_pandas_input_discloses_a_native_handle_fallback(native_conf
9595

9696

9797
def test_empty_zero_column_pandas_input_stays_native(native_config):
98-
out, report = fd.clean(pd.DataFrame(), config=native_config, engine="polars",
98+
out, report = fd.clean(pd.DataFrame(index=pd.RangeIndex(0)),
99+
config=native_config, engine="polars",
99100
return_report=True)
100101
assert out.shape == (0, 0)
101102
assert report.rows_before == report.rows_after == 0

0 commit comments

Comments
 (0)