Skip to content

Commit 5433544

Browse files
Merge pull request #518 from WilliamK112/codex/polars-zero-column-463
fix(polars): preserve rows in zero-column pandas inputs
2 parents ba3f01f + a5a34d0 commit 5433544

6 files changed

Lines changed: 69 additions & 3 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ adheres to [Semantic Versioning](https://semver.org/).
88

99
### Fixed
1010

11+
- The Polars engine no longer silently loses rows from a zero-column pandas
12+
input. It uses the recorded pandas fallback, preserving the input index and
13+
row counts with pandas output; `fallback_policy="error"` rejects the run
14+
before ingestion instead. Empty inputs with no rows stay native (#463).
15+
1116
- A repeated index label no longer breaks cleaning. `fd.clean(df)` and
1217
`fd.suggest_plan(df)` raised on ordinary input whenever the frame's index
1318
carried the same label twice — the shape you get from concatenating two

‎docs/fallback-matrix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ FreshCore also runs its own config and data checks in
3939
| semantic cleaning | native-distinct | native-distinct | pandas | pandas | polars/duckdb run it over a natively extracted distinct table; non-default semantic backends force pandas |
4040
| pandas input with a mixed-type object column (e.g. numbers and strings) | pandas | pandas | — | pandas | native ingestion would reject the column (polars) or cast every value to text (duckdb) |
4141
| pandas input with duplicate column labels | pandas | pandas | — | pandas | native frames need unique column names; the pandas pipeline deduplicates them (`"x", "x"` → `"x", "x_2"`) |
42+
| pandas input with zero columns and one or more rows | pandas | pandas | — | — | the pandas reference preserves the rows and index; Polars 1.36 loses these rows during ingestion, and DuckDB cannot register a zero-column frame. Use pandas output to retain the row count |
4243
| pandas input whose column labels collide once stringified (e.g. `1` and `"1"`) | native | native | — | pandas | FreshCore names columns by `str(label)`, so one column would overwrite the other; distinct non-string labels (e.g. `0`, `1`) stay native and come back unchanged |
4344
| pandas input with datetime / timedelta / categorical columns | native | native | — | pandas | FreshCore v1 carries only float, bool and string columns, so these dtypes would come back as strings |
4445
| pandas input with period or interval columns | pandas | pandas | — | pandas | DuckDB rejects both dtypes outright; Polars ingests a period as its raw int64 ordinal (`2020-01` → `600`) and an interval as a `{left, right}` struct |

‎src/freshdata/execution/_ingest.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ def pandas_ingest_fallback_reason(source: Any, engine: str | None = None) -> str
3030

3131
if not isinstance(source, pd.DataFrame):
3232
return None
33+
if engine == "polars" and len(source) > 0 and len(source.columns) == 0:
34+
# Polars 1.36 loses these rows during ingestion. Even where ingestion
35+
# preserves them, per-column metadata cannot recover their input count.
36+
return "zero-column pandas source requires the pandas reference to preserve its rows"
3337
if source.columns.duplicated().any():
3438
return "duplicate input column labels require the pandas reference path"
3539
for i, dtype in enumerate(source.dtypes):

‎src/freshdata/execution/backends/_polars.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,10 @@ def _stage_drop_duplicates(
472472
pl: Any,
473473
engine_config: EngineConfig | None = None,
474474
) -> Any:
475+
# With no columns, the pandas reference skips duplicate handling. Newer
476+
# Polars versions retain height here but unique() can collapse the rows.
477+
if not lf.collect_schema().names():
478+
return lf
475479
# Counts go through the streaming engine: a plain .collect() re-evaluates
476480
# the whole upstream plan in memory, defeating out-of-core execution.
477481
n_before = self._count(lf, engine_config, pl)

‎tests/test_execution/test_engine_parity_edge_cases.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,11 @@ def _all_empty() -> pd.DataFrame:
108108

109109

110110
@pytest.mark.parametrize("engine", ["polars", "duckdb"])
111-
def test_all_empty_columns_match_pandas(engine):
112-
ref_out, ref = fd.clean(_all_empty(), config=_config(), engine="pandas", return_report=True)
113-
out, report = fd.clean(_all_empty(), config=_config(), engine=engine, return_report=True)
111+
@pytest.mark.parametrize("drop_duplicates", [False, True])
112+
def test_all_empty_columns_match_pandas(engine, drop_duplicates):
113+
config = _config(drop_duplicates=drop_duplicates)
114+
ref_out, ref = fd.clean(_all_empty(), config=config, engine="pandas", return_report=True)
115+
out, report = fd.clean(_all_empty(), config=config, engine=engine, return_report=True)
114116
assert list(out.columns) == list(ref_out.columns) == []
115117
assert len(out) == len(ref_out) == 2
116118
assert (report.rows_after, report.cols_after) == (ref.rows_after, ref.cols_after) == (2, 0)

‎tests/test_execution/test_polars_engine.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,56 @@ def test_drop_empty_column(native_config):
5454
assert "empty" not in out.columns
5555

5656

57+
def test_zero_column_pandas_input_keeps_rows_and_report_counts(native_config):
58+
df = pd.DataFrame(index=range(3))
59+
out, report = fd.clean(df, config=native_config, engine="polars", return_report=True)
60+
assert out.shape == (3, 0)
61+
assert report.rows_before == report.rows_after == report.rows_materialized == 3
62+
assert report.requested_backend == "polars"
63+
assert report.backend == "pandas"
64+
(event,) = report.fallback_events
65+
assert "zero-column" in event["fallback_reason"]
66+
67+
68+
def test_zero_column_pandas_input_preserves_its_index(native_config):
69+
df = pd.DataFrame(index=pd.RangeIndex(7, 10, name="record"))
70+
out = fd.clean(df, config=native_config, engine="polars")
71+
reference = fd.clean(df, config=native_config, engine="pandas")
72+
pd.testing.assert_frame_equal(out, reference, check_frame_type=False)
73+
74+
75+
def test_zero_column_pandas_input_honours_error_policy(native_config):
76+
with pytest.raises(FallbackError, match="zero-column"):
77+
fd.clean(pd.DataFrame(index=range(3)), config=native_config, engine="polars",
78+
fallback_policy="error")
79+
80+
81+
def test_zero_column_pandas_input_warns_when_requested(native_config):
82+
with pytest.warns(fd.FallbackWarning, match="zero-column"):
83+
out = fd.clean(pd.DataFrame(index=range(3)), config=native_config, engine="polars",
84+
fallback_policy="warn")
85+
assert out.shape == (3, 0)
86+
87+
88+
def test_zero_column_pandas_input_discloses_a_native_handle_fallback(native_config):
89+
out, report = fd.clean(pd.DataFrame(index=range(3)), config=native_config, engine="polars",
90+
output_format="polars-lazy", return_report=True)
91+
assert isinstance(out, pd.DataFrame)
92+
assert out.shape == (3, 0)
93+
assert report.materialized
94+
assert report.fallback_events
95+
96+
97+
def test_empty_zero_column_pandas_input_stays_native(native_config):
98+
out, report = fd.clean(pd.DataFrame(index=pd.RangeIndex(0)),
99+
config=native_config, engine="polars",
100+
return_report=True)
101+
assert out.shape == (0, 0)
102+
assert report.rows_before == report.rows_after == 0
103+
assert report.backend == "polars"
104+
assert report.fallback_events == []
105+
106+
57107
def test_fallback_on_knn_like_config_warns(small_df, caplog):
58108
"""A config needing the decision engine falls back to pandas with a warning."""
59109
import logging

0 commit comments

Comments
 (0)