Skip to content

Commit c49cb46

Browse files
fix(duckdb): route a zero-column source through the disclosed pandas fallback
The zero-column short-circuit ran ahead of the backend's fallback chain, so it bypassed both `pandas_ingest_fallback_reason` and `_pandas_index_forces_fallback`. Three consequences: - A zero-column frame with a non-RangeIndex worked before this change (the index guard routed it to pandas and the labels survived) and came back with a rebuilt RangeIndex after it, silently dropping the labels and breaking the parity the `native_config` fixture exists to enforce. - `output_format="duckdb"` still raised, now a RuntimeError from `_require_recorded_fallback`, since a materialized frame may stand in for a native handle only after a fallback the report discloses. - `rows_before=len(source)` is only meaningful for pandas: a zero-column LazyFrame raised TypeError and a parquet path would have passed its character count off as a row count. A source DuckDB cannot register is an ingest limitation, so name it as one and let the existing chain handle it. The pandas reference already keeps the rows and the index on a zero-column frame, and the fallback is disclosed in the report and still blocked by `fallback_policy="error"`. Fixes #457 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 6b2d0a6 commit c49cb46

2 files changed

Lines changed: 43 additions & 14 deletions

File tree

‎src/freshdata/execution/backends/_duckdb.py‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,14 @@
4242
)
4343
from .._plan import PlanGenerator
4444
from .._report import finalize_report, finalize_report_native, init_report, zero_column_frame
45-
from ...report import CleanReport
4645
from .._spill import create_run_spill_dir, remove_run_spill_dir
4746
from ._pandas import materialize_to_pandas
4847

4948
if TYPE_CHECKING: # pragma: no cover - typing only
5049
import pandas as pd
5150

5251
from ...config import CleanConfig
52+
from ...report import CleanReport
5353
from .._config import EngineConfig
5454

5555
log = logging.getLogger("freshdata.execution.duckdb")
@@ -147,20 +147,13 @@ def execute(
147147

148148
plan_cols = self._peek_columns(source)
149149
plan = PlanGenerator(config).plan(plan_cols)
150-
151-
if not plan_cols:
152-
started = time.perf_counter()
153-
report = CleanReport(
154-
rows_before=len(source),
155-
cols_before=0,
156-
memory_before=self._memory_before(source),
157-
)
158-
report.backend = "duckdb"
159-
cleaned = zero_column_frame("duckdb", engine_config.output_format, report)
160-
finalize_report(report, cleaned, started)
161-
return cleaned, report
162-
163150
reason = plan.fallback_reason or pandas_ingest_fallback_reason(source, self.name)
151+
if reason is None and not plan_cols:
152+
# DuckDB cannot register a frame without columns ("Need a DataFrame
153+
# with at least one column"). The pandas reference keeps the rows and
154+
# the index on a zero-column frame, so disclose the fallback and let
155+
# it produce the result.
156+
reason = "zero-column source"
164157
if reason is None and self._pandas_index_forces_fallback(source):
165158
reason = "pandas index semantics"
166159
if reason is not None:

‎tests/test_execution/test_duckdb_engine.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,42 @@ def test_zero_column_dataframe(native_config):
5858
assert out.shape == (3, 0)
5959

6060

61+
def test_zero_column_dataframe_discloses_the_pandas_fallback(native_config):
62+
df = pd.DataFrame(index=range(3))
63+
out, report = fd.clean(df, config=native_config, engine="duckdb", return_report=True)
64+
assert out.shape == (3, 0)
65+
(event,) = report.fallback_events
66+
assert "zero-column source" in event["fallback_reason"]
67+
68+
69+
def test_zero_column_dataframe_keeps_a_non_range_index(native_config):
70+
# The pandas reference keeps the rows *and* the index labels on a
71+
# zero-column frame; the native backend must agree with it rather than
72+
# rebuilding a RangeIndex.
73+
df = pd.DataFrame(index=["a", "b", "c"])
74+
out = fd.clean(df, config=native_config, engine="duckdb")
75+
reference = fd.clean(df, config=native_config, engine="pandas")
76+
# check_frame_type=False: the pandas engine hands back a CleanResult wrapper
77+
# while the fallback path unwraps to a plain DataFrame. Only the contents matter.
78+
pd.testing.assert_frame_equal(out, reference, check_frame_type=False)
79+
assert list(out.index) == ["a", "b", "c"]
80+
81+
82+
def test_zero_column_dataframe_with_a_native_handle_request(native_config):
83+
# A zero-column source cannot become a DuckDB relation at all; the
84+
# disclosed fallback is what lets a materialized pandas frame through.
85+
out, report = fd.clean(
86+
pd.DataFrame(index=range(3)),
87+
config=native_config,
88+
engine="duckdb",
89+
output_format="duckdb",
90+
return_report=True,
91+
)
92+
assert isinstance(out, pd.DataFrame)
93+
assert out.shape == (3, 0)
94+
assert report.fallback_events
95+
96+
6197
def test_drop_duplicates(native_config):
6298
df = pd.DataFrame({"a": [1, 1, 2, 2, 3], "b": ["x", "x", "y", "y", "z"]})
6399
out = fd.clean(df, config=native_config, engine="duckdb", drop_duplicates=True)

0 commit comments

Comments
 (0)