diff --git a/dataframely/_deprecation.py b/dataframely/_deprecation.py index 82367834..d8378153 100644 --- a/dataframely/_deprecation.py +++ b/dataframely/_deprecation.py @@ -1,27 +1,58 @@ # Copyright (c) QuantCo 2025-2026 # SPDX-License-Identifier: BSD-3-Clause -import os -from collections.abc import Callable -from functools import wraps +from __future__ import annotations -TRUTHY_VALUES = ["1", "true"] +import sys +import warnings +from functools import wraps +from typing import TYPE_CHECKING, TypeVar +if TYPE_CHECKING: + from collections.abc import Callable + from typing import ParamSpec -def skip_if(env: str) -> Callable: - """Decorator to skip warnings based on environment variable. + P = ParamSpec("P") + T = TypeVar("T") - If the environment variable is equivalent to any of TRUTHY_VALUES, the wrapped - function is skipped. - """ - def decorator(fun: Callable) -> Callable: - @wraps(fun) - def wrapper() -> None: - if os.getenv(env, "").lower() in TRUTHY_VALUES: - return - fun() +def issue_deprecation_warning(message: str, *, version: str = "") -> None: + """Issue a deprecation warning pointing at the caller of the deprecated method. - return wrapper + This must be called directly from the body of the deprecated (public) method so + that the warning points at the user's code rather than at dataframely internals. - return decorator + Args: + message: The message associated with the warning. + version: The dataframely version in which the deprecation occurred (if not + already part of ``message``). + """ + if version: + message = f"{message.strip()}\n(Deprecated in dataframely {version})" + # `stacklevel=2` blames the caller of the deprecated method (one frame up from this + # function). All call sites invoke this directly from the deprecated method body. + warnings.warn(message, DeprecationWarning, stacklevel=2) + + +if sys.version_info >= (3, 13): + from warnings import deprecated +else: + try: + from typing_extensions import deprecated + except ImportError: # pragma: no cover + + def deprecated( # type: ignore[no-redef] + message: str, + ) -> Callable[[Callable[P, T]], Callable[P, T]]: + """Fallback for :func:`warnings.deprecated` without :pep:`702` support.""" + + def decorate(function: Callable[P, T]) -> Callable[P, T]: + @wraps(function) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + issue_deprecation_warning(message) + return function(*args, **kwargs) + + wrapper.__deprecated__ = message # type: ignore[attr-defined] + return wrapper + + return decorate diff --git a/dataframely/collection/collection.py b/dataframely/collection/collection.py index 900f3fb8..cf59836e 100644 --- a/dataframely/collection/collection.py +++ b/dataframely/collection/collection.py @@ -28,6 +28,7 @@ import polars.exceptions as plexc from dataframely._compat import deltalake +from dataframely._deprecation import deprecated, issue_deprecation_warning from dataframely._filter import Filter from dataframely._native import format_rule_failures from dataframely._plugin import all_rules_required @@ -63,6 +64,15 @@ _FILTER_COLUMN_PREFIX = "__DATAFRAMELY_FILTER_COLUMN__" +#: Deprecation message emitted when reading a collection with implicit validation, i.e. +#: with any ``validation`` other than ``"skip"`` (see #367). +_IMPLICIT_VALIDATION_DEPRECATION = ( + "Reading a collection with `validation != 'skip'` is deprecated. Starting with " + "dataframely v3, data is read without inspecting schema metadata and without " + "running validation. Pass `validation='skip'` to opt into the future behavior, or " + "call `validate` explicitly if you require validation." +) + P = ParamSpec("P") T = TypeVar("T") @@ -1061,7 +1071,16 @@ def read_parquet( Attention: Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + Reading with `validation != "skip"` is deprecated. Starting with + dataframely v3, this method reads the data without inspecting any schema + metadata and without running validation. Pass `validation="skip"` to opt + into this behavior, or call :meth:`validate` explicitly if you require + validation. """ + if validation != "skip": + issue_deprecation_warning(_IMPLICIT_VALIDATION_DEPRECATION) return cls._read( backend=ParquetStorageBackend(), validation=validation, @@ -1118,7 +1137,16 @@ def scan_parquet( Attention: Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + Reading with `validation != "skip"` is deprecated. Starting with + dataframely v3, this method reads the data without inspecting any schema + metadata and without running validation. Pass `validation="skip"` to opt + into this behavior, or call :meth:`validate` explicitly if you require + validation. """ + if validation != "skip": + issue_deprecation_warning(_IMPLICIT_VALIDATION_DEPRECATION) return cls._read( backend=ParquetStorageBackend(), validation=validation, @@ -1127,6 +1155,10 @@ def scan_parquet( **kwargs, ) + @deprecated( + "`Collection.write_delta` is deprecated and will be removed in dataframely v3. " + "Write the individual members with `polars.DataFrame.write_delta` instead." + ) def write_delta( self, target: str | Path | deltalake.DeltaTable, **kwargs: Any ) -> None: @@ -1153,6 +1185,10 @@ def write_delta( break your schema. This method suffers from the same limitations as :meth:`~dataframely.Schema.serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Write the + individual members with :meth:`polars.DataFrame.write_delta` instead. """ self._write( backend=DeltaStorageBackend(), @@ -1161,6 +1197,11 @@ def write_delta( ) @classmethod + @deprecated( + "`Collection.scan_delta` is deprecated and will be removed in dataframely v3. " + "Read the individual members with `polars.scan_delta` and call `validate` " + "explicitly instead." + ) def scan_delta( cls, source: str | Path | deltalake.DeltaTable, @@ -1214,6 +1255,11 @@ def scan_delta( break your schema. Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Read the + individual members with :meth:`polars.scan_delta` and call :meth:`validate` + explicitly instead. """ return cls._read( backend=DeltaStorageBackend(), @@ -1223,6 +1269,11 @@ def scan_delta( ) @classmethod + @deprecated( + "`Collection.read_delta` is deprecated and will be removed in dataframely v3. " + "Read the individual members with `polars.read_delta` and call `validate` " + "explicitly instead." + ) def read_delta( cls, source: str | Path | deltalake.DeltaTable, @@ -1275,6 +1326,11 @@ def read_delta( break your schema. Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Read the + individual members with :meth:`polars.read_delta` and call :meth:`validate` + explicitly instead. """ return cls._read( backend=DeltaStorageBackend(), diff --git a/dataframely/schema.py b/dataframely/schema.py index 9d2e04e8..dab8d151 100644 --- a/dataframely/schema.py +++ b/dataframely/schema.py @@ -18,6 +18,7 @@ from ._base_schema import ORIGINAL_COLUMN_PREFIX, BaseSchema from ._compat import PartitionSchemeOrSinkDirectory, deltalake, pa, pydantic, sa +from ._deprecation import deprecated from ._match_to_schema import match_to_schema from ._native import format_rule_failures from ._plugin import all_rules, all_rules_horizontal, all_rules_required @@ -897,6 +898,10 @@ def _as_dict(cls) -> dict[str, Any]: # ------------------------------------ PARQUET ----------------------------------- # @classmethod + @deprecated( + "`Schema.write_parquet` is deprecated and will be removed in dataframely v3. " + "Use `polars.DataFrame.write_parquet` directly instead." + ) def write_parquet( cls, df: DataFrame[Self], /, file: str | Path | IO[bytes], **kwargs: Any ) -> None: @@ -919,10 +924,18 @@ def write_parquet( Attention: Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.DataFrame.write_parquet` directly instead. """ cls._write(df=df, backend=ParquetStorageBackend(), file=file, **kwargs) @classmethod + @deprecated( + "`Schema.sink_parquet` is deprecated and will be removed in dataframely v3. " + "Use `polars.LazyFrame.sink_parquet` directly instead." + ) def sink_parquet( cls, lf: LazyFrame[Self], @@ -947,10 +960,18 @@ def sink_parquet( Attention: Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.LazyFrame.sink_parquet` directly instead. """ cls._sink(lf=lf, backend=ParquetStorageBackend(), file=file, **kwargs) @classmethod + @deprecated( + "`Schema.read_parquet` is deprecated and will be removed in dataframely v3. " + "Use `polars.read_parquet` and call `validate` explicitly instead." + ) def read_parquet( cls, source: FileSource, @@ -997,6 +1018,10 @@ def read_parquet( Attention: Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.read_parquet` and call :meth:`validate` explicitly instead. """ return cls._read( ParquetStorageBackend(), @@ -1007,6 +1032,10 @@ def read_parquet( ) @classmethod + @deprecated( + "`Schema.scan_parquet` is deprecated and will be removed in dataframely v3. " + "Use `polars.scan_parquet` and call `validate` explicitly instead." + ) def scan_parquet( cls, source: FileSource, @@ -1053,6 +1082,10 @@ def scan_parquet( Attention: Be aware that this method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.scan_parquet` and call :meth:`validate` explicitly instead. """ return cls._read( ParquetStorageBackend(), @@ -1099,6 +1132,10 @@ def _requires_validation_for_reading_parquet( # --------------------------------- Delta -----------------------------------------# @classmethod + @deprecated( + "`Schema.write_delta` is deprecated and will be removed in dataframely v3. " + "Use `polars.DataFrame.write_delta` directly instead." + ) def write_delta( cls, df: DataFrame[Self], @@ -1127,6 +1164,10 @@ def write_delta( in violation of group constraints that dataframely cannot catch without re-validating. Only use appends if you are certain that they do not break your schema. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.DataFrame.write_delta` directly instead. """ DeltaStorageBackend().write_frame( df=df, @@ -1135,6 +1176,10 @@ def write_delta( ) @classmethod + @deprecated( + "`Schema.scan_delta` is deprecated and will be removed in dataframely v3. " + "Use `polars.scan_delta` and call `validate` explicitly instead." + ) def scan_delta( cls, source: str | Path | deltalake.DeltaTable, @@ -1182,6 +1227,10 @@ def scan_delta( that are not through dataframely will result in losing the metadata. This method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.scan_delta` and call :meth:`validate` explicitly instead. """ return cls._read( DeltaStorageBackend(), @@ -1192,6 +1241,10 @@ def scan_delta( ) @classmethod + @deprecated( + "`Schema.read_delta` is deprecated and will be removed in dataframely v3. " + "Use `polars.read_delta` and call `validate` explicitly instead." + ) def read_delta( cls, source: str | Path | deltalake.DeltaTable, @@ -1244,6 +1297,10 @@ def read_delta( break your schema. This method suffers from the same limitations as :meth:`serialize`. + + .. deprecated:: 3.0.0 + This method is deprecated and will be removed in dataframely v3. Use + :meth:`polars.read_delta` and call :meth:`validate` explicitly instead. """ return cls._read( DeltaStorageBackend(), diff --git a/docs/guides/features/serialization.md b/docs/guides/features/serialization.md index 7e7296ac..e22a7f4d 100644 --- a/docs/guides/features/serialization.md +++ b/docs/guides/features/serialization.md @@ -1,5 +1,27 @@ # Serialization +```{warning} +Most of the I/O functionality described on this page is **deprecated** and will be +removed in dataframely v3 (see [#367](https://github.com/Quantco/dataframely/issues/367)). +Calling any of these methods now emits a {class}`DeprecationWarning`. Specifically: + +- All I/O methods on {class}`~dataframely.Schema` + ({meth}`~dataframely.Schema.write_parquet`, {meth}`~dataframely.Schema.sink_parquet`, + {meth}`~dataframely.Schema.read_parquet`, {meth}`~dataframely.Schema.scan_parquet`, + {meth}`~dataframely.Schema.write_delta`, {meth}`~dataframely.Schema.read_delta`, + {meth}`~dataframely.Schema.scan_delta`) are deprecated. Use the corresponding + `polars` functions directly and call {meth}`~dataframely.Schema.validate` explicitly + where validation is required. +- The `deltalake` I/O methods on {class}`~dataframely.Collection` + ({meth}`~dataframely.Collection.write_delta`, {meth}`~dataframely.Collection.read_delta`, + {meth}`~dataframely.Collection.scan_delta`) are deprecated. +- {meth}`~dataframely.Collection.read_parquet` and + {meth}`~dataframely.Collection.scan_parquet` continue to exist, but reading with + `validation != "skip"` is deprecated: metadata will no longer be inspected and + validation will no longer run implicitly. Pass `validation="skip"` to opt into the + future behavior, or call {meth}`~dataframely.Collection.validate` explicitly. +``` + `dataframely` provides support for easily storing and reading validated data. `polars` already provides native support for serializing data frames into different storage backends. For the storage of the data itself, `dataframely` usually dispatches to polars-native diff --git a/docs/guides/migration/index.md b/docs/guides/migration/index.md index 167d06c7..f32c0ffc 100644 --- a/docs/guides/migration/index.md +++ b/docs/guides/migration/index.md @@ -26,17 +26,16 @@ As always, automated testing is useful here, but we also recommend checking the [published on GitHub](https://github.com/Quantco/dataframely/releases). In order to give users a heads-up before breaking changes are released, we introduce -[FutureWarnings](https://docs.python.org/3/library/exceptions.html#FutureWarning). +[DeprecationWarnings](https://docs.python.org/3/library/exceptions.html#DeprecationWarning). Warnings are the most direct and effective tool at our disposal for reaching users directly. We therefore generally recommend that users do not silence such warnings explicitly, but instead migrate their code proactively, whenever possible. However, we also understand that the need for migration may catch users at an inconvenient time, and a temporary band aid solution might be required. -Users can disable `FutureWarnings` either through -[python builtins](https://docs.python.org/3/library/warnings.html#warnings.filterwarnings), -builtins from tools -like [pytest](https://docs.pytest.org/en/stable/how-to/capture-warnings.html#controlling-warnings), -or by setting the `DATAFRAMELY_NO_FUTURE_WARNINGS` environment variable to `true` or `1`. +Users can disable `DeprecationWarnings` either through +[python builtins](https://docs.python.org/3/library/warnings.html#warnings.filterwarnings) +or through builtins from tools +like [pytest](https://docs.pytest.org/en/stable/how-to/capture-warnings.html#controlling-warnings). ## Experimental features diff --git a/pyproject.toml b/pyproject.toml index 87093b1e..f1b2c256 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,10 +107,13 @@ addopts = "--import-mode=importlib --benchmark-skip -m 'not s3'" filterwarnings = [ # By default, all warnings should yield errors "error", - # Almost all tests are oblivious to the value of `nullable`. Let's ignore the warning as long as it exists. - "ignore:The 'nullable' argument was not explicitly set:FutureWarning", # boto3 still uses .utcnow() "ignore::DeprecationWarning:botocore.*:", + # I/O deprecation warnings (see #367) are asserted in dedicated tests via + # `pytest.warns`; silence them elsewhere so existing tests keep exercising the + # still-present functionality. + "ignore:.*Reading a collection with .validation.*:DeprecationWarning", + "ignore:.*deprecated and will be removed in dataframely v3.*:DeprecationWarning", ] markers = [ "s3: tests that run against and S3 backend", diff --git a/tests/test_deprecation.py b/tests/test_deprecation.py index a9ba658a..eaff3360 100644 --- a/tests/test_deprecation.py +++ b/tests/test_deprecation.py @@ -1,24 +1,149 @@ # Copyright (c) QuantCo 2025-2026 # SPDX-License-Identifier: BSD-3-Clause +import warnings +from pathlib import Path + +import polars as pl import pytest -from dataframely._deprecation import skip_if +import dataframely as dy +from dataframely._deprecation import deprecated, issue_deprecation_warning + +# ------------------------------------ COMMON ------------------------------------- # + + +def test_issue_deprecation_warning() -> None: + with pytest.warns(DeprecationWarning, match="my message"): + issue_deprecation_warning("my message") + + +def test_issue_deprecation_warning_with_version() -> None: + with pytest.warns(DeprecationWarning, match=r"Deprecated in dataframely 3\.0\.0"): + issue_deprecation_warning("my message", version="3.0.0") + + +def test_issue_deprecation_warning_points_at_caller() -> None: + # The warning should point at this test module, not at dataframely internals. + with pytest.warns(DeprecationWarning) as record: + issue_deprecation_warning("my message") + assert record[0].filename == __file__ + + +def test_deprecated_decorator_warns_and_calls() -> None: + @deprecated("`foo` is deprecated.") + def foo(x: int) -> int: + return x + 1 + + with pytest.warns(DeprecationWarning, match="`foo` is deprecated"): + assert foo(1) == 2 + + +# ------------------------------------ SCHEMA ------------------------------------- # + + +class MySchema(dy.Schema): + a = dy.Int64() + + +@pytest.mark.parametrize("method", ["write_parquet", "sink_parquet"]) +def test_schema_write_parquet_deprecated(tmp_path: Path, method: str) -> None: + df = MySchema.create_empty() + frame = df.lazy() if method == "sink_parquet" else df + with pytest.warns(DeprecationWarning, match=f"Schema.{method}"): + getattr(MySchema, method)(frame, tmp_path / "df.parquet") + + +@pytest.mark.parametrize("method", ["read_parquet", "scan_parquet"]) +def test_schema_read_parquet_deprecated(tmp_path: Path, method: str) -> None: + # Arrange: write with plain polars to avoid the write deprecation warning. + file = tmp_path / "df.parquet" + pl.DataFrame({"a": []}, schema={"a": pl.Int64}).write_parquet(file) + + # Act / Assert + with pytest.warns(DeprecationWarning, match=f"Schema.{method}"): + getattr(MySchema, method)(file, validation="skip") + + +def test_schema_write_delta_deprecated(tmp_path: Path) -> None: + pytest.importorskip("deltalake") + with pytest.warns(DeprecationWarning, match="Schema.write_delta"): + MySchema.write_delta(MySchema.create_empty(), str(tmp_path / "table")) + + +@pytest.mark.parametrize("method", ["read_delta", "scan_delta"]) +def test_schema_read_delta_deprecated(tmp_path: Path, method: str) -> None: + pytest.importorskip("deltalake") + target = str(tmp_path / "table") + # Write with the (deprecated) writer, silencing its warning. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + MySchema.write_delta(MySchema.create_empty(), target) + + with pytest.warns(DeprecationWarning, match=f"Schema.{method}"): + getattr(MySchema, method)(target, validation="skip") + + +# ---------------------------------- COLLECTION ----------------------------------- # + + +class MyCollection(dy.Collection): + first: dy.LazyFrame[MySchema] + + +def _empty_collection() -> MyCollection: + return MyCollection.create_empty() + + +@pytest.mark.parametrize("method", ["read_parquet", "scan_parquet"]) +@pytest.mark.parametrize("validation", ["allow", "warn", "forbid"]) +def test_collection_read_parquet_implicit_validation_deprecated( + tmp_path: Path, method: str, validation: str +) -> None: + # Arrange + _empty_collection().write_parquet(tmp_path) + + # Act / Assert: implicit validation is deprecated for all but "skip". + with pytest.warns(DeprecationWarning, match="validation != 'skip'"): + getattr(MyCollection, method)(tmp_path, validation=validation) + + +@pytest.mark.parametrize("method", ["read_parquet", "scan_parquet"]) +def test_collection_read_parquet_skip_not_deprecated( + tmp_path: Path, method: str +) -> None: + # Arrange + _empty_collection().write_parquet(tmp_path) + + # Act / Assert: `validation="skip"` does not warn. + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + getattr(MyCollection, method)(tmp_path, validation="skip") + + +@pytest.mark.parametrize("method", ["write_parquet", "sink_parquet"]) +def test_collection_write_parquet_not_deprecated(tmp_path: Path, method: str) -> None: + # Act / Assert: writing parquet collections is retained without warning. + collection = _empty_collection() + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + getattr(collection, method)(tmp_path) -# ------------------------- Common ---------------------------------# +def test_collection_write_delta_deprecated(tmp_path: Path) -> None: + pytest.importorskip("deltalake") + with pytest.warns(DeprecationWarning, match="Collection.write_delta"): + _empty_collection().write_delta(str(tmp_path / "table")) -@pytest.mark.parametrize("env_var", ["1", "True", "true"]) -def test_skip_if(monkeypatch: pytest.MonkeyPatch, env_var: str) -> None: - """The skip_if decorator should allow us to prevent execution of a wrapped - function.""" - variable_name = "DATAFRAMELY_NO_FUTURE_WARNINGS" - @skip_if(variable_name) - def callable() -> None: - raise ValueError() +@pytest.mark.parametrize("method", ["read_delta", "scan_delta"]) +def test_collection_read_delta_deprecated(tmp_path: Path, method: str) -> None: + pytest.importorskip("deltalake") + target = str(tmp_path / "table") + # Write with the (deprecated) writer, silencing its warning. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + _empty_collection().write_delta(target) - with pytest.raises(ValueError): - callable() - monkeypatch.setenv(variable_name, env_var) - callable() + with pytest.warns(DeprecationWarning, match=f"Collection.{method}"): + getattr(MyCollection, method)(target, validation="skip")