Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions natsort/compat/fake_fastnumbers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from __future__ import annotations

import unicodedata
from decimal import Decimal, InvalidOperation
from typing import Callable, Union

from natsort.unicode_numbers import decimal_chars
Expand Down Expand Up @@ -34,10 +35,31 @@
ASCII_NUMS = "0123456789+-"
POTENTIAL_FIRST_CHAR = frozenset(decimal_chars + list(ASCII_NUMS + "."))

StrOrFloat = Union[str, float]
StrOrFloat = Union[str, float, Decimal]
StrOrInt = Union[str, int]


def widen_if_overflowed(x: str, ret: float) -> StrOrFloat:
"""
Widen a float that overflowed to +/-inf back into something orderable.

float() silently rounds a finite number with too large a magnitude to
+/-inf instead of raising, so two different numbers like "1e400" and
"1e500" would otherwise both come out as inf and compare equal, losing
their relative order. Decimal has a much larger exponent range, so use
it to keep such values distinguishable. A literal "inf"/"nan" spelling
is left alone as a plain float, since it isn't the result of an overflow.
"""
if ret not in (float("inf"), float("-inf")):

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we pre-define float("inf") and float("-inf") so that we don't have to create this object on each call?

return ret
if x.strip().lstrip("+-")[:3].lower() in ("inf", "nan"):

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use the _NAN_INF set for this.

return ret
try:
return Decimal(x)
except InvalidOperation: # pragma: no cover
return ret


def fast_float(
x: str,
key: Callable[[str], str] = lambda x: x,
Expand Down Expand Up @@ -75,7 +97,7 @@ def fast_float(
except TypeError: # pragma: no cover
return key(x)
else:
return nan if ret != ret else ret
return nan if ret != ret else widen_if_overflowed(x, ret)
else:
try:
return _uni(x, key(x)) if len(x) == 1 else key(x)
Expand Down
69 changes: 50 additions & 19 deletions natsort/compat/fastnumbers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,20 @@
from __future__ import annotations

import re
from decimal import Decimal
from typing import TYPE_CHECKING, Callable, Union

from natsort.compat.fake_fastnumbers import widen_if_overflowed
from natsort.compat.locale import StrOrBytes

if TYPE_CHECKING:
from collections.abc import Iterable, Iterator

StrOrFloat = Union[str, float]
# str/bytes can come back out when on_fail is given and the input could not
# be converted (e.g. groupletters or a locale-aware strxfrm, either of
# which may return bytes). Decimal can come back out when a magnitude was
# too large for a float and got widened to keep its relative order.
StrOrFloat = Union[StrOrBytes, float, Decimal]
StrOrInt = Union[str, int]

__all__ = ["try_float", "try_int"]
Expand Down Expand Up @@ -45,6 +53,16 @@ def ensure_minimum_fastnumbers(fastnumbers_version: str) -> None:

# If the user has fastnumbers installed, they will get great speed
# benefits. If not, we use the simulated functions that come with natsort.
#
# Note that whichever float-conversion primitive we end up with here
# (fast_float, or the newer mapping-capable try_float) is only used as
# the *raw* building block for this module's own try_float below - none
# of them are re-exported directly. That raw primitive rounds a finite
# number with too large a magnitude to +/-inf, same as the float()
# builtin, so on its own it can't tell two overflowing values like
# "1e400" and "1e500" apart. This module's try_float always widens such
# results to Decimal afterwards, so that behavior is consistent no matter
# which backend produced the raw float.
try:
# noinspection PyPackageRequirements
from fastnumbers import __version__ as fn_ver
Expand All @@ -56,30 +74,14 @@ def ensure_minimum_fastnumbers(fastnumbers_version: str) -> None:
# For versions of fastnumbers with mapping capability, use that
if is_supported_fastnumbers(fn_ver, (5, 0, 0)):
del fast_float, fast_int
from fastnumbers import try_float, try_int
from fastnumbers import try_float as _raw_try_float_map
from fastnumbers import try_int
except ImportError:
from natsort.compat.fake_fastnumbers import ( # type: ignore[no-redef]
fast_float,
fast_int,
)

# Re-map the old-or-compatibility functions fast_float/fast_int to the
# newer API of try_float/try_int. If we already imported try_float/try_int
# then there is nothing to do.
if "try_float" not in globals():

def try_float( # type: ignore[no-redef]
x: Iterable[str],
*,
map: bool,
nan: float = float("inf"),
on_fail: Callable[[str], str] = lambda x: x,
) -> Iterator[StrOrFloat]:
"""Attempt to convert a string to a float."""
assert map is True # noqa: S101
return (fast_float(y, nan=nan, key=on_fail) for y in x)


if "try_int" not in globals():

def try_int( # type: ignore[no-redef]
Expand All @@ -91,3 +93,32 @@ def try_int( # type: ignore[no-redef]
"""Attempt to convert a string to an int."""
assert map is True # noqa: S101
return (fast_int(y, key=on_fail) for y in x)


def _raw_try_float(
x: list[str],
*,
nan: float,
on_fail: Callable[[str], StrOrBytes],
) -> Iterable[StrOrFloat]:
"""Convert strings to floats using whichever backend is available."""
if "_raw_try_float_map" in globals():
return _raw_try_float_map(x, nan=nan, on_fail=on_fail, map=True)
return (fast_float(y, nan=nan, key=on_fail) for y in x)


def try_float(
x: Iterable[str],
*,
map: bool,
nan: float = float("inf"),
on_fail: Callable[[str], StrOrBytes] = lambda x: x,
) -> Iterator[StrOrFloat]:
"""Attempt to convert a string to a float."""
assert map is True # noqa: S101
values = list(x)
raw_results = _raw_try_float(values, nan=nan, on_fail=on_fail)
return (
widen_if_overflowed(orig, ret) if isinstance(ret, float) else ret
for orig, ret in zip(values, raw_results)
)
15 changes: 13 additions & 2 deletions natsort/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

import re
from collections.abc import Iterable, Iterator
from decimal import Decimal
from functools import partial, reduce
from itertools import chain as ichain
from operator import methodcaller
Expand Down Expand Up @@ -104,7 +105,9 @@ def __gt__(self, __other: Any) -> bool: ... # noqa: ANN401, D105, PYI063
NumTransformer = Callable[[Any], NumTransform]

# For the string component transform factory
StrBytesNum = Union[str, bytes, float, int]
# Decimal is included because a magnitude too large for a float (e.g.
# "1e500") is widened to Decimal instead of silently collapsing to +/-inf.
StrBytesNum = Union[str, bytes, float, int, Decimal]
StrTransformer = Callable[[Iterable[str]], Iterator[StrBytesNum]]

# For the final data transform factory
Expand Down Expand Up @@ -703,7 +706,15 @@ def string_component_transform_factory(alg: NSType) -> StrTransformer:
kwargs["map"] = True
if alg & ns.FLOAT:
kwargs["nan"] = nan_val
return cast("StrTransformer", partial(try_float, **kwargs))
# The values of kwargs are a heterogeneous union (bool/float/on_fail
# callable), and mypy cannot distribute a dict of that shape across
# try_float's individual keyword-only parameter types when unpacked
# with **kwargs, even though each key always carries a value of the
# correct type for that parameter at runtime.
return cast(
"StrTransformer",
partial(try_float, **cast("dict[str, Any]", kwargs)),
)
return cast("StrTransformer", partial(try_int, **kwargs))


Expand Down
48 changes: 48 additions & 0 deletions tests/test_compat_fastnumbers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Test the fastnumbers compatibility layer."""

from __future__ import annotations

from decimal import Decimal

from natsort.compat.fastnumbers import try_float


def test_try_float_widens_overflowing_magnitude_to_decimal_example() -> None:
# "1e400" and "1e500" both overflow float()'s range to inf, so if the
# backend's raw result were used as-is they would compare equal and
# lose their real order. This must hold regardless of whether the
# real fastnumbers package (any supported version) or natsort's own
# fallback is providing the underlying conversion.
small, large = try_float(["1e400", "1e500"], map=True)
assert isinstance(small, Decimal)
assert isinstance(large, Decimal)
assert small != large
assert small < large


def test_try_float_orders_reversed_overflowing_magnitudes_too() -> None:
# Same as above, but make sure the result doesn't just happen to match
# because of the order the values were given in.
large, small = try_float(["1e500", "1e400"], map=True)
assert isinstance(small, Decimal)
assert isinstance(large, Decimal)
assert small < large


def test_try_float_still_returns_a_plain_float_for_literal_infinity() -> None:
# A string that spells out infinity is not the result of an overflow,
# so it should still come back as a plain float, same as before.
pos_inf, neg_inf = try_float(["inf", "-inf"], map=True)
assert pos_inf == float("inf")
assert neg_inf == float("-inf")
assert isinstance(pos_inf, float)
assert isinstance(neg_inf, float)


def test_try_float_nan_substitution_is_not_treated_as_overflow() -> None:
# The "nan" substitution value can legitimately be +/-inf (depending on
# ns.NANLAST), and that is a deliberate substitution, not an overflow,
# so it must not get widened to Decimal.
(result,) = try_float(["nan"], map=True, nan=float("-inf"))
assert result == float("-inf")
assert isinstance(result, float)
21 changes: 21 additions & 0 deletions tests/test_fake_fastnumbers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from __future__ import annotations

import unicodedata
from decimal import Decimal
from math import isinf
from typing import cast

Expand Down Expand Up @@ -77,6 +78,26 @@ def test_fast_float_converts_float_string_to_float(x: float) -> None:
assert fast_float(repr(x)) == x


def test_fast_float_widens_overflowing_magnitude_to_decimal_example() -> None:
# "1e400" and "1e500" both overflow float()'s range to inf, so on
# their own they would compare equal and lose their real order.
# Widening the result to Decimal keeps them distinguishable.
small = fast_float("1e400")
large = fast_float("1e500")
assert isinstance(small, Decimal)
assert isinstance(large, Decimal)
assert small != large
assert small < large


def test_fast_float_still_returns_a_plain_float_for_literal_infinity() -> None:
# A string that spells out infinity is not the result of an overflow,
# so it should still come back as a plain float, same as before.
assert fast_float("inf") == float("inf")
assert fast_float("-inf") == float("-inf")
assert isinstance(fast_float("inf"), float)


def test_fast_float_leaves_string_as_is_example() -> None:
assert fast_float("invalid") == "invalid"
assert fast_float("") == ""
Expand Down
10 changes: 10 additions & 0 deletions tests/test_natsorted.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ def test_natsorted_can_sort_as_signed_floats_with_exponents(
assert natsorted(float_list, alg=ns.REAL) == expected


def test_natsorted_orders_magnitudes_too_large_for_a_float() -> None:
# "1e400" and "1e500" both overflow to inf if parsed with a plain
# float(), so they used to compare equal and the result depended on
# which order they were given in rather than their actual size.
given = ["1e400", "1e500"]
expected = ["1e400", "1e500"]
assert natsorted(given, alg=ns.REAL) == expected
assert natsorted(reversed(given), alg=ns.REAL) == expected


@pytest.mark.parametrize(
# UNSIGNED is default
"alg",
Expand Down