Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ Unreleased

### Fixed

- Recognize secondary digit grouping in locales such as Indian English when
sorting numbers with `LOCALENUM` or `LOCALE`.
- Recognize locale thousands separators when `COMPATIBILITYNORMALIZE` is enabled,
including non-breaking spaces used by French locales.
- CLI no longer strips leading and trailing whitespace from entries, which
Expand Down
8 changes: 8 additions & 0 deletions mypy_stubs/icu.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ class DecimalFormatSymbols:
def __init__(self, locale: str) -> None: ...
def getSymbol(self, symbol: int) -> str: ...

class NumberFormat:
@classmethod
def createInstance(cls, locale: str) -> NumberFormat: ...

class DecimalFormat(NumberFormat):
def getGroupingSize(self) -> int: ...
def getSecondaryGroupingSize(self) -> int: ...

class Collator:
@classmethod
def createInstance(cls, locale: str) -> Collator: ...
Expand Down
21 changes: 21 additions & 0 deletions natsort/compat/locale.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,16 @@ def get_decimal_point() -> str:
sep = icu.DecimalFormatSymbols.kDecimalSeparatorSymbol
return icu.DecimalFormatSymbols(get_icu_locale()).getSymbol(sep)

def get_grouping() -> tuple[int, int]:
"""Return the primary and secondary digit group sizes for this locale."""
formatter = icu.NumberFormat.createInstance(get_icu_locale())
if not isinstance(formatter, icu.DecimalFormat):
return 3, 3
primary = formatter.getGroupingSize()
if primary <= 0:
return 3, 3
return primary, formatter.getSecondaryGroupingSize() or primary

except ImportError:
import locale
from locale import strxfrm
Expand Down Expand Up @@ -132,3 +142,14 @@ def get_thousands_sep() -> str:
def get_decimal_point() -> str:
"""Return the appropriate decimal point for this locale."""
return locale.localeconv()["decimal_point"]

def get_grouping() -> tuple[int, int]:
"""Return the primary and secondary digit group sizes for this locale."""
grouping = locale.localeconv()["grouping"]
if not grouping or not 0 < grouping[0] < locale.CHAR_MAX:
return 3, 3
primary = grouping[0]
secondary = grouping[1] if len(grouping) > 1 else 0
if not 0 < secondary < locale.CHAR_MAX:
secondary = primary
return primary, secondary
15 changes: 11 additions & 4 deletions natsort/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
from natsort.compat.locale import (
StrOrBytes,
get_decimal_point,
get_grouping,
get_strxfrm,
get_thousands_sep,
)
Expand Down Expand Up @@ -632,9 +633,7 @@ def input_string_transform_factory(alg: NSType) -> StrToStr:
(?<![0-9]{{4}}) # No more than 3 numbers
{nodecimal} # Cannot follow decimal
{thou} # The thousands separator
(?=[0-9]{{3}} # Three numbers must follow
([^0-9]|$) # But a non-number after that
)
(?={grouped_digits}) # A complete digit group must follow
"""
nodecimal = r""
if alg & ns.FLOAT:
Expand All @@ -646,9 +645,17 @@ def input_string_transform_factory(alg: NSType) -> StrToStr:
nodecimal += r"(?<!" + d + r"[0-9]{3})"
# Input is already normalized when this expression is applied.
thousands_sep = _normalize_input_factory(alg)(get_thousands_sep())
escaped_sep = re.escape(thousands_sep)
grouped_digits = r"[0-9]{3}([^0-9]|$)"
primary, secondary = get_grouping()
if thousands_sep and secondary != primary:
# Keep three-digit groups accepted, but also recognize secondary
# groups before another separator (e.g. 1,23,45,678 in en_IN).
grouped_digits += rf"|[0-9]{{{secondary}}}{escaped_sep}"
strip_thousands = strip_thousands.format(
thou=re.escape(thousands_sep),
thou=escaped_sep,
nodecimal=nodecimal,
grouped_digits=grouped_digits,
)
strip_thousands_re = re.compile(strip_thousands, flags=re.VERBOSE)
function_chain.append(partial(strip_thousands_re.sub, ""))
Expand Down
14 changes: 14 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,20 @@ def with_locale_de_de() -> Iterator[None]:
locale.setlocale(locale.LC_ALL, orig)


@pytest.fixture
def with_locale_en_in() -> Iterator[None]:
"""Load the Indian English locale, skipping if it is unavailable."""
orig = locale.getlocale()
try:
load_locale("en_IN")
except locale.Error:
pytest.skip("requires en_IN locale to be installed")
else:
yield
finally:
locale.setlocale(locale.LC_ALL, orig)


@pytest.fixture
def with_locale_cs_cz() -> Iterator[None]:
"""
Expand Down
73 changes: 73 additions & 0 deletions tests/test_compat_locale.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import importlib.util
import locale
import sys
import types
from typing import TYPE_CHECKING
Expand Down Expand Up @@ -43,3 +44,75 @@ def test_icu_null_string_locale_max_sorts_after_any_bytes(
# after every sort key PyICU can return.
assert icu_locale_module.null_string_locale_max > b"\xff" * 49
assert icu_locale_module.null_string_locale_max > b"\xfe" * 100


@pytest.fixture
def stdlib_locale_module(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType:
"""Load the standard-library backend even when PyICU is installed."""
monkeypatch.setitem(sys.modules, "icu", None)
spec = importlib.util.spec_from_file_location(
"_stdlib_locale", natsort.compat.locale.__file__
)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module


@pytest.mark.parametrize(
("grouping", "expected"),
[
([], (3, 3)),
([0], (3, 3)),
([locale.CHAR_MAX], (3, 3)),
([3], (3, 3)),
([3, 0], (3, 3)),
([3, locale.CHAR_MAX], (3, 3)),
([3, 3, 0], (3, 3)),
([3, 2, 0], (3, 2)),
],
)
def test_get_grouping_stdlib(
grouping: list[int],
expected: tuple[int, int],
stdlib_locale_module: types.ModuleType,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(locale, "localeconv", lambda: {"grouping": grouping})
assert stdlib_locale_module.get_grouping() == expected


@pytest.mark.parametrize(
("primary", "secondary", "expected"),
[(3, 2, (3, 2)), (3, 0, (3, 3)), (3, 3, (3, 3)), (0, 0, (3, 3))],
)
def test_get_grouping_icu(
primary: int,
secondary: int,
expected: tuple[int, int],
icu_locale_module: types.ModuleType,
monkeypatch: pytest.MonkeyPatch,
) -> None:
formatter = types.SimpleNamespace(
getGroupingSize=lambda: primary,
getSecondaryGroupingSize=lambda: secondary,
)
icu_locale_module.icu.DecimalFormat = types.SimpleNamespace
icu_locale_module.icu.NumberFormat = types.SimpleNamespace(
createInstance=lambda _: formatter
)
monkeypatch.setattr(icu_locale_module, "get_icu_locale", lambda: "en_IN")
assert icu_locale_module.get_grouping() == expected


def test_get_grouping_icu_falls_back_for_non_decimal_format(
icu_locale_module: types.ModuleType,
monkeypatch: pytest.MonkeyPatch,
) -> None:
icu_locale_module.icu.DecimalFormat = types.SimpleNamespace
icu_locale_module.icu.NumberFormat = types.SimpleNamespace(
createInstance=lambda _: object()
)
monkeypatch.setattr(icu_locale_module, "get_icu_locale", lambda: "en_IN")
assert icu_locale_module.get_grouping() == (3, 3)
27 changes: 27 additions & 0 deletions tests/test_input_string_transform_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,33 @@ def test_input_string_transform_factory_handles_us_locale(
assert input_string_transform_func(x) == expected


@pytest.mark.parametrize(
("x", "expected"),
[
("1,00,000", "100000"),
("12,34,567", "1234567"),
("1,23,45,678", "12345678"),
("-12,34,567.89", "-1234567.89"),
("1,23,456e2", "123456e2"),
("value12,34,567.txt", "value1234567.txt"),
("12,345", "12345"),
("1,234,567", "1234567"),
("12,3456", "12,3456"),
("12.34,567", "12.34,567"),
],
)
def test_input_string_transform_factory_handles_secondary_grouping(
x: str,
expected: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr("natsort.utils.get_thousands_sep", lambda: ",")
monkeypatch.setattr("natsort.utils.get_decimal_point", lambda: ".")
monkeypatch.setattr("natsort.utils.get_grouping", lambda: (3, 2), raising=False)
transform = input_string_transform_factory(ns.LOCALENUM | ns.FLOAT)
assert transform(x) == expected


@pytest.mark.parametrize(
("x", "expected"),
[
Expand Down
9 changes: 9 additions & 0 deletions tests/test_natsorted_convenience.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,15 @@ def test_humansorted_is_identical_to_natsorted_with_locale_alg(
assert humansorted(fruit_list) == natsorted(fruit_list, alg=ns.LOCALE)


@pytest.mark.parametrize(
"alg", [ns.LOCALENUM, ns.LOCALE, ns.LOCALENUM | ns.FLOAT, ns.LOCALE | ns.FLOAT]
)
@pytest.mark.usefixtures("with_locale_en_in")
def test_natsorted_respects_secondary_locale_grouping(alg: int) -> None:
values = ["12,34,567", "20,000", "1,00,000", "1,234"]
assert natsorted(values, alg=alg) == ["1,234", "20,000", "1,00,000", "12,34,567"]


def test_index_natsorted_returns_integer_list_of_sort_order_for_input_list() -> None:
given = ["num3", "num5", "num2"]
other = ["foo", "bar", "baz"]
Expand Down