From fe29adb1c406d874adf11200c3bc0b1e58741e90 Mon Sep 17 00:00:00 2001 From: KennyMcSimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:15:56 +0800 Subject: [PATCH] Respect secondary digit grouping in locale-aware sorting --- CHANGELOG.md | 2 + mypy_stubs/icu.pyi | 8 +++ natsort/compat/locale.py | 21 ++++++ natsort/utils.py | 15 ++-- tests/conftest.py | 14 ++++ tests/test_compat_locale.py | 73 ++++++++++++++++++++ tests/test_input_string_transform_factory.py | 27 ++++++++ tests/test_natsorted_convenience.py | 9 +++ 8 files changed, 165 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e118995..93a1aec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ Unreleased ### Fixed +- Recognize secondary digit grouping in locales such as Indian English when + sorting numbers with `LOCALENUM` or `LOCALE`. - Recognize locale thousands separators when `COMPATIBILITYNORMALIZE` is enabled, including non-breaking spaces used by French locales. - CLI no longer strips leading and trailing whitespace from entries, which diff --git a/mypy_stubs/icu.pyi b/mypy_stubs/icu.pyi index f7ac204..285dd6a 100644 --- a/mypy_stubs/icu.pyi +++ b/mypy_stubs/icu.pyi @@ -17,6 +17,14 @@ class DecimalFormatSymbols: def __init__(self, locale: str) -> None: ... def getSymbol(self, symbol: int) -> str: ... +class NumberFormat: + @classmethod + def createInstance(cls, locale: str) -> NumberFormat: ... + +class DecimalFormat(NumberFormat): + def getGroupingSize(self) -> int: ... + def getSecondaryGroupingSize(self) -> int: ... + class Collator: @classmethod def createInstance(cls, locale: str) -> Collator: ... diff --git a/natsort/compat/locale.py b/natsort/compat/locale.py index 6713fcc..c7b85e6 100644 --- a/natsort/compat/locale.py +++ b/natsort/compat/locale.py @@ -63,6 +63,16 @@ def get_decimal_point() -> str: sep = icu.DecimalFormatSymbols.kDecimalSeparatorSymbol return icu.DecimalFormatSymbols(get_icu_locale()).getSymbol(sep) + def get_grouping() -> tuple[int, int]: + """Return the primary and secondary digit group sizes for this locale.""" + formatter = icu.NumberFormat.createInstance(get_icu_locale()) + if not isinstance(formatter, icu.DecimalFormat): + return 3, 3 + primary = formatter.getGroupingSize() + if primary <= 0: + return 3, 3 + return primary, formatter.getSecondaryGroupingSize() or primary + except ImportError: import locale from locale import strxfrm @@ -132,3 +142,14 @@ def get_thousands_sep() -> str: def get_decimal_point() -> str: """Return the appropriate decimal point for this locale.""" return locale.localeconv()["decimal_point"] + + def get_grouping() -> tuple[int, int]: + """Return the primary and secondary digit group sizes for this locale.""" + grouping = locale.localeconv()["grouping"] + if not grouping or not 0 < grouping[0] < locale.CHAR_MAX: + return 3, 3 + primary = grouping[0] + secondary = grouping[1] if len(grouping) > 1 else 0 + if not 0 < secondary < locale.CHAR_MAX: + secondary = primary + return primary, secondary diff --git a/natsort/utils.py b/natsort/utils.py index 012a03a..7bd3cf7 100644 --- a/natsort/utils.py +++ b/natsort/utils.py @@ -61,6 +61,7 @@ from natsort.compat.locale import ( StrOrBytes, get_decimal_point, + get_grouping, get_strxfrm, get_thousands_sep, ) @@ -632,9 +633,7 @@ def input_string_transform_factory(alg: NSType) -> StrToStr: (? StrToStr: nodecimal += r"(? Iterator[None]: locale.setlocale(locale.LC_ALL, orig) +@pytest.fixture +def with_locale_en_in() -> Iterator[None]: + """Load the Indian English locale, skipping if it is unavailable.""" + orig = locale.getlocale() + try: + load_locale("en_IN") + except locale.Error: + pytest.skip("requires en_IN locale to be installed") + else: + yield + finally: + locale.setlocale(locale.LC_ALL, orig) + + @pytest.fixture def with_locale_cs_cz() -> Iterator[None]: """ diff --git a/tests/test_compat_locale.py b/tests/test_compat_locale.py index f0f9f1e..fd6e30f 100644 --- a/tests/test_compat_locale.py +++ b/tests/test_compat_locale.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import locale import sys import types from typing import TYPE_CHECKING @@ -43,3 +44,75 @@ def test_icu_null_string_locale_max_sorts_after_any_bytes( # after every sort key PyICU can return. assert icu_locale_module.null_string_locale_max > b"\xff" * 49 assert icu_locale_module.null_string_locale_max > b"\xfe" * 100 + + +@pytest.fixture +def stdlib_locale_module(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + """Load the standard-library backend even when PyICU is installed.""" + monkeypatch.setitem(sys.modules, "icu", None) + spec = importlib.util.spec_from_file_location( + "_stdlib_locale", natsort.compat.locale.__file__ + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("grouping", "expected"), + [ + ([], (3, 3)), + ([0], (3, 3)), + ([locale.CHAR_MAX], (3, 3)), + ([3], (3, 3)), + ([3, 0], (3, 3)), + ([3, locale.CHAR_MAX], (3, 3)), + ([3, 3, 0], (3, 3)), + ([3, 2, 0], (3, 2)), + ], +) +def test_get_grouping_stdlib( + grouping: list[int], + expected: tuple[int, int], + stdlib_locale_module: types.ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(locale, "localeconv", lambda: {"grouping": grouping}) + assert stdlib_locale_module.get_grouping() == expected + + +@pytest.mark.parametrize( + ("primary", "secondary", "expected"), + [(3, 2, (3, 2)), (3, 0, (3, 3)), (3, 3, (3, 3)), (0, 0, (3, 3))], +) +def test_get_grouping_icu( + primary: int, + secondary: int, + expected: tuple[int, int], + icu_locale_module: types.ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + formatter = types.SimpleNamespace( + getGroupingSize=lambda: primary, + getSecondaryGroupingSize=lambda: secondary, + ) + icu_locale_module.icu.DecimalFormat = types.SimpleNamespace + icu_locale_module.icu.NumberFormat = types.SimpleNamespace( + createInstance=lambda _: formatter + ) + monkeypatch.setattr(icu_locale_module, "get_icu_locale", lambda: "en_IN") + assert icu_locale_module.get_grouping() == expected + + +def test_get_grouping_icu_falls_back_for_non_decimal_format( + icu_locale_module: types.ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + icu_locale_module.icu.DecimalFormat = types.SimpleNamespace + icu_locale_module.icu.NumberFormat = types.SimpleNamespace( + createInstance=lambda _: object() + ) + monkeypatch.setattr(icu_locale_module, "get_icu_locale", lambda: "en_IN") + assert icu_locale_module.get_grouping() == (3, 3) diff --git a/tests/test_input_string_transform_factory.py b/tests/test_input_string_transform_factory.py index df3a160..2f1810b 100644 --- a/tests/test_input_string_transform_factory.py +++ b/tests/test_input_string_transform_factory.py @@ -85,6 +85,33 @@ def test_input_string_transform_factory_handles_us_locale( assert input_string_transform_func(x) == expected +@pytest.mark.parametrize( + ("x", "expected"), + [ + ("1,00,000", "100000"), + ("12,34,567", "1234567"), + ("1,23,45,678", "12345678"), + ("-12,34,567.89", "-1234567.89"), + ("1,23,456e2", "123456e2"), + ("value12,34,567.txt", "value1234567.txt"), + ("12,345", "12345"), + ("1,234,567", "1234567"), + ("12,3456", "12,3456"), + ("12.34,567", "12.34,567"), + ], +) +def test_input_string_transform_factory_handles_secondary_grouping( + x: str, + expected: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("natsort.utils.get_thousands_sep", lambda: ",") + monkeypatch.setattr("natsort.utils.get_decimal_point", lambda: ".") + monkeypatch.setattr("natsort.utils.get_grouping", lambda: (3, 2), raising=False) + transform = input_string_transform_factory(ns.LOCALENUM | ns.FLOAT) + assert transform(x) == expected + + @pytest.mark.parametrize( ("x", "expected"), [ diff --git a/tests/test_natsorted_convenience.py b/tests/test_natsorted_convenience.py index 296e14f..6525855 100644 --- a/tests/test_natsorted_convenience.py +++ b/tests/test_natsorted_convenience.py @@ -69,6 +69,15 @@ def test_humansorted_is_identical_to_natsorted_with_locale_alg( assert humansorted(fruit_list) == natsorted(fruit_list, alg=ns.LOCALE) +@pytest.mark.parametrize( + "alg", [ns.LOCALENUM, ns.LOCALE, ns.LOCALENUM | ns.FLOAT, ns.LOCALE | ns.FLOAT] +) +@pytest.mark.usefixtures("with_locale_en_in") +def test_natsorted_respects_secondary_locale_grouping(alg: int) -> None: + values = ["12,34,567", "20,000", "1,00,000", "1,234"] + assert natsorted(values, alg=alg) == ["1,234", "20,000", "1,00,000", "12,34,567"] + + def test_index_natsorted_returns_integer_list_of_sort_order_for_input_list() -> None: given = ["num3", "num5", "num2"] other = ["foo", "bar", "baz"]