diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index f8c165b77903..99053c170322 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -17,6 +17,7 @@ def Fbeta( recall: Recall | None = None, output_transform: Callable | None = None, device: str | torch.device | None = None, + class_names: list[str] | None = None, ) -> MetricsLambda: r"""Calculates F-beta score. @@ -42,6 +43,7 @@ def Fbeta( device: specifies which device updates are accumulated on. Setting the metric's device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By default, CPU. + class_names: list of class name strings used to label per-class output. Default: ``None``. Returns: MetricsLambda, F-beta metric @@ -140,6 +142,9 @@ def thresholded_output_transform(output): .. testoutput:: 4 0.7499... + + .. versionchanged:: 0.6.0 + ``class_names`` argument is added. """ if not (beta > 0): raise ValueError(f"Beta should be a positive integer, but given {beta}") @@ -159,23 +164,61 @@ def thresholded_output_transform(output): if precision is None and recall is None and device is None: device = torch.device("cpu") + if class_names is not None: + if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): + raise ValueError("class_names must be a list of strings") + if average not in (False, None): + raise ValueError( + f"class_names is only applicable when average=False or average=None, got average={average!r}." + ) + + if precision is not None and precision._average: + raise ValueError("Input precision metric should have average=False") + + if recall is not None and recall._average: + raise ValueError("Input recall metric should have average=False") + + active_metrics = [m for m in (precision, recall) if m is not None] + + if class_names is not None and any(m._class_names != class_names for m in active_metrics): + raise ValueError("precision and recall metric class_names must match Fbeta class_names") + + if len(active_metrics) == 2 and active_metrics[0]._class_names != active_metrics[1]._class_names: + raise ValueError("precision and recall class_names must match") + + if class_names is None and precision is not None: + class_names = precision._class_names + if class_names is None and recall is not None: + class_names = recall._class_names + if precision is None: precision = Precision( output_transform=(lambda x: x) if output_transform is None else output_transform, average=False, device=cast(str | torch.device, recall._device if recall else device), + class_names=class_names, ) - elif precision._average: - raise ValueError("Input precision metric should have average=False") if recall is None: recall = Recall( output_transform=(lambda x: x) if output_transform is None else output_transform, average=False, device=cast(str | torch.device, precision._device if precision else device), + class_names=class_names, ) - elif recall._average: - raise ValueError("Input recall metric should have average=False") + + if class_names is not None: + + def _fbeta_with_class_names(p: dict, r: dict) -> dict: + p_vals = torch.tensor(list(p.values())) + r_vals = torch.tensor(list(r.values())) + scores = (1.0 + beta**2) * p_vals * r_vals / (beta**2 * p_vals + r_vals + 1e-15) + values = scores.tolist() + if not isinstance(values, list): + values = [values] + return dict(zip(class_names, values)) + + return MetricsLambda(_fbeta_with_class_names, precision, recall) fbeta = (1.0 + beta**2) * precision * recall / (beta**2 * precision + recall + 1e-15) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index be33b268b15e..6fdc019084d7 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -23,7 +23,17 @@ def __init__( is_multilabel: bool = False, device: str | torch.device = torch.device("cpu"), skip_unrolling: bool = False, + class_names: list[str] | None = None, ): + if class_names is not None: + if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): + raise ValueError("class_names must be a list of strings") + if average not in (False, None): + raise ValueError( + f"class_names is only applicable when average=False or average=None, got average={average!r}." + ) + self._class_names = class_names + if not (average is None or isinstance(average, bool) or average in ["macro", "micro", "weighted", "samples"]): raise ValueError( "Argument average should be None or a boolean or one of values" @@ -125,7 +135,7 @@ def reset(self) -> None: super().reset() @sync_all_reduce("_numerator", "_denominator") - def compute(self) -> torch.Tensor | float: + def compute(self) -> torch.Tensor | float | dict[str, float]: r""" Return value of the metric for `average` options `'weighted'` and `'macro'` is computed as follows. @@ -157,6 +167,11 @@ def compute(self) -> torch.Tensor | float: elif self._average == "macro": return cast(torch.Tensor, fraction).mean().item() else: + if self._class_names is not None: + values = cast(torch.Tensor, fraction).tolist() + if not isinstance(values, list): + values = [values] + return dict(zip(self._class_names, values)) return fraction @@ -246,6 +261,10 @@ class Precision(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: @@ -379,6 +398,9 @@ def thresholded_output_transform(output): .. versionchanged:: 0.5.1 ``skip_unrolling`` argument is added. + + .. versionchanged:: 0.6.0 + ``class_names`` argument is added. """ @reinit__is_reduced @@ -428,5 +450,11 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) - + if self._class_names is not None: + num_classes = 1 if self._numerator.ndim == 0 else self._numerator.shape[0] + if len(self._class_names) != num_classes: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{num_classes} classes." + ) self._updated = True diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index 34da36ced48f..3194d49e60ba 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -97,6 +97,10 @@ class Recall(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: @@ -218,6 +222,9 @@ def thresholded_output_transform(output): .. versionchanged:: 0.5.1 ``skip_unrolling`` argument is added. + + .. versionchanged:: 0.6.0 + ``class_names`` argument is added. """ @reinit__is_reduced @@ -240,5 +247,12 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) + if self._class_names is not None: + num_classes = 1 if self._numerator.ndim == 0 else self._numerator.shape[0] + if len(self._class_names) != num_classes: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{num_classes} classes." + ) self._updated = True diff --git a/tests/ignite/metrics/test_fbeta.py b/tests/ignite/metrics/test_fbeta.py index 251ff590cfd4..3330838a5794 100644 --- a/tests/ignite/metrics/test_fbeta.py +++ b/tests/ignite/metrics/test_fbeta.py @@ -39,6 +39,26 @@ def test_wrong_inputs(): r = Recall(average=False) Fbeta(1.0, recall=r, output_transform=lambda x: x) + # class_names validation + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Fbeta(beta=1.0, average=False, class_names=[1, 2]) + + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Fbeta(beta=1.0, average=True, class_names=["cat", "dog"]) + + p_no_cn = Precision(average=False) + with pytest.raises(ValueError, match="precision and recall metric class_names must match Fbeta class_names"): + Fbeta(beta=1.0, average=False, class_names=["cat", "dog"], precision=p_no_cn) + + r_no_cn = Recall(average=False) + with pytest.raises(ValueError, match="precision and recall metric class_names must match Fbeta class_names"): + Fbeta(beta=1.0, average=False, class_names=["cat", "dog"], recall=r_no_cn) + + p = Precision(average=False, class_names=["cat", "dog"]) + r = Recall(average=False, class_names=["a", "b"]) + with pytest.raises(ValueError, match="precision and recall class_names must match"): + Fbeta(beta=1.0, average=False, precision=p, recall=r) + def _output_transform(output): return output["y_pred"], output["y"] @@ -233,3 +253,36 @@ def test_multinode_distrib_gloo_cpu_or_gpu(distributed_context_multi_node_gloo): def test_multinode_distrib_nccl_gpu(distributed_context_multi_node_nccl): device = idist.device() _test_distrib_integration(device) + + +@pytest.mark.parametrize( + "precision_cls, recall_cls, class_names", + [ + (None, None, ["cat", "dog", "bird"]), + ( + lambda device: Precision(average=False, device=device, class_names=["cat", "dog", "bird"]), + lambda device: Recall(average=False, device=device, class_names=["cat", "dog", "bird"]), + None, + ), + ], +) +def test_class_names_integration(precision_cls, recall_cls, class_names, available_device): + p = precision_cls(available_device) if precision_cls else None + r = recall_cls(available_device) if recall_cls else None + + y_true = torch.tensor([0, 1, 2]) + y_pred = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + + device = None if p is not None and r is not None else available_device + f1 = Fbeta(beta=1.0, average=False, precision=p, recall=r, class_names=class_names, device=device) + f1.update((y_pred, y_true)) + res = f1.compute() + assert isinstance(res, dict) + assert list(res.keys()) == ["cat", "dog", "bird"] + assert res == {"cat": 1.0, "dog": 1.0, "bird": 1.0} diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index 7bd80540f5de..441670c46353 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -325,6 +325,86 @@ def test_incorrect_y_classes(average): assert pr._updated is False +@pytest.mark.parametrize( + "invalid_class_names", + [ + [0, 1, 2], # list of ints + "cat", # string instead of list + [0.1, 0.2], # list of floats + ["cat", 1], # mixed + ], +) +def test_class_names_invalid_type(invalid_class_names): + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Precision(average=False, class_names=invalid_class_names) + + +@pytest.mark.parametrize("average", ["macro", "micro", "weighted", "samples", True]) +def test_class_names_incompatible_average(average): + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Precision(average=average, class_names=["cat", "dog", "horse"]) + + +@pytest.mark.parametrize("average", [False, None]) +def test_class_names_multiclass(average): + class_names = ["cat", "dog", "horse"] + pr = Precision(average=average, class_names=class_names) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, dict) + assert list(result.keys()) == class_names + assert result == pytest.approx({"cat": 0.5, "dog": 0.0, "horse": 0.3333333333333333}) + + +def test_class_names_length_mismatch(): + pr = Precision(average=False, class_names=["cat", "dog"]) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + ] + ) + y = torch.tensor([2, 0, 1]) + + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes"): + pr.update((y_pred, y)) + + +def test_class_names_none_returns_tensor(): + pr = Precision(average=False) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, torch.Tensor) + + @pytest.mark.usefixtures("distributed") class TestDistributed: @pytest.mark.parametrize("average", [False, "macro", "weighted", "micro"]) @@ -503,3 +583,33 @@ def test_multilabel_accumulator_device(self, average): if average == "weighted": assert pr._weight.device == metric_device, f"{type(pr._weight.device)}:{pr._weight.device} vs " f"{type(metric_device)}:{metric_device}" + + +def test_class_names(): + # Invalid class_names type + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Precision(average=False, class_names=[1, 2]) + + # Incompatible average mode + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Precision(average="macro", class_names=["cat", "dog"]) + + # Correct computation returning dict + pr = Precision(average=False, class_names=["cat", "dog", "bird"]) + y_true = torch.tensor([0, 1, 2]) + y_pred = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + pr.update((y_pred, y_true)) + res = pr.compute() + assert isinstance(res, dict) + assert res == {"cat": 1.0, "dog": 1.0, "bird": 1.0} + + # Class names length mismatch + pr_mismatch = Precision(average=False, class_names=["cat", "dog"]) + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes."): + pr_mismatch.update((y_pred, y_true)) diff --git a/tests/ignite/metrics/test_recall.py b/tests/ignite/metrics/test_recall.py index 90e78d1a82ad..df29278a7c79 100644 --- a/tests/ignite/metrics/test_recall.py +++ b/tests/ignite/metrics/test_recall.py @@ -505,3 +505,33 @@ def test_multilabel_accumulator_device(self, average): if average == "weighted": assert re._weight.device == metric_device, f"{type(re._weight.device)}:{re._weight.device} vs " f"{type(metric_device)}:{metric_device}" + + +def test_class_names(): + # Invalid class_names type + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Recall(average=False, class_names=[1, 2]) + + # Incompatible average mode + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Recall(average="macro", class_names=["cat", "dog"]) + + # Correct computation returning dict + re = Recall(average=False, class_names=["cat", "dog", "bird"]) + y_true = torch.tensor([0, 1, 2]) + y_pred = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + re.update((y_pred, y_true)) + res = re.compute() + assert isinstance(res, dict) + assert res == {"cat": 1.0, "dog": 1.0, "bird": 1.0} + + # Class names length mismatch + re_mismatch = Recall(average=False, class_names=["cat", "dog"]) + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes."): + re_mismatch.update((y_pred, y_true))