From cb75dc7960da25fb72401f36a33c6712304645fd Mon Sep 17 00:00:00 2001 From: velu-97 Date: Sat, 19 Sep 2026 14:45:01 +0800 Subject: [PATCH 1/2] Add macro precision, recall and F1 to the classification validator --- CHANGELOG.md | 11 ++ docs/classification_training.md | 16 +++ libreyolo/cli/commands/val.py | 18 ++-- libreyolo/models/base/model.py | 6 ++ libreyolo/validation/classify_validator.py | 63 ++++++++++- tests/unit/cli/test_command_utils.py | 115 +++++++++++++++++++++ tests/unit/test_classification.py | 103 ++++++++++++++++++ 7 files changed, 324 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bcba746..3b6c2e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ before 1.4.0 are documented in the ### Added +- **Classification validation reports macro precision, recall and F1 (#852).** + `ClassifyValidator` accumulates a confusion matrix and adds + `metrics/precision`, `metrics/recall` and `metrics/f1` next to the top-1 and + top-5 accuracies: the unweighted mean over classes present in the validation + targets, with precision 0 for a class that is never predicted. `fitness` and + best-checkpoint selection stay top-1, and the existing keys are unchanged. + `libreyolo val` prints the new values and reports them in `--json` as + `precision`, `recall` and `f1`. Anything iterating classification metric keys + (custom loggers, `results.csv` headers) sees three new columns. Definition: + `docs/classification_training.md`. + - **Validation sample-plot count is configurable (#830).** `plot_samples` sets how many validated images appear in the sample-image plot, on `val()`, `train()` and both CLI commands. `0` disables that plot, `-1` keeps every diff --git a/docs/classification_training.md b/docs/classification_training.md index c754d7f4..67f6bbd0 100644 --- a/docs/classification_training.md +++ b/docs/classification_training.md @@ -92,3 +92,19 @@ The four CNN trainers use the weighted loss in both eager and CUDA-graph assembly paths. DINOv2's RF-DETR classification trainer retains its existing eager fallback when CUDA-graph training is requested. Hardware and convergence validation evidence is recorded with the change, separately from this API. + +## Validation metrics + +Classification validation reports `metrics/accuracy_top1`, +`metrics/accuracy_top5`, and macro-averaged `metrics/precision`, +`metrics/recall` and `metrics/f1`. The macro metrics come from a confusion +matrix accumulated over the validation split: per class, precision is +`tp / (tp + fp)` (zero when the class is never predicted), recall is +`tp / (tp + fn)`, and F1 is their harmonic mean. The reported value is the +unweighted mean over classes that appear in the validation targets, so a rare +class counts as much as a common one. Classes absent from the validation +targets are excluded from the mean, so the values can differ from a macro +average computed over the union of target and predicted labels, for example +when validating a 1000-class head on a subset split. `fitness` and +best-checkpoint selection remain top-1 accuracy. `libreyolo val --json` +reports the same values as `precision`, `recall` and `f1`. diff --git a/libreyolo/cli/commands/val.py b/libreyolo/cli/commands/val.py index fac5cc03..e8595849 100644 --- a/libreyolo/cli/commands/val.py +++ b/libreyolo/cli/commands/val.py @@ -177,22 +177,28 @@ def val_cmd( if getattr(loaded_model, "task", "detect") == "classify": top1 = metrics.get("metrics/accuracy_top1", 0.0) top5 = metrics.get("metrics/accuracy_top5", 0.0) + classify_metrics = { + "accuracy_top1": round(float(top1), 4), + "accuracy_top5": round(float(top5), 4), + } + human_line = f" top1: {float(top1):.4f} top5: {float(top5):.4f}" + for metric_name in ("precision", "recall", "f1"): + value = _rounded_metric(metrics, f"metrics/{metric_name}") + if value is not None: + classify_metrics[metric_name] = value + human_line += f" {metric_name}: {value:.4f}" data_out = { "model": model, "model_family": loaded_model.FAMILY, "data": data, "split": split, "device": str(loaded_model.device), - "metrics": { - "accuracy_top1": round(float(top1), 4), - "accuracy_top5": round(float(top5), 4), - }, + "metrics": classify_metrics, } if not json_output: data_out["_human_text"] = ( f"Validating {loaded_model.FAMILY}-{loaded_model.size} " - f"on {data} ({split}):\n" - f" top1: {float(top1):.4f} top5: {float(top5):.4f}" + f"on {data} ({split}):\n" + human_line ) out.result(data_out) return diff --git a/libreyolo/models/base/model.py b/libreyolo/models/base/model.py index 4bded096..b4c32542 100644 --- a/libreyolo/models/base/model.py +++ b/libreyolo/models/base/model.py @@ -2255,6 +2255,12 @@ def val( deployment instead of a folklore default. Entries are NaN for classes where no threshold reaches F1 > 0 (no predictions, no ground truth, or all false positives). + + For ``task="classify"``, the dictionary instead holds + ``metrics/accuracy_top1``, ``metrics/accuracy_top5``, + macro-averaged ``metrics/precision``, ``metrics/recall`` and + ``metrics/f1`` (the mean over classes present in the validation + targets), and ``fitness`` (top-1 accuracy). """ from libreyolo.validation import ( ClassifyValidator, diff --git a/libreyolo/validation/classify_validator.py b/libreyolo/validation/classify_validator.py index 4c6b482d..3023a7d6 100644 --- a/libreyolo/validation/classify_validator.py +++ b/libreyolo/validation/classify_validator.py @@ -1,6 +1,7 @@ """Image-classification validator for LibreYOLO. -Computes top-1 and top-5 accuracy over an ImageFolder-style validation split, +Computes top-1 and top-5 accuracy, plus macro-averaged precision, recall and +F1 from a confusion matrix, over an ImageFolder-style validation split, reusing the :class:`BaseValidator` template (setup -> iterate -> finalize). """ @@ -31,9 +32,16 @@ class ClassifyValidator(ValidationLossMixin, BaseValidator): - """Top-1/top-5 accuracy validator for the classification task.""" + """Top-1/top-5 accuracy validator for the classification task. + + Also reports macro-averaged precision, recall and F1 from a confusion + matrix accumulated over the validation split. + """ task = "classify" + # Confusion matrix, rows = targets, cols = top-1 predictions. Class-level + # default so metric code is safe on instances that skip _init_metrics. + _confusion: Optional[torch.Tensor] = None def __init__( self, @@ -168,6 +176,7 @@ def _init_metrics(self) -> None: self._top1_correct = 0 self._top5_correct = 0 self._total = 0 + self._confusion = None self._reset_validation_loss() def _preprocess_batch(self, batch: Any) -> tuple: @@ -206,22 +215,72 @@ def _update_metrics( self._top5_correct += int(correct.any(dim=1).sum().item()) self._total += int(targets.numel()) + # Sized lazily from the logits width. Targets outside the head's class + # range (dataset/head mismatch) are skipped here; they already count + # as wrong for top-1 above. + pred = topk[:, 0] + if self._confusion is None: + self._confusion = torch.zeros(num_classes, num_classes, dtype=torch.long) + valid = (targets >= 0) & (targets < num_classes) + flat = targets[valid].long() * num_classes + pred[valid].long() + self._confusion.view(-1).index_add_(0, flat, torch.ones_like(flat)) + def _compute_metrics(self) -> Dict[str, float]: total = max(self._total, 1) top1 = self._top1_correct / total top5 = self._top5_correct / total + precision, recall, f1 = self._macro_precision_recall_f1() return { "metrics/accuracy_top1": top1, "metrics/accuracy_top5": top5, + "metrics/precision": precision, + "metrics/recall": recall, + "metrics/f1": f1, "fitness": top1, **self._validation_loss_metrics(), } + def _macro_precision_recall_f1(self) -> tuple[float, float, float]: + """Macro-averaged precision, recall and F1 over classes present in the targets. + + Per class: precision = tp / (tp + fp) (0 when the class was never + predicted), recall = tp / (tp + fn), f1 = 2PR / (P + R) (0 when both + are 0). Classes with no ground-truth samples are excluded from the + mean. Returns zeros when nothing was accumulated. + """ + if self._confusion is None: + return 0.0, 0.0, 0.0 + confusion = self._confusion.double() + tp = confusion.diag() + fp = confusion.sum(dim=0) - tp + fn = confusion.sum(dim=1) - tp + present = confusion.sum(dim=1) > 0 + if not bool(present.any()): + return 0.0, 0.0, 0.0 + zeros = torch.zeros_like(tp) + precision = torch.where(tp + fp > 0, tp / (tp + fp).clamp(min=1), zeros) + recall = torch.where(tp + fn > 0, tp / (tp + fn).clamp(min=1), zeros) + denom = precision + recall + f1 = torch.where( + denom > 0, 2 * precision * recall / denom.clamp(min=1e-12), zeros + ) + return ( + float(precision[present].mean()), + float(recall[present].mean()), + float(f1[present].mean()), + ) + def _print_results(self, metrics: Dict[str, float]) -> None: logger.info("=" * 50) logger.info("Classification Validation Results") logger.info("=" * 50) logger.info(" top-1 accuracy: %.4f", metrics.get("metrics/accuracy_top1", 0.0)) logger.info(" top-5 accuracy: %.4f", metrics.get("metrics/accuracy_top5", 0.0)) + logger.info( + " macro precision: %.4f recall: %.4f f1: %.4f", + metrics.get("metrics/precision", 0.0), + metrics.get("metrics/recall", 0.0), + metrics.get("metrics/f1", 0.0), + ) logger.info(" images: %d", self._total) logger.info("=" * 50) diff --git a/tests/unit/cli/test_command_utils.py b/tests/unit/cli/test_command_utils.py index 8bef5c80..01fb3af0 100644 --- a/tests/unit/cli/test_command_utils.py +++ b/tests/unit/cli/test_command_utils.py @@ -686,6 +686,121 @@ def val(self, **kwargs): assert "box_metrics" not in data +def test_val_json_reports_classification_macro_metrics(monkeypatch): + app = _make_app([("val", val.val_cmd), ("info", special.info_cmd)]) + captured = {} + + class _ClassifyModel: + FAMILY = "yolo9" + task = "classify" + size = "t" + device = "cpu" + + def val(self, **kwargs): + captured.update(kwargs) + return { + "metrics/accuracy_top1": 0.81234, + "metrics/accuracy_top5": 0.98765, + "metrics/precision": 0.71234, + "metrics/recall": 0.65432, + "metrics/f1": 0.68111, + } + + monkeypatch.setattr( + "libreyolo.cli.commands.val.resolve_model_or_exit", + lambda out, model: model, + ) + monkeypatch.setattr( + "libreyolo.cli.commands.val.load_model_or_exit", + lambda out, model, model_path, device: _ClassifyModel(), + ) + monkeypatch.setattr( + "libreyolo.utils.general.increment_path", + lambda path, exist_ok=False, mkdir=False: Path(path), + ) + + result = runner.invoke( + app, + [ + "val", + "data=smoke10", + "model=LibreYOLO9t-cls.pt", + "imgsz=224", + "batch=8", + "workers=0", + "--json", + ], + ) + + assert result.exit_code == 0 + assert captured["data"] == "smoke10" + assert captured["imgsz"] == 224 + assert captured["batch"] == 8 + assert captured["workers"] == 0 + data = json.loads(result.stdout) + assert data["model_family"] == "yolo9" + assert data["metrics"] == { + "accuracy_top1": 0.8123, + "accuracy_top5": 0.9877, + "precision": 0.7123, + "recall": 0.6543, + "f1": 0.6811, + } + assert "mAP50" not in data["metrics"] + assert "box_metrics" not in data + + +def test_val_text_reports_classification_macro_metrics(monkeypatch): + app = _make_app([("val", val.val_cmd), ("info", special.info_cmd)]) + captured = {} + + class _ClassifyModel: + FAMILY = "yolo9" + task = "classify" + size = "t" + device = "cpu" + + def val(self, **kwargs): + captured.update(kwargs) + return { + "metrics/accuracy_top1": 0.81234, + "metrics/accuracy_top5": 0.98765, + "metrics/precision": 0.71234, + "metrics/recall": 0.65432, + "metrics/f1": 0.68111, + } + + monkeypatch.setattr( + "libreyolo.cli.commands.val.resolve_model_or_exit", + lambda out, model: model, + ) + monkeypatch.setattr( + "libreyolo.cli.commands.val.load_model_or_exit", + lambda out, model, model_path, device: _ClassifyModel(), + ) + monkeypatch.setattr( + "libreyolo.utils.general.increment_path", + lambda path, exist_ok=False, mkdir=False: Path(path), + ) + + result = runner.invoke( + app, + [ + "val", + "data=smoke10", + "model=LibreYOLO9t-cls.pt", + "imgsz=224", + "batch=8", + "workers=0", + ], + ) + + assert result.exit_code == 0 + assert "precision: 0.7123" in result.stdout + assert "recall: 0.6543" in result.stdout + assert "f1: 0.6811" in result.stdout + + def test_export_runtime_error_includes_stage_context(failing_app): result = runner.invoke( failing_app, diff --git a/tests/unit/test_classification.py b/tests/unit/test_classification.py index 206e124f..ea281d34 100644 --- a/tests/unit/test_classification.py +++ b/tests/unit/test_classification.py @@ -309,6 +309,7 @@ def test_classify_family_train_end_to_end(tmp_path): val = epoch_metrics[-1].get("val_metrics") or {} scalars = val.get("metrics", val) assert "metrics/accuracy_top1" in scalars + assert "metrics/f1" in scalars # The saved checkpoint reloads as a 2-class classifier and predicts. reloaded = LibreMobileNetV4(str(best), device="cpu") @@ -317,6 +318,108 @@ def test_classify_family_train_end_to_end(tmp_path): assert result.probs.data.shape[0] == 2 +def _bare_classify_validator(): + from contextlib import nullcontext + + from libreyolo.validation.classify_validator import ClassifyValidator + + validator = object.__new__(ClassifyValidator) + validator.loss_adapter = None + validator._autocast_context = nullcontext + validator._init_metrics() + return validator + + +def _one_hot_logits(preds, nc): + out = torch.full((len(preds), nc), -5.0) + for row, cls in enumerate(preds): + out[row, cls] = 5.0 + return out + + +def test_classify_validator_macro_precision_recall_f1(): + """Macro P/R/F1 from the confusion matrix, over classes present in targets.""" + validator = _bare_classify_validator() + + # Batch 1: targets [0, 0, 1, 1], preds [0, 1, 1, 1] + validator._update_metrics( + _one_hot_logits([0, 1, 1, 1], 4), torch.tensor([0, 0, 1, 1]), None + ) + # Batch 2: targets [2, 2], preds [2, 0] + validator._update_metrics(_one_hot_logits([2, 0], 4), torch.tensor([2, 2]), None) + + metrics = validator._compute_metrics() + + # Confusion (rows=target, cols=pred): + # c0: tp=1 fp=1 fn=1 -> P=0.5 R=0.5 F1=0.5 + # c1: tp=2 fp=1 fn=0 -> P=2/3 R=1.0 F1=0.8 + # c2: tp=1 fp=0 fn=1 -> P=1.0 R=0.5 F1=2/3 + # c3: absent from targets -> excluded from the macro mean + assert metrics["metrics/accuracy_top1"] == pytest.approx(4 / 6) + assert metrics["metrics/precision"] == pytest.approx((0.5 + 2 / 3 + 1.0) / 3) + assert metrics["metrics/recall"] == pytest.approx((0.5 + 1.0 + 0.5) / 3) + assert metrics["metrics/f1"] == pytest.approx((0.5 + 0.8 + 2 / 3) / 3) + assert metrics["fitness"] == pytest.approx(metrics["metrics/accuracy_top1"]) + + +def test_classify_validator_confusion_accumulates_across_batches_and_repeats(): + """A repeated (target, pred) pair within one batch must count every time. + + Regression guard for the index_add_ scatter: a naive + ``confusion[t, p] += 1`` written as a fancy-indexing assignment silently + drops repeats within the same batch, unlike bincount/index_add_. + """ + validator = _bare_classify_validator() + + # Batch 1: index (0, 0) repeats three times, (1, 1) once. + validator._update_metrics( + _one_hot_logits([0, 0, 0, 1], 3), torch.tensor([0, 0, 0, 1]), None + ) + # Batch 2: targets [1], preds [2]. + validator._update_metrics(_one_hot_logits([2], 3), torch.tensor([1]), None) + + assert validator._confusion.tolist() == [[3, 0, 0], [0, 1, 1], [0, 0, 0]] + + +def test_classify_validator_macro_metrics_are_zero_without_samples(): + metrics = _bare_classify_validator()._compute_metrics() + + assert metrics["metrics/precision"] == 0.0 + assert metrics["metrics/recall"] == 0.0 + assert metrics["metrics/f1"] == 0.0 + + +def test_classify_validator_precision_is_zero_for_never_predicted_class(): + """A class present in targets but never predicted scores 0, not NaN.""" + validator = _bare_classify_validator() + validator._update_metrics( + _one_hot_logits([0, 0, 0, 0], 3), torch.tensor([0, 0, 1, 2]), None + ) + + metrics = validator._compute_metrics() + + # c0: P=2/4 R=1 F1=2/3; c1 and c2: never predicted -> 0, 0, 0. + assert metrics["metrics/precision"] == pytest.approx(0.5 / 3) + assert metrics["metrics/recall"] == pytest.approx(1.0 / 3) + assert metrics["metrics/f1"] == pytest.approx((2 / 3) / 3) + for key in ("metrics/precision", "metrics/recall", "metrics/f1"): + assert metrics[key] == metrics[key] # not NaN + + +def test_classify_validator_ignores_out_of_range_targets_in_confusion(): + """A target outside the head's class range still counts as wrong, never crashes.""" + validator = _bare_classify_validator() + validator._update_metrics(_one_hot_logits([0, 0], 2), torch.tensor([2, 0]), None) + + metrics = validator._compute_metrics() + + assert metrics["metrics/accuracy_top1"] == pytest.approx(0.5) + assert validator._confusion.tolist() == [[1, 0], [0, 0]] + assert metrics["metrics/precision"] == pytest.approx(1.0) + assert metrics["metrics/recall"] == pytest.approx(1.0) + assert metrics["metrics/f1"] == pytest.approx(1.0) + + # --------------------------------------------------------------------------- # Classification augmentation pack: auto_augment / erasing / mixup / cutmix. # --------------------------------------------------------------------------- From 92b7aad47b3ccc123a89d93b22e25cca7c43c26b Mon Sep 17 00:00:00 2001 From: velu-97 Date: Sat, 19 Sep 2026 15:58:48 +0800 Subject: [PATCH 2/2] Accumulate per-class confusion counts instead of a dense matrix --- CHANGELOG.md | 3 +- docs/classification_training.md | 25 +++++----- libreyolo/validation/classify_validator.py | 56 ++++++++++++++-------- tests/unit/test_classification.py | 28 ++++++++++- 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b6c2e1b..2a667178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ before 1.4.0 are documented in the ### Added - **Classification validation reports macro precision, recall and F1 (#852).** - `ClassifyValidator` accumulates a confusion matrix and adds + `ClassifyValidator` accumulates per-class confusion counts (the confusion + matrix's diagonal and marginals, linear memory in the class count) and adds `metrics/precision`, `metrics/recall` and `metrics/f1` next to the top-1 and top-5 accuracies: the unweighted mean over classes present in the validation targets, with precision 0 for a class that is never predicted. `fitness` and diff --git a/docs/classification_training.md b/docs/classification_training.md index 67f6bbd0..bef47f34 100644 --- a/docs/classification_training.md +++ b/docs/classification_training.md @@ -97,14 +97,17 @@ validation evidence is recorded with the change, separately from this API. Classification validation reports `metrics/accuracy_top1`, `metrics/accuracy_top5`, and macro-averaged `metrics/precision`, -`metrics/recall` and `metrics/f1`. The macro metrics come from a confusion -matrix accumulated over the validation split: per class, precision is -`tp / (tp + fp)` (zero when the class is never predicted), recall is -`tp / (tp + fn)`, and F1 is their harmonic mean. The reported value is the -unweighted mean over classes that appear in the validation targets, so a rare -class counts as much as a common one. Classes absent from the validation -targets are excluded from the mean, so the values can differ from a macro -average computed over the union of target and predicted labels, for example -when validating a 1000-class head on a subset split. `fitness` and -best-checkpoint selection remain top-1 accuracy. `libreyolo val --json` -reports the same values as `precision`, `recall` and `f1`. +`metrics/recall` and `metrics/f1`. The macro metrics come from per-class +confusion counts accumulated over the validation split (true positives, +predictions and targets per class, which are the confusion matrix's diagonal +and marginals), so memory stays linear in the number of classes: per class, +precision is `tp / (tp + fp)` (zero when the class is never predicted), +recall is `tp / (tp + fn)`, and F1 is their harmonic mean. The reported +value is the unweighted mean over classes that appear in the validation +targets, so a rare class counts as much as a common one. Classes absent +from the validation targets are excluded from the mean, so the values can +differ from a macro average computed over the union of target and +predicted labels, for example when validating a 1000-class head on a +subset split. `fitness` and best-checkpoint selection remain top-1 accuracy. +`libreyolo val --json` reports the same values as `precision`, `recall` and +`f1`. diff --git a/libreyolo/validation/classify_validator.py b/libreyolo/validation/classify_validator.py index 3023a7d6..3abaebc7 100644 --- a/libreyolo/validation/classify_validator.py +++ b/libreyolo/validation/classify_validator.py @@ -1,8 +1,9 @@ """Image-classification validator for LibreYOLO. Computes top-1 and top-5 accuracy, plus macro-averaged precision, recall and -F1 from a confusion matrix, over an ImageFolder-style validation split, -reusing the :class:`BaseValidator` template (setup -> iterate -> finalize). +F1 from per-class confusion counts (the confusion matrix's diagonal and +marginals), over an ImageFolder-style validation split, reusing the +:class:`BaseValidator` template (setup -> iterate -> finalize). """ from __future__ import annotations @@ -34,14 +35,19 @@ class ClassifyValidator(ValidationLossMixin, BaseValidator): """Top-1/top-5 accuracy validator for the classification task. - Also reports macro-averaged precision, recall and F1 from a confusion - matrix accumulated over the validation split. + Also reports macro-averaged precision, recall and F1 from per-class + confusion counts (the confusion matrix's diagonal and marginals) + accumulated over the validation split. """ task = "classify" - # Confusion matrix, rows = targets, cols = top-1 predictions. Class-level - # default so metric code is safe on instances that skip _init_metrics. - _confusion: Optional[torch.Tensor] = None + # Per-class confusion counts: the confusion matrix's diagonal and its + # marginals. Kept as three length-nc vectors so memory stays linear in the + # class count. Class-level defaults keep metric code safe on instances + # that skip _init_metrics. + _class_tp: torch.Tensor | None = None + _class_pred: torch.Tensor | None = None + _class_target: torch.Tensor | None = None def __init__( self, @@ -176,7 +182,9 @@ def _init_metrics(self) -> None: self._top1_correct = 0 self._top5_correct = 0 self._total = 0 - self._confusion = None + self._class_tp = None + self._class_pred = None + self._class_target = None self._reset_validation_loss() def _preprocess_batch(self, batch: Any) -> tuple: @@ -219,11 +227,18 @@ def _update_metrics( # range (dataset/head mismatch) are skipped here; they already count # as wrong for top-1 above. pred = topk[:, 0] - if self._confusion is None: - self._confusion = torch.zeros(num_classes, num_classes, dtype=torch.long) + if self._class_tp is None: + self._class_tp = torch.zeros(num_classes, dtype=torch.long) + self._class_pred = torch.zeros(num_classes, dtype=torch.long) + self._class_target = torch.zeros(num_classes, dtype=torch.long) valid = (targets >= 0) & (targets < num_classes) - flat = targets[valid].long() * num_classes + pred[valid].long() - self._confusion.view(-1).index_add_(0, flat, torch.ones_like(flat)) + target_idx = targets[valid].long() + pred_idx = pred[valid].long() + ones = torch.ones_like(target_idx) + self._class_target.index_add_(0, target_idx, ones) + self._class_pred.index_add_(0, pred_idx, ones) + hit = target_idx == pred_idx + self._class_tp.index_add_(0, target_idx[hit], ones[hit]) def _compute_metrics(self) -> Dict[str, float]: total = max(self._total, 1) @@ -241,25 +256,24 @@ def _compute_metrics(self) -> Dict[str, float]: } def _macro_precision_recall_f1(self) -> tuple[float, float, float]: - """Macro-averaged precision, recall and F1 over classes present in the targets. + """Macro-averaged precision, recall and F1 from per-class confusion counts. Per class: precision = tp / (tp + fp) (0 when the class was never predicted), recall = tp / (tp + fn), f1 = 2PR / (P + R) (0 when both are 0). Classes with no ground-truth samples are excluded from the mean. Returns zeros when nothing was accumulated. """ - if self._confusion is None: + if self._class_tp is None: return 0.0, 0.0, 0.0 - confusion = self._confusion.double() - tp = confusion.diag() - fp = confusion.sum(dim=0) - tp - fn = confusion.sum(dim=1) - tp - present = confusion.sum(dim=1) > 0 + tp = self._class_tp.double() + predicted = self._class_pred.double() + support = self._class_target.double() + present = support > 0 if not bool(present.any()): return 0.0, 0.0, 0.0 zeros = torch.zeros_like(tp) - precision = torch.where(tp + fp > 0, tp / (tp + fp).clamp(min=1), zeros) - recall = torch.where(tp + fn > 0, tp / (tp + fn).clamp(min=1), zeros) + precision = torch.where(predicted > 0, tp / predicted.clamp(min=1), zeros) + recall = torch.where(support > 0, tp / support.clamp(min=1), zeros) denom = precision + recall f1 = torch.where( denom > 0, 2 * precision * recall / denom.clamp(min=1e-12), zeros diff --git a/tests/unit/test_classification.py b/tests/unit/test_classification.py index ea281d34..f9b2a21d 100644 --- a/tests/unit/test_classification.py +++ b/tests/unit/test_classification.py @@ -378,7 +378,29 @@ def test_classify_validator_confusion_accumulates_across_batches_and_repeats(): # Batch 2: targets [1], preds [2]. validator._update_metrics(_one_hot_logits([2], 3), torch.tensor([1]), None) - assert validator._confusion.tolist() == [[3, 0, 0], [0, 1, 1], [0, 0, 0]] + assert validator._class_target.tolist() == [3, 2, 0] + assert validator._class_pred.tolist() == [3, 1, 1] + assert validator._class_tp.tolist() == [3, 1, 0] + + +def test_classify_validator_confusion_memory_is_linear_in_class_count(): + """Per-class vectors, not an nc x nc matrix: memory must stay linear in nc. + + Regression guard for the quadratic confusion-matrix allocation (#852 + follow-up): a dense nc x nc int64 matrix costs ~3.5 GB at ImageNet-21k + width. Every tensor the validator stores must be at most length nc. + """ + validator = _bare_classify_validator() + nc = 4096 + + validator._update_metrics(_one_hot_logits([0, 5], nc), torch.tensor([0, 7]), None) + + for name, value in vars(validator).items(): + if isinstance(value, torch.Tensor): + assert value.numel() <= nc, f"{name} has {value.numel()} elements" + + metrics = validator._compute_metrics() + assert metrics["metrics/recall"] == pytest.approx(0.5) def test_classify_validator_macro_metrics_are_zero_without_samples(): @@ -414,7 +436,9 @@ def test_classify_validator_ignores_out_of_range_targets_in_confusion(): metrics = validator._compute_metrics() assert metrics["metrics/accuracy_top1"] == pytest.approx(0.5) - assert validator._confusion.tolist() == [[1, 0], [0, 0]] + assert validator._class_target.tolist() == [1, 0] + assert validator._class_pred.tolist() == [1, 0] + assert validator._class_tp.tolist() == [1, 0] assert metrics["metrics/precision"] == pytest.approx(1.0) assert metrics["metrics/recall"] == pytest.approx(1.0) assert metrics["metrics/f1"] == pytest.approx(1.0)