Add macro precision, recall and F1 to the classification validator

CHANGELOG.md

@@ -9,6 +9,18 @@ before 1.4.0 are documented in the
 
 ### Added
 
+- **Classification validation reports macro precision, recall and F1 (#852).**
+  `ClassifyValidator` accumulates per-class confusion counts (the confusion
+  matrix's diagonal and marginals, linear memory in the class count) and adds
+  `metrics/precision`, `metrics/recall` and `metrics/f1` next to the top-1 and
+  top-5 accuracies: the unweighted mean over classes present in the validation
+  targets, with precision 0 for a class that is never predicted. `fitness` and
+  best-checkpoint selection stay top-1, and the existing keys are unchanged.
+  `libreyolo val` prints the new values and reports them in `--json` as
+  `precision`, `recall` and `f1`. Anything iterating classification metric keys
+  (custom loggers, `results.csv` headers) sees three new columns. Definition:
+  `docs/classification_training.md`.
+
 - **Validation sample-plot count is configurable (#830).** `plot_samples` sets
   how many validated images appear in the sample-image plot, on `val()`,
   `train()` and both CLI commands. `0` disables that plot, `-1` keeps every

docs/classification_training.md

@@ -92,3 +92,22 @@ The four CNN trainers use the weighted loss in both eager and CUDA-graph
 assembly paths. DINOv2's RF-DETR classification trainer retains its existing
 eager fallback when CUDA-graph training is requested. Hardware and convergence
 validation evidence is recorded with the change, separately from this API.
+
+## Validation metrics
+
+Classification validation reports `metrics/accuracy_top1`,
+`metrics/accuracy_top5`, and macro-averaged `metrics/precision`,
+`metrics/recall` and `metrics/f1`. The macro metrics come from per-class
+confusion counts accumulated over the validation split (true positives,
+predictions and targets per class, which are the confusion matrix's diagonal
+and marginals), so memory stays linear in the number of classes: per class,
+precision is `tp / (tp + fp)` (zero when the class is never predicted),
+recall is `tp / (tp + fn)`, and F1 is their harmonic mean. The reported
+value is the unweighted mean over classes that appear in the validation
+targets, so a rare class counts as much as a common one. Classes absent
+from the validation targets are excluded from the mean, so the values can
+differ from a macro average computed over the union of target and
+predicted labels, for example when validating a 1000-class head on a
+subset split. `fitness` and best-checkpoint selection remain top-1 accuracy.
+`libreyolo val --json` reports the same values as `precision`, `recall` and
+`f1`.

libreyolo/cli/commands/val.py

@@ -177,22 +177,28 @@ def val_cmd(
     if getattr(loaded_model, "task", "detect") == "classify":
         top1 = metrics.get("metrics/accuracy_top1", 0.0)
         top5 = metrics.get("metrics/accuracy_top5", 0.0)
+        classify_metrics = {
+            "accuracy_top1": round(float(top1), 4),
+            "accuracy_top5": round(float(top5), 4),
+        }
+        human_line = f"  top1: {float(top1):.4f}  top5: {float(top5):.4f}"
+        for metric_name in ("precision", "recall", "f1"):
+            value = _rounded_metric(metrics, f"metrics/{metric_name}")
+            if value is not None:
+                classify_metrics[metric_name] = value
+                human_line += f"  {metric_name}: {value:.4f}"
         data_out = {
             "model": model,
             "model_family": loaded_model.FAMILY,
             "data": data,
             "split": split,
             "device": str(loaded_model.device),
-            "metrics": {
-                "accuracy_top1": round(float(top1), 4),
-                "accuracy_top5": round(float(top5), 4),
-            },
+            "metrics": classify_metrics,
         }
         if not json_output:
             data_out["_human_text"] = (
                 f"Validating {loaded_model.FAMILY}-{loaded_model.size} "
-                f"on {data} ({split}):\n"
-                f"  top1: {float(top1):.4f}  top5: {float(top5):.4f}"
+                f"on {data} ({split}):\n" + human_line
             )
         out.result(data_out)
         return

libreyolo/models/base/model.py

@@ -2255,6 +2255,12 @@ def val(
             deployment instead of a folklore default. Entries are NaN for
             classes where no threshold reaches F1 > 0 (no predictions, no
             ground truth, or all false positives).
+
+            For ``task="classify"``, the dictionary instead holds
+            ``metrics/accuracy_top1``, ``metrics/accuracy_top5``,
+            macro-averaged ``metrics/precision``, ``metrics/recall`` and
+            ``metrics/f1`` (the mean over classes present in the validation
+            targets), and ``fitness`` (top-1 accuracy).
         """
         from libreyolo.validation import (
             ClassifyValidator,

libreyolo/validation/classify_validator.py

@@ -1,7 +1,9 @@
 """Image-classification validator for LibreYOLO.
 
-Computes top-1 and top-5 accuracy over an ImageFolder-style validation split,
-reusing the :class:`BaseValidator` template (setup -> iterate -> finalize).
+Computes top-1 and top-5 accuracy, plus macro-averaged precision, recall and
+F1 from per-class confusion counts (the confusion matrix's diagonal and
+marginals), over an ImageFolder-style validation split, reusing the
+:class:`BaseValidator` template (setup -> iterate -> finalize).
 """
 
 from __future__ import annotations
@@ -31,9 +33,21 @@
 
 
 class ClassifyValidator(ValidationLossMixin, BaseValidator):
-    """Top-1/top-5 accuracy validator for the classification task."""
+    """Top-1/top-5 accuracy validator for the classification task.
+
+    Also reports macro-averaged precision, recall and F1 from per-class
+    confusion counts (the confusion matrix's diagonal and marginals)
+    accumulated over the validation split.
+    """
 
     task = "classify"
+    # Per-class confusion counts: the confusion matrix's diagonal and its
+    # marginals. Kept as three length-nc vectors so memory stays linear in the
+    # class count. Class-level defaults keep metric code safe on instances
+    # that skip _init_metrics.
+    _class_tp: torch.Tensor | None = None
+    _class_pred: torch.Tensor | None = None
+    _class_target: torch.Tensor | None = None
 
     def __init__(
         self,
@@ -168,6 +182,9 @@ def _init_metrics(self) -> None:
         self._top1_correct = 0
         self._top5_correct = 0
         self._total = 0
+        self._class_tp = None
+        self._class_pred = None
+        self._class_target = None
         self._reset_validation_loss()
 
     def _preprocess_batch(self, batch: Any) -> tuple:
@@ -206,22 +223,78 @@ def _update_metrics(
         self._top5_correct += int(correct.any(dim=1).sum().item())
         self._total += int(targets.numel())
 
+        # Sized lazily from the logits width. Targets outside the head's class
+        # range (dataset/head mismatch) are skipped here; they already count
+        # as wrong for top-1 above.
+        pred = topk[:, 0]
+        if self._class_tp is None:
+            self._class_tp = torch.zeros(num_classes, dtype=torch.long)
+            self._class_pred = torch.zeros(num_classes, dtype=torch.long)
+            self._class_target = torch.zeros(num_classes, dtype=torch.long)
+        valid = (targets >= 0) & (targets < num_classes)
+        target_idx = targets[valid].long()
+        pred_idx = pred[valid].long()
+        ones = torch.ones_like(target_idx)
+        self._class_target.index_add_(0, target_idx, ones)
+        self._class_pred.index_add_(0, pred_idx, ones)
+        hit = target_idx == pred_idx
+        self._class_tp.index_add_(0, target_idx[hit], ones[hit])
+
     def _compute_metrics(self) -> Dict[str, float]:
         total = max(self._total, 1)
         top1 = self._top1_correct / total
         top5 = self._top5_correct / total
+        precision, recall, f1 = self._macro_precision_recall_f1()
         return {
             "metrics/accuracy_top1": top1,
             "metrics/accuracy_top5": top5,
+            "metrics/precision": precision,
+            "metrics/recall": recall,
+            "metrics/f1": f1,
             "fitness": top1,
             **self._validation_loss_metrics(),
         }
 
+    def _macro_precision_recall_f1(self) -> tuple[float, float, float]:
+        """Macro-averaged precision, recall and F1 from per-class confusion counts.
+
+        Per class: precision = tp / (tp + fp) (0 when the class was never
+        predicted), recall = tp / (tp + fn), f1 = 2PR / (P + R) (0 when both
+        are 0). Classes with no ground-truth samples are excluded from the
+        mean. Returns zeros when nothing was accumulated.
+        """
+        if self._class_tp is None:
+            return 0.0, 0.0, 0.0
+        tp = self._class_tp.double()
+        predicted = self._class_pred.double()
+        support = self._class_target.double()
+        present = support > 0
+        if not bool(present.any()):
+            return 0.0, 0.0, 0.0
+        zeros = torch.zeros_like(tp)
+        precision = torch.where(predicted > 0, tp / predicted.clamp(min=1), zeros)
+        recall = torch.where(support > 0, tp / support.clamp(min=1), zeros)
+        denom = precision + recall
+        f1 = torch.where(
+            denom > 0, 2 * precision * recall / denom.clamp(min=1e-12), zeros
+        )
+        return (
+            float(precision[present].mean()),
+            float(recall[present].mean()),
+            float(f1[present].mean()),
+        )
+
     def _print_results(self, metrics: Dict[str, float]) -> None:
         logger.info("=" * 50)
         logger.info("Classification Validation Results")
         logger.info("=" * 50)
         logger.info("  top-1 accuracy: %.4f", metrics.get("metrics/accuracy_top1", 0.0))
         logger.info("  top-5 accuracy: %.4f", metrics.get("metrics/accuracy_top5", 0.0))
+        logger.info(
+            "  macro precision: %.4f  recall: %.4f  f1: %.4f",
+            metrics.get("metrics/precision", 0.0),
+            metrics.get("metrics/recall", 0.0),
+            metrics.get("metrics/f1", 0.0),
+        )
         logger.info("  images: %d", self._total)
         logger.info("=" * 50)

tests/unit/cli/test_command_utils.py

@@ -686,6 +686,121 @@ def val(self, **kwargs):
     assert "box_metrics" not in data
 
 
+def test_val_json_reports_classification_macro_metrics(monkeypatch):
+    app = _make_app([("val", val.val_cmd), ("info", special.info_cmd)])
+    captured = {}
+
+    class _ClassifyModel:
+        FAMILY = "yolo9"
+        task = "classify"
+        size = "t"
+        device = "cpu"
+
+        def val(self, **kwargs):
+            captured.update(kwargs)
+            return {
+                "metrics/accuracy_top1": 0.81234,
+                "metrics/accuracy_top5": 0.98765,
+                "metrics/precision": 0.71234,
+                "metrics/recall": 0.65432,
+                "metrics/f1": 0.68111,
+            }
+
+    monkeypatch.setattr(
+        "libreyolo.cli.commands.val.resolve_model_or_exit",
+        lambda out, model: model,
+    )
+    monkeypatch.setattr(
+        "libreyolo.cli.commands.val.load_model_or_exit",
+        lambda out, model, model_path, device: _ClassifyModel(),
+    )
+    monkeypatch.setattr(
+        "libreyolo.utils.general.increment_path",
+        lambda path, exist_ok=False, mkdir=False: Path(path),
+    )
+
+    result = runner.invoke(
+        app,
+        [
+            "val",
+            "data=smoke10",
+            "model=LibreYOLO9t-cls.pt",
+            "imgsz=224",
+            "batch=8",
+            "workers=0",
+            "--json",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert captured["data"] == "smoke10"
+    assert captured["imgsz"] == 224
+    assert captured["batch"] == 8
+    assert captured["workers"] == 0
+    data = json.loads(result.stdout)
+    assert data["model_family"] == "yolo9"
+    assert data["metrics"] == {
+        "accuracy_top1": 0.8123,
+        "accuracy_top5": 0.9877,
+        "precision": 0.7123,
+        "recall": 0.6543,
+        "f1": 0.6811,
+    }
+    assert "mAP50" not in data["metrics"]
+    assert "box_metrics" not in data
+
+
+def test_val_text_reports_classification_macro_metrics(monkeypatch):
+    app = _make_app([("val", val.val_cmd), ("info", special.info_cmd)])
+    captured = {}
+
+    class _ClassifyModel:
+        FAMILY = "yolo9"
+        task = "classify"
+        size = "t"
+        device = "cpu"
+
+        def val(self, **kwargs):
+            captured.update(kwargs)
+            return {
+                "metrics/accuracy_top1": 0.81234,
+                "metrics/accuracy_top5": 0.98765,
+                "metrics/precision": 0.71234,
+                "metrics/recall": 0.65432,
+                "metrics/f1": 0.68111,
+            }
+
+    monkeypatch.setattr(
+        "libreyolo.cli.commands.val.resolve_model_or_exit",
+        lambda out, model: model,
+    )
+    monkeypatch.setattr(
+        "libreyolo.cli.commands.val.load_model_or_exit",
+        lambda out, model, model_path, device: _ClassifyModel(),
+    )
+    monkeypatch.setattr(
+        "libreyolo.utils.general.increment_path",
+        lambda path, exist_ok=False, mkdir=False: Path(path),
+    )
+
+    result = runner.invoke(
+        app,
+        [
+            "val",
+            "data=smoke10",
+            "model=LibreYOLO9t-cls.pt",
+            "imgsz=224",
+            "batch=8",
+            "workers=0",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert "precision: 0.7123" in result.stdout
+    assert "recall: 0.6543" in result.stdout
+    assert "f1: 0.6811" in result.stdout
+
+
 def test_export_runtime_error_includes_stage_context(failing_app):
     result = runner.invoke(
         failing_app,