diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bcba746..2a667178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ before 1.4.0 are documented in the ### Added +- **Classification validation reports macro precision, recall and F1 (#852).** + `ClassifyValidator` accumulates per-class confusion counts (the confusion + matrix's diagonal and marginals, linear memory in the class count) and adds + `metrics/precision`, `metrics/recall` and `metrics/f1` next to the top-1 and + top-5 accuracies: the unweighted mean over classes present in the validation + targets, with precision 0 for a class that is never predicted. `fitness` and + best-checkpoint selection stay top-1, and the existing keys are unchanged. + `libreyolo val` prints the new values and reports them in `--json` as + `precision`, `recall` and `f1`. Anything iterating classification metric keys + (custom loggers, `results.csv` headers) sees three new columns. Definition: + `docs/classification_training.md`. + - **Validation sample-plot count is configurable (#830).** `plot_samples` sets how many validated images appear in the sample-image plot, on `val()`, `train()` and both CLI commands. `0` disables that plot, `-1` keeps every diff --git a/docs/classification_training.md b/docs/classification_training.md index c754d7f4..bef47f34 100644 --- a/docs/classification_training.md +++ b/docs/classification_training.md @@ -92,3 +92,22 @@ The four CNN trainers use the weighted loss in both eager and CUDA-graph assembly paths. DINOv2's RF-DETR classification trainer retains its existing eager fallback when CUDA-graph training is requested. Hardware and convergence validation evidence is recorded with the change, separately from this API. + +## Validation metrics + +Classification validation reports `metrics/accuracy_top1`, +`metrics/accuracy_top5`, and macro-averaged `metrics/precision`, +`metrics/recall` and `metrics/f1`. The macro metrics come from per-class +confusion counts accumulated over the validation split (true positives, +predictions and targets per class, which are the confusion matrix's diagonal +and marginals), so memory stays linear in the number of classes: per class, +precision is `tp / (tp + fp)` (zero when the class is never predicted), +recall is `tp / (tp + fn)`, and F1 is their harmonic mean. The reported +value is the unweighted mean over classes that appear in the validation +targets, so a rare class counts as much as a common one. Classes absent +from the validation targets are excluded from the mean, so the values can +differ from a macro average computed over the union of target and +predicted labels, for example when validating a 1000-class head on a +subset split. `fitness` and best-checkpoint selection remain top-1 accuracy. +`libreyolo val --json` reports the same values as `precision`, `recall` and +`f1`. diff --git a/libreyolo/cli/commands/val.py b/libreyolo/cli/commands/val.py index fac5cc03..e8595849 100644 --- a/libreyolo/cli/commands/val.py +++ b/libreyolo/cli/commands/val.py @@ -177,22 +177,28 @@ def val_cmd( if getattr(loaded_model, "task", "detect") == "classify": top1 = metrics.get("metrics/accuracy_top1", 0.0) top5 = metrics.get("metrics/accuracy_top5", 0.0) + classify_metrics = { + "accuracy_top1": round(float(top1), 4), + "accuracy_top5": round(float(top5), 4), + } + human_line = f" top1: {float(top1):.4f} top5: {float(top5):.4f}" + for metric_name in ("precision", "recall", "f1"): + value = _rounded_metric(metrics, f"metrics/{metric_name}") + if value is not None: + classify_metrics[metric_name] = value + human_line += f" {metric_name}: {value:.4f}" data_out = { "model": model, "model_family": loaded_model.FAMILY, "data": data, "split": split, "device": str(loaded_model.device), - "metrics": { - "accuracy_top1": round(float(top1), 4), - "accuracy_top5": round(float(top5), 4), - }, + "metrics": classify_metrics, } if not json_output: data_out["_human_text"] = ( f"Validating {loaded_model.FAMILY}-{loaded_model.size} " - f"on {data} ({split}):\n" - f" top1: {float(top1):.4f} top5: {float(top5):.4f}" + f"on {data} ({split}):\n" + human_line ) out.result(data_out) return diff --git a/libreyolo/models/base/model.py b/libreyolo/models/base/model.py index 4bded096..b4c32542 100644 --- a/libreyolo/models/base/model.py +++ b/libreyolo/models/base/model.py @@ -2255,6 +2255,12 @@ def val( deployment instead of a folklore default. Entries are NaN for classes where no threshold reaches F1 > 0 (no predictions, no ground truth, or all false positives). + + For ``task="classify"``, the dictionary instead holds + ``metrics/accuracy_top1``, ``metrics/accuracy_top5``, + macro-averaged ``metrics/precision``, ``metrics/recall`` and + ``metrics/f1`` (the mean over classes present in the validation + targets), and ``fitness`` (top-1 accuracy). """ from libreyolo.validation import ( ClassifyValidator, diff --git a/libreyolo/validation/classify_validator.py b/libreyolo/validation/classify_validator.py index 4c6b482d..3abaebc7 100644 --- a/libreyolo/validation/classify_validator.py +++ b/libreyolo/validation/classify_validator.py @@ -1,7 +1,9 @@ """Image-classification validator for LibreYOLO. -Computes top-1 and top-5 accuracy over an ImageFolder-style validation split, -reusing the :class:`BaseValidator` template (setup -> iterate -> finalize). +Computes top-1 and top-5 accuracy, plus macro-averaged precision, recall and +F1 from per-class confusion counts (the confusion matrix's diagonal and +marginals), over an ImageFolder-style validation split, reusing the +:class:`BaseValidator` template (setup -> iterate -> finalize). """ from __future__ import annotations @@ -31,9 +33,21 @@ class ClassifyValidator(ValidationLossMixin, BaseValidator): - """Top-1/top-5 accuracy validator for the classification task.""" + """Top-1/top-5 accuracy validator for the classification task. + + Also reports macro-averaged precision, recall and F1 from per-class + confusion counts (the confusion matrix's diagonal and marginals) + accumulated over the validation split. + """ task = "classify" + # Per-class confusion counts: the confusion matrix's diagonal and its + # marginals. Kept as three length-nc vectors so memory stays linear in the + # class count. Class-level defaults keep metric code safe on instances + # that skip _init_metrics. + _class_tp: torch.Tensor | None = None + _class_pred: torch.Tensor | None = None + _class_target: torch.Tensor | None = None def __init__( self, @@ -168,6 +182,9 @@ def _init_metrics(self) -> None: self._top1_correct = 0 self._top5_correct = 0 self._total = 0 + self._class_tp = None + self._class_pred = None + self._class_target = None self._reset_validation_loss() def _preprocess_batch(self, batch: Any) -> tuple: @@ -206,22 +223,78 @@ def _update_metrics( self._top5_correct += int(correct.any(dim=1).sum().item()) self._total += int(targets.numel()) + # Sized lazily from the logits width. Targets outside the head's class + # range (dataset/head mismatch) are skipped here; they already count + # as wrong for top-1 above. + pred = topk[:, 0] + if self._class_tp is None: + self._class_tp = torch.zeros(num_classes, dtype=torch.long) + self._class_pred = torch.zeros(num_classes, dtype=torch.long) + self._class_target = torch.zeros(num_classes, dtype=torch.long) + valid = (targets >= 0) & (targets < num_classes) + target_idx = targets[valid].long() + pred_idx = pred[valid].long() + ones = torch.ones_like(target_idx) + self._class_target.index_add_(0, target_idx, ones) + self._class_pred.index_add_(0, pred_idx, ones) + hit = target_idx == pred_idx + self._class_tp.index_add_(0, target_idx[hit], ones[hit]) + def _compute_metrics(self) -> Dict[str, float]: total = max(self._total, 1) top1 = self._top1_correct / total top5 = self._top5_correct / total + precision, recall, f1 = self._macro_precision_recall_f1() return { "metrics/accuracy_top1": top1, "metrics/accuracy_top5": top5, + "metrics/precision": precision, + "metrics/recall": recall, + "metrics/f1": f1, "fitness": top1, **self._validation_loss_metrics(), } + def _macro_precision_recall_f1(self) -> tuple[float, float, float]: + """Macro-averaged precision, recall and F1 from per-class confusion counts. + + Per class: precision = tp / (tp + fp) (0 when the class was never + predicted), recall = tp / (tp + fn), f1 = 2PR / (P + R) (0 when both + are 0). Classes with no ground-truth samples are excluded from the + mean. Returns zeros when nothing was accumulated. + """ + if self._class_tp is None: + return 0.0, 0.0, 0.0 + tp = self._class_tp.double() + predicted = self._class_pred.double() + support = self._class_target.double() + present = support > 0 + if not bool(present.any()): + return 0.0, 0.0, 0.0 + zeros = torch.zeros_like(tp) + precision = torch.where(predicted > 0, tp / predicted.clamp(min=1), zeros) + recall = torch.where(support > 0, tp / support.clamp(min=1), zeros) + denom = precision + recall + f1 = torch.where( + denom > 0, 2 * precision * recall / denom.clamp(min=1e-12), zeros + ) + return ( + float(precision[present].mean()), + float(recall[present].mean()), + float(f1[present].mean()), + ) + def _print_results(self, metrics: Dict[str, float]) -> None: logger.info("=" * 50) logger.info("Classification Validation Results") logger.info("=" * 50) logger.info(" top-1 accuracy: %.4f", metrics.get("metrics/accuracy_top1", 0.0)) logger.info(" top-5 accuracy: %.4f", metrics.get("metrics/accuracy_top5", 0.0)) + logger.info( + " macro precision: %.4f recall: %.4f f1: %.4f", + metrics.get("metrics/precision", 0.0), + metrics.get("metrics/recall", 0.0), + metrics.get("metrics/f1", 0.0), + ) logger.info(" images: %d", self._total) logger.info("=" * 50) diff --git a/tests/unit/cli/test_command_utils.py b/tests/unit/cli/test_command_utils.py index 8bef5c80..01fb3af0 100644 --- a/tests/unit/cli/test_command_utils.py +++ b/tests/unit/cli/test_command_utils.py @@ -686,6 +686,121 @@ def val(self, **kwargs): assert "box_metrics" not in data +def test_val_json_reports_classification_macro_metrics(monkeypatch): + app = _make_app([("val", val.val_cmd), ("info", special.info_cmd)]) + captured = {} + + class _ClassifyModel: + FAMILY = "yolo9" + task = "classify" + size = "t" + device = "cpu" + + def val(self, **kwargs): + captured.update(kwargs) + return { + "metrics/accuracy_top1": 0.81234, + "metrics/accuracy_top5": 0.98765, + "metrics/precision": 0.71234, + "metrics/recall": 0.65432, + "metrics/f1": 0.68111, + } + + monkeypatch.setattr( + "libreyolo.cli.commands.val.resolve_model_or_exit", + lambda out, model: model, + ) + monkeypatch.setattr( + "libreyolo.cli.commands.val.load_model_or_exit", + lambda out, model, model_path, device: _ClassifyModel(), + ) + monkeypatch.setattr( + "libreyolo.utils.general.increment_path", + lambda path, exist_ok=False, mkdir=False: Path(path), + ) + + result = runner.invoke( + app, + [ + "val", + "data=smoke10", + "model=LibreYOLO9t-cls.pt", + "imgsz=224", + "batch=8", + "workers=0", + "--json", + ], + ) + + assert result.exit_code == 0 + assert captured["data"] == "smoke10" + assert captured["imgsz"] == 224 + assert captured["batch"] == 8 + assert captured["workers"] == 0 + data = json.loads(result.stdout) + assert data["model_family"] == "yolo9" + assert data["metrics"] == { + "accuracy_top1": 0.8123, + "accuracy_top5": 0.9877, + "precision": 0.7123, + "recall": 0.6543, + "f1": 0.6811, + } + assert "mAP50" not in data["metrics"] + assert "box_metrics" not in data + + +def test_val_text_reports_classification_macro_metrics(monkeypatch): + app = _make_app([("val", val.val_cmd), ("info", special.info_cmd)]) + captured = {} + + class _ClassifyModel: + FAMILY = "yolo9" + task = "classify" + size = "t" + device = "cpu" + + def val(self, **kwargs): + captured.update(kwargs) + return { + "metrics/accuracy_top1": 0.81234, + "metrics/accuracy_top5": 0.98765, + "metrics/precision": 0.71234, + "metrics/recall": 0.65432, + "metrics/f1": 0.68111, + } + + monkeypatch.setattr( + "libreyolo.cli.commands.val.resolve_model_or_exit", + lambda out, model: model, + ) + monkeypatch.setattr( + "libreyolo.cli.commands.val.load_model_or_exit", + lambda out, model, model_path, device: _ClassifyModel(), + ) + monkeypatch.setattr( + "libreyolo.utils.general.increment_path", + lambda path, exist_ok=False, mkdir=False: Path(path), + ) + + result = runner.invoke( + app, + [ + "val", + "data=smoke10", + "model=LibreYOLO9t-cls.pt", + "imgsz=224", + "batch=8", + "workers=0", + ], + ) + + assert result.exit_code == 0 + assert "precision: 0.7123" in result.stdout + assert "recall: 0.6543" in result.stdout + assert "f1: 0.6811" in result.stdout + + def test_export_runtime_error_includes_stage_context(failing_app): result = runner.invoke( failing_app, diff --git a/tests/unit/test_classification.py b/tests/unit/test_classification.py index 206e124f..f9b2a21d 100644 --- a/tests/unit/test_classification.py +++ b/tests/unit/test_classification.py @@ -309,6 +309,7 @@ def test_classify_family_train_end_to_end(tmp_path): val = epoch_metrics[-1].get("val_metrics") or {} scalars = val.get("metrics", val) assert "metrics/accuracy_top1" in scalars + assert "metrics/f1" in scalars # The saved checkpoint reloads as a 2-class classifier and predicts. reloaded = LibreMobileNetV4(str(best), device="cpu") @@ -317,6 +318,132 @@ def test_classify_family_train_end_to_end(tmp_path): assert result.probs.data.shape[0] == 2 +def _bare_classify_validator(): + from contextlib import nullcontext + + from libreyolo.validation.classify_validator import ClassifyValidator + + validator = object.__new__(ClassifyValidator) + validator.loss_adapter = None + validator._autocast_context = nullcontext + validator._init_metrics() + return validator + + +def _one_hot_logits(preds, nc): + out = torch.full((len(preds), nc), -5.0) + for row, cls in enumerate(preds): + out[row, cls] = 5.0 + return out + + +def test_classify_validator_macro_precision_recall_f1(): + """Macro P/R/F1 from the confusion matrix, over classes present in targets.""" + validator = _bare_classify_validator() + + # Batch 1: targets [0, 0, 1, 1], preds [0, 1, 1, 1] + validator._update_metrics( + _one_hot_logits([0, 1, 1, 1], 4), torch.tensor([0, 0, 1, 1]), None + ) + # Batch 2: targets [2, 2], preds [2, 0] + validator._update_metrics(_one_hot_logits([2, 0], 4), torch.tensor([2, 2]), None) + + metrics = validator._compute_metrics() + + # Confusion (rows=target, cols=pred): + # c0: tp=1 fp=1 fn=1 -> P=0.5 R=0.5 F1=0.5 + # c1: tp=2 fp=1 fn=0 -> P=2/3 R=1.0 F1=0.8 + # c2: tp=1 fp=0 fn=1 -> P=1.0 R=0.5 F1=2/3 + # c3: absent from targets -> excluded from the macro mean + assert metrics["metrics/accuracy_top1"] == pytest.approx(4 / 6) + assert metrics["metrics/precision"] == pytest.approx((0.5 + 2 / 3 + 1.0) / 3) + assert metrics["metrics/recall"] == pytest.approx((0.5 + 1.0 + 0.5) / 3) + assert metrics["metrics/f1"] == pytest.approx((0.5 + 0.8 + 2 / 3) / 3) + assert metrics["fitness"] == pytest.approx(metrics["metrics/accuracy_top1"]) + + +def test_classify_validator_confusion_accumulates_across_batches_and_repeats(): + """A repeated (target, pred) pair within one batch must count every time. + + Regression guard for the index_add_ scatter: a naive + ``confusion[t, p] += 1`` written as a fancy-indexing assignment silently + drops repeats within the same batch, unlike bincount/index_add_. + """ + validator = _bare_classify_validator() + + # Batch 1: index (0, 0) repeats three times, (1, 1) once. + validator._update_metrics( + _one_hot_logits([0, 0, 0, 1], 3), torch.tensor([0, 0, 0, 1]), None + ) + # Batch 2: targets [1], preds [2]. + validator._update_metrics(_one_hot_logits([2], 3), torch.tensor([1]), None) + + assert validator._class_target.tolist() == [3, 2, 0] + assert validator._class_pred.tolist() == [3, 1, 1] + assert validator._class_tp.tolist() == [3, 1, 0] + + +def test_classify_validator_confusion_memory_is_linear_in_class_count(): + """Per-class vectors, not an nc x nc matrix: memory must stay linear in nc. + + Regression guard for the quadratic confusion-matrix allocation (#852 + follow-up): a dense nc x nc int64 matrix costs ~3.5 GB at ImageNet-21k + width. Every tensor the validator stores must be at most length nc. + """ + validator = _bare_classify_validator() + nc = 4096 + + validator._update_metrics(_one_hot_logits([0, 5], nc), torch.tensor([0, 7]), None) + + for name, value in vars(validator).items(): + if isinstance(value, torch.Tensor): + assert value.numel() <= nc, f"{name} has {value.numel()} elements" + + metrics = validator._compute_metrics() + assert metrics["metrics/recall"] == pytest.approx(0.5) + + +def test_classify_validator_macro_metrics_are_zero_without_samples(): + metrics = _bare_classify_validator()._compute_metrics() + + assert metrics["metrics/precision"] == 0.0 + assert metrics["metrics/recall"] == 0.0 + assert metrics["metrics/f1"] == 0.0 + + +def test_classify_validator_precision_is_zero_for_never_predicted_class(): + """A class present in targets but never predicted scores 0, not NaN.""" + validator = _bare_classify_validator() + validator._update_metrics( + _one_hot_logits([0, 0, 0, 0], 3), torch.tensor([0, 0, 1, 2]), None + ) + + metrics = validator._compute_metrics() + + # c0: P=2/4 R=1 F1=2/3; c1 and c2: never predicted -> 0, 0, 0. + assert metrics["metrics/precision"] == pytest.approx(0.5 / 3) + assert metrics["metrics/recall"] == pytest.approx(1.0 / 3) + assert metrics["metrics/f1"] == pytest.approx((2 / 3) / 3) + for key in ("metrics/precision", "metrics/recall", "metrics/f1"): + assert metrics[key] == metrics[key] # not NaN + + +def test_classify_validator_ignores_out_of_range_targets_in_confusion(): + """A target outside the head's class range still counts as wrong, never crashes.""" + validator = _bare_classify_validator() + validator._update_metrics(_one_hot_logits([0, 0], 2), torch.tensor([2, 0]), None) + + metrics = validator._compute_metrics() + + assert metrics["metrics/accuracy_top1"] == pytest.approx(0.5) + assert validator._class_target.tolist() == [1, 0] + assert validator._class_pred.tolist() == [1, 0] + assert validator._class_tp.tolist() == [1, 0] + assert metrics["metrics/precision"] == pytest.approx(1.0) + assert metrics["metrics/recall"] == pytest.approx(1.0) + assert metrics["metrics/f1"] == pytest.approx(1.0) + + # --------------------------------------------------------------------------- # Classification augmentation pack: auto_augment / erasing / mixup / cutmix. # ---------------------------------------------------------------------------