From f489485e383fec839325c2b2180676c155db226b Mon Sep 17 00:00:00 2001 From: Xuban <59646791+EHxuban11@users.noreply.github.com> Date: Sat, 15 Aug 2026 18:42:06 +0200 Subject: [PATCH 01/20] Add VLM confidence scoring foundation --- docs/adr/0002-librevlm-contract.md | 42 ++-- docs/librevlm_design.md | 25 ++- libreyolo/models/modus/model.py | 1 + libreyolo/models/vlm/base.py | 137 +++++++++++-- libreyolo/models/vlm/confidence.py | 296 +++++++++++++++++++++++++++ libreyolo/models/vlm/parsing.py | 85 +++++++- libreyolo/models/vlm/qwen3vl.py | 3 + tests/unit/test_modus.py | 1 + tests/unit/test_vlm_confidence.py | 311 +++++++++++++++++++++++++++++ tests/unit/test_vlm_parsing.py | 133 ++++++++++++ 10 files changed, 991 insertions(+), 43 deletions(-) create mode 100644 libreyolo/models/vlm/confidence.py create mode 100644 tests/unit/test_vlm_confidence.py diff --git a/docs/adr/0002-librevlm-contract.md b/docs/adr/0002-librevlm-contract.md index 541a32f0c..83a6de179 100644 --- a/docs/adr/0002-librevlm-contract.md +++ b/docs/adr/0002-librevlm-contract.md @@ -103,8 +103,10 @@ shared `InferenceRunner` drives: - `_preprocess(image, ...)` builds the chat-template inputs from the image plus the detection prompt; returns `(inputs, pil_image, (W, H), ratio=1.0)`. Boxes come back normalized to the image, so there is no letterbox/unpad math. -- `_forward(inputs)` runs `model.generate(...)` greedily and returns only the - newly generated tokens. +- `_forward(inputs)` runs `model.generate(...)` greedily and returns the newly + generated tokens. A family-gated scoring path can also attach one + selected-token log-probability per step without retaining vocabulary-sized + score tensors. - `_postprocess(output, conf, ...)` decodes, tolerantly parses the JSON, scales the coordinates per `BBOX_KEY`/`COORD_DIVISOR`, and returns the standard detection dict `{boxes, scores, classes, num_detections}` that @@ -126,18 +128,31 @@ in [`../librevlm_design.md`](../librevlm_design.md). ## Confidence -Generated detections carry no calibrated per-box score. The tier assigns a -constant placeholder (`DEFAULT_SCORE = 1.0`), so `predict`/draw/`track` behave -normally and `conf=` filtering still functions mechanically. Consequences: +Generated detections carry no calibrated per-box score. The generic VLM families +currently assign a constant placeholder (`DEFAULT_SCORE = 1.0`). A bounded-memory +candidate for Qwen3-VL can derive a ranking signal from the geometric mean of +generated label-token and coordinate-token probabilities. It records one +selected-token log-probability after the configured generation processors per +step, rather than retaining a vocabulary-sized score tensor for every token. +The candidate remains disabled until its real-data gate passes, so ordinary +`predict()` keeps the established constant-score behavior. LibreMODUS separately +uses the minimum constrained-token probability for each detection. -- `conf=` thresholds and ranking are soft, not calibrated. -- `track()` runs, but because every box is scored 1.0, ByteTrack's two-stage, - score-stratified association is inert (no separate low-confidence recovery - stage and `new_track_thresh` never bites) until a real score lands. -- `val()` (mAP) is intentionally unsupported; it would be misleading. +`model.confidence_method` reports the configured source (`constant` today for +Qwen3-VL, and `constrained_token_min` for LibreMODUS). -`_score_detections(items)` is the documented override point for a real signal -(decoder token log-probs or self-consistency) in a later iteration. +Consequences: + +- On constant-score families, `conf=` filtering is mechanical and ByteTrack's + score-stratified association remains inert (no separate low-confidence + recovery stage). +- `val()` (mAP) remains unsupported until the candidate score orders correct + detections better than the constant baseline, behaves safely with the public + confidence threshold, and is reproducible. Unit tests establish plumbing, not + score quality. + +`_score_detections(items)` remains the scalar fallback for custom generation +paths. Scored greedy generations use the additive per-item scoring path. ## Licensing @@ -180,7 +195,8 @@ executing mutable upstream model-repository code under the same alias. ### Negative -- Confidence is synthetic until the log-prob path lands. +- Generic-family confidence remains constant until each score path passes its + real-data quality gate; LibreMODUS's constrained-token score is uncalibrated. - Generation is slower and less deterministic than a detector forward. - Adds `transformers` (already an optional extra) to the `vlm` extra. diff --git a/docs/librevlm_design.md b/docs/librevlm_design.md index e064b7ef3..4facffd1d 100644 --- a/docs/librevlm_design.md +++ b/docs/librevlm_design.md @@ -126,13 +126,16 @@ The tier returns the standard `Results` (`boxes.xyxy`, `boxes.cls`, `boxes.conf`, `.plot()`, `.save()`), so folders, video, tracking, and drawing all work unchanged. No new output type is invented. -But these models emit no calibrated per-box score, so `conf` is a placeholder. -We do not pretend otherwise: +These models emit no calibrated detector score. The generic VLM families retain +`1.0`. A bounded-memory Qwen3-VL candidate can derive a ranking signal from +generated label-token and coordinate-token probabilities, but it remains off +until the real-data quality gate passes. Inspect `model.confidence_method` for +the configured source. We do not pretend any token-derived score is calibrated: -- `conf=` filtering and ranking are soft, not calibrated. -- `val()` (mAP) is intentionally unsupported, because it would be misleading. -- `_score_detections()` is the documented hook for a real signal later (decoder - token log-probabilities or self-consistency). +- On constant-score families, `conf=` filtering is mechanical. +- `val()` (mAP) remains unsupported until a real Qwen benchmark demonstrates + useful ordering, safe threshold behavior, and reproducibility. +- `_score_detections()` remains the scalar fallback for custom generation paths. This is the honest boundary of the tier: it gives you boxes and labels, not a calibrated detector contract. For calibrated scores and tight boxes, the @@ -286,11 +289,11 @@ precision tiers, and terms distinction are documented in These are deliberate v1 scoping choices, called out so behavior matches expectations: -- **Confidence is usually synthetic.** Chat-model adapters assign every box - `1.0`; `conf=` filtering is mechanical, and ByteTrack's score-stratified - association is inert. LibreMODUS derives a sequence score from constrained - token probabilities, but it is still not calibrated detector confidence. - Generic `val()`/mAP remains unsupported. +- **Confidence is not calibrated.** Local chat VLM families currently backfill + `1.0`; the bounded-memory Qwen3-VL scoring candidate is staged but disabled. + LibreMODUS derives a separate per-box minimum from constrained token + probabilities. Generic `val()`/mAP remains unsupported until each score path + passes its real-data quality gate. - **`batch=` does not speed up VLMs.** `predict("folder/")` works, but generation runs one image at a time, so a larger `batch=` gives no throughput gain in v1. - **Python-API only.** The `libreyolo` CLI does not resolve VLM aliases yet; use diff --git a/libreyolo/models/modus/model.py b/libreyolo/models/modus/model.py index 8ec1c9b98..6756d427f 100644 --- a/libreyolo/models/modus/model.py +++ b/libreyolo/models/modus/model.py @@ -76,6 +76,7 @@ class LibreMODUS(LibreVLMModel): DEFAULT_TASK = "detect" SUPPORTS_BATCHED_PREDICT = False TTA_ENABLED = False + CONFIDENCE_METHOD = "constrained_token_min" def __init__( self, diff --git a/libreyolo/models/vlm/base.py b/libreyolo/models/vlm/base.py index 16067d0c8..5e20ae568 100644 --- a/libreyolo/models/vlm/base.py +++ b/libreyolo/models/vlm/base.py @@ -23,6 +23,7 @@ import logging import re from collections.abc import MutableMapping +from dataclasses import dataclass from pathlib import Path from typing import Any, ClassVar, Dict, Optional, Tuple @@ -31,6 +32,7 @@ from ...utils.image_loader import ImageInput, ImageLoader from ..base.model import BaseModel +from .confidence import TokenSpan, decode_token_spans, score_detection_items from .parsing import build_detection_dict, extract_detections logger = logging.getLogger(__name__) @@ -43,6 +45,37 @@ _COMMIT_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") +@dataclass(frozen=True) +class _ScoredGeneration: + """Generated token ids plus one generation-policy log-probability per step.""" + + token_ids: torch.Tensor + token_logprobs: torch.Tensor + + +class _GreedyTokenLogprobRecorder: + """Record greedy post-processor log-probabilities without retaining logits.""" + + def __init__(self) -> None: + self._steps: list[torch.Tensor] = [] + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + # This processor is appended after repetition/constraint processors. With + # do_sample=False and num_beams=1, generate selects scores.argmax(-1). + # Keep only one scalar per batch row instead of a vocabulary-sized tensor + # for each generation step (MAX_NEW_TOKENS is 1024). + del input_ids + logits = scores.float() + selected_logprob = logits.amax(dim=-1) - torch.logsumexp(logits, dim=-1) + self._steps.append(selected_logprob.detach()) + return scores + + def values(self) -> torch.Tensor: + if not self._steps: + return torch.empty((1, 0), dtype=torch.float32) + return torch.stack(self._steps, dim=1).to(device="cpu", dtype=torch.float32) + + class LibreVLMModel(BaseModel): """Generative VLM repurposed as a closed-set object detector.""" @@ -57,10 +90,14 @@ class LibreVLMModel(BaseModel): SUPPORTED_TASKS: ClassVar[tuple] = ("detect",) DEFAULT_TASK: ClassVar[str] = "detect" - # Generative output has no calibrated per-box confidence. v1 assigns a - # constant placeholder so predict/draw/track behave; ``conf=`` filtering and - # mAP are therefore soft. Override ``_score_detections`` for a real signal. + # Generative output has no calibrated per-box confidence. Families that have + # not verified a ranking signal use a constant placeholder so predict/draw/ + # track behave. ``confidence_method`` makes that provenance inspectable. DEFAULT_SCORE: ClassVar[float] = 1.0 + CONFIDENCE_METHOD: ClassVar[str] = "constant" + # Opt-in only after a family has verified both its greedy generation path + # and score quality on real detection data. Other families keep DEFAULT_SCORE. + TOKEN_LOGPROB_CONFIDENCE: ClassVar[bool] = False MAX_NEW_TOKENS: ClassVar[int] = 1024 # Output coordinate convention. LFM2-VL emits ``bbox`` normalized to [0, 1]; # Qwen-style models emit ``bbox_2d`` on a 0-1000 scale. Families override. @@ -202,6 +239,12 @@ def set_classes(self, classes: list) -> "LibreVLMModel": self._name_to_id = {v.strip().lower(): k for k, v in self.names.items()} return self + @property + def confidence_method(self) -> str: + """Name the configured source of ``boxes.conf`` values for this family.""" + + return self.CONFIDENCE_METHOD + def set_task(self, task: str) -> "LibreVLMModel": """Switch the active task without reloading the model. @@ -566,22 +609,76 @@ def _preprocess( # normalized to the image, so no letterbox/unpad bookkeeping is needed. return inputs, img, img.size, 1.0 - def _forward(self, inputs: Any) -> torch.Tensor: + def _forward(self, inputs: Any) -> Any: inputs = self._prepare_generation_inputs(inputs) input_len = inputs["input_ids"].shape[1] - generated = self.model.generate( - **inputs, - max_new_tokens=self.MAX_NEW_TOKENS, - do_sample=False, - repetition_penalty=self.REPETITION_PENALTY, - ) + generate_kwargs = { + "max_new_tokens": self.MAX_NEW_TOKENS, + "do_sample": False, + "repetition_penalty": self.REPETITION_PENALTY, + } + recorder = None + if self.TOKEN_LOGPROB_CONFIDENCE: + recorder = _GreedyTokenLogprobRecorder() + # The recorder relies on the selected token being argmax(scores). + generate_kwargs.update(num_beams=1, logits_processor=[recorder]) + generated = self.model.generate(**inputs, **generate_kwargs) + sequences = getattr(generated, "sequences", generated) # Strip the prompt tokens; keep only what the model generated. - return generated[:, input_len:] + new_tokens = sequences[:, input_len:] + if recorder is None: + return new_tokens + token_logprobs = recorder.values() + if token_logprobs.shape != new_tokens.shape: + logger.warning( + "Token-confidence alignment failed for %s: %s token ids vs %s " + "log-probabilities; using fallback scores.", + self.FAMILY, + tuple(new_tokens.shape), + tuple(token_logprobs.shape), + ) + return new_tokens + return _ScoredGeneration(new_tokens, token_logprobs) def _score_detections(self, items: list) -> float: """Per-call confidence for parsed detections (placeholder in v1).""" return self.DEFAULT_SCORE + def _decode_token_ids(self, token_ids) -> str: + """Decode generated ids without whitespace cleanup that shifts spans.""" + + payload = token_ids + if isinstance(token_ids, (list, tuple)): + payload = [list(token_ids)] + kwargs = {"skip_special_tokens": True, "clean_up_tokenization_spaces": False} + try: + return self.processor.batch_decode(payload, **kwargs)[0] + except TypeError: + kwargs.pop("clean_up_tokenization_spaces") + return self.processor.batch_decode(payload, **kwargs)[0] + + def _scores_for_detections( + self, + output: Any, + text: str, + items: list, + token_spans: list[TokenSpan], + ) -> Optional[list[float]]: + """Return per-item scores when generation carries aligned policy logprobs.""" + + if not isinstance(output, _ScoredGeneration) or not token_spans: + return None + raw_scores = score_detection_items( + text, items, token_spans, bbox_key=self.BBOX_KEY + ) + # Never mix a maximum-valued placeholder with real scores: that would + # rank an unaligned item above every successfully scored detection. If + # any source object cannot be aligned, keep the established constant + # behavior for the whole response. + if any(score is None for score in raw_scores): + return None + return [float(score) for score in raw_scores if score is not None] + def _postprocess( self, output: Any, @@ -592,8 +689,19 @@ def _postprocess( ratio: float = 1.0, **kwargs, ) -> Dict: - text = self.processor.batch_decode(output, skip_special_tokens=True)[0] + token_spans: list[TokenSpan] = [] + if isinstance(output, _ScoredGeneration): + text, token_spans = decode_token_spans( + output.token_ids, + output.token_logprobs, + self._decode_token_ids, + ) + else: + # Preserve the established family decode behavior when scoring is + # disabled; whitespace-stable decoding is needed only for spans. + text = self.processor.batch_decode(output, skip_special_tokens=True)[0] items = extract_detections(text) + item_scores = self._scores_for_detections(output, text, items, token_spans) return build_detection_dict( items, self._name_to_id, @@ -602,6 +710,7 @@ def _postprocess( max_det=max_det, classes=kwargs.get("classes"), default_score=self._score_detections(items), + item_scores=item_scores, bbox_key=self.BBOX_KEY, coord_divisor=self.COORD_DIVISOR, box_format=self.BOX_FORMAT, @@ -651,8 +760,8 @@ def train(self, data: Optional[str] = None, **kwargs): def val(self, *args, **kwargs): raise NotImplementedError( f"Dataset validation is not supported for {type(self).__name__}: " - "generated boxes carry only a placeholder confidence, so COCO mAP " - "would be misleading. Evaluate qualitatively via predict()." + "per-box score ordering has not passed the real-data validation gate, " + "so publishing COCO mAP would be premature. Evaluate via predict()." ) def export(self, format: str = "onnx", **kwargs) -> str: diff --git a/libreyolo/models/vlm/confidence.py b/libreyolo/models/vlm/confidence.py new file mode 100644 index 000000000..609c5a44f --- /dev/null +++ b/libreyolo/models/vlm/confidence.py @@ -0,0 +1,296 @@ +"""Token-to-detection confidence helpers for autoregressive VLM output. + +The helpers in this module are deliberately model-agnostic. They align selected +token log-probabilities with the decoded text, locate the label and coordinate +values for each parsed detection, and reduce those values to one ranking score +per box. No score is treated as calibrated probability. +""" + +from __future__ import annotations + +import math +import re +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Optional + +from .parsing import locate_detection_spans + +__all__ = [ + "TokenSpan", + "decode_token_spans", + "score_detection_items", +] + + +@dataclass(frozen=True) +class TokenSpan: + """Character range and selected-token log-probability in decoded text.""" + + start: int + end: int + logprob: float + + +_NUMBER = re.compile(r"-?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?") + + +def _single_row(values, name: str) -> list: + if hasattr(values, "detach"): + values = values.detach() + if hasattr(values, "cpu"): + values = values.cpu() + if hasattr(values, "tolist"): + values = values.tolist() + values = list(values) + if values and isinstance(values[0], (list, tuple)): + if len(values) != 1: + raise ValueError(f"{name} must contain exactly one generated sequence.") + values = list(values[0]) + return values + + +def decode_token_spans( + token_ids, + token_logprobs, + decode: Callable[[Sequence[int]], str], +) -> tuple[str, list[TokenSpan]]: + """Decode one generated sequence and align each token to character offsets. + + Most tokenizers compose when each token is decoded separately, which gives a + linear-time path. Tokenizers whose whitespace handling is context-dependent + fall back to decoding prefixes. If prefix decoding rewrites earlier text, the + function returns the full text with no spans so callers can safely retain + their constant-score fallback instead of attaching scores to the wrong box. + """ + + ids = [int(value) for value in _single_row(token_ids, "token_ids")] + logprobs = [ + float(value) for value in _single_row(token_logprobs, "token_logprobs") + ] + if len(ids) != len(logprobs): + raise ValueError( + "token_ids and token_logprobs must have the same generated length." + ) + + full_text = decode(ids) + pieces = [decode([token_id]) for token_id in ids] + if "".join(pieces) == full_text: + spans = [] + cursor = 0 + for piece, logprob in zip(pieces, logprobs): + end = cursor + len(piece) + spans.append(TokenSpan(cursor, end, logprob)) + cursor = end + return full_text, spans + + spans = [] + previous = "" + for index, logprob in enumerate(logprobs, 1): + current = decode(ids[:index]) + if not current.startswith(previous): + return full_text, [] + spans.append(TokenSpan(len(previous), len(current), logprob)) + previous = current + if previous != full_text: + return full_text, [] + return full_text, spans + + +def _quoted_end(blob: str, start: int) -> Optional[int]: + if start >= len(blob) or blob[start] not in {'"', "'"}: + return None + quote = blob[start] + escaped = False + for index in range(start + 1, len(blob)): + char = blob[index] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == quote: + return index + 1 + return None + + +def _value_end(blob: str, start: int) -> Optional[int]: + if start >= len(blob): + return None + if blob[start] in {'"', "'"}: + return _quoted_end(blob, start) + pairs = {"[": "]", "{": "}"} + if blob[start] in pairs: + stack = [pairs[blob[start]]] + index = start + 1 + while index < len(blob): + char = blob[index] + if char in {'"', "'"}: + string_end = _quoted_end(blob, index) + if string_end is None: + return None + index = string_end + continue + if char in pairs: + stack.append(pairs[char]) + elif char in "]}": + if not stack or char != stack.pop(): + return None + if not stack: + return index + 1 + index += 1 + return None + index = start + while index < len(blob) and blob[index] not in ",}": + index += 1 + return index + + +def _object_members(blob: str) -> dict[str, list[tuple[int, int]]]: + """Return top-level object value spans, or an empty map when ambiguous.""" + + if len(blob) < 2 or blob[0] != "{" or blob[-1] != "}": + return {} + members: dict[str, list[tuple[int, int]]] = {} + index = 1 + while index < len(blob) - 1: + while index < len(blob) - 1 and (blob[index].isspace() or blob[index] == ","): + index += 1 + if index >= len(blob) - 1: + break + key_end = _quoted_end(blob, index) + if key_end is None: + return {} + key = blob[index + 1 : key_end - 1] + # The scoring keys are ASCII literals. Reject escaped keys rather than + # guessing how their decoded spelling maps back to source characters. + if "\\" in key: + return {} + index = key_end + while index < len(blob) - 1 and blob[index].isspace(): + index += 1 + if index >= len(blob) - 1 or blob[index] != ":": + return {} + index += 1 + while index < len(blob) - 1 and blob[index].isspace(): + index += 1 + value_start = index + value_end = _value_end(blob, value_start) + if value_end is None: + return {} + members.setdefault(key, []).append((value_start, value_end)) + index = value_end + while index < len(blob) - 1 and blob[index].isspace(): + index += 1 + if index < len(blob) - 1 and blob[index] not in ",}": + return {} + return members + + +def _unique_member( + members: dict[str, list[tuple[int, int]]], key: str +) -> Optional[tuple[int, int]]: + regions = members.get(key, []) + return regions[0] if len(regions) == 1 else None + + +def _string_value_region( + blob: str, + members: dict[str, list[tuple[int, int]]], + key: str, + offset: int, +) -> list[tuple[int, int]]: + region = _unique_member(members, key) + if region is None: + return [] + start, end = region + if blob[start] not in {'"', "'"} or end <= start + 1: + return [] + return [(offset + start + 1, offset + end - 1)] + + +def _number_value_regions( + blob: str, + members: dict[str, list[tuple[int, int]]], + key: str, + offset: int, +) -> list[tuple[int, int]]: + region = _unique_member(members, key) + if region is None: + return [] + start, end = region + if blob[start] != "[" or blob[end - 1] != "]": + return [] + return [ + (offset + match.start(), offset + match.end()) + for match in _NUMBER.finditer(blob, start + 1, end - 1) + ] + + +def _mean_logprob( + regions: Sequence[tuple[int, int]], token_spans: Sequence[TokenSpan] +) -> Optional[float]: + selected = { + index + for index, token in enumerate(token_spans) + if token.end > token.start + and any(token.start < end and token.end > start for start, end in regions) + } + values = [token_spans[index].logprob for index in sorted(selected)] + if not values or any(not math.isfinite(value) for value in values): + return None + return sum(values) / len(values) + + +def score_detection_items( + text: str, + items: Sequence[dict], + token_spans: Sequence[TokenSpan], + *, + bbox_key: str, +) -> list[Optional[float]]: + """Return one token-logprob ranking score for every parsed detection. + + Coordinate-number tokens and label-value tokens are reduced separately by + their mean log-probability, then given equal weight. This prevents a label's + score from being drowned out when four coordinates each split into several + tokens. Both components are required. Missing or ambiguous keys return + ``None`` so the caller can safely fall back instead of ranking a box using a + different source value. + """ + + object_spans = locate_detection_spans(text, items) + scores: list[Optional[float]] = [] + for item, object_span in zip(items, object_spans): + if object_span is None: + scores.append(None) + continue + object_start, object_end = object_span + blob = text[object_start:object_end] + members = _object_members(blob) + label_regions = _string_value_region(blob, members, "label", object_start) + + # Mirror ``build_detection_dict`` exactly: a present, non-null preferred + # key wins even when its source mapping is ambiguous. Only a null/missing + # preferred value permits the builder's first-present alias fallback. + coord_key = bbox_key if item.get(bbox_key) is not None else None + if coord_key is None: + for alias in ("bbox", "bbox_2d"): + if alias != bbox_key and alias in item: + coord_key = alias + break + coord_regions = ( + _number_value_regions(blob, members, coord_key, object_start) + if coord_key is not None + else [] + ) + + label_logprob = _mean_logprob(label_regions, token_spans) + coord_logprob = _mean_logprob(coord_regions, token_spans) + if label_logprob is None or coord_logprob is None: + scores.append(None) + continue + # A normalized geometric mean stays in [0, 1]. Log-probabilities should + # be <= 0; clamp tiny positive numerical noise before exponentiating. + score = math.exp(min(0.0, (label_logprob + coord_logprob) / 2.0)) + scores.append(score) + return scores diff --git a/libreyolo/models/vlm/parsing.py b/libreyolo/models/vlm/parsing.py index ef2ca5c7b..0e671743d 100644 --- a/libreyolo/models/vlm/parsing.py +++ b/libreyolo/models/vlm/parsing.py @@ -15,12 +15,14 @@ from __future__ import annotations import json +import math import re from typing import Dict, List, Optional, Tuple __all__ = [ "extract_detections", "extract_bare_boxes", + "locate_detection_spans", "normalize_bbox", "to_xyxy", "resolve_label", @@ -139,6 +141,44 @@ def extract_detections(text: str) -> List[dict]: return recovered +def locate_detection_spans( + text: str, items: List[dict] +) -> List[Optional[Tuple[int, int]]]: + """Locate parsed detection objects in the original generated text. + + Results stay aligned one-for-one with ``items``. Repeated identical objects + consume successive source occurrences, and an object that cannot be matched + returns ``None`` rather than borrowing another detection's token span. + """ + + if not isinstance(text, str): + return [None] * len(items) + occurrences = [] + for match in _OBJECT.finditer(text): + parsed = _loads_object(match.group(0)) + if parsed is not None: + occurrences.append((parsed, match.span())) + + used = set() + spans: List[Optional[Tuple[int, int]]] = [] + for item in items: + matching_occurrences = [ + index for index, (parsed, _span) in enumerate(occurrences) if parsed == item + ] + matching_items = sum(candidate == item for candidate in items) + if len(matching_occurrences) != matching_items: + spans.append(None) + continue + found = None + for index in matching_occurrences: + if index not in used: + used.add(index) + found = occurrences[index][1] + break + spans.append(found) + return spans + + _BARE_QUAD = re.compile( r"\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*," r"\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]" @@ -247,6 +287,7 @@ def build_detection_dict( coord_divisor: float = 1.0, box_format: str = "xyxy", iou_thres: Optional[float] = None, + item_scores: Optional[List[Optional[float]]] = None, ) -> dict: """Turn parsed items into the ``InferenceRunner`` detection dict. @@ -258,9 +299,15 @@ def build_detection_dict( malformed boxes are skipped. If ``classes`` is provided, that class filter is applied before the ``max_det`` cap so requested classes are not dropped by an earlier out-of-filter prediction. ``default_score`` is the synthetic per-box - confidence (the VLM emits none); rows below ``conf_thres`` are dropped so - ``conf=`` still filters. + confidence fallback. ``item_scores``, when supplied, must align one-for-one + with ``items``. A missing, non-finite, or out-of-range entry fails the whole + list back to ``default_score`` and source order. Valid scored detections are + processed from highest to lowest so duplicate/IoU suppression and + ``max_det`` retain the strongest candidate. Rows below ``conf_thres`` are + dropped so ``conf=`` still filters. """ + if item_scores is not None and len(item_scores) != len(items): + raise ValueError("item_scores must have one entry per detection item.") if max_det <= 0: return { "boxes": [], @@ -280,7 +327,35 @@ def build_detection_dict( # class + box rounded to ~0.1% of the image). seen = set() - for item in items: + try: + fallback_score = float(default_score) + except (TypeError, ValueError): + fallback_score = 1.0 + if not math.isfinite(fallback_score): + fallback_score = 1.0 + fallback_score = min(1.0, max(0.0, fallback_score)) + + resolved_scores = None + if item_scores is not None: + try: + candidates = [float(value) for value in item_scores] + except (TypeError, ValueError): + candidates = [] + if len(candidates) == len(items) and all( + math.isfinite(score) and 0.0 <= score <= 1.0 for score in candidates + ): + resolved_scores = candidates + + indexed = list(enumerate(items)) + if resolved_scores is not None: + indexed.sort(key=lambda pair: resolved_scores[pair[0]], reverse=True) + + for item_index, item in indexed: + score = ( + resolved_scores[item_index] + if resolved_scores is not None + else fallback_score + ) class_id = resolve_label(item.get("label"), name_to_id) if class_id is None: continue @@ -305,7 +380,7 @@ def build_detection_dict( box = normalize_bbox(to_xyxy(scaled, box_format)) if scaled else None if box is None: continue - if default_score < conf_thres: + if score < conf_thres: continue key = (class_id, *(round(v, 3) for v in box)) if key in seen: @@ -319,7 +394,7 @@ def build_detection_dict( x1, y1, x2, y2 = box norm_boxes.append(box) boxes.append([x1 * width, y1 * height, x2 * width, y2 * height]) - scores.append(default_score) + scores.append(score) class_ids.append(class_id) if len(boxes) >= max_det: break diff --git a/libreyolo/models/vlm/qwen3vl.py b/libreyolo/models/vlm/qwen3vl.py index b20527e9e..b7d764c90 100644 --- a/libreyolo/models/vlm/qwen3vl.py +++ b/libreyolo/models/vlm/qwen3vl.py @@ -39,6 +39,9 @@ class LibreQwen3VL(LibreVLMModel): # Qwen emits {"bbox_2d": [x1,y1,x2,y2], "label": ...} on a 0-1000 scale. BBOX_KEY = "bbox_2d" COORD_DIVISOR = 1000.0 + # The bounded-memory scoring candidate remains off until a real-data gate + # establishes useful ordering and a safe interaction with ``conf=``. + TOKEN_LOGPROB_CONFIDENCE = False # First trainable family: grounding-pretrained, Apache-2.0, native # transformers classes, and an official upstream fine-tuning recipe this diff --git a/tests/unit/test_modus.py b/tests/unit/test_modus.py index 1eb0db67f..e9498ced4 100644 --- a/tests/unit/test_modus.py +++ b/tests/unit/test_modus.py @@ -732,6 +732,7 @@ def test_factory_aliases_and_lazy_exports(monkeypatch): assert vlm.LibreMODUS is LibreMODUS assert vlm.LibreModus is LibreMODUS assert vlm._MODUS_ALIASES["libremodus-14b-a7b"] == "14b-a7b" + assert LibreMODUS.CONFIDENCE_METHOD == "constrained_token_min" class Sentinel: def __init__(self, size, **kwargs): diff --git a/tests/unit/test_vlm_confidence.py b/tests/unit/test_vlm_confidence.py new file mode 100644 index 000000000..d0302301b --- /dev/null +++ b/tests/unit/test_vlm_confidence.py @@ -0,0 +1,311 @@ +"""Offline tests for VLM token-logprob confidence scoring.""" + +import math +import re + +import pytest +import torch + +from libreyolo.models.vlm.base import ( + LibreVLMModel, + _GreedyTokenLogprobRecorder, + _ScoredGeneration, +) +from libreyolo.models.vlm.confidence import ( + TokenSpan, + decode_token_spans, + score_detection_items, +) +from libreyolo.models.vlm.parsing import build_detection_dict, extract_detections +from libreyolo.models.vlm.qwen3vl import LibreQwen3VL + +pytestmark = pytest.mark.unit + + +def _char_spans(text: str, values: dict[int, float]) -> list[TokenSpan]: + return [ + TokenSpan(index, index + 1, values.get(index, -100.0)) + for index in range(len(text)) + ] + + +class TestDecodeTokenSpans: + def test_composable_token_pieces_take_linear_path(self): + pieces = {1: "red", 2: " car", 3: ""} + + def decode(ids): + return "".join(pieces[token_id] for token_id in ids) + + text, spans = decode_token_spans( + [[1, 2, 3]], [[math.log(0.8), math.log(0.6), math.log(0.9)]], decode + ) + assert text == "red car" + assert [(span.start, span.end) for span in spans] == [(0, 3), (3, 7), (7, 7)] + + def test_context_dependent_piece_uses_monotonic_prefix_fallback(self): + prefixes = {(1,): "red", (2,): "car", (1, 2): "red car"} + text, spans = decode_token_spans( + [1, 2], [-0.1, -0.2], lambda ids: prefixes[tuple(ids)] + ) + assert text == "red car" + assert [(span.start, span.end) for span in spans] == [(0, 3), (3, 7)] + + def test_non_monotonic_decode_fails_closed(self): + values = {(1,): "a", (2,): "b", (1, 2): "B"} + text, spans = decode_token_spans( + [1, 2], [-0.1, -0.2], lambda ids: values[tuple(ids)] + ) + assert text == "B" + assert spans == [] + + def test_length_mismatch_raises(self): + with pytest.raises(ValueError, match="same generated length"): + decode_token_spans([1, 2], [-0.1], lambda ids: "x" * len(ids)) + + +class TestScoreDetectionItems: + def test_equal_weights_label_and_coordinate_components(self): + text = '[{"bbox_2d":[10,20,30,40],"label":"red car"}]' + items = extract_detections(text) + values = {} + for match in re.finditer(r"10|20|30|40", text): + values.update( + {index: math.log(0.25) for index in range(match.start(), match.end())} + ) + label_start = text.index("red car") + values.update( + { + index: math.log(0.81) + for index in range(label_start, label_start + len("red car")) + } + ) + scores = score_detection_items( + text, items, _char_spans(text, values), bbox_key="bbox_2d" + ) + assert scores == pytest.approx([0.45]) + + def test_punctuation_and_key_logprobs_are_excluded(self): + text = '[{"bbox_2d":[1,2,3,4],"label":"cat"}]' + items = extract_detections(text) + values = { + index: math.log(0.64) + for index, char in enumerate(text) + if char.isdigit() + } + start = text.index("cat") + values.update({index: math.log(0.64) for index in range(start, start + 3)}) + scores = score_detection_items( + text, items, _char_spans(text, values), bbox_key="bbox_2d" + ) + assert scores == pytest.approx([0.64]) + + def test_repeated_objects_use_successive_token_spans(self): + obj = '{"bbox_2d":[1,2,3,4],"label":"cat"}' + text = f"[{obj},{obj}]" + items = extract_detections(text) + spans = _char_spans(text, {}) + first_start = text.index(obj) + second_start = text.index(obj, first_start + 1) + mutable = list(spans) + for object_start, probability in ((first_start, 0.8), (second_start, 0.2)): + for index in range(object_start, object_start + len(obj)): + mutable[index] = TokenSpan(index, index + 1, math.log(probability)) + scores = score_detection_items(text, items, mutable, bbox_key="bbox_2d") + assert scores == pytest.approx([0.8, 0.2]) + + def test_missing_source_span_returns_none(self): + scores = score_detection_items( + "[]", + [{"bbox_2d": [1, 2, 3, 4], "label": "cat"}], + [], + bbox_key="bbox_2d", + ) + assert scores == [None] + + @pytest.mark.parametrize( + "text", + [ + '[{"bbox_2d":[1,2,3,4],"bbox_2d":[5,6,7,8],"label":"cat"}]', + ( + '[{"bbox_2d":[1,2,3,4],"bbox":[9,9,10,10],' + '"bbox_2d":[5,6,7,8],"label":"cat"}]' + ), + '[{"bbox_2d":[1,2,3,4],"label":"dog","label":"cat"}]', + ], + ) + def test_duplicate_scoring_keys_fail_closed(self, text): + items = extract_detections(text) + spans = _char_spans(text, {index: math.log(0.8) for index in range(len(text))}) + assert score_detection_items(text, items, spans, bbox_key="bbox_2d") == [ + None + ] + + def test_key_like_text_inside_string_is_not_a_member(self): + text = ( + '[{"note":"fake \'bbox_2d\': [1,2,3,4]",' + '"bbox_2d":[5,6,7,8],"label":"cat"}]' + ) + items = extract_detections(text) + values = {} + actual_box = text.index("[5,6,7,8]") + for index in range(actual_box + 1, actual_box + len("5,6,7,8") + 1): + if text[index].isdigit(): + values[index] = math.log(0.25) + label = text.index("cat") + values.update({index: math.log(0.25) for index in range(label, label + 3)}) + scores = score_detection_items( + text, items, _char_spans(text, values), bbox_key="bbox_2d" + ) + assert scores == pytest.approx([0.25]) + + def test_missing_label_or_coordinate_component_fails_closed(self): + text = '[{"bbox_2d":[1,2,3,4]}]' + items = extract_detections(text) + spans = _char_spans(text, {index: math.log(0.8) for index in range(len(text))}) + assert score_detection_items(text, items, spans, bbox_key="bbox_2d") == [ + None + ] + + def test_non_finite_component_token_fails_closed(self): + text = '[{"bbox_2d":[1,2,3,4],"label":"cat"}]' + items = extract_detections(text) + spans = _char_spans(text, {index: math.log(0.8) for index in range(len(text))}) + label_start = text.index("cat") + spans[label_start] = TokenSpan(label_start, label_start + 1, float("nan")) + assert score_detection_items(text, items, spans, bbox_key="bbox_2d") == [ + None + ] + + +class TestGreedyRecorder: + def test_records_normalized_argmax_only(self): + recorder = _GreedyTokenLogprobRecorder() + probabilities = torch.tensor([[0.1, 0.6, 0.3]]) + logits = probabilities.log() + returned = recorder(torch.tensor([[1, 2]]), logits) + assert returned is logits + assert recorder.values().shape == (1, 1) + assert recorder.values()[0, 0].item() == pytest.approx(math.log(0.6)) + + def test_empty_recorder_has_bounded_empty_shape(self): + assert _GreedyTokenLogprobRecorder().values().shape == (1, 0) + + +class _StubGenerateModel: + def __init__(self, step_probabilities): + self.step_probabilities = step_probabilities + self.kwargs = None + + def generate(self, input_ids, **kwargs): + self.kwargs = kwargs + sequence = input_ids + for probabilities in self.step_probabilities: + scores = torch.tensor([probabilities], dtype=torch.float32).log() + for processor in kwargs.get("logits_processor", []): + scores = processor(sequence, scores) + selected = scores.argmax(dim=-1, keepdim=True) + sequence = torch.cat((sequence, selected), dim=1) + return sequence + + +class _StubDecodeProcessor: + pieces = { + 0: '[{"bbox_2d":[', + 1: "10,20,30,40", + 2: '],"label":"cat"}]', + } + + def __init__(self): + self.calls = [] + + def batch_decode(self, rows, **kwargs): + self.calls.append(kwargs) + if isinstance(rows, torch.Tensor): + rows = rows.tolist() + return ["".join(self.pieces[token] for token in row) for row in rows] + + +class TestBaseConfidenceIntegration: + def _model(self): + model = object.__new__(LibreVLMModel) + model.FAMILY = "stub" + model.TOKEN_LOGPROB_CONFIDENCE = True + model.MAX_NEW_TOKENS = 3 + model.REPETITION_PENALTY = 1.0 + model.BBOX_KEY = "bbox_2d" + model.COORD_DIVISOR = 1000.0 + model.BOX_FORMAT = "xyxy" + model.DEFAULT_SCORE = 1.0 + model._model_dtype = None + model._name_to_id = {"cat": 0} + model.processor = _StubDecodeProcessor() + model.model = _StubGenerateModel( + [[0.7, 0.2, 0.1], [0.1, 0.8, 0.1], [0.1, 0.2, 0.7]] + ) + return model + + def test_forward_returns_only_scalar_per_generated_step(self): + model = self._model() + output = model._forward({"input_ids": torch.tensor([[8, 9]])}) + assert isinstance(output, _ScoredGeneration) + assert output.token_ids.shape == (1, 3) + assert output.token_logprobs.shape == (1, 3) + assert model.model.kwargs["num_beams"] == 1 + assert "output_scores" not in model.model.kwargs + + def test_confidence_method_exposes_score_provenance(self): + assert LibreVLMModel.CONFIDENCE_METHOD == "constant" + assert LibreQwen3VL.CONFIDENCE_METHOD == "constant" + assert LibreQwen3VL.TOKEN_LOGPROB_CONFIDENCE is False + + def test_postprocess_emits_per_box_token_score(self): + model = self._model() + output = model._forward({"input_ids": torch.tensor([[8, 9]])}) + result = model._postprocess( + output, + conf_thres=0.0, + iou_thres=0.7, + original_size=(1000, 1000), + ) + assert result["num_detections"] == 1 + assert 0.0 < result["scores"][0] < 1.0 + + def test_unscored_output_keeps_constant_fallback(self): + model = self._model() + tokens = torch.tensor([[0, 1, 2]]) + result = model._postprocess( + tokens, + conf_thres=0.0, + iou_thres=0.7, + original_size=(1000, 1000), + ) + assert result["scores"] == [1.0] + assert model.processor.calls == [{"skip_special_tokens": True}] + + def test_one_unaligned_object_falls_whole_response_back_to_constant(self): + model = self._model() + model._name_to_id = {"cat": 0, "dog {x}": 1} + text = ( + '[{"bbox_2d":[10,20,30,40],"label":"cat"},' + '{"bbox_2d":[50,60,70,80],"label":"dog {x}"}]' + ) + items = extract_detections(text) + output = _ScoredGeneration(torch.tensor([[0]]), torch.tensor([[-0.1]])) + token_spans = _char_spans( + text, {index: math.log(0.8) for index in range(len(text))} + ) + item_scores = model._scores_for_detections( + output, text, items, token_spans + ) + assert item_scores is None + result = build_detection_dict( + items, + model._name_to_id, + (1000, 1000), + default_score=model.DEFAULT_SCORE, + item_scores=item_scores, + bbox_key="bbox_2d", + coord_divisor=1000.0, + ) + assert result["classes"] == [0, 1] + assert result["scores"] == [1.0, 1.0] diff --git a/tests/unit/test_vlm_parsing.py b/tests/unit/test_vlm_parsing.py index 8841313cf..606f7de9e 100644 --- a/tests/unit/test_vlm_parsing.py +++ b/tests/unit/test_vlm_parsing.py @@ -5,6 +5,7 @@ from libreyolo.models.vlm.parsing import ( build_detection_dict, extract_detections, + locate_detection_spans, normalize_bbox, resolve_label, ) @@ -103,6 +104,34 @@ def test_truncated_real_array_behind_preamble(self): assert "ship" in labels +class TestLocateDetectionSpans: + def test_repeated_objects_consume_successive_occurrences(self): + obj = '{"label":"person","bbox":[0.1,0.2,0.3,0.4]}' + text = f"prefix [{obj}, {obj}]" + items = extract_detections(text) + spans = locate_detection_spans(text, items) + assert spans == [ + (text.index(obj), text.index(obj) + len(obj)), + (text.rindex(obj), text.rindex(obj) + len(obj)), + ] + + def test_truncated_array_object_still_locates(self): + text = 'answer: [{"label":"ship","bbox":[0.1,0.2,0.3,0.4]}' + items = extract_detections(text) + span = locate_detection_spans(text, items)[0] + assert text[slice(*span)] == '{"label":"ship","bbox":[0.1,0.2,0.3,0.4]}' + + def test_unmatched_item_fails_closed(self): + assert locate_detection_spans("[]", [{"label": "ship"}]) == [None] + + def test_more_identical_source_occurrences_than_items_is_ambiguous(self): + obj = '{"label":"ship","bbox":[0.1,0.2,0.3,0.4]}' + text = f"example {obj}; final [{obj}]" + items = extract_detections(text) + assert len(items) == 1 + assert locate_detection_spans(text, items) == [None] + + class TestNormalizeBbox: def test_passthrough(self): assert normalize_bbox([0.1, 0.2, 0.3, 0.4]) == (0.1, 0.2, 0.3, 0.4) @@ -229,6 +258,110 @@ def test_empty_items(self): "num_detections": 0, } + def test_item_scores_remain_aligned_after_invalid_rows(self): + items = [ + {"label": "unknown", "bbox": [0.0, 0.0, 1.0, 1.0]}, + {"label": "person", "bbox": [0.1, 0.1, 0.2, 0.2]}, + {"label": "ship", "bbox": [0.3, 0.3, 0.3, 0.5]}, + ] + result = build_detection_dict( + items, + NAME_TO_ID, + (100, 100), + item_scores=[0.99, 0.42, 0.8], + ) + assert result["classes"] == [0] + assert result["scores"] == [0.42] + + def test_item_confidence_filter_is_per_row(self): + items = [ + {"label": "person", "bbox": [0.1, 0.1, 0.2, 0.2]}, + {"label": "ship", "bbox": [0.3, 0.3, 0.4, 0.4]}, + ] + result = build_detection_dict( + items, + NAME_TO_ID, + (100, 100), + conf_thres=0.5, + item_scores=[0.2, 0.8], + ) + assert result["classes"] == [8] + assert result["scores"] == [0.8] + + def test_max_det_retains_highest_score(self): + items = [ + {"label": "person", "bbox": [0.1, 0.1, 0.2, 0.2]}, + {"label": "ship", "bbox": [0.3, 0.3, 0.4, 0.4]}, + ] + result = build_detection_dict( + items, + NAME_TO_ID, + (100, 100), + max_det=1, + item_scores=[0.2, 0.8], + ) + assert result["classes"] == [8] + assert result["scores"] == [0.8] + + def test_duplicate_and_iou_suppression_retain_highest_score(self): + items = [ + {"label": "person", "bbox": [0.1, 0.1, 0.5, 0.5]}, + {"label": "person", "bbox": [0.1, 0.1, 0.5, 0.5]}, + {"label": "person", "bbox": [0.11, 0.11, 0.51, 0.51]}, + ] + result = build_detection_dict( + items, + NAME_TO_ID, + (100, 100), + item_scores=[0.2, 0.9, 0.7], + iou_thres=0.5, + ) + assert result["num_detections"] == 1 + assert result["scores"] == [0.9] + + def test_score_sanitization_is_explicit(self): + items = [ + {"label": "person", "bbox": [0.0, 0.0, 0.1, 0.1]}, + {"label": "person", "bbox": [0.2, 0.2, 0.3, 0.3]}, + {"label": "person", "bbox": [0.4, 0.4, 0.5, 0.5]}, + {"label": "person", "bbox": [0.6, 0.6, 0.7, 0.7]}, + ] + result = build_detection_dict( + items, + NAME_TO_ID, + (100, 100), + default_score=0.4, + item_scores=[None, float("nan"), 2.0, -1.0], + ) + assert result["scores"] == [0.4, 0.4, 0.4, 0.4] + + def test_item_score_length_mismatch_raises(self): + with pytest.raises(ValueError, match="one entry per detection"): + build_detection_dict( + [{"label": "person", "bbox": [0.1, 0.1, 0.2, 0.2]}], + NAME_TO_ID, + (100, 100), + item_scores=[], + ) + + def test_legacy_positional_arguments_keep_their_meaning(self): + items = [{"label": "ship", "bbox_2d": [100, 200, 300, 400]}] + result = build_detection_dict( + items, + NAME_TO_ID, + (100, 100), + 0.0, + 300, + None, + 0.75, + "bbox_2d", + 1000.0, + "xyxy", + 0.5, + ) + assert result["boxes"] == [[10.0, 20.0, 30.0, 40.0]] + assert result["scores"] == [0.75] + def test_box_format_xywh(self): # x,y,w,h: [0.25,0.25,0.25,0.5] -> xyxy [0.25,0.25,0.5,0.75] -> px items = [{"label": "ship", "bbox": [0.25, 0.25, 0.25, 0.5]}] From fce359f5aea20570c06f0c0266c7c73840107a8a Mon Sep 17 00:00:00 2001 From: Xuban <59646791+EHxuban11@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:14:45 +0200 Subject: [PATCH 02/20] Harden VLM validation gates --- docs/librevlm_design.md | 11 +- docs/vlm_training.md | 23 +- libreyolo/models/vlm/base.py | 264 +++- libreyolo/models/vlm/parsing.py | 9 +- libreyolo/models/vlm/qwen3vl.py | 13 +- libreyolo/models/vlm/training/checkpoint.py | 92 +- libreyolo/models/vlm/training/recipes.py | 5 +- libreyolo/validation/vlm_confidence.py | 869 +++++++++++ .../validation/vlm_confidence_validator.py | 1365 +++++++++++++++++ tests/e2e/test_vlm_train_qwen3vl.py | 21 +- tests/unit/test_vlm_api.py | 56 + tests/unit/test_vlm_confidence.py | 138 +- tests/unit/test_vlm_confidence_quality.py | 322 ++++ tests/unit/test_vlm_confidence_validator.py | 744 +++++++++ tests/unit/test_vlm_training.py | 244 ++- 15 files changed, 4103 insertions(+), 73 deletions(-) create mode 100644 libreyolo/validation/vlm_confidence.py create mode 100644 libreyolo/validation/vlm_confidence_validator.py create mode 100644 tests/unit/test_vlm_confidence_quality.py create mode 100644 tests/unit/test_vlm_confidence_validator.py diff --git a/docs/librevlm_design.md b/docs/librevlm_design.md index 4facffd1d..88020c83f 100644 --- a/docs/librevlm_design.md +++ b/docs/librevlm_design.md @@ -201,12 +201,11 @@ output but grounds single-class queries extremely well, so its family runs one "Locate every