update dataset

Author: benjibc

examples/vision_food_reasoning_dataset/data/vision_food_reasoning_full.jsonl

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""
+Utility script to convert the raw fireworks vision-food-reasoning dataset into native
+Eval Protocol EvaluationRow JSONL files so that the default dataset adapter can be used.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from typing import Any, Iterable
+
+DATASET_SOURCE_ID = "fireworks-ai/vision-food-reasoning-dataset"
+
+_BOLD_LABEL_PATTERN = re.compile(r"\*\*(?P<label>[^*]+)\*\*")
+_APPEARS_PATTERN = re.compile(r"appears to be\s+(?P<label>[A-Za-z0-9_\- ]+)", re.IGNORECASE)
+_IS_PATTERN = re.compile(r"is\s+(?:a|an|the)?\s*(?P<label>[A-Za-z0-9_\- ]+)", re.IGNORECASE)
+_SECTION_HEADINGS = {
+    "visual characteristics",
+    "texture and shape",
+    "texture",
+    "shape",
+    "cooking method or preparation style",
+    "cooking method",
+    "preparation style",
+    "cultural context or typical presentation",
+    "cultural context",
+    "presentation",
+    "distinguishing features",
+    "ingredients",
+    "aroma",
+    "flavor profile",
+}
+
+
+def _normalize_label(label: str | None) -> str:
+    if not label:
+        return ""
+    cleaned = re.sub(r"[^a-z0-9]+", "_", label.lower())
+    cleaned = re.sub(r"_+", "_", cleaned).strip("_")
+    return cleaned
+
+
+def _content_to_text(content: Any) -> str:
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, Iterable):
+        parts: list[str] = []
+        for part in content:
+            if isinstance(part, dict) and part.get("type") == "text":
+                text_val = part.get("text")
+                if isinstance(text_val, str):
+                    parts.append(text_val)
+        return "\n".join(parts)
+    return ""
+
+
+def _extract_label_from_text(text: str) -> str | None:
+    if not text:
+        return None
+    bold_matches = _BOLD_LABEL_PATTERN.findall(text)
+    if bold_matches:
+        for candidate in reversed(bold_matches):
+            normalized = candidate.strip().lower()
+            if normalized not in _SECTION_HEADINGS:
+                return candidate.strip()
+    for pattern in (_APPEARS_PATTERN, _IS_PATTERN):
+        match = pattern.search(text)
+        if match:
+            label = match.group("label").strip()
+            if len(label.split()) <= 5:
+                return label
+    sentences = [segment.strip() for segment in re.split(r"[.!?\n]+", text) if segment.strip()]
+    if sentences:
+        tail = sentences[-1]
+        tokens = re.findall(r"[A-Za-z][A-Za-z0-9_\- ]+", tail)
+        if tokens:
+            return tokens[-1].strip()
+    return None
+
+
+def convert_dataset(input_path: Path, output_path: Path) -> None:
+    rows: list[dict[str, Any]] = []
+    with input_path.open() as infile:
+        for line in infile:
+            line = line.strip()
+            if not line:
+                continue
+            rows.append(json.loads(line))
+
+    converted_rows: list[dict[str, Any]] = []
+    skipped = 0
+
+    for idx, raw in enumerate(rows):
+        messages_payload = raw.get("messages")
+        if not isinstance(messages_payload, list) or len(messages_payload) < 2:
+            skipped += 1
+            continue
+
+        assistant_reference = messages_payload[-1]
+        prompt_messages = [
+            message
+            for message in messages_payload[:-1]
+            if isinstance(message, dict) and message.get("role") in {"system", "user"}
+        ]
+        if not prompt_messages:
+            skipped += 1
+            continue
+
+        reference_text = _content_to_text(assistant_reference.get("content"))
+        raw_label = _extract_label_from_text(reference_text)
+        normalized_label = _normalize_label(raw_label)
+        if not normalized_label:
+            skipped += 1
+            continue
+
+        row_id = str(raw.get("id") or f"vision_food_reasoning_{idx}")
+        converted_rows.append(
+            {
+                "messages": prompt_messages,
+                "ground_truth": {
+                    "label": normalized_label,
+                    "raw_label": raw_label or "",
+                    "reference_answer": reference_text,
+                },
+                "input_metadata": {
+                    "row_id": row_id,
+                    "dataset_info": {
+                        "source": DATASET_SOURCE_ID,
+                        "normalized_label": normalized_label,
+                    },
+                },
+            }
+        )
+
+    with output_path.open("w") as outfile:
+        for row in converted_rows:
+            outfile.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    print(f"Converted {len(converted_rows)} rows (skipped {skipped}) from {input_path} -> {output_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Convert raw vision food reasoning dataset.")
+    parser.add_argument("--input", required=True, type=Path, help="Path to the raw JSONL dataset.")
+    parser.add_argument(
+        "--output",
+        required=True,
+        type=Path,
+        help="Destination JSONL path for the converted EvaluationRow dataset.",
+    )
+    args = parser.parse_args()
+    convert_dataset(args.input, args.output)
+
+
+if __name__ == "__main__":
+    main()

examples/vision_food_reasoning_dataset/data/vision_food_reasoning_raw_full.jsonl

@@ -1,21 +1,22 @@
+import json
+import os
 import re
 from pathlib import Path
 from typing import Any
 
+import litellm
+
 from eval_protocol.models import (
     EvaluateResult,
     EvaluationRow,
     Message,
     MetricResult,
     ChatCompletionContentPartTextParam,
 )
-from eval_protocol.pytest.default_single_turn_rollout_process import (
-    SingleTurnRolloutProcessor,
-)
+from eval_protocol.pytest.default_single_turn_rollout_process import SingleTurnRolloutProcessor
 from eval_protocol.pytest.evaluation_test import evaluation_test
 
 DATASET_PATH = Path(__file__).resolve().parents[1] / "data" / "vision_food_reasoning_sample.jsonl"
-DATASET_SOURCE_ID = "fireworks-ai/vision-food-reasoning-dataset"
 
 _BOLD_LABEL_PATTERN = re.compile(r"\*\*(?P<label>[^*]+)\*\*")
 _APPEARS_PATTERN = re.compile(r"appears to be\s+(?P<label>[A-Za-z0-9_\- ]+)", re.IGNORECASE)
@@ -84,35 +85,6 @@ def _extract_label_from_text(text: str) -> str | None:
     return None
 
 
-def vision_food_reasoning_dataset_adapter(rows: list[dict[str, Any]]) -> list[EvaluationRow]:
-    adapted: list[EvaluationRow] = []
-    for idx, raw in enumerate(rows):
-        messages_payload = raw.get("messages")
-        if not isinstance(messages_payload, list) or len(messages_payload) < 2:
-            continue
-        try:
-            user_message = Message.model_validate(messages_payload[0])
-            assistant_reference = Message.model_validate(messages_payload[-1])
-        except Exception:
-            continue
-        reference_text = _content_to_text(assistant_reference.content)
-        raw_label = _extract_label_from_text(reference_text)
-        normalized_label = _normalize_label(raw_label)
-        if not normalized_label:
-            continue
-        row = EvaluationRow(messages=[user_message])
-        row.input_metadata.row_id = str(raw.get("id") or f"vision_food_reasoning_{idx}")
-        row.input_metadata.dataset_info = {
-            "source": DATASET_SOURCE_ID,
-            "normalized_label": normalized_label,
-        }
-        row.ground_truth = {"label": normalized_label, "raw_label": raw_label or ""}
-        adapted.append(row)
-    if not adapted:
-        raise RuntimeError("Vision food reasoning adapter returned no usable rows.")
-    return adapted
-
-
 def _extract_prediction(row: EvaluationRow) -> tuple[str, str]:
     assistant_messages = [m for m in row.messages if m.role == "assistant"]
     if not assistant_messages:
@@ -122,6 +94,80 @@ def _extract_prediction(row: EvaluationRow) -> tuple[str, str]:
     return label, text
 
 
+def _llm_equivalence_check(
+    ground_truth_label: str,
+    prediction_text: str,
+    *,
+    judge_model: str | None = None,
+) -> tuple[bool, str]:
+    model_name = (
+        judge_model
+        or os.getenv("VISION_FOOD_REASONING_JUDGE_MODEL")
+        or "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"
+    )
+    prediction_text = prediction_text.strip()
+    if not prediction_text:
+        return False, "LLM judge skipped: prediction text is empty."
+
+    system_prompt = (
+        "You are a strict food classification judge. "
+        "Given the ground-truth dish label and a model response, decide whether the response "
+        "unambiguously identifies the same dish. "
+        "Only consider the final answer portion; ignore speculation or unrelated commentary. "
+        'Respond with compact JSON like {"equivalent": true, "reason": "..."}.'
+    )
+    user_prompt = (
+        "GROUND TRUTH LABEL: {label}\n"
+        'MODEL RESPONSE:\n"""\n{response}\n"""\n\n'
+        "If the model clearly identifies the same dish, set equivalent=true, otherwise false. "
+        "Explain the decision in the reason."
+    ).format(label=ground_truth_label, response=prediction_text)
+
+    try:
+        completion = litellm.completion(
+            model=model_name,
+            temperature=0,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+        )
+        raw_content = completion["choices"][0]["message"]["content"]
+    except Exception as exc:  # pragma: no cover - depends on external service
+        return False, f"LLM judge failed: {exc}"
+
+    if not raw_content:
+        return False, "LLM judge returned empty response."
+
+    parsed = _parse_json_blob(raw_content)
+    if not isinstance(parsed, dict):
+        return False, f"LLM judge did not return JSON: {raw_content}"
+
+    decision = parsed.get("equivalent")
+    reason = parsed.get("reason") or "LLM judge provided no reason."
+    if isinstance(decision, str):
+        decision = decision.strip().lower() in {"true", "yes", "1"}
+    elif not isinstance(decision, bool):
+        decision = False
+        reason = f"LLM judge missing boolean decision. Raw: {raw_content}"
+    return bool(decision), reason
+
+
+def _parse_json_blob(blob: str) -> Any:
+    try:
+        return json.loads(blob)
+    except json.JSONDecodeError:
+        start = blob.find("{")
+        end = blob.rfind("}")
+        if start != -1 and end != -1 and start < end:
+            snippet = blob[start : end + 1]
+            try:
+                return json.loads(snippet)
+            except json.JSONDecodeError:
+                return None
+    return None
+
+
 def _ground_truth_label(row: EvaluationRow) -> str:
     if isinstance(row.ground_truth, dict):
         return _normalize_label(row.ground_truth.get("label"))
@@ -132,10 +178,9 @@ def _ground_truth_label(row: EvaluationRow) -> str:
 
 @evaluation_test(
     input_dataset=[str(DATASET_PATH)],
-    dataset_adapter=vision_food_reasoning_dataset_adapter,
     completion_params=[
         {
-            "model": "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-instruct",
+            "model": "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-instruct",
             # "max_tokens": 512,
             # "model": "openrouter/qwen/qwen3-vl-30b-a3b-instruct",
             # "model": "gpt-4.1-mini",
@@ -151,8 +196,8 @@ def _ground_truth_label(row: EvaluationRow) -> str:
     rollout_processor=SingleTurnRolloutProcessor(),
     aggregation_method="mean",
     passed_threshold=None,
-    num_runs=1,
     max_dataset_rows=10,
+    num_runs=1,
     mode="pointwise",
 )
 def test_vision_food_reasoning(row: EvaluationRow) -> EvaluationRow:
@@ -161,11 +206,23 @@ def test_vision_food_reasoning(row: EvaluationRow) -> EvaluationRow:
 
     is_valid = bool(predicted_label)
     is_correct = is_valid and predicted_label == ground_truth_label and bool(ground_truth_label)
+
+    llm_equivalent = False
+    llm_reason = "LLM judge not triggered."
+    if not is_correct and ground_truth_label:
+        llm_equivalent, llm_reason = _llm_equivalence_check(ground_truth_label, raw_prediction)
+        if llm_equivalent:
+            is_correct = True
+            is_valid = True
+
     score = 1.0 if is_correct else 0.0
+    reason = "Prediction matches ground truth" if is_correct else "Prediction did not match"
+    if llm_equivalent:
+        reason = "LLM judge considered the prediction equivalent to the ground truth."
 
     row.evaluation_result = EvaluateResult(
         score=score,
-        reason="Prediction matches ground truth" if is_correct else "Prediction did not match",
+        reason=reason,
         is_score_valid=is_valid,
         metrics={
             "exact_match": MetricResult(
@@ -177,7 +234,12 @@ def test_vision_food_reasoning(row: EvaluationRow) -> EvaluationRow:
                     "ground_truth_label": ground_truth_label,
                     "raw_prediction": raw_prediction,
                 },
-            )
+            ),
+            "llm_equivalence": MetricResult(
+                score=1.0 if llm_equivalent else 0.0,
+                is_score_valid=llm_equivalent,
+                reason=llm_reason,
+            ),
         },
     )
     return row