Move validator business logic into every_eval_ever and update to de-duplication logic

.gitignore

@@ -223,4 +223,5 @@ vendor/
 _site/
 .sass-cache/
 .jekyll-cache/
-.jekyll-metadata
+.jekyll-metadataUSAGE_*
+USAGE_*

docs/data-structure/validation.md

@@ -48,3 +48,39 @@ uv run python -m every_eval_ever validate --format github data/
 | `--max-errors N` | `50` | Maximum errors reported per JSONL file |
 
 Exit code is `0` if all files pass and `1` if any fail.
+
+## Duplicate Check
+
+Run duplicate detection separately for aggregate JSON records:
+
+```sh
+uv run every_eval_ever check-duplicates data/benchmark/
+```
+
+This command uses the same semantic fingerprint as the validator Space. It
+ignores non-identity fields such as UUIDs, timestamps, free-form details, paths,
+and source metadata, then compares model, evaluation library, dataset identity,
+metric identity, score, and generation config within each `data/<collection>`.
+
+## Semantic Warnings
+
+The CLI and Space share the same non-blocking semantic warnings:
+
+- Datastore path hierarchy and UUID4 filename checks.
+- Missing aggregate `.jsonl` companions when detailed results reference them.
+- Missing `score_type`, `min_score`, or `max_score`, and scores outside bounds.
+- Non-integer count fields such as `num_samples`.
+- Model deployment metadata under `model_info.additional_details`:
+  `deployment_type` is `api`, `local`, or `unknown`; `api` models use
+  `model_availability` values `closed_source`, `open_weights_deployment`, or
+  `other`; `local` models use `hf`, `unavailable`, or `other`.
+- Required Hugging Face model checks when `model_availability` is `hf`.
+- Required Hugging Face dataset checks when `source_data.source_type` is
+  `hf_dataset`, plus warnings for weak `other` dataset provenance.
+
+## PR Bot
+
+The Hugging Face datastore PR bot validates changed `data/**/*.json` and
+`data/**/*.jsonl` files through the package validation core, checks paths,
+compares aggregate candidates against accepted records, and posts a visible PR
+report.

every_eval_ever/__init__.py

@@ -5,11 +5,16 @@
 import importlib
 from typing import Any
 
-__all__ = ['eval_types', 'instance_level_types']
+__all__ = ['dedup', 'eval_types', 'instance_level_types', 'validation_core']
 
 
 def __getattr__(name: str) -> Any:
-    if name in {'eval_types', 'instance_level_types'}:
+    if name in {
+        'dedup',
+        'eval_types',
+        'instance_level_types',
+        'validation_core',
+    }:
         module = importlib.import_module(f'.{name}', __name__)
         globals()[name] = module
         return module

every_eval_ever/check_duplicate_entries.py

@@ -4,6 +4,8 @@
 import os
 from typing import Any, Dict, List
 
+from every_eval_ever.dedup import check_duplicates
+
 IGNORE_KEYS = {'retrieved_timestamp', 'evaluation_id'}
 
 
@@ -81,59 +83,29 @@ def main(argv: List[str] | None = None) -> int:
     print(f'Checking {len(file_paths)} JSON files for duplicates...')
     print()
 
-    groups: Dict[str, List[Dict[str, Any]]] = {}
-    for file_path in file_paths:
-        try:
-            with open(file_path, 'r') as f:
-                payload = json.load(f)
-        except json.JSONDecodeError as e:
-            message = f'JSONDecodeError: {str(e)}'
-            annotate_error(
-                file_path,
-                message,
-                title='JSONDecodeError',
-                col=e.colno,
-                line=e.lineno,
-            )
-            print(f'{file_path}')
-            print('  ' + message)
-            print()
-            raise
-
-        entry_hash = normalized_hash(payload)
-        groups.setdefault(entry_hash, []).append(
-            {
-                'path': file_path,
-                'evaluation_id': payload.get('evaluation_id'),
-                'retrieved_timestamp': payload.get('retrieved_timestamp'),
-            }
-        )
-
-    duplicate_groups = [
-        entries for entries in groups.values() if len(entries) > 1
+    local_paths = {file_path: file_path for file_path in file_paths}
+    dedup_report = check_duplicates(file_paths, local_paths, {'files': {}})
+    duplicate_results = [
+        result for result in dedup_report.results if result.duplicate_of
     ]
-    if not duplicate_groups:
+
+    if not duplicate_results:
         print('No duplicates found.')
         print()
         return 0
 
-    ignore_label = ', '.join(f'`{key}`' for key in sorted(IGNORE_KEYS))
-    print(f'Found duplicate entries (ignoring keys: {ignore_label}).')
+    print('Found duplicate entries (semantic fingerprint match).')
     print()
 
-    for index, entries in enumerate(duplicate_groups, start=1):
-        print(f'Duplicate group {index} ({len(entries)} files):')
-        for entry in entries:
-            print(f'  - {entry["path"]}')
-            print(f'    evaluation_id: {entry.get("evaluation_id")}')
-            print(
-                f'    retrieved_timestamp: {entry.get("retrieved_timestamp")}'
-            )
-            annotate_error(
-                entry['path'],
-                'Duplicate entry detected (ignoring `evaluation_id` and `retrieved_timestamp`).',
-                title='DuplicateEntry',
-            )
+    for index, result in enumerate(duplicate_results, start=1):
+        print(f'Duplicate group {index}:')
+        print(f'  - {result.file_path}')
+        print(f'    duplicate_of: {result.duplicate_of}')
+        annotate_error(
+            result.file_path,
+            f'Duplicate entry detected; semantic fingerprint matches {result.duplicate_of}.',
+            title='DuplicateEntry',
+        )
         print()
 
     return 1

every_eval_ever/cli.py

@@ -215,6 +215,7 @@ def _cmd_convert_alpaca_eval(args: argparse.Namespace) -> int:
                 )
             if args.evaluator_relationship != 'third_party':
                 from every_eval_ever.eval_types import EvaluatorRelationship
+
                 log.source_metadata.evaluator_relationship = (
                     EvaluatorRelationship(args.evaluator_relationship)
                 )
@@ -279,8 +280,8 @@ def build_parser() -> argparse.ArgumentParser:
         'check-duplicates',
         help='Detect duplicate evaluation JSON entries',
         description=(
-            'Detect duplicate evaluation entries while ignoring scrape-specific '
-            'keys (evaluation_id and retrieved_timestamp).'
+            'Detect duplicate aggregate evaluation records using the shared '
+            'semantic fingerprint used by the datastore validator.'
         ),
     )
     check_duplicates_parser.add_argument(
@@ -397,11 +398,11 @@ def main(argv: list[str] | None = None) -> int:
 
         return validate_main(
             [
-                *args.paths,
                 '--max-errors',
                 str(args.max_errors),
                 '--format',
                 args.output_format,
+                *args.paths,
             ]
         )
 

every_eval_ever/converters/alpaca_eval/__main__.py

@@ -6,8 +6,6 @@
 import uuid
 from pathlib import Path
 
-from every_eval_ever.converters import SCHEMA_VERSION
-
 from .adapter import LEADERBOARDS, AlpacaEvalAdapter
 
 

every_eval_ever/converters/inspect/utils.py

@@ -1,10 +1,7 @@
 import json
 import re
-from pathlib import Path
 from typing import Any, Dict, List, Type
 
-from pydantic import BaseModel
-
 from every_eval_ever.converters.common.utils import get_model_organization_info
 from every_eval_ever.converters.inspect.supplemental_eval_details import (
     SupplementalAgenticEvalConfig,
@@ -335,14 +332,14 @@ def extract_model_info_from_model_path(model_path: str) -> ModelInfo:
 
 
 SYNTHETIC_METRIC_CONFIG_FIELDS = {
-    "evaluation_description",
-    "lower_is_better",
-    "score_type",
-    "level_names",
-    "level_metadata",
-    "has_unknown_level",
-    "min_score",
-    "max_score",
+    'evaluation_description',
+    'lower_is_better',
+    'score_type',
+    'level_names',
+    'level_metadata',
+    'has_unknown_level',
+    'min_score',
+    'max_score',
 }
 
 
@@ -356,14 +353,18 @@ def parse_supplemental_eval_details(
         return raw_supplemental_eval_details
 
     if isinstance(raw_supplemental_eval_details, dict):
-        return SupplementalEvalDetails.model_validate(raw_supplemental_eval_details)
+        return SupplementalEvalDetails.model_validate(
+            raw_supplemental_eval_details
+        )
 
     raise ValueError(
         "metadata_args['supplemental_eval_details'] must be a dict or SupplementalEvalDetails instance."
     )
 
 
-def convert_to_string_dict(data: dict[str, Any] | None) -> dict[str, str] | None:
+def convert_to_string_dict(
+    data: dict[str, Any] | None,
+) -> dict[str, str] | None:
     if data is None:
         return None
     return {
@@ -395,7 +396,10 @@ def apply_model_info_supplement(
     model_info: ModelInfo,
     supplemental_eval_details: SupplementalEvalDetails | None,
 ) -> None:
-    if supplemental_eval_details is None or supplemental_eval_details.model_info is None:
+    if (
+        supplemental_eval_details is None
+        or supplemental_eval_details.model_info is None
+    ):
         return
 
     model_info.additional_details = extend_additional_details(
@@ -429,13 +433,13 @@ def apply_generation_config_supplement(
         generation_config.generation_args = GenerationArgs()
 
     if generation_config.generation_args.agentic_eval_config is None:
-        generation_config.generation_args.agentic_eval_config = AgenticEvalConfig()
-
-    generation_config.generation_args.agentic_eval_config.additional_details = (
-        extend_additional_details(
-            generation_config.generation_args.agentic_eval_config.additional_details,
-            agentic_supplement.additional_details,
+        generation_config.generation_args.agentic_eval_config = (
+            AgenticEvalConfig()
         )
+
+    generation_config.generation_args.agentic_eval_config.additional_details = extend_additional_details(
+        generation_config.generation_args.agentic_eval_config.additional_details,
+        agentic_supplement.additional_details,
     )
 
 
@@ -446,9 +450,11 @@ def apply_source_data_supplement(
     if source_data_supplement is None:
         return
 
-    evaluation_result.source_data.additional_details = extend_additional_details(
-        evaluation_result.source_data.additional_details,
-        source_data_supplement.additional_details,
+    evaluation_result.source_data.additional_details = (
+        extend_additional_details(
+            evaluation_result.source_data.additional_details,
+            source_data_supplement.additional_details,
+        )
     )
 
 
@@ -460,19 +466,19 @@ def apply_metric_config_supplement(
     if metric_supplement is None:
         return
 
-    current = evaluation_result.metric_config.model_dump(mode="python")
-    supplemental = metric_supplement.model_dump(mode="python", exclude_none=True)
+    current = evaluation_result.metric_config.model_dump(mode='python')
+    supplemental = metric_supplement.model_dump(
+        mode='python', exclude_none=True
+    )
 
-    additional_details = supplemental.pop("additional_details", None)
+    additional_details = supplemental.pop('additional_details', None)
 
     for field_name, field_value in supplemental.items():
-        if (
-            field_name in SYNTHETIC_METRIC_CONFIG_FIELDS
-        ):
+        if field_name in SYNTHETIC_METRIC_CONFIG_FIELDS:
             current[field_name] = field_value
 
-    current["additional_details"] = extend_additional_details(
-        current.get("additional_details"),
+    current['additional_details'] = extend_additional_details(
+        current.get('additional_details'),
         additional_details,
     )
 
@@ -526,10 +532,12 @@ def apply_supplemental_eval_details(
         [s for s in result_supplements if s.evaluation_name is not None]
     ):
         raise ValueError(
-            "Duplicate evaluation_name values in supplemental_eval_details.evaluation_results."
+            'Duplicate evaluation_name values in supplemental_eval_details.evaluation_results.'
         )
     unnamed_supplements = [
-        supplement for supplement in result_supplements if supplement.evaluation_name is None
+        supplement
+        for supplement in result_supplements
+        if supplement.evaluation_name is None
     ]
     unnamed_idx = 0