diff --git a/.gitignore b/.gitignore index b925edb9a..04bf61ac5 100644 --- a/.gitignore +++ b/.gitignore @@ -223,4 +223,5 @@ vendor/ _site/ .sass-cache/ .jekyll-cache/ -.jekyll-metadata \ No newline at end of file +.jekyll-metadataUSAGE_* +USAGE_* diff --git a/docs/data-structure/validation.md b/docs/data-structure/validation.md index 6234f1258..51fdccd5a 100644 --- a/docs/data-structure/validation.md +++ b/docs/data-structure/validation.md @@ -48,3 +48,39 @@ uv run python -m every_eval_ever validate --format github data/ | `--max-errors N` | `50` | Maximum errors reported per JSONL file | Exit code is `0` if all files pass and `1` if any fail. + +## Duplicate Check + +Run duplicate detection separately for aggregate JSON records: + +```sh +uv run every_eval_ever check-duplicates data/benchmark/ +``` + +This command uses the same semantic fingerprint as the validator Space. It +ignores non-identity fields such as UUIDs, timestamps, free-form details, paths, +and source metadata, then compares model, evaluation library, dataset identity, +metric identity, score, and generation config within each `data/`. + +## Semantic Warnings + +The CLI and Space share the same non-blocking semantic warnings: + +- Datastore path hierarchy and UUID4 filename checks. +- Missing aggregate `.jsonl` companions when detailed results reference them. +- Missing `score_type`, `min_score`, or `max_score`, and scores outside bounds. +- Non-integer count fields such as `num_samples`. +- Model deployment metadata under `model_info.additional_details`: + `deployment_type` is `api`, `local`, or `unknown`; `api` models use + `model_availability` values `closed_source`, `open_weights_deployment`, or + `other`; `local` models use `hf`, `unavailable`, or `other`. +- Required Hugging Face model checks when `model_availability` is `hf`. +- Required Hugging Face dataset checks when `source_data.source_type` is + `hf_dataset`, plus warnings for weak `other` dataset provenance. + +## PR Bot + +The Hugging Face datastore PR bot validates changed `data/**/*.json` and +`data/**/*.jsonl` files through the package validation core, checks paths, +compares aggregate candidates against accepted records, and posts a visible PR +report. diff --git a/every_eval_ever/__init__.py b/every_eval_ever/__init__.py index 7d41cde78..d0b3a7384 100644 --- a/every_eval_ever/__init__.py +++ b/every_eval_ever/__init__.py @@ -5,11 +5,16 @@ import importlib from typing import Any -__all__ = ['eval_types', 'instance_level_types'] +__all__ = ['dedup', 'eval_types', 'instance_level_types', 'validation_core'] def __getattr__(name: str) -> Any: - if name in {'eval_types', 'instance_level_types'}: + if name in { + 'dedup', + 'eval_types', + 'instance_level_types', + 'validation_core', + }: module = importlib.import_module(f'.{name}', __name__) globals()[name] = module return module diff --git a/every_eval_ever/check_duplicate_entries.py b/every_eval_ever/check_duplicate_entries.py index 5f6c53342..19a5a4831 100644 --- a/every_eval_ever/check_duplicate_entries.py +++ b/every_eval_ever/check_duplicate_entries.py @@ -4,6 +4,8 @@ import os from typing import Any, Dict, List +from every_eval_ever.dedup import check_duplicates + IGNORE_KEYS = {'retrieved_timestamp', 'evaluation_id'} @@ -81,59 +83,29 @@ def main(argv: List[str] | None = None) -> int: print(f'Checking {len(file_paths)} JSON files for duplicates...') print() - groups: Dict[str, List[Dict[str, Any]]] = {} - for file_path in file_paths: - try: - with open(file_path, 'r') as f: - payload = json.load(f) - except json.JSONDecodeError as e: - message = f'JSONDecodeError: {str(e)}' - annotate_error( - file_path, - message, - title='JSONDecodeError', - col=e.colno, - line=e.lineno, - ) - print(f'{file_path}') - print(' ' + message) - print() - raise - - entry_hash = normalized_hash(payload) - groups.setdefault(entry_hash, []).append( - { - 'path': file_path, - 'evaluation_id': payload.get('evaluation_id'), - 'retrieved_timestamp': payload.get('retrieved_timestamp'), - } - ) - - duplicate_groups = [ - entries for entries in groups.values() if len(entries) > 1 + local_paths = {file_path: file_path for file_path in file_paths} + dedup_report = check_duplicates(file_paths, local_paths, {'files': {}}) + duplicate_results = [ + result for result in dedup_report.results if result.duplicate_of ] - if not duplicate_groups: + + if not duplicate_results: print('No duplicates found.') print() return 0 - ignore_label = ', '.join(f'`{key}`' for key in sorted(IGNORE_KEYS)) - print(f'Found duplicate entries (ignoring keys: {ignore_label}).') + print('Found duplicate entries (semantic fingerprint match).') print() - for index, entries in enumerate(duplicate_groups, start=1): - print(f'Duplicate group {index} ({len(entries)} files):') - for entry in entries: - print(f' - {entry["path"]}') - print(f' evaluation_id: {entry.get("evaluation_id")}') - print( - f' retrieved_timestamp: {entry.get("retrieved_timestamp")}' - ) - annotate_error( - entry['path'], - 'Duplicate entry detected (ignoring `evaluation_id` and `retrieved_timestamp`).', - title='DuplicateEntry', - ) + for index, result in enumerate(duplicate_results, start=1): + print(f'Duplicate group {index}:') + print(f' - {result.file_path}') + print(f' duplicate_of: {result.duplicate_of}') + annotate_error( + result.file_path, + f'Duplicate entry detected; semantic fingerprint matches {result.duplicate_of}.', + title='DuplicateEntry', + ) print() return 1 diff --git a/every_eval_ever/cli.py b/every_eval_ever/cli.py index c215c89cd..084404272 100644 --- a/every_eval_ever/cli.py +++ b/every_eval_ever/cli.py @@ -215,6 +215,7 @@ def _cmd_convert_alpaca_eval(args: argparse.Namespace) -> int: ) if args.evaluator_relationship != 'third_party': from every_eval_ever.eval_types import EvaluatorRelationship + log.source_metadata.evaluator_relationship = ( EvaluatorRelationship(args.evaluator_relationship) ) @@ -279,8 +280,8 @@ def build_parser() -> argparse.ArgumentParser: 'check-duplicates', help='Detect duplicate evaluation JSON entries', description=( - 'Detect duplicate evaluation entries while ignoring scrape-specific ' - 'keys (evaluation_id and retrieved_timestamp).' + 'Detect duplicate aggregate evaluation records using the shared ' + 'semantic fingerprint used by the datastore validator.' ), ) check_duplicates_parser.add_argument( @@ -397,11 +398,11 @@ def main(argv: list[str] | None = None) -> int: return validate_main( [ - *args.paths, '--max-errors', str(args.max_errors), '--format', args.output_format, + *args.paths, ] ) diff --git a/every_eval_ever/converters/alpaca_eval/__main__.py b/every_eval_ever/converters/alpaca_eval/__main__.py index 5d7b527ed..3cdb5b134 100644 --- a/every_eval_ever/converters/alpaca_eval/__main__.py +++ b/every_eval_ever/converters/alpaca_eval/__main__.py @@ -6,8 +6,6 @@ import uuid from pathlib import Path -from every_eval_ever.converters import SCHEMA_VERSION - from .adapter import LEADERBOARDS, AlpacaEvalAdapter diff --git a/every_eval_ever/converters/inspect/utils.py b/every_eval_ever/converters/inspect/utils.py index 8538b1928..ab3a6aa5d 100644 --- a/every_eval_ever/converters/inspect/utils.py +++ b/every_eval_ever/converters/inspect/utils.py @@ -1,10 +1,7 @@ import json import re -from pathlib import Path from typing import Any, Dict, List, Type -from pydantic import BaseModel - from every_eval_ever.converters.common.utils import get_model_organization_info from every_eval_ever.converters.inspect.supplemental_eval_details import ( SupplementalAgenticEvalConfig, @@ -335,14 +332,14 @@ def extract_model_info_from_model_path(model_path: str) -> ModelInfo: SYNTHETIC_METRIC_CONFIG_FIELDS = { - "evaluation_description", - "lower_is_better", - "score_type", - "level_names", - "level_metadata", - "has_unknown_level", - "min_score", - "max_score", + 'evaluation_description', + 'lower_is_better', + 'score_type', + 'level_names', + 'level_metadata', + 'has_unknown_level', + 'min_score', + 'max_score', } @@ -356,14 +353,18 @@ def parse_supplemental_eval_details( return raw_supplemental_eval_details if isinstance(raw_supplemental_eval_details, dict): - return SupplementalEvalDetails.model_validate(raw_supplemental_eval_details) + return SupplementalEvalDetails.model_validate( + raw_supplemental_eval_details + ) raise ValueError( "metadata_args['supplemental_eval_details'] must be a dict or SupplementalEvalDetails instance." ) -def convert_to_string_dict(data: dict[str, Any] | None) -> dict[str, str] | None: +def convert_to_string_dict( + data: dict[str, Any] | None, +) -> dict[str, str] | None: if data is None: return None return { @@ -395,7 +396,10 @@ def apply_model_info_supplement( model_info: ModelInfo, supplemental_eval_details: SupplementalEvalDetails | None, ) -> None: - if supplemental_eval_details is None or supplemental_eval_details.model_info is None: + if ( + supplemental_eval_details is None + or supplemental_eval_details.model_info is None + ): return model_info.additional_details = extend_additional_details( @@ -429,13 +433,13 @@ def apply_generation_config_supplement( generation_config.generation_args = GenerationArgs() if generation_config.generation_args.agentic_eval_config is None: - generation_config.generation_args.agentic_eval_config = AgenticEvalConfig() - - generation_config.generation_args.agentic_eval_config.additional_details = ( - extend_additional_details( - generation_config.generation_args.agentic_eval_config.additional_details, - agentic_supplement.additional_details, + generation_config.generation_args.agentic_eval_config = ( + AgenticEvalConfig() ) + + generation_config.generation_args.agentic_eval_config.additional_details = extend_additional_details( + generation_config.generation_args.agentic_eval_config.additional_details, + agentic_supplement.additional_details, ) @@ -446,9 +450,11 @@ def apply_source_data_supplement( if source_data_supplement is None: return - evaluation_result.source_data.additional_details = extend_additional_details( - evaluation_result.source_data.additional_details, - source_data_supplement.additional_details, + evaluation_result.source_data.additional_details = ( + extend_additional_details( + evaluation_result.source_data.additional_details, + source_data_supplement.additional_details, + ) ) @@ -460,19 +466,19 @@ def apply_metric_config_supplement( if metric_supplement is None: return - current = evaluation_result.metric_config.model_dump(mode="python") - supplemental = metric_supplement.model_dump(mode="python", exclude_none=True) + current = evaluation_result.metric_config.model_dump(mode='python') + supplemental = metric_supplement.model_dump( + mode='python', exclude_none=True + ) - additional_details = supplemental.pop("additional_details", None) + additional_details = supplemental.pop('additional_details', None) for field_name, field_value in supplemental.items(): - if ( - field_name in SYNTHETIC_METRIC_CONFIG_FIELDS - ): + if field_name in SYNTHETIC_METRIC_CONFIG_FIELDS: current[field_name] = field_value - current["additional_details"] = extend_additional_details( - current.get("additional_details"), + current['additional_details'] = extend_additional_details( + current.get('additional_details'), additional_details, ) @@ -526,10 +532,12 @@ def apply_supplemental_eval_details( [s for s in result_supplements if s.evaluation_name is not None] ): raise ValueError( - "Duplicate evaluation_name values in supplemental_eval_details.evaluation_results." + 'Duplicate evaluation_name values in supplemental_eval_details.evaluation_results.' ) unnamed_supplements = [ - supplement for supplement in result_supplements if supplement.evaluation_name is None + supplement + for supplement in result_supplements + if supplement.evaluation_name is None ] unnamed_idx = 0 diff --git a/every_eval_ever/dedup.py b/every_eval_ever/dedup.py new file mode 100644 index 000000000..4abe5eff1 --- /dev/null +++ b/every_eval_ever/dedup.py @@ -0,0 +1,320 @@ +"""Semantic duplicate detection for Every Eval Ever aggregate JSON files.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError + +MANIFEST_PATH = 'manifest.json' +DEFAULT_DATASET_REPO_ID = 'evaleval/EEE_datastore' +_QUANT_DECIMALS = 6 +_QUANT_FORMAT = f'.{_QUANT_DECIMALS}f' +_TRIM_CHARS = '"\'.,;:!?()[]{} ' + + +class ManifestError(RuntimeError): + """Raised when manifest.json cannot be safely loaded or used.""" + + +def _norm_str(value: Any) -> str | None: + if value is None: + return None + text = ' '.join(str(value).lower().split()) + return text.strip(_TRIM_CHARS) or None + + +def _quantize(value: Any) -> Any: + if isinstance(value, bool): + return value + if isinstance(value, int): + return str(value) + if isinstance(value, float): + text = format(value, _QUANT_FORMAT).rstrip('0').rstrip('.') + return '0' if text in {'', '-0'} else text + return value + + +def _canon(obj: Any) -> Any: + if obj is None: + return None + if isinstance(obj, bool): + return obj + if isinstance(obj, (int, float)): + return _quantize(obj) + if isinstance(obj, str): + return _norm_str(obj) + if isinstance(obj, dict): + return {key: _canon(value) for key, value in obj.items()} + if isinstance(obj, list): + return [_canon(value) for value in obj] + return obj + + +def _source_identity(source_data: Any) -> dict[str, Any]: + if not isinstance(source_data, dict): + return {} + identity = { + 'type': _norm_str(source_data.get('source_type')), + 'name': _norm_str(source_data.get('dataset_name')), + } + if source_data.get('source_type') == 'hf_dataset': + repo = _norm_str(source_data.get('hf_repo')) + if repo: + identity['hf'] = repo + return identity + + +def _metric_identity(metric_config: Any) -> dict[str, Any]: + if not isinstance(metric_config, dict): + return {} + metric_id = metric_config.get('metric_id') or metric_config.get( + 'metric_name' + ) + return { + 'id': _norm_str(metric_id), + 'kind': _norm_str(metric_config.get('metric_kind')), + 'params': _canon(metric_config.get('metric_parameters') or {}), + 'score_type': _norm_str(metric_config.get('score_type')), + 'unit': _norm_str(metric_config.get('metric_unit')), + 'lower_is_better': metric_config.get('lower_is_better'), + } + + +def _generation_config_identity( + generation_config: Any, +) -> dict[str, Any] | None: + if not isinstance(generation_config, dict): + return None + args = generation_config.get('generation_args') + if not isinstance(args, dict): + return None + plan = args.get('eval_plan') or {} + limits = args.get('eval_limits') or {} + return { + 'temp': _quantize(args.get('temperature')), + 'top_p': _quantize(args.get('top_p')), + 'top_k': args.get('top_k'), + 'max_tokens': args.get('max_tokens'), + 'reasoning': args.get('reasoning'), + 'plan': _norm_str(plan.get('name')) if isinstance(plan, dict) else None, + 'time_limit': ( + limits.get('time_limit') if isinstance(limits, dict) else None + ), + 'msg_limit': ( + limits.get('message_limit') if isinstance(limits, dict) else None + ), + 'token_limit': ( + limits.get('token_limit') if isinstance(limits, dict) else None + ), + 'max_attempts': args.get('max_attempts'), + } + + +def _result_identity(result: dict[str, Any]) -> dict[str, Any]: + score_details = result.get('score_details') or {} + return { + 'eval': _norm_str(result.get('evaluation_name')), + 'src': _source_identity(result.get('source_data')), + 'metric': _metric_identity(result.get('metric_config')), + 'score': _quantize(score_details.get('score')), + 'gen': _generation_config_identity(result.get('generation_config')), + } + + +def compute_aggregate_identity(data: dict[str, Any]) -> str: + """Hash the normalized semantic identity of an aggregate eval record.""" + model_info = data.get('model_info') or {} + eval_library = data.get('eval_library') or {} + raw_results = data.get('evaluation_results') or [] + if not isinstance(raw_results, list): + raise ValueError('aggregate evaluation_results must be a list') + result_items: list[dict[str, Any]] = [] + for item in raw_results: + if item is None: + continue + if not isinstance(item, dict): + raise ValueError( + 'aggregate evaluation_results entries must be objects' + ) + result_items.append(item) + identity = { + 'model': _norm_str(model_info.get('id')), + 'lib': _norm_str(eval_library.get('name')), + 'results': [_result_identity(item) for item in result_items], + } + canonical = _canon(identity) + canonical['results'] = sorted( + canonical['results'], + key=lambda item: json.dumps(item, sort_keys=True), + ) + payload = json.dumps(canonical, sort_keys=True, ensure_ascii=True).encode() + return hashlib.sha256(payload).hexdigest() + + +def compute_fingerprint(content: bytes) -> str: + """Compute the semantic duplicate fingerprint for aggregate JSON bytes.""" + try: + data = json.loads(content) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ValueError( + 'dedup fingerprint requires aggregate JSON content' + ) from exc + if isinstance(data, dict) and isinstance( + data.get('evaluation_results'), list + ): + return compute_aggregate_identity(data) + raise ValueError('dedup fingerprint requires aggregate JSON content') + + +def compute_file_fingerprint(local_path: str | Path) -> str: + with Path(local_path).open('rb') as handle: + return compute_fingerprint(handle.read()) + + +def collection_key(file_path: str) -> str: + """Return the datastore collection root for scoped duplicate comparison.""" + parts = file_path.split('/') + if len(parts) >= 2 and parts[0] == 'data' and parts[1]: + return f'data/{parts[1]}' + return '' + + +@dataclass +class DedupResult: + """Deduplication result for one aggregate file.""" + + file_path: str + fingerprint: str + duplicate_of: str | None = None + + +@dataclass +class DedupReport: + """Aggregated deduplication report.""" + + results: Sequence[DedupResult] = field(default_factory=list) + warnings: Sequence[str] = field(default_factory=list) + + +def load_manifest( + api: HfApi, + *, + dataset_repo_id: str = DEFAULT_DATASET_REPO_ID, + manifest_path: str = MANIFEST_PATH, + revision: str = 'main', +) -> dict[str, Any]: + """Download and validate datastore manifest.json from Hugging Face.""" + try: + manifest_file = hf_hub_download( + repo_id=dataset_repo_id, + filename=manifest_path, + repo_type='dataset', + revision=revision, + ) + with Path(manifest_file).open(encoding='utf-8') as handle: + manifest = json.load(handle) + except (EntryNotFoundError, RepositoryNotFoundError) as exc: + raise ManifestError( + f'{manifest_path} not found in {dataset_repo_id}' + ) from exc + except Exception as exc: + raise ManifestError( + f'Failed to load {manifest_path} from {dataset_repo_id}' + ) from exc + + validate_manifest(manifest, manifest_path=manifest_path) + return manifest + + +def validate_manifest( + manifest: dict[str, Any], *, manifest_path: str = MANIFEST_PATH +) -> None: + if not isinstance(manifest, dict): + raise ManifestError(f'{manifest_path} must contain a JSON object') + files = manifest.get('files') + if not isinstance(files, dict): + raise ManifestError( + f"{manifest_path} must contain an object field 'files'" + ) + for path, entry in files.items(): + if not isinstance(path, str) or not isinstance(entry, dict): + raise ManifestError( + f'{manifest_path} has invalid file entry {path!r}' + ) + if not isinstance(entry.get('fingerprint'), str): + raise ManifestError( + f'{manifest_path} entry {path!r} is missing fingerprint' + ) + + +def build_dedup_report( + file_fingerprints: dict[str, str], + manifest: dict[str, Any], +) -> DedupReport: + """Compare candidate fingerprints against manifest and same-batch files.""" + validate_manifest(manifest) + results: list[DedupResult] = [] + manifest_files: dict[str, dict[str, Any]] = manifest['files'] + fingerprint_to_path: dict[tuple[str, str], str] = { + (collection_key(path), entry['fingerprint']): path + for path, entry in manifest_files.items() + } + + for file_path in sorted(file_fingerprints): + fingerprint = file_fingerprints[file_path] + key = (collection_key(file_path), fingerprint) + existing_path = fingerprint_to_path.get(key) + duplicate_of = None + if existing_path is not None and existing_path != file_path: + duplicate_of = existing_path + results.append( + DedupResult( + file_path=file_path, + fingerprint=fingerprint, + duplicate_of=duplicate_of, + ) + ) + fingerprint_to_path.setdefault(key, file_path) + + return DedupReport(results=results) + + +def check_duplicates( + file_paths: list[str], + local_paths: dict[str, str | Path], + manifest: dict[str, Any], +) -> DedupReport: + """Compute fingerprints for aggregate JSON files and compare to manifest.""" + file_fingerprints: dict[str, str] = {} + warnings: list[str] = [] + for file_path in sorted(file_paths): + if not file_path.endswith('.json'): + raise ValueError( + f'Duplicate check only accepts .json files: {file_path}' + ) + local_path = local_paths.get(file_path) + if local_path is None: + warnings.append( + f'Duplicate check skipped {file_path}: local path was not provided' + ) + continue + try: + file_fingerprints[file_path] = compute_file_fingerprint(local_path) + except Exception as exc: + warnings.append( + f'Duplicate check skipped {file_path}: ' + f'{type(exc).__name__}: {exc}' + ) + + report = build_dedup_report(file_fingerprints, manifest) + return DedupReport( + results=report.results, + warnings=[*report.warnings, *warnings], + ) diff --git a/every_eval_ever/helpers/eee_stats.py b/every_eval_ever/helpers/eee_stats.py index f4556c1b8..9f3dc6ed2 100644 --- a/every_eval_ever/helpers/eee_stats.py +++ b/every_eval_ever/helpers/eee_stats.py @@ -344,7 +344,6 @@ def create_visualisations(con, schema_table, instance_table, csv_path) -> None: try: import matplotlib.pyplot as plt import seaborn as sns - from matplotlib.colors import LinearSegmentedColormap except ModuleNotFoundError: raise ImportError('seaborn or matplotlib not installed') diff --git a/every_eval_ever/validate.py b/every_eval_ever/validate.py index 46f4d0a3b..cba4e1502 100644 --- a/every_eval_ever/validate.py +++ b/every_eval_ever/validate.py @@ -1,15 +1,7 @@ -""" -Pydantic-based validation for EEE schema files. - -Validates aggregate (.json) files against EvaluationLog and -instance-level (_samples.jsonl) files against InstanceLevelEvaluationLog. +"""CLI and compatibility wrapper for EEE validation. -Usage: - uv run python -m every_eval_ever validate data/benchmark/dev/model/uuid.json - uv run python -m every_eval_ever validate data/benchmark/dev/model/uuid_samples.jsonl - uv run python -m every_eval_ever validate data/benchmark/dev/model/ # directory recurse - uv run python -m every_eval_ever validate --format json data/*.json - uv run python -m every_eval_ever validate --max-errors 10 data/*_samples.jsonl +The validation rules live in :mod:`every_eval_ever.validation_core` so the +local CLI and the datastore validator Space run the same checks. """ from __future__ import annotations @@ -17,200 +9,68 @@ import argparse import json import sys +from collections import OrderedDict from dataclasses import dataclass, field from pathlib import Path -from pydantic import ValidationError from rich.console import Console from rich.panel import Panel from rich.text import Text -from every_eval_ever.eval_types import EvaluationLog -from every_eval_ever.instance_level_types import InstanceLevelEvaluationLog - -DEFAULT_MAX_ERRORS = 50 +from every_eval_ever.validation_core import ( + DEFAULT_MAX_ERRORS, + ValidationReport, + check_companion_exists, + check_dataset_provenance, + check_integer_counts, + check_model_deployment, + check_path_structure, + check_score_metadata, + format_error, + format_warning, + get_schema_fingerprint, + get_schema_version, + repo_path_from_path, + validate_aggregate, + validate_file, + validate_instance_file, + validate_many, +) + +__all__ = [ + 'DEFAULT_MAX_ERRORS', + 'ValidationReport', + 'check_companion_exists', + 'check_dataset_provenance', + 'check_integer_counts', + 'check_model_deployment', + 'check_path_structure', + 'check_score_metadata', + 'expand_paths', + 'format_error', + 'format_warning', + 'get_schema_fingerprint', + 'get_schema_version', + 'main', + 'render_report_github', + 'render_report_json', + 'render_report_rich', + 'render_summary_rich', + 'repo_path_from_path', + 'validate_aggregate', + 'validate_file', + 'validate_instance_file', + 'validate_many', +] + +DEFAULT_WARNING_EXAMPLES = 3 +WARNING_GROUP_CAP = 15 @dataclass -class ValidationReport: - """Result of validating a single file.""" - - file_path: Path - valid: bool - errors: list[dict] = field(default_factory=list) - file_type: str = '' # "aggregate" or "instance" - line_count: int = 0 # for JSONL files - - -def _format_loc(loc: tuple) -> str: - """Format a Pydantic error location tuple as a readable path.""" - parts = [] - for part in loc: - if isinstance(part, int): - parts.append(f'[{part}]') - else: - if parts: - parts.append(f' -> {part}') - else: - parts.append(str(part)) - return ''.join(parts) if parts else '(root)' - - -def _pydantic_errors_to_dicts(exc: ValidationError) -> list[dict]: - """Convert Pydantic ValidationError to a list of error dicts.""" - errors = [] - for err in exc.errors(): - errors.append( - { - 'loc': _format_loc(err['loc']), - 'msg': err['msg'], - 'type': err['type'], - 'input': err.get('input'), - } - ) - return errors - - -def validate_aggregate(file_path: Path) -> ValidationReport: - """Validate a .json file as an EvaluationLog.""" - report = ValidationReport( - file_path=file_path, valid=True, file_type='aggregate' - ) - - try: - raw = file_path.read_text(encoding='utf-8') - except OSError as e: - report.valid = False - report.errors.append( - {'loc': '(file)', 'msg': str(e), 'type': 'io_error'} - ) - return report - - try: - data = json.loads(raw) - except json.JSONDecodeError as e: - report.valid = False - report.errors.append( - { - 'loc': f'line {e.lineno}, col {e.colno}', - 'msg': e.msg, - 'type': 'json_parse_error', - } - ) - return report - - try: - EvaluationLog.model_validate(data) - except ValidationError as e: - report.valid = False - report.errors = _pydantic_errors_to_dicts(e) - - return report - - -def _validate_instance_line(line: str, line_num: int) -> list[dict]: - """Validate a single JSONL line. Returns list of error dicts.""" - try: - data = json.loads(line) - except json.JSONDecodeError as e: - return [ - { - 'loc': f'line {line_num}, col {e.colno}', - 'msg': e.msg, - 'type': 'json_parse_error', - } - ] - - try: - InstanceLevelEvaluationLog.model_validate(data) - except ValidationError as e: - errors = _pydantic_errors_to_dicts(e) - for err in errors: - err['loc'] = f'line {line_num} -> {err["loc"]}' - return errors - - return [] - - -def validate_instance_file( - file_path: Path, max_errors: int = DEFAULT_MAX_ERRORS -) -> ValidationReport: - """Validate a .jsonl file as InstanceLevelEvaluationLog (line-by-line).""" - report = ValidationReport( - file_path=file_path, valid=True, file_type='instance' - ) - - try: - f = file_path.open(encoding='utf-8') - except OSError as e: - report.valid = False - report.errors.append( - {'loc': '(file)', 'msg': str(e), 'type': 'io_error'} - ) - return report - - with f: - for line_num, line in enumerate(f, start=1): - stripped = line.strip() - if not stripped: - continue - - report.line_count += 1 - line_errors = _validate_instance_line(stripped, line_num) - - if line_errors: - report.valid = False - - # Respect max_errors by truncating line_errors to the remaining budget - remaining = max_errors - len(report.errors) - if remaining <= 0: - report.errors.append( - { - 'loc': '(truncated)', - 'msg': f'Error limit reached ({max_errors}). Use --max-errors to increase.', - 'type': 'truncated', - } - ) - break - - if len(line_errors) > remaining: - line_errors = line_errors[:remaining] - - report.errors.extend(line_errors) - - if len(report.errors) >= max_errors: - report.errors.append( - { - 'loc': '(truncated)', - 'msg': f'Error limit reached ({max_errors}). Use --max-errors to increase.', - 'type': 'truncated', - } - ) - break - - return report - - -def validate_file( - file_path: Path, max_errors: int = DEFAULT_MAX_ERRORS -) -> ValidationReport: - """Dispatch validation by file extension.""" - if file_path.suffix == '.json': - return validate_aggregate(file_path) - elif file_path.suffix == '.jsonl': - return validate_instance_file(file_path, max_errors) - else: - report = ValidationReport( - file_path=file_path, valid=False, file_type='unsupported' - ) - report.errors.append( - { - 'loc': '(file)', - 'msg': f"Unsupported file extension '{file_path.suffix}'. Expected .json or .jsonl", - 'type': 'unsupported_extension', - } - ) - return report +class _FindingGroup: + count: int = 0 + examples: list[str] = field(default_factory=list) def expand_paths(paths: list[str]) -> list[Path]: @@ -224,21 +84,49 @@ def expand_paths(paths: list[str]) -> list[Path]: for ext in ('*.json', '*.jsonl'): result.extend(sorted(path.rglob(ext))) else: - result.append(path) # let validate_file report the error + result.append(path) return result def _truncate(value: object, max_len: int = 80) -> str: - """Truncate a repr for display.""" s = repr(value) if len(s) > max_len: return s[: max_len - 3] + '...' return s -# --------------------------------------------------------------------------- -# Output renderers -# --------------------------------------------------------------------------- +def _group_findings( + reports: list[ValidationReport], + *, + kind: str, + cap: int = WARNING_GROUP_CAP, + example_cap: int = DEFAULT_WARNING_EXAMPLES, +) -> tuple[OrderedDict[str, _FindingGroup], int, int]: + groups: OrderedDict[str, _FindingGroup] = OrderedDict() + distinct: set[str] = set() + total = 0 + + for report in reports: + findings = report.warnings if kind == 'warning' else report.errors + for finding in findings: + signature = ( + format_warning(finding) + if kind == 'warning' + else format_error(finding) + ) + total += 1 + distinct.add(signature) + group = groups.get(signature) + if group is None: + if len(groups) >= cap: + continue + group = _FindingGroup() + groups[signature] = group + group.count += 1 + if len(group.examples) < example_cap: + group.examples.append(str(report.file_path)) + + return groups, total, len(distinct) def render_report_rich(report: ValidationReport, console: Console) -> None: @@ -250,105 +138,155 @@ def render_report_rich(report: ValidationReport, console: Console) -> None: if report.file_type == 'aggregate' else f'Instance (InstanceLevelEvaluationLog, {report.line_count} lines)' ) + if report.warnings: + kind += f', {len(report.warnings)} warning(s)' header = Text.assemble(label, ' ', (kind, 'dim')) + border_style = 'yellow' if report.warnings else 'green' console.print( Panel( header, title=f'[blue underline]{report.file_path}[/]', title_align='left', - border_style='green', + border_style=border_style, ) ) - else: - label = Text(' FAIL ', style='bold white on red') - kind = ( - 'Aggregate (EvaluationLog)' - if report.file_type == 'aggregate' - else 'Instance (InstanceLevelEvaluationLog)' - ) - header_line = Text.assemble(label, ' ', (kind, 'dim')) - - lines = [header_line, Text('')] - for i, err in enumerate(report.errors, 1): - loc_text = Text(f' {i}. {err["loc"]}', style='cyan') - msg_text = Text(f' {err["msg"]}', style='default') - lines.append(loc_text) - lines.append(msg_text) - if 'input' in err and err['input'] is not None: - got_text = Text( - f' Got: {_truncate(err["input"])}', style='dim' - ) - lines.append(got_text) - lines.append(Text('')) - - body = Text('\n').join(lines) - console.print( - Panel( - body, - title=f'[blue underline]{report.file_path}[/]', - title_align='left', - border_style='red', + return + + label = Text(' FAIL ', style='bold white on red') + kind = ( + 'Aggregate (EvaluationLog)' + if report.file_type == 'aggregate' + else 'Instance (InstanceLevelEvaluationLog)' + ) + header_line = Text.assemble(label, ' ', (kind, 'dim')) + + lines = [header_line, Text('')] + for index, err in enumerate(report.errors, 1): + loc_text = Text(f' {index}. {err["loc"]}', style='cyan') + msg_text = Text(f' {err["msg"]}', style='default') + lines.append(loc_text) + lines.append(msg_text) + if 'input' in err and err['input'] is not None: + lines.append( + Text(f' Got: {_truncate(err["input"])}', style='dim') ) + lines.append(Text('')) + + body = Text('\n').join(lines) + console.print( + Panel( + body, + title=f'[blue underline]{report.file_path}[/]', + title_align='left', + border_style='red', ) + ) + + +def _render_grouped_warnings( + reports: list[ValidationReport], console: Console +) -> None: + groups, total, distinct = _group_findings(reports, kind='warning') + if not total: + return + + console.print() + console.print( + Panel( + Text( + f'{total} warning(s) across {distinct} warning pattern(s)', + style='bold yellow', + ), + title='Warnings', + border_style='yellow', + ) + ) + for signature, group in groups.items(): + console.print(f'\n{group.count} file(s)') + console.print(f'Warning: {signature}') + examples = ', '.join(group.examples) + if group.count > len(group.examples): + examples += f', ... +{group.count - len(group.examples)} more' + console.print(f'Examples: {examples}') + + remaining = distinct - len(groups) + if remaining > 0: + console.print(f'\n... and {remaining} more warning pattern(s)') def render_summary_rich( reports: list[ValidationReport], console: Console ) -> None: - """Render a summary panel.""" - passed = sum(1 for r in reports if r.valid) + """Render a summary panel and grouped semantic warnings.""" + passed = sum(1 for report in reports if report.valid) failed = len(reports) - passed - total_errors = sum(len(r.errors) for r in reports) + total_errors = sum(len(report.errors) for report in reports) if failed == 0: style = 'bold green' msg = f'All {passed} file(s) passed validation' else: style = 'bold red' - msg = f'{failed} file(s) failed, {passed} passed ({total_errors} total errors)' + msg = ( + f'{failed} file(s) failed, {passed} passed ' + f'({total_errors} total errors)' + ) console.print() console.print( Panel(Text(msg, style=style), title='Summary', border_style='dim') ) + _render_grouped_warnings(reports, console) def render_report_json(reports: list[ValidationReport]) -> str: """Render all reports as a JSON array.""" output = [] - for r in reports: + for report in reports: output.append( { - 'file': str(r.file_path), - 'valid': r.valid, - 'file_type': r.file_type, - 'line_count': r.line_count, - 'errors': r.errors, + 'file': str(report.file_path), + 'valid': report.valid, + 'file_type': report.file_type, + 'line_count': report.line_count, + 'errors': report.errors, + 'warnings': report.warnings, } ) return json.dumps(output, indent=2, default=str) def render_report_github(reports: list[ValidationReport]) -> str: - """Render errors as GitHub Actions annotations.""" + """Render errors and warnings as GitHub Actions annotations.""" lines = [] - for r in reports: - for err in r.errors: + for report in reports: + for err in report.errors: + lines.append( + f'::error file={report.file_path}::{err["loc"]}: {err["msg"]}' + ) + for warning in report.warnings: lines.append( - f'::error file={r.file_path}::{err["loc"]}: {err["msg"]}' + f'::warning file={report.file_path}::{format_warning(warning)}' ) return '\n'.join(lines) -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- +def _build_hf_api(): + """Create HfApi for mandatory HF checks when HF metadata is present.""" + try: + from huggingface_hub import HfApi + except Exception: + return None + try: + return HfApi() + except Exception: + return None def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( prog='eee-validate', - description='Validate EEE schema files using Pydantic models', + description='Validate EEE schema files using shared package checks', ) parser.add_argument( 'paths', @@ -375,9 +313,14 @@ def main(argv: list[str] | None = None) -> int: print('No files found to validate.', file=sys.stderr) return 1 - reports = [ - validate_file(fp, max_errors=args.max_errors) for fp in file_paths - ] + pairs = [(repo_path_from_path(path), path) for path in file_paths] + available_files = {repo_path for repo_path, _ in pairs} + reports = validate_many( + pairs, + max_errors=args.max_errors, + available_files=available_files, + hf_api=_build_hf_api(), + ) if args.output_format == 'rich': console = Console() @@ -393,7 +336,7 @@ def main(argv: list[str] | None = None) -> int: if output: print(output) - return 1 if any(not r.valid for r in reports) else 0 + return 1 if any(not report.valid for report in reports) else 0 if __name__ == '__main__': diff --git a/every_eval_ever/validation_core.py b/every_eval_ever/validation_core.py new file mode 100644 index 000000000..99f3f637d --- /dev/null +++ b/every_eval_ever/validation_core.py @@ -0,0 +1,727 @@ +"""Shared validation checks for Every Eval Ever data. + +This module is the source of truth for package CLI validation and the +datastore validator Space. It intentionally keeps orchestration out: callers +provide local files, repo-relative paths, available companion files, and an +optional ``HfApi`` for required Hugging Face existence checks. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Callable, Container +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +from huggingface_hub.errors import RepositoryNotFoundError +from pydantic import ValidationError + +from every_eval_ever.eval_types import EvaluationLog +from every_eval_ever.instance_level_types import InstanceLevelEvaluationLog +from every_eval_ever.schema import schema_json, schema_text + +DEFAULT_MAX_ERRORS = 50 + +_EXPECTED_PATH_PARTS = 5 # data / benchmark / developer / model / filename +_UUID_FILE_RE = re.compile( + r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}' + r'(?:_samples)?\.jsonl?$', + re.IGNORECASE, +) +_COUNT_FIELDS = frozenset( + {'num_samples', 'num_bootstrap_samples', 'samples_number'} +) + +_DEPLOYMENT_TYPES = ('api', 'local', 'unknown') +_AVAILABILITY_BY_DEPLOYMENT: dict[str, tuple[str, ...]] = { + 'api': ('closed_source', 'open_weights_deployment', 'other'), + 'local': ('hf', 'unavailable', 'other'), +} + +_existence_cache: dict[tuple[str, str], tuple[bool | None, str | None]] = {} + + +@dataclass +class ValidationReport: + """Result of validating a single file.""" + + file_path: Path + valid: bool + errors: list[dict[str, Any]] = field(default_factory=list) + warnings: list[dict[str, Any]] = field(default_factory=list) + file_type: str = '' + line_count: int = 0 + + +@dataclass(frozen=True) +class ValidationContext: + """Context supplied by CLI or Space orchestration for semantic checks.""" + + local_path: Path + repo_path: str + available_files: Container[str] = field(default_factory=frozenset) + hf_api: Any = None + + +CheckScope = Literal['aggregate', 'instance', 'file'] + + +@dataclass(frozen=True) +class ValidationCheck: + """A named validation check registered with the shared runner.""" + + name: str + scope: CheckScope + run: Callable[[ValidationContext, dict[str, Any] | None], list[str]] + + +def get_schema_version() -> str: + """Read the bundled aggregate schema version.""" + data = schema_json('eval.schema.json') + version = data.get('version') + if not isinstance(version, str) or not version.strip(): + raise ValueError("eval.schema.json missing or empty 'version' field") + return version.strip() + + +def get_schema_fingerprint() -> str: + """SHA-256 of the bundled aggregate and instance schema files.""" + hasher = hashlib.sha256() + hasher.update(schema_text('eval.schema.json').encode()) + hasher.update(schema_text('instance_level_eval.schema.json').encode()) + return hasher.hexdigest() + + +def repo_path_from_path(path: Path) -> str: + """Best-effort repo-relative path for local CLI use. + + If an absolute local path contains a ``data`` component, warnings should use + the datastore path from that point onward. Otherwise the supplied path is + used as-is. + """ + raw = path.as_posix() + parts = list(path.parts) + if 'data' in parts: + data_index = parts.index('data') + return '/'.join(parts[data_index:]) + return raw + + +def _format_loc(loc: tuple[Any, ...]) -> str: + parts = [] + for part in loc: + if isinstance(part, int): + parts.append(f'[{part}]') + else: + if parts: + parts.append(f' -> {part}') + else: + parts.append(str(part)) + return ''.join(parts) if parts else '(root)' + + +def pydantic_errors_to_dicts(exc: ValidationError) -> list[dict[str, Any]]: + """Convert Pydantic errors to the report format used by the CLI and Space.""" + errors: list[dict[str, Any]] = [] + for err in exc.errors(): + errors.append( + { + 'loc': _format_loc(err['loc']), + 'msg': err['msg'], + 'type': err['type'], + 'input': err.get('input'), + } + ) + return errors + + +def warning_to_dict(message: str) -> dict[str, str]: + """Convert a grouped warning string into a structured report warning.""" + if ': ' in message: + loc, msg = message.split(': ', 1) + return {'loc': loc, 'msg': msg, 'type': 'semantic_warning'} + return {'loc': '', 'msg': message, 'type': 'semantic_warning'} + + +def format_warning(warning: dict[str, Any]) -> str: + """Format a warning dict as the signature used for grouping.""" + loc = warning.get('loc') + msg = warning.get('msg', '') + return f'{loc}: {msg}' if loc else str(msg) + + +def format_error(error: dict[str, Any]) -> str: + loc = error.get('loc') + msg = error.get('msg', '') + return f'{loc}: {msg}' if loc else str(msg) + + +def check_path_structure(repo_path: str) -> list[str]: + """Warn unless path matches data/{benchmark}/{developer}/{model}/{uuid}.json[l].""" + parts = [p for p in repo_path.split('/') if p] + + if len(parts) != _EXPECTED_PATH_PARTS: + return [ + 'Unexpected path depth: expected ' + "'data/benchmark/developer/model/uuid.json[l]', " + f"got {len(parts)} components in '{repo_path}'" + ] + + if parts[0] != 'data': + return [f"Path does not start with 'data/': '{repo_path}'"] + + if not _UUID_FILE_RE.match(parts[4]): + return [ + f"Filename '{parts[4]}' does not match " + f"'{{UUID4}}[_samples].json[l]' in '{repo_path}'" + ] + + return [] + + +def check_companion_exists( + repo_path: str, + aggregate_data: dict[str, Any], + available_files: Container[str], +) -> list[str]: + """Warn when aggregate detailed results point to a missing JSONL companion.""" + detail = aggregate_data.get('detailed_evaluation_results') + if not isinstance(detail, dict) or not detail.get('file_path'): + return [] + + folder = Path(repo_path).parent + uuid = Path(repo_path).stem + expected = { + str(folder / f'{uuid}.jsonl'), + str(folder / f'{uuid}_samples.jsonl'), + } + if not any(path in available_files for path in expected): + return [ + f"Companion .jsonl for '{Path(repo_path).name}' not found " + 'in the dataset or this PR' + ] + return [] + + +def check_score_metadata(data: dict[str, Any]) -> list[str]: + """Warn on missing score type/bounds and score values outside bounds.""" + warnings: list[str] = [] + results = data.get('evaluation_results') + if not isinstance(results, list): + return warnings + + for index, result in enumerate(results): + if not isinstance(result, dict): + continue + metric = result.get('metric_config') + if not isinstance(metric, dict): + continue + for key in ('score_type', 'min_score', 'max_score'): + if key not in metric: + warnings.append( + f"evaluation_results[{index}].metric_config: missing '{key}'" + ) + + score_details = result.get('score_details') + if not isinstance(score_details, dict): + continue + score = score_details.get('score') + lo = metric.get('min_score') + hi = metric.get('max_score') + if ( + isinstance(score, (int, float)) + and not isinstance(score, bool) + and isinstance(lo, (int, float)) + and not isinstance(lo, bool) + and isinstance(hi, (int, float)) + and not isinstance(hi, bool) + and (score < lo or score > hi) + ): + warnings.append( + f'evaluation_results[{index}]: score {score} is outside ' + f'[min_score={lo}, max_score={hi}]' + ) + return warnings + + +def check_integer_counts(data: dict[str, Any]) -> list[str]: + """Warn when count fields are present but not plain integers.""" + warnings: list[str] = [] + + def walk(obj: Any, path: str) -> None: + if isinstance(obj, dict): + for key, value in obj.items(): + child = f'{path}.{key}' + if key in _COUNT_FIELDS and value is not None: + if isinstance(value, bool) or not isinstance(value, int): + warnings.append( + f'{child}: expected integer count, got {value!r}' + ) + walk(value, child) + elif isinstance(obj, list): + for index, value in enumerate(obj): + walk(value, f'{path}[{index}]') + + walk(data, '$') + return warnings + + +def _hf_exists( + api: Any, kind: str, repo_id: str +) -> tuple[bool | None, str | None]: + """Return (exists, error); exists is None when verification failed.""" + key = (kind, repo_id) + if key in _existence_cache: + return _existence_cache[key] + try: + if kind == 'model': + api.model_info(repo_id) + else: + api.dataset_info(repo_id) + except RepositoryNotFoundError: + result = (False, None) + except Exception as exc: + detail = f'{type(exc).__name__}: {exc}' + result = (None, detail) + else: + result = (True, None) + _existence_cache[key] = result + return result + + +def check_model_deployment(data: dict[str, Any], api: Any = None) -> list[str]: + """Warn on missing/invalid model deployment metadata. + + ``model_info.additional_details.deployment_type`` is required and must be + ``api``, ``local``, or ``unknown``. For ``api`` and ``local``, + ``model_availability`` is also required. When availability is ``hf``, + Hugging Face model existence verification is required. + """ + warnings: list[str] = [] + model_info = data.get('model_info') + if not isinstance(model_info, dict): + return warnings + + details = model_info.get('additional_details') + if not isinstance(details, dict): + details = {} + + deployment_type = details.get('deployment_type') + if deployment_type is None: + warnings.append( + "model_info.additional_details: missing 'deployment_type' " + '(expected api|local|unknown)' + ) + return warnings + if deployment_type not in _DEPLOYMENT_TYPES: + warnings.append( + 'model_info.additional_details.deployment_type: expected one of ' + f'{list(_DEPLOYMENT_TYPES)}, got {deployment_type!r}' + ) + return warnings + + availability = details.get('model_availability') + allowed = _AVAILABILITY_BY_DEPLOYMENT.get(deployment_type) + if allowed is not None: + if availability is None: + warnings.append( + "model_info.additional_details: missing 'model_availability' " + f'for deployment_type={deployment_type!r} ' + f'(expected one of {list(allowed)})' + ) + elif availability not in allowed: + warnings.append( + 'model_info.additional_details.model_availability: expected ' + f'one of {list(allowed)} for deployment_type={deployment_type!r}, ' + f'got {availability!r}' + ) + + if availability == 'hf': + model_id = model_info.get('id') + if not isinstance(model_id, str) or not model_id: + warnings.append( + "model_info.id: missing model id for model_availability='hf'" + ) + elif api is None: + warnings.append( + f'model_info.id {model_id!r}: HuggingFace model existence ' + "check required because model_availability is 'hf', but no " + 'HfApi was provided' + ) + else: + exists, error = _hf_exists(api, 'model', model_id) + if exists is False: + warnings.append( + f'model_info.id {model_id!r}: not found on HuggingFace ' + "(model_availability is 'hf')" + ) + elif exists is None: + warnings.append( + f'model_info.id {model_id!r}: HuggingFace model ' + f'existence check did not complete: {error}' + ) + return warnings + + +def check_dataset_provenance( + data: dict[str, Any], api: Any = None +) -> list[str]: + """Warn on weak dataset provenance and verify HF dataset repos.""" + warnings: list[str] = [] + results = data.get('evaluation_results') + if not isinstance(results, list): + return warnings + + other_count = 0 + for index, result in enumerate(results): + if not isinstance(result, dict): + continue + source_data = result.get('source_data') + if not isinstance(source_data, dict): + continue + source_type = source_data.get('source_type') + if source_type == 'hf_dataset': + repo = source_data.get('hf_repo') + if not isinstance(repo, str) or not repo: + warnings.append( + f'evaluation_results[{index}].source_data: source_type ' + "is 'hf_dataset' but 'hf_repo' is missing" + ) + elif api is None: + warnings.append( + f'evaluation_results[{index}].source_data: HuggingFace ' + f'dataset existence check required for {repo!r}, but no ' + 'HfApi was provided' + ) + else: + exists, error = _hf_exists(api, 'dataset', repo) + if exists is False: + warnings.append( + f'evaluation_results[{index}].source_data: HF dataset ' + f'{repo!r} not found' + ) + elif exists is None: + warnings.append( + f'evaluation_results[{index}].source_data: HF dataset ' + f'existence check for {repo!r} did not complete: {error}' + ) + elif source_type == 'other': + other_count += 1 + + if other_count: + warnings.append( + f"{other_count} evaluation_results use dataset source_type 'other' " + '(no URL/HF repo provenance)' + ) + return warnings + + +def _file_check_path( + context: ValidationContext, data: dict[str, Any] | None +) -> list[str]: + return check_path_structure(context.repo_path) + + +def _aggregate_check_companion( + context: ValidationContext, data: dict[str, Any] | None +) -> list[str]: + if data is None: + return [] + return check_companion_exists( + context.repo_path, data, context.available_files + ) + + +def _aggregate_check_score_metadata( + context: ValidationContext, data: dict[str, Any] | None +) -> list[str]: + return check_score_metadata(data or {}) + + +def _aggregate_check_integer_counts( + context: ValidationContext, data: dict[str, Any] | None +) -> list[str]: + return check_integer_counts(data or {}) + + +def _aggregate_check_model_deployment( + context: ValidationContext, data: dict[str, Any] | None +) -> list[str]: + return check_model_deployment(data or {}, context.hf_api) + + +def _aggregate_check_dataset_provenance( + context: ValidationContext, data: dict[str, Any] | None +) -> list[str]: + return check_dataset_provenance(data or {}, context.hf_api) + + +REGISTERED_CHECKS: tuple[ValidationCheck, ...] = ( + ValidationCheck('path structure', 'file', _file_check_path), + ValidationCheck('companion file', 'aggregate', _aggregate_check_companion), + ValidationCheck( + 'score metadata', 'aggregate', _aggregate_check_score_metadata + ), + ValidationCheck( + 'integer counts', 'aggregate', _aggregate_check_integer_counts + ), + ValidationCheck( + 'model deployment', 'aggregate', _aggregate_check_model_deployment + ), + ValidationCheck( + 'dataset provenance', 'aggregate', _aggregate_check_dataset_provenance + ), +) + + +def run_registered_checks( + context: ValidationContext, + *, + file_type: Literal['aggregate', 'instance'], + data: dict[str, Any] | None, + checks: tuple[ValidationCheck, ...] = REGISTERED_CHECKS, +) -> list[dict[str, Any]]: + """Run registered semantic checks and return structured warnings.""" + warnings: list[dict[str, Any]] = [] + for check in checks: + if check.scope not in {'file', file_type}: + continue + try: + messages = check.run(context, data) + except Exception as exc: + messages = [ + f'{check.name} check did not complete: ' + f'{type(exc).__name__}: {exc or ""}' + ] + warnings.extend(warning_to_dict(message) for message in messages) + return warnings + + +def validate_aggregate( + file_path: Path, + *, + repo_path: str | None = None, + available_files: Container[str] | None = None, + hf_api: Any = None, + run_semantic_checks: bool = True, +) -> ValidationReport: + """Validate a .json file as an EvaluationLog plus semantic warnings.""" + report = ValidationReport( + file_path=file_path, valid=True, file_type='aggregate' + ) + repo_path = repo_path or repo_path_from_path(file_path) + if available_files is None: + available_files = frozenset({repo_path}) + + try: + raw = file_path.read_text(encoding='utf-8') + except OSError as exc: + report.valid = False + report.errors.append( + {'loc': '(file)', 'msg': str(exc), 'type': 'io_error'} + ) + return report + + try: + loaded = json.loads(raw) + except json.JSONDecodeError as exc: + report.valid = False + report.errors.append( + { + 'loc': f'line {exc.lineno}, col {exc.colno}', + 'msg': exc.msg, + 'type': 'json_parse_error', + } + ) + return report + + data = loaded if isinstance(loaded, dict) else None + try: + EvaluationLog.model_validate(loaded) + except ValidationError as exc: + report.valid = False + report.errors = pydantic_errors_to_dicts(exc) + + if run_semantic_checks: + context = ValidationContext( + local_path=file_path, + repo_path=repo_path, + available_files=available_files, + hf_api=hf_api, + ) + report.warnings = run_registered_checks( + context, file_type='aggregate', data=data + ) + + return report + + +def _validate_instance_line(line: str, line_num: int) -> list[dict[str, Any]]: + try: + data = json.loads(line) + except json.JSONDecodeError as exc: + return [ + { + 'loc': f'line {line_num}, col {exc.colno}', + 'msg': exc.msg, + 'type': 'json_parse_error', + } + ] + + try: + InstanceLevelEvaluationLog.model_validate(data) + except ValidationError as exc: + errors = pydantic_errors_to_dicts(exc) + for error in errors: + error['loc'] = f'line {line_num} -> {error["loc"]}' + return errors + + return [] + + +def validate_instance_file( + file_path: Path, + max_errors: int = DEFAULT_MAX_ERRORS, + *, + repo_path: str | None = None, + available_files: Container[str] | None = None, + run_semantic_checks: bool = True, +) -> ValidationReport: + """Validate a .jsonl file as InstanceLevelEvaluationLog line-by-line.""" + report = ValidationReport( + file_path=file_path, valid=True, file_type='instance' + ) + repo_path = repo_path or repo_path_from_path(file_path) + if available_files is None: + available_files = frozenset({repo_path}) + + try: + handle = file_path.open(encoding='utf-8') + except OSError as exc: + report.valid = False + report.errors.append( + {'loc': '(file)', 'msg': str(exc), 'type': 'io_error'} + ) + return report + + with handle: + for line_num, line in enumerate(handle, start=1): + stripped = line.strip() + if not stripped: + continue + + report.line_count += 1 + line_errors = _validate_instance_line(stripped, line_num) + if not line_errors: + continue + + report.valid = False + remaining = max_errors - len(report.errors) + if remaining <= 0: + report.errors.append( + { + 'loc': '(truncated)', + 'msg': ( + f'Error limit reached ({max_errors}). ' + 'Use --max-errors to increase.' + ), + 'type': 'truncated', + } + ) + break + report.errors.extend(line_errors[:remaining]) + if len(report.errors) >= max_errors: + report.errors.append( + { + 'loc': '(truncated)', + 'msg': ( + f'Error limit reached ({max_errors}). ' + 'Use --max-errors to increase.' + ), + 'type': 'truncated', + } + ) + break + + if run_semantic_checks: + context = ValidationContext( + local_path=file_path, + repo_path=repo_path, + available_files=available_files, + ) + report.warnings = run_registered_checks( + context, file_type='instance', data=None + ) + + return report + + +def validate_file( + file_path: Path, + max_errors: int = DEFAULT_MAX_ERRORS, + *, + repo_path: str | None = None, + available_files: Container[str] | None = None, + hf_api: Any = None, + run_semantic_checks: bool = True, +) -> ValidationReport: + """Dispatch validation by extension.""" + if file_path.suffix == '.json': + return validate_aggregate( + file_path, + repo_path=repo_path, + available_files=available_files, + hf_api=hf_api, + run_semantic_checks=run_semantic_checks, + ) + if file_path.suffix == '.jsonl': + return validate_instance_file( + file_path, + max_errors=max_errors, + repo_path=repo_path, + available_files=available_files, + run_semantic_checks=run_semantic_checks, + ) + + report = ValidationReport( + file_path=file_path, valid=False, file_type='unsupported' + ) + report.errors.append( + { + 'loc': '(file)', + 'msg': ( + f"Unsupported file extension '{file_path.suffix}'. " + 'Expected .json or .jsonl' + ), + 'type': 'unsupported_extension', + } + ) + return report + + +def validate_many( + files: list[tuple[str, Path]], + *, + max_errors: int = DEFAULT_MAX_ERRORS, + available_files: Container[str] | None = None, + hf_api: Any = None, +) -> list[ValidationReport]: + """Validate repo-path/local-path pairs with a shared context.""" + available = ( + frozenset(repo_path for repo_path, _ in files) + if available_files is None + else available_files + ) + return [ + validate_file( + local_path, + max_errors=max_errors, + repo_path=repo_path, + available_files=available, + hf_api=hf_api, + ) + for repo_path, local_path in files + ] diff --git a/tests/test_dedup.py b/tests/test_dedup.py new file mode 100644 index 000000000..d244cf308 --- /dev/null +++ b/tests/test_dedup.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import copy +import json + +import pytest + +from every_eval_ever.dedup import ( + check_duplicates, + compute_aggregate_identity, + compute_fingerprint, +) + + +def _base() -> dict: + return { + 'schema_version': '0.2.2', + 'evaluation_id': 'gsm8k/llama/111', + 'retrieved_timestamp': '111', + 'evaluation_timestamp': '2025-01-01T00:00:00Z', + 'source_metadata': { + 'source_type': 'evaluation_run', + 'source_organization_name': 'Org', + 'evaluator_relationship': 'third_party', + }, + 'model_info': { + 'id': 'meta-llama/Llama-3.1-8B-Instruct', + 'name': 'Llama', + }, + 'eval_library': {'name': 'lm_eval', 'version': '0.4'}, + 'detailed_evaluation_results': {'file_path': 'x.jsonl'}, + 'additional_details': {'freeform': 'ignored'}, + 'evaluation_results': [ + { + 'evaluation_name': 'GSM8K', + 'source_data': { + 'source_type': 'hf_dataset', + 'dataset_name': 'gsm8k', + 'hf_repo': 'openai/gsm8k', + }, + 'metric_config': { + 'metric_id': 'accuracy', + 'metric_kind': 'accuracy', + 'metric_unit': 'proportion', + 'score_type': 'continuous', + 'lower_is_better': False, + 'metric_parameters': {'k': 1}, + }, + 'score_details': {'score': 0.95, 'details': {'n': '500'}}, + 'generation_config': { + 'generation_args': { + 'temperature': 0.0, + 'reasoning': False, + 'max_tokens': 512, + } + }, + 'additional_details': {'note': 'ignored'}, + } + ], + } + + +def _first_result(data: dict) -> dict: + return data['evaluation_results'][0] + + +def _write(tmp_path, spec: dict[str, dict]) -> dict[str, str]: + local_paths = {} + for repo_path, payload in spec.items(): + local_path = tmp_path / repo_path.replace('/', '__') + local_path.write_text(json.dumps(payload), encoding='utf-8') + local_paths[repo_path] = str(local_path) + return local_paths + + +def test_non_identity_fields_do_not_change_fingerprint(): + baseline = compute_aggregate_identity(_base()) + + data = _base() + data['evaluation_id'] = 'new-id' + data['retrieved_timestamp'] = '999' + data['schema_version'] = '0.3.0' + data['source_metadata'] = {'source_type': 'documentation'} + data['additional_details'] = {'anything': 'else'} + _first_result(data)['additional_details'] = {'different': 'freeform'} + _first_result(data)['score_details']['details'] = {'different': 'freeform'} + + assert compute_aggregate_identity(data) == baseline + + +def test_identity_fields_change_fingerprint(): + baseline = compute_aggregate_identity(_base()) + cases = [] + + changed_score = _base() + _first_result(changed_score)['score_details']['score'] = 0.96 + cases.append(changed_score) + + changed_temp = _base() + _first_result(changed_temp)['generation_config']['generation_args'][ + 'temperature' + ] = 0.5 + cases.append(changed_temp) + + changed_model = _base() + changed_model['model_info']['id'] = 'openai/gpt-4o' + cases.append(changed_model) + + for case in cases: + assert compute_aggregate_identity(case) != baseline + + +def test_results_order_is_invariant(): + data = _base() + second = copy.deepcopy(_first_result(data)) + second['evaluation_name'] = 'MMLU' + second['metric_config']['metric_id'] = 'mmlu_accuracy' + data['evaluation_results'].append(second) + forward = compute_aggregate_identity(data) + data['evaluation_results'].reverse() + assert compute_aggregate_identity(data) == forward + + +def test_none_results_are_ignored_but_non_object_results_are_rejected(): + data = _base() + baseline = compute_aggregate_identity(data) + + data_with_none = _base() + data_with_none['evaluation_results'].append(None) + assert compute_aggregate_identity(data_with_none) == baseline + + data_with_bad_result = _base() + data_with_bad_result['evaluation_results'].append('not a result') + with pytest.raises(ValueError, match='entries must be objects'): + compute_aggregate_identity(data_with_bad_result) + + +def test_large_integer_identity_fields_are_not_float_coerced(): + first = _base() + second = _base() + first_args = _first_result(first)['generation_config']['generation_args'] + second_args = _first_result(second)['generation_config']['generation_args'] + first_args['max_tokens'] = 2**53 + second_args['max_tokens'] = 2**53 + 1 + + assert compute_aggregate_identity(first) != compute_aggregate_identity( + second + ) + + +def test_compute_fingerprint_rejects_non_aggregate_json(): + with pytest.raises(ValueError, match='aggregate JSON'): + compute_fingerprint(b'{"x": 1}') + + +def test_manifest_and_intra_batch_dedup_are_collection_scoped(tmp_path): + existing = _base() + candidate = _base() + manifest = { + 'files': { + 'data/gsm8k/openai/model/existing.json': { + 'fingerprint': compute_fingerprint( + json.dumps(existing).encode() + ) + }, + 'data/other/openai/model/existing.json': { + 'fingerprint': compute_fingerprint( + json.dumps(existing).encode() + ) + }, + } + } + local = _write( + tmp_path, + { + 'data/gsm8k/openai/model/candidate.json': candidate, + 'data/gsm8k/openai/model/second.json': candidate, + }, + ) + + report = check_duplicates(list(local), local, manifest) + by_path = {result.file_path: result for result in report.results} + + assert ( + by_path['data/gsm8k/openai/model/candidate.json'].duplicate_of + == 'data/gsm8k/openai/model/existing.json' + ) + assert ( + by_path['data/gsm8k/openai/model/second.json'].duplicate_of + == 'data/gsm8k/openai/model/existing.json' + ) + + +def test_distinct_scores_are_not_duplicates(tmp_path): + first = _base() + second = _base() + _first_result(second)['score_details']['score'] = 0.71 + local = _write( + tmp_path, + { + 'data/gsm8k/model/one.json': first, + 'data/gsm8k/model/two.json': second, + }, + ) + + report = check_duplicates(list(local), local, {'files': {}}) + + assert all(result.duplicate_of is None for result in report.results) + + +def test_missing_local_path_is_reported_as_warning(): + report = check_duplicates( + ['data/gsm8k/openai/model/missing.json'], {}, {'files': {}} + ) + + assert report.results == [] + assert report.warnings == [ + 'Duplicate check skipped data/gsm8k/openai/model/missing.json: ' + 'local path was not provided' + ] + + +def test_check_duplicates_rejects_non_json_paths(): + with pytest.raises(ValueError, match='only accepts .json files'): + check_duplicates(['data/gsm8k/model/readme.txt'], {}, {'files': {}}) diff --git a/tests/test_helm_adapter.py b/tests/test_helm_adapter.py index e08f48dcc..a7ec34f45 100644 --- a/tests/test_helm_adapter.py +++ b/tests/test_helm_adapter.py @@ -162,8 +162,9 @@ def test_missing_model_deployment_falls_back_to_model(): Copies a helm data item and explicitly removes a field to test robustness to model_deployment missing. Regression test for #112 """ - import shutil import json + import shutil + src = Path( 'tests/data/helm/' 'mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' diff --git a/tests/test_validate.py b/tests/test_validate.py index edb2d5b6a..78892fc99 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -6,12 +6,19 @@ from pathlib import Path from every_eval_ever.validate import ( + check_companion_exists, + check_dataset_provenance, + check_integer_counts, + check_model_deployment, + check_path_structure, + check_score_metadata, expand_paths, render_report_github, render_report_json, validate_aggregate, validate_file, validate_instance_file, + validate_many, ) # --------------------------------------------------------------------------- @@ -361,7 +368,7 @@ def test_github_output_empty_on_pass(self, tmp_path: Path): fp = _write_json(tmp_path, 'pass.json', VALID_AGGREGATE) report = validate_file(fp) output = render_report_github([report]) - assert output == '' + assert output.startswith('::warning file=') class TestExitCode: @@ -376,3 +383,130 @@ def test_exit_code_1_on_failure(self, tmp_path: Path): fp = _write_json(tmp_path, 'fail.json', data) report = validate_file(fp) assert report.valid is False + + +class TestSemanticWarnings: + def test_path_structure_matches_validator_bot(self): + good = ( + 'data/gsm8k/openai/gpt-4o/550e8400-e29b-41d4-a716-446655440000.json' + ) + bad = 'data/gsm8k/file.json' + assert check_path_structure(good) == [] + assert 'Unexpected path depth' in check_path_structure(bad)[0] + + def test_companion_warning_uses_available_files(self): + uuid = '550e8400-e29b-41d4-a716-446655440000' + repo_path = f'data/bench/dev/model/{uuid}.json' + data = {'detailed_evaluation_results': {'file_path': f'{uuid}.jsonl'}} + assert ( + check_companion_exists( + repo_path, data, {f'data/bench/dev/model/{uuid}.jsonl'} + ) + == [] + ) + warnings = check_companion_exists(repo_path, data, {repo_path}) + assert 'Companion .jsonl' in warnings[0] + + def test_score_metadata_missing_and_bounds_warn(self): + data = json.loads(json.dumps(VALID_AGGREGATE)) + warnings = check_score_metadata(data) + assert any("missing 'min_score'" in warning for warning in warnings) + assert any("missing 'max_score'" in warning for warning in warnings) + + data['evaluation_results'][0]['metric_config'].update( + {'score_type': 'continuous', 'min_score': 0, 'max_score': 1} + ) + data['evaluation_results'][0]['score_details']['score'] = 1.5 + warnings = check_score_metadata(data) + assert any( + 'outside [min_score=0, max_score=1]' in warning + for warning in warnings + ) + + def test_integer_count_warning(self): + warnings = check_integer_counts( + {'score_details': {'uncertainty': {'num_samples': 10.0}}} + ) + assert any('num_samples' in warning for warning in warnings) + + def test_model_deployment_two_field_taxonomy(self): + base = {'model_info': {'id': 'org/model', 'additional_details': {}}} + assert 'deployment_type' in check_model_deployment(base)[0] + + api_record = { + 'model_info': { + 'id': 'org/model', + 'additional_details': { + 'deployment_type': 'api', + 'model_availability': 'closed_source', + }, + } + } + assert check_model_deployment(api_record) == [] + + local_closed = { + 'model_info': { + 'id': 'org/model', + 'additional_details': { + 'deployment_type': 'local', + 'model_availability': 'closed_source', + }, + } + } + assert 'model_availability' in check_model_deployment(local_closed)[0] + + def test_hf_model_availability_requires_api(self): + data = { + 'model_info': { + 'id': 'org/model', + 'additional_details': { + 'deployment_type': 'local', + 'model_availability': 'hf', + }, + } + } + warnings = check_model_deployment(data) + assert any('no HfApi was provided' in warning for warning in warnings) + + def test_dataset_provenance_requires_hf_api_for_hf_dataset(self): + data = { + 'evaluation_results': [ + { + 'source_data': { + 'source_type': 'hf_dataset', + 'hf_repo': 'org/dataset', + } + }, + {'source_data': {'source_type': 'other'}}, + ] + } + warnings = check_dataset_provenance(data) + assert any('no HfApi was provided' in warning for warning in warnings) + assert any("source_type 'other'" in warning for warning in warnings) + + def test_validate_many_preserves_explicit_empty_available_files( + self, tmp_path: Path + ): + uuid = '550e8400-e29b-41d4-a716-446655440000' + aggregate = json.loads(json.dumps(VALID_AGGREGATE)) + aggregate['detailed_evaluation_results'] = { + 'format': 'jsonl', + 'file_path': f'{uuid}.jsonl', + } + json_path = _write_json(tmp_path, f'{uuid}.json', aggregate) + jsonl_path = _write_jsonl( + tmp_path, f'{uuid}.jsonl', [VALID_SINGLE_TURN] + ) + reports = validate_many( + [ + (f'data/bench/dev/model/{uuid}.json', json_path), + (f'data/bench/dev/model/{uuid}.jsonl', jsonl_path), + ], + available_files=set(), + ) + + aggregate_report = reports[0] + assert any( + 'Companion .jsonl' in warning['msg'] + for warning in aggregate_report.warnings + ) diff --git a/utils/exgentic/adapter.py b/utils/exgentic/adapter.py index b7a4c02b5..53966082e 100644 --- a/utils/exgentic/adapter.py +++ b/utils/exgentic/adapter.py @@ -47,17 +47,17 @@ SourceMetadata, Uncertainty, ) -from helpers import save_evaluation_log, sanitize_filename +from helpers import sanitize_filename, save_evaluation_log -SCHEMA_VERSION = "0.2.2" -OUTPUT_DIR = "data/exgentic" -HF_DATASET = "Exgentic/open-agent-leaderboard-results" +SCHEMA_VERSION = '0.2.2' +OUTPUT_DIR = 'data/exgentic' +HF_DATASET = 'Exgentic/open-agent-leaderboard-results' # Map model name prefixes to developer organizations MODEL_DEVELOPER_MAP = { - "claude": ("Anthropic", "anthropic"), - "gpt": ("OpenAI", "openai"), - "gemini": ("Google", "google"), + 'claude': ('Anthropic', 'anthropic'), + 'gpt': ('OpenAI', 'openai'), + 'gemini': ('Google', 'google'), } @@ -70,11 +70,11 @@ def parse_model_info(model_name: str) -> tuple[str, str, str]: Returns: (developer_display, developer_slug, model_slug) """ - parts = model_name.split("/") + parts = model_name.split('/') raw_model = parts[-1] if parts else model_name - developer_display = "unknown" - developer_slug = "unknown" + developer_display = 'unknown' + developer_slug = 'unknown' lower = raw_model.lower() for prefix, (display, slug) in MODEL_DEVELOPER_MAP.items(): if lower.startswith(prefix): @@ -87,57 +87,67 @@ def parse_model_info(model_name: str) -> tuple[str, str, str]: def make_agent_slug(agent_name: str) -> str: """Convert agent display name to a URL-safe slug.""" - return re.sub(r"[^a-z0-9]+", "-", agent_name.lower()).strip("-") + return re.sub(r'[^a-z0-9]+', '-', agent_name.lower()).strip('-') def convert_result(result: dict, retrieved_timestamp: str) -> EvaluationLog: """Convert a single exgentic result dict to an EvaluationLog.""" - model_name_raw = result.get("model_name") or "unknown" - developer_display, developer_slug, model_slug = parse_model_info(model_name_raw) - model_id = f"{developer_slug}/{model_slug}" + model_name_raw = result.get('model_name') or 'unknown' + developer_display, developer_slug, model_slug = parse_model_info( + model_name_raw + ) + model_id = f'{developer_slug}/{model_slug}' - benchmark = result.get("benchmark_name") or result.get("benchmark") or "unknown" - agent_name = result.get("agent_name") or result.get("agent") or "unknown" - agent_framework = result.get("agent") or make_agent_slug(agent_name) + benchmark = ( + result.get('benchmark_name') or result.get('benchmark') or 'unknown' + ) + agent_name = result.get('agent_name') or result.get('agent') or 'unknown' + agent_framework = result.get('agent') or make_agent_slug(agent_name) agent_slug = make_agent_slug(agent_name) - subset = result.get("subset_name") + subset = result.get('subset_name') - eval_name = benchmark.lower().replace(" ", "-") + eval_name = benchmark.lower().replace(' ', '-') if subset: - eval_name = f"{eval_name}/{subset}" + eval_name = f'{eval_name}/{subset}' - score = result.get("benchmark_score") + score = result.get('benchmark_score') if score is None: - score = result.get("average_score", 0.0) + score = result.get('average_score', 0.0) # Build uncertainty from session counts - total = result.get("total_sessions") + total = result.get('total_sessions') uncertainty = None if total and int(total) > 0: uncertainty = Uncertainty(num_samples=int(total)) # Build score details details: dict[str, str] = {} - if result.get("average_agent_cost") is not None: - details["average_agent_cost"] = str(round(float(result["average_agent_cost"]), 2)) - if result.get("total_run_cost") is not None: - details["total_run_cost"] = str(round(float(result["total_run_cost"]), 2)) - if result.get("average_steps") is not None: - details["average_steps"] = str(round(float(result["average_steps"]), 2)) - if result.get("percent_finished") is not None: - details["percent_finished"] = str(round(float(result["percent_finished"]), 4)) + if result.get('average_agent_cost') is not None: + details['average_agent_cost'] = str( + round(float(result['average_agent_cost']), 2) + ) + if result.get('total_run_cost') is not None: + details['total_run_cost'] = str( + round(float(result['total_run_cost']), 2) + ) + if result.get('average_steps') is not None: + details['average_steps'] = str(round(float(result['average_steps']), 2)) + if result.get('percent_finished') is not None: + details['percent_finished'] = str( + round(float(result['percent_finished']), 4) + ) eval_result = EvaluationResult( evaluation_name=eval_name, source_data=SourceDataUrl( dataset_name=eval_name, - source_type="url", - url=["https://github.com/Exgentic/exgentic"], + source_type='url', + url=['https://github.com/Exgentic/exgentic'], ), evaluation_timestamp=retrieved_timestamp, metric_config=MetricConfig( - evaluation_description=f"{benchmark} benchmark evaluation" - + (f" ({subset} subset)" if subset else ""), + evaluation_description=f'{benchmark} benchmark evaluation' + + (f' ({subset} subset)' if subset else ''), lower_is_better=False, score_type=ScoreType.continuous, min_score=0.0, @@ -152,39 +162,41 @@ def convert_result(result: dict, retrieved_timestamp: str) -> EvaluationLog: generation_args=GenerationArgs( agentic_eval_config=AgenticEvalConfig( additional_details={ - "agent_name": agent_name, - "agent_framework": agent_framework, + 'agent_name': agent_name, + 'agent_framework': agent_framework, }, ), ), ), ) - sanitized_model_id = model_id.replace("/", "_") - evaluation_id = f"{eval_name}/{agent_slug}__{sanitized_model_id}/{retrieved_timestamp}" + sanitized_model_id = model_id.replace('/', '_') + evaluation_id = ( + f'{eval_name}/{agent_slug}__{sanitized_model_id}/{retrieved_timestamp}' + ) return EvaluationLog( schema_version=SCHEMA_VERSION, evaluation_id=evaluation_id, retrieved_timestamp=retrieved_timestamp, source_metadata=SourceMetadata( - source_name="Exgentic Open Agent Leaderboard", - source_type="evaluation_run", - source_organization_name="Exgentic", - source_organization_url="https://github.com/Exgentic", + source_name='Exgentic Open Agent Leaderboard', + source_type='evaluation_run', + source_organization_name='Exgentic', + source_organization_url='https://github.com/Exgentic', evaluator_relationship=EvaluatorRelationship.third_party, ), eval_library=EvalLibrary( - name="exgentic", - version="0.1.0", + name='exgentic', + version='0.1.0', ), model_info=ModelInfo( name=model_slug, id=model_id, developer=developer_display, additional_details={ - "agent_name": agent_name, - "agent_framework": agent_framework, + 'agent_name': agent_name, + 'agent_framework': agent_framework, }, ), evaluation_results=[eval_result], @@ -196,21 +208,21 @@ def load_results_from_dir(results_dir: str) -> list[dict]: results = [] base = Path(results_dir) - for config_path in sorted(base.rglob("config.json")): + for config_path in sorted(base.rglob('config.json')): try: config = json.loads(config_path.read_text()) - run_id = config.get("run_id") + run_id = config.get('run_id') if not run_id: continue - results_path = config_path.parent / run_id / "results.json" + results_path = config_path.parent / run_id / 'results.json' if not results_path.is_file(): continue payload = json.loads(results_path.read_text()) - if "benchmark_score" not in payload: + if 'benchmark_score' not in payload: continue results.append(payload) except (json.JSONDecodeError, OSError) as e: - print(f"Warning: skipping {config_path}: {e}") + print(f'Warning: skipping {config_path}: {e}') return results @@ -219,35 +231,37 @@ def load_results_from_hf() -> list[dict]: try: from datasets import load_dataset except ImportError: - print("Error: 'datasets' package required. Install with: pip install datasets") + print( + "Error: 'datasets' package required. Install with: pip install datasets" + ) sys.exit(1) - ds = load_dataset(HF_DATASET, split="train") + ds = load_dataset(HF_DATASET, split='train') return list(ds) def main(): parser = argparse.ArgumentParser( - description="Convert Exgentic results to Every Eval Ever format" + description='Convert Exgentic results to Every Eval Ever format' ) parser.add_argument( - "--results-dir", - help="Path to exgentic experiments directory containing config.json files", + '--results-dir', + help='Path to exgentic experiments directory containing config.json files', ) parser.add_argument( - "--from-hf", - action="store_true", - help=f"Load results from HuggingFace dataset ({HF_DATASET})", + '--from-hf', + action='store_true', + help=f'Load results from HuggingFace dataset ({HF_DATASET})', ) parser.add_argument( - "--output-dir", + '--output-dir', default=OUTPUT_DIR, - help=f"Output directory for EEE JSON files (default: {OUTPUT_DIR})", + help=f'Output directory for EEE JSON files (default: {OUTPUT_DIR})', ) args = parser.parse_args() if not args.results_dir and not args.from_hf: - parser.error("Specify either --results-dir or --from-hf") + parser.error('Specify either --results-dir or --from-hf') if args.results_dir: results = load_results_from_dir(args.results_dir) @@ -255,10 +269,10 @@ def main(): results = load_results_from_hf() if not results: - print("No results found.") + print('No results found.') sys.exit(1) - print(f"Loaded {len(results)} result(s)") + print(f'Loaded {len(results)} result(s)') retrieved_timestamp = str(time.time()) count = 0 @@ -267,21 +281,23 @@ def main(): try: eval_log = convert_result(result, retrieved_timestamp) model_info = eval_log.model_info - developer_slug = sanitize_filename(model_info.developer or "unknown") + developer_slug = sanitize_filename( + model_info.developer or 'unknown' + ) model_name = sanitize_filename(model_info.name) filepath = save_evaluation_log( eval_log, args.output_dir, developer_slug, model_name ) - print(f" {filepath}") + print(f' {filepath}') count += 1 except Exception as e: - benchmark = result.get("benchmark", "?") - agent = result.get("agent", "?") - model = result.get("model_name", "?") - print(f"Error processing {benchmark}/{agent}/{model}: {e}") + benchmark = result.get('benchmark', '?') + agent = result.get('agent', '?') + model = result.get('model_name', '?') + print(f'Error processing {benchmark}/{agent}/{model}: {e}') - print(f"\nGenerated {count} file(s) in {args.output_dir}/") + print(f'\nGenerated {count} file(s) in {args.output_dir}/') -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/utils/hal/adapter.py b/utils/hal/adapter.py index 977670216..445eb6040 100644 --- a/utils/hal/adapter.py +++ b/utils/hal/adapter.py @@ -29,20 +29,21 @@ import time import uuid from dataclasses import dataclass, field +from html import unescape from pathlib import Path from typing import Optional -from urllib.request import urlopen, Request from urllib.error import URLError -from html import unescape +from urllib.request import Request, urlopen from every_eval_ever.helpers import SCHEMA_VERSION -HAL_BASE_URL = "https://hal.cs.princeton.edu" +HAL_BASE_URL = 'https://hal.cs.princeton.edu' # --------------------------------------------------------------------------- # Benchmark definitions # --------------------------------------------------------------------------- + @dataclass class ToolDef: name: str @@ -51,13 +52,13 @@ class ToolDef: @dataclass class BenchmarkDef: - slug: str # URL path segment on hal.cs.princeton.edu - name: str # Human-readable name - output_name: str # Directory name under data/ - category: str # Category label - dataset_url: str # Canonical dataset/benchmark URL - description: str # Short description of what is measured - metric_description: str # Description of the primary accuracy metric + slug: str # URL path segment on hal.cs.princeton.edu + name: str # Human-readable name + output_name: str # Directory name under data/ + category: str # Category label + dataset_url: str # Canonical dataset/benchmark URL + description: str # Short description of what is measured + metric_description: str # Description of the primary accuracy metric # Tools available to agents in this benchmark tools: list[ToolDef] = field(default_factory=list) extra_metrics: list[dict] = field(default_factory=list) # GAIA levels etc. @@ -67,168 +68,183 @@ class BenchmarkDef: BENCHMARKS: list[BenchmarkDef] = [ BenchmarkDef( - slug="assistantbench", - name="AssistantBench", - output_name="hal-assistantbench", - category="Web Assistance", - dataset_url="https://assistantbench.github.io", + slug='assistantbench', + name='AssistantBench', + output_name='hal-assistantbench', + category='Web Assistance', + dataset_url='https://assistantbench.github.io', description=( - "HAL evaluates AI agents on a 33-task split of AssistantBench, " - "a benchmark of realistic, time-consuming, and automatically verifiable " - "web assistance tasks based on real human needs. " - "The full benchmark contains 214 tasks." + 'HAL evaluates AI agents on a 33-task split of AssistantBench, ' + 'a benchmark of realistic, time-consuming, and automatically verifiable ' + 'web assistance tasks based on real human needs. ' + 'The full benchmark contains 214 tasks.' ), metric_description="Accuracy on HAL's 33-task AssistantBench split (0.0–1.0)", tools=[ - ToolDef("browser", "Navigate and interact with live web pages"), - ToolDef("web_search", "Search the web for information"), + ToolDef('browser', 'Navigate and interact with live web pages'), + ToolDef('web_search', 'Search the web for information'), ], source_data_details={ - "tasks_evaluated": "33", - "full_benchmark_size": "214", - "note": "HAL evaluates on a 33-task subset; full AssistantBench has 214 tasks", + 'tasks_evaluated': '33', + 'full_benchmark_size': '214', + 'note': 'HAL evaluates on a 33-task subset; full AssistantBench has 214 tasks', }, ), BenchmarkDef( - slug="corebench_hard", - name="CORE-Bench Hard", - output_name="hal-corebench-hard", - category="Scientific Programming", - dataset_url="https://github.com/siegelz/core-bench", + slug='corebench_hard', + name='CORE-Bench Hard', + output_name='hal-corebench-hard', + category='Scientific Programming', + dataset_url='https://github.com/siegelz/core-bench', description=( - "CORE-Bench Hard tests agents on hard computational reproducibility tasks " - "drawn from published scientific papers." + 'CORE-Bench Hard tests agents on hard computational reproducibility tasks ' + 'drawn from published scientific papers.' ), - metric_description="Fraction of CORE-Bench Hard tasks solved (0.0–1.0)", + metric_description='Fraction of CORE-Bench Hard tasks solved (0.0–1.0)', tools=[ - ToolDef("bash", "Execute shell commands"), - ToolDef("python", "Execute Python code"), - ToolDef("read_file", "Read files from the filesystem"), - ToolDef("write_file", "Write files to the filesystem"), + ToolDef('bash', 'Execute shell commands'), + ToolDef('python', 'Execute Python code'), + ToolDef('read_file', 'Read files from the filesystem'), + ToolDef('write_file', 'Write files to the filesystem'), ], ), BenchmarkDef( - slug="gaia", - name="GAIA", - output_name="hal-gaia", - category="Web Assistance", - dataset_url="https://huggingface.co/datasets/gaia-benchmark/GAIA", + slug='gaia', + name='GAIA', + output_name='hal-gaia', + category='Web Assistance', + dataset_url='https://huggingface.co/datasets/gaia-benchmark/GAIA', description=( - "GAIA (General AI Assistants) measures whether AI agents can answer " - "real-world questions requiring multi-step reasoning and tool use. " - "Questions are divided into 3 difficulty levels." + 'GAIA (General AI Assistants) measures whether AI agents can answer ' + 'real-world questions requiring multi-step reasoning and tool use. ' + 'Questions are divided into 3 difficulty levels.' ), - metric_description="Overall accuracy on GAIA validation set (0.0–1.0)", + metric_description='Overall accuracy on GAIA validation set (0.0–1.0)', tools=[ - ToolDef("web_search", "Search the web for information"), - ToolDef("browser", "Navigate and interact with live web pages"), - ToolDef("python", "Execute Python code for computation"), - ToolDef("read_file", "Read and process files"), + ToolDef('web_search', 'Search the web for information'), + ToolDef('browser', 'Navigate and interact with live web pages'), + ToolDef('python', 'Execute Python code for computation'), + ToolDef('read_file', 'Read and process files'), ], extra_metrics=[ - {"key": "level1", "name": "GAIA Level 1", "description": "Accuracy on Level 1 questions (simplest)"}, - {"key": "level2", "name": "GAIA Level 2", "description": "Accuracy on Level 2 questions (moderate)"}, - {"key": "level3", "name": "GAIA Level 3", "description": "Accuracy on Level 3 questions (hardest)"}, + { + 'key': 'level1', + 'name': 'GAIA Level 1', + 'description': 'Accuracy on Level 1 questions (simplest)', + }, + { + 'key': 'level2', + 'name': 'GAIA Level 2', + 'description': 'Accuracy on Level 2 questions (moderate)', + }, + { + 'key': 'level3', + 'name': 'GAIA Level 3', + 'description': 'Accuracy on Level 3 questions (hardest)', + }, ], ), BenchmarkDef( - slug="online_mind2web", - name="Online Mind2Web", - output_name="hal-online-mind2web", - category="Web Assistance", - dataset_url="https://osu-nlp-group.github.io/Mind2Web/", + slug='online_mind2web', + name='Online Mind2Web', + output_name='hal-online-mind2web', + category='Web Assistance', + dataset_url='https://osu-nlp-group.github.io/Mind2Web/', description=( - "Online Mind2Web evaluates web agents on live website tasks " - "requiring multi-step navigation and interaction." + 'Online Mind2Web evaluates web agents on live website tasks ' + 'requiring multi-step navigation and interaction.' ), - metric_description="Task success rate on Online Mind2Web (0.0–1.0)", + metric_description='Task success rate on Online Mind2Web (0.0–1.0)', tools=[ - ToolDef("browser", "Navigate and interact with live web pages"), - ToolDef("click", "Click on web page elements"), - ToolDef("type", "Type text into web page inputs"), - ToolDef("scroll", "Scroll web pages"), + ToolDef('browser', 'Navigate and interact with live web pages'), + ToolDef('click', 'Click on web page elements'), + ToolDef('type', 'Type text into web page inputs'), + ToolDef('scroll', 'Scroll web pages'), ], ), BenchmarkDef( - slug="scicode", - name="Scicode", - output_name="hal-scicode", - category="Scientific Programming", - dataset_url="https://scicode-bench.github.io", + slug='scicode', + name='Scicode', + output_name='hal-scicode', + category='Scientific Programming', + dataset_url='https://scicode-bench.github.io', description=( - "Scicode tests agents on scientific coding problems spanning " - "mathematics, physics, chemistry, biology, and material science." + 'Scicode tests agents on scientific coding problems spanning ' + 'mathematics, physics, chemistry, biology, and material science.' ), - metric_description="Fraction of Scicode problems solved (0.0–1.0)", + metric_description='Fraction of Scicode problems solved (0.0–1.0)', tools=[ - ToolDef("python", "Execute Python code for scientific computation"), - ToolDef("bash", "Execute shell commands"), + ToolDef('python', 'Execute Python code for scientific computation'), + ToolDef('bash', 'Execute shell commands'), ], ), BenchmarkDef( - slug="scienceagentbench", - name="ScienceAgentBench", - output_name="hal-scienceagentbench", - category="Scientific Programming", - dataset_url="https://osu-nlp-group.github.io/ScienceAgentBench/", + slug='scienceagentbench', + name='ScienceAgentBench', + output_name='hal-scienceagentbench', + category='Scientific Programming', + dataset_url='https://osu-nlp-group.github.io/ScienceAgentBench/', description=( - "ScienceAgentBench evaluates language agents on end-to-end data-driven " - "scientific discovery tasks drawn from peer-reviewed publications." + 'ScienceAgentBench evaluates language agents on end-to-end data-driven ' + 'scientific discovery tasks drawn from peer-reviewed publications.' ), - metric_description="Success rate on ScienceAgentBench tasks (0.0–1.0)", + metric_description='Success rate on ScienceAgentBench tasks (0.0–1.0)', tools=[ - ToolDef("python", "Execute Python code for data analysis"), - ToolDef("bash", "Execute shell commands"), - ToolDef("read_file", "Read datasets and files"), - ToolDef("write_file", "Write output files and results"), + ToolDef('python', 'Execute Python code for data analysis'), + ToolDef('bash', 'Execute shell commands'), + ToolDef('read_file', 'Read datasets and files'), + ToolDef('write_file', 'Write output files and results'), ], ), BenchmarkDef( - slug="swebench_verified_mini", - name="SWE-bench Verified Mini", - output_name="hal-swebench-verified-mini", - category="Software Engineering", - dataset_url="https://www.swebench.com", + slug='swebench_verified_mini', + name='SWE-bench Verified Mini', + output_name='hal-swebench-verified-mini', + category='Software Engineering', + dataset_url='https://www.swebench.com', description=( - "SWE-bench Verified Mini is a 50-instance subset of SWE-bench Verified, " - "requiring agents to resolve real GitHub issues." + 'SWE-bench Verified Mini is a 50-instance subset of SWE-bench Verified, ' + 'requiring agents to resolve real GitHub issues.' ), - metric_description="Fraction of 50 verified GitHub issues resolved (0.0–1.0)", + metric_description='Fraction of 50 verified GitHub issues resolved (0.0–1.0)', tools=[ - ToolDef("bash", "Execute shell commands"), - ToolDef("edit_file", "Edit files in the repository"), - ToolDef("read_file", "Read files from the repository"), + ToolDef('bash', 'Execute shell commands'), + ToolDef('edit_file', 'Edit files in the repository'), + ToolDef('read_file', 'Read files from the repository'), ], ), BenchmarkDef( - slug="taubench_airline", - name="TAU-bench Airline", - output_name="hal-taubench-airline", - category="Customer Service", - dataset_url="https://github.com/sierra-research/tau-bench", + slug='taubench_airline', + name='TAU-bench Airline', + output_name='hal-taubench-airline', + category='Customer Service', + dataset_url='https://github.com/sierra-research/tau-bench', description=( - "TAU-bench Airline tests tool-augmented language models on realistic " - "airline customer service tasks with complex policies." + 'TAU-bench Airline tests tool-augmented language models on realistic ' + 'airline customer service tasks with complex policies.' ), - metric_description="Task success rate on TAU-bench Airline (0.0–1.0)", + metric_description='Task success rate on TAU-bench Airline (0.0–1.0)', tools=[ - ToolDef("function_calling", "Call predefined airline service API functions"), + ToolDef( + 'function_calling', + 'Call predefined airline service API functions', + ), ], ), BenchmarkDef( - slug="usaco", - name="USACO", - output_name="hal-usaco", - category="Programming", - dataset_url="https://usaco.guide", + slug='usaco', + name='USACO', + output_name='hal-usaco', + category='Programming', + dataset_url='https://usaco.guide', description=( - "USACO evaluates agents on competitive programming problems from the " - "USA Computing Olympiad across multiple difficulty levels." + 'USACO evaluates agents on competitive programming problems from the ' + 'USA Computing Olympiad across multiple difficulty levels.' ), - metric_description="Fraction of USACO problems solved (0.0–1.0)", + metric_description='Fraction of USACO problems solved (0.0–1.0)', tools=[ - ToolDef("bash", "Execute shell commands and compile/run code"), - ToolDef("python", "Execute Python code"), + ToolDef('bash', 'Execute shell commands and compile/run code'), + ToolDef('python', 'Execute Python code'), ], ), ] @@ -243,57 +259,57 @@ class BenchmarkDef: # These override the automatic pattern-matching in helpers/developer.py. MODEL_DEVELOPER_MAP: dict[str, str] = { # OpenAI - "o1": "openai", - "o3": "openai", - "o4": "openai", - "gpt": "openai", + 'o1': 'openai', + 'o3': 'openai', + 'o4': 'openai', + 'gpt': 'openai', # Anthropic - "claude": "anthropic", + 'claude': 'anthropic', # Google - "gemini": "google", - "gemma": "google", + 'gemini': 'google', + 'gemma': 'google', # Meta - "llama": "meta", + 'llama': 'meta', # DeepSeek - "deepseek": "deepseek", + 'deepseek': 'deepseek', # Mistral - "mistral": "mistralai", - "mixtral": "mistralai", + 'mistral': 'mistralai', + 'mixtral': 'mistralai', # Qwen / Alibaba — use "qwen" to match all other adapters in this repo - "qwen": "qwen", + 'qwen': 'qwen', # Microsoft - "phi": "microsoft", + 'phi': 'microsoft', } # Maps cleaned model name → canonical EEE model ID # This handles date-stripped versions of model names. MODEL_ID_OVERRIDES: dict[str, str] = { - "claude-3-7-sonnet": "anthropic/claude-3-7-sonnet-20250219", - "claude-3-5-sonnet": "anthropic/claude-3-5-sonnet-20241022", - "claude-3-5-haiku": "anthropic/claude-3-5-haiku-20241022", - "claude-3-opus": "anthropic/claude-3-opus-20240229", - "claude opus 4.5": "anthropic/claude-opus-4-5", - "claude sonnet 4.5": "anthropic/claude-sonnet-4-5", - "claude opus 4.1": "anthropic/claude-opus-4-1", - "claude sonnet 4.1": "anthropic/claude-sonnet-4-1", - "claude haiku 4.1": "anthropic/claude-haiku-4-1", - "claude-3.7 sonnet": "anthropic/claude-3-7-sonnet-20250219", - "gemini 2.0 flash": "google/gemini-2.0-flash", - "gemini 2.5 pro": "google/gemini-2.5-pro", - "gemini 2.5 flash": "google/gemini-2.5-flash", - "gemini 1.5 pro": "google/gemini-1.5-pro", - "deepseek r1": "deepseek/deepseek-r1", - "deepseek v3": "deepseek/deepseek-v3", - "gpt-5": "openai/gpt-5", - "gpt-4.1": "openai/gpt-4.1", - "gpt-4o": "openai/gpt-4o", - "o1": "openai/o1", - "o3": "openai/o3", - "o4-mini": "openai/o4-mini", - "llama-4-maverick": "meta-llama/llama-4-maverick", - "llama-4-scout": "meta-llama/llama-4-scout", - "qwen3-235b": "qwen/qwen3-235b", - "qwen3-32b": "qwen/qwen3-32b", + 'claude-3-7-sonnet': 'anthropic/claude-3-7-sonnet-20250219', + 'claude-3-5-sonnet': 'anthropic/claude-3-5-sonnet-20241022', + 'claude-3-5-haiku': 'anthropic/claude-3-5-haiku-20241022', + 'claude-3-opus': 'anthropic/claude-3-opus-20240229', + 'claude opus 4.5': 'anthropic/claude-opus-4-5', + 'claude sonnet 4.5': 'anthropic/claude-sonnet-4-5', + 'claude opus 4.1': 'anthropic/claude-opus-4-1', + 'claude sonnet 4.1': 'anthropic/claude-sonnet-4-1', + 'claude haiku 4.1': 'anthropic/claude-haiku-4-1', + 'claude-3.7 sonnet': 'anthropic/claude-3-7-sonnet-20250219', + 'gemini 2.0 flash': 'google/gemini-2.0-flash', + 'gemini 2.5 pro': 'google/gemini-2.5-pro', + 'gemini 2.5 flash': 'google/gemini-2.5-flash', + 'gemini 1.5 pro': 'google/gemini-1.5-pro', + 'deepseek r1': 'deepseek/deepseek-r1', + 'deepseek v3': 'deepseek/deepseek-v3', + 'gpt-5': 'openai/gpt-5', + 'gpt-4.1': 'openai/gpt-4.1', + 'gpt-4o': 'openai/gpt-4o', + 'o1': 'openai/o1', + 'o3': 'openai/o3', + 'o4-mini': 'openai/o4-mini', + 'llama-4-maverick': 'meta-llama/llama-4-maverick', + 'llama-4-scout': 'meta-llama/llama-4-scout', + 'qwen3-235b': 'qwen/qwen3-235b', + 'qwen3-32b': 'qwen/qwen3-32b', } @@ -325,7 +341,7 @@ def get_developer_for_model(raw_model: str) -> str: for prefix, dev in MODEL_DEVELOPER_MAP.items(): if lower.startswith(prefix) or f' {prefix}' in lower: return dev - return "unknown" + return 'unknown' def get_model_id(raw_model: str) -> str: @@ -343,11 +359,13 @@ def get_model_id(raw_model: str) -> str: # A prefix match is only accepted when the remaining suffix is solely a # known inference-effort token (low / medium / high). _EFFORT_SUFFIX = re.compile(r'^(low|medium|high)$', re.IGNORECASE) - for key, model_id in sorted(MODEL_ID_OVERRIDES.items(), key=lambda kv: len(kv[0]), reverse=True): + for key, model_id in sorted( + MODEL_ID_OVERRIDES.items(), key=lambda kv: len(kv[0]), reverse=True + ): if cleaned == key: return model_id - if cleaned.startswith(f"{key} "): - suffix = cleaned[len(key):].strip() + if cleaned.startswith(f'{key} '): + suffix = cleaned[len(key) :].strip() if _EFFORT_SUFFIX.fullmatch(suffix): return model_id @@ -357,9 +375,9 @@ def get_model_id(raw_model: str) -> str: slug = re.sub(r'\b(low|medium|high)\b', '', cleaned, flags=re.IGNORECASE) slug = re.sub(r'\s+', '-', slug.strip()).strip('-') slug = re.sub(r'-+', '-', slug) - if developer != "unknown": - return f"{developer}/{slug}" - return f"unknown/{slug}" + if developer != 'unknown': + return f'{developer}/{slug}' + return f'unknown/{slug}' def slugify(text: str) -> str: @@ -374,11 +392,12 @@ def slugify(text: str) -> str: # HTML parsing # --------------------------------------------------------------------------- + def _fetch_page(url: str) -> str: """Fetch a URL and return the HTML body as a string.""" - req = Request(url, headers={"User-Agent": "Mozilla/5.0 (EEE-adapter)"}) + req = Request(url, headers={'User-Agent': 'Mozilla/5.0 (EEE-adapter)'}) with urlopen(req, timeout=30) as resp: - return resp.read().decode("utf-8", errors="replace") + return resp.read().decode('utf-8', errors='replace') def _clean_cell(html_fragment: str) -> str: @@ -417,7 +436,7 @@ class LeaderboardRow: model_raw: str verified: bool is_pareto: bool - accuracy: float # 0.0–1.0 + accuracy: float # 0.0–1.0 accuracy_ci: Optional[str] cost_usd: Optional[float] cost_ci: Optional[str] @@ -434,12 +453,14 @@ def _parse_agent_cell(raw: str) -> tuple[str, bool, bool, Optional[str]]: Returns: (agent_name, is_pareto, has_submitter_link, submitter_info) """ - is_pareto = "Pareto optimal" in raw + is_pareto = 'Pareto optimal' in raw # Remove marker text - name = raw.replace("Pareto optimal", "").strip() + name = raw.replace('Pareto optimal', '').strip() # Remove "Submitted by ... Download main.py" patterns submitter = None - m = re.search(r'Submitted by (.+?)(?:Download\s+\w+\.?\w*)?$', name, re.IGNORECASE) + m = re.search( + r'Submitted by (.+?)(?:Download\s+\w+\.?\w*)?$', name, re.IGNORECASE + ) if m: submitter = m.group(1).strip() name = re.sub(r'Submitted by .+', '', name).strip() @@ -449,7 +470,9 @@ def _parse_agent_cell(raw: str) -> tuple[str, bool, bool, Optional[str]]: return name, is_pareto, bool(submitter), submitter -def _parse_accuracy_cell(raw: str) -> tuple[float, Optional[str], Optional[str]]: +def _parse_accuracy_cell( + raw: str, +) -> tuple[float, Optional[str], Optional[str]]: """ Parse accuracy cell, which may include notes like '(95.5% w/ manual validation)'. Returns: (accuracy_fraction, confidence_interval, notes_string) @@ -470,7 +493,7 @@ def parse_table(html: str, benchmark: BenchmarkDef) -> list[LeaderboardRow]: """Parse the main leaderboard table from a HAL benchmark page.""" m = re.search(r']*>(.*?)', html, re.DOTALL) if not m: - raise ValueError(f"No found on {benchmark.slug} page") + raise ValueError(f'No found on {benchmark.slug} page') tbody = m.group(1) raw_rows = re.findall(r']*>(.*?)', tbody, re.DOTALL) @@ -508,10 +531,10 @@ def parse_table(html: str, benchmark: BenchmarkDef) -> list[LeaderboardRow]: for i, em in enumerate(benchmark.extra_metrics): cell_idx = 5 + i if cell_idx < len(cells): - extra_scores[em["key"]] = _parse_percent(cells[cell_idx]) + extra_scores[em['key']] = _parse_percent(cells[cell_idx]) cost_idx = 5 + has_extra - cost_raw = cells[cost_idx] if cost_idx < len(cells) else "" + cost_raw = cells[cost_idx] if cost_idx < len(cells) else '' cost_usd = _parse_cost(cost_raw) cost_ci = None ci_m = re.search(r'\(([+-][\d.]+/[+-][\d.]+)\)', cost_raw) @@ -526,20 +549,22 @@ def parse_table(html: str, benchmark: BenchmarkDef) -> list[LeaderboardRow]: except ValueError: runs = 1 - results.append(LeaderboardRow( - rank=rank, - agent_name=agent_name, - model_raw=model_raw, - verified=verified, - is_pareto=is_pareto, - accuracy=accuracy, - accuracy_ci=accuracy_ci, - cost_usd=cost_usd, - cost_ci=cost_ci, - runs=runs, - extra_scores=extra_scores, - notes=notes, - )) + results.append( + LeaderboardRow( + rank=rank, + agent_name=agent_name, + model_raw=model_raw, + verified=verified, + is_pareto=is_pareto, + accuracy=accuracy, + accuracy_ci=accuracy_ci, + cost_usd=cost_usd, + cost_ci=cost_ci, + runs=runs, + extra_scores=extra_scores, + notes=notes, + ) + ) return results @@ -548,17 +573,18 @@ def parse_table(html: str, benchmark: BenchmarkDef) -> list[LeaderboardRow]: # EEE record building # --------------------------------------------------------------------------- + def build_source_data(benchmark: BenchmarkDef) -> dict: result: dict = { - "source_type": "url", - "dataset_name": benchmark.name, - "url": [ + 'source_type': 'url', + 'dataset_name': benchmark.name, + 'url': [ benchmark.dataset_url, - f"{HAL_BASE_URL}/{benchmark.slug}", + f'{HAL_BASE_URL}/{benchmark.slug}', ], } if benchmark.source_data_details: - result["additional_details"] = benchmark.source_data_details + result['additional_details'] = benchmark.source_data_details return result @@ -570,42 +596,59 @@ def build_evaluation_result( row: LeaderboardRow, details: Optional[dict] = None, ) -> dict: - score_details: dict = {"score": round(score, 6)} + score_details: dict = {'score': round(score, 6)} if details: - score_details["details"] = {k: str(v) for k, v in details.items() if v is not None} + score_details['details'] = { + k: str(v) for k, v in details.items() if v is not None + } available_tools = [ - {"name": t.name, **({"description": t.description} if t.description else {})} + { + 'name': t.name, + **({'description': t.description} if t.description else {}), + } for t in benchmark.tools ] return { - "evaluation_name": evaluation_name, - "source_data": build_source_data(benchmark), - "metric_config": { - "evaluation_description": description, - "lower_is_better": False, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0, + 'evaluation_name': evaluation_name, + 'source_data': build_source_data(benchmark), + 'metric_config': { + 'evaluation_description': description, + 'lower_is_better': False, + 'score_type': 'continuous', + 'min_score': 0.0, + 'max_score': 1.0, }, - "score_details": score_details, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "available_tools": available_tools, + 'score_details': score_details, + 'generation_config': { + 'generation_args': { + 'agentic_eval_config': { + 'available_tools': available_tools, }, }, - "additional_details": { - "agent_scaffold": row.agent_name, - "hal_rank": str(row.rank), - "runs": str(row.runs), - "verified": str(row.verified), - "is_pareto": str(row.is_pareto), - **({"total_cost_usd": str(row.cost_usd)} if row.cost_usd is not None else {}), - **({"cost_confidence_interval": row.cost_ci} if row.cost_ci else {}), - **({"accuracy_confidence_interval": row.accuracy_ci} if row.accuracy_ci else {}), - **({"notes": row.notes} if row.notes else {}), + 'additional_details': { + 'agent_scaffold': row.agent_name, + 'hal_rank': str(row.rank), + 'runs': str(row.runs), + 'verified': str(row.verified), + 'is_pareto': str(row.is_pareto), + **( + {'total_cost_usd': str(row.cost_usd)} + if row.cost_usd is not None + else {} + ), + **( + {'cost_confidence_interval': row.cost_ci} + if row.cost_ci + else {} + ), + **( + {'accuracy_confidence_interval': row.accuracy_ci} + if row.accuracy_ci + else {} + ), + **({'notes': row.notes} if row.notes else {}), }, }, } @@ -622,8 +665,10 @@ def build_eee_record( Returns: (record_dict, developer_slug, model_slug) """ model_id = get_model_id(row.model_raw) - developer = model_id.split("/")[0] if "/" in model_id else "unknown" - model_slug_clean = model_id.split("/", 1)[-1] if "/" in model_id else model_id + developer = model_id.split('/')[0] if '/' in model_id else 'unknown' + model_slug_clean = ( + model_id.split('/', 1)[-1] if '/' in model_id else model_id + ) effort = _effort_level(row.model_raw) agent_slug = slugify(row.agent_name) @@ -633,8 +678,8 @@ def build_eee_record( raw_model_slug = slugify(row.model_raw) eval_id = ( - f"{benchmark.output_name}/{agent_slug}" - f"/{raw_model_slug}/{retrieved_timestamp}" + f'{benchmark.output_name}/{agent_slug}' + f'/{raw_model_slug}/{retrieved_timestamp}' ) # Primary evaluation result @@ -646,20 +691,20 @@ def build_eee_record( benchmark=benchmark, row=row, details={ - "accuracy_raw": f"{row.accuracy * 100:.2f}%", + 'accuracy_raw': f'{row.accuracy * 100:.2f}%', }, ) ] # Extra metrics (e.g. GAIA levels) for em in benchmark.extra_metrics: - score = row.extra_scores.get(em["key"]) + score = row.extra_scores.get(em['key']) if score is not None: eval_results.append( build_evaluation_result( - evaluation_name=f"{benchmark.name} - {em['name'].split(' - ')[-1] if ' - ' in em['name'] else em['name']}", + evaluation_name=f'{benchmark.name} - {em["name"].split(" - ")[-1] if " - " in em["name"] else em["name"]}', score=score, - description=em["description"] + " (0.0–1.0)", + description=em['description'] + ' (0.0–1.0)', benchmark=benchmark, row=row, ) @@ -667,42 +712,42 @@ def build_eee_record( # Model info additional details model_additional: dict[str, str] = { - "hal_model_name": row.model_raw, - "agent_scaffold": row.agent_name, - "benchmark": benchmark.name, + 'hal_model_name': row.model_raw, + 'agent_scaffold': row.agent_name, + 'benchmark': benchmark.name, } if effort: - model_additional["inference_effort"] = effort + model_additional['inference_effort'] = effort if row.cost_usd is not None: - model_additional["total_cost_usd"] = str(row.cost_usd) + model_additional['total_cost_usd'] = str(row.cost_usd) record = { - "schema_version": SCHEMA_VERSION, - "evaluation_id": eval_id, - "retrieved_timestamp": retrieved_timestamp, - "source_metadata": { - "source_name": f"HAL Leaderboard — {benchmark.name}", - "source_type": "documentation", - "source_organization_name": "Princeton SAgE Team", - "source_organization_url": HAL_BASE_URL, - "evaluator_relationship": "third_party", - "additional_details": { - "paper": "https://arxiv.org/pdf/2510.11977", - "benchmark_category": benchmark.category, - "benchmark_slug": benchmark.slug, + 'schema_version': SCHEMA_VERSION, + 'evaluation_id': eval_id, + 'retrieved_timestamp': retrieved_timestamp, + 'source_metadata': { + 'source_name': f'HAL Leaderboard — {benchmark.name}', + 'source_type': 'documentation', + 'source_organization_name': 'Princeton SAgE Team', + 'source_organization_url': HAL_BASE_URL, + 'evaluator_relationship': 'third_party', + 'additional_details': { + 'paper': 'https://arxiv.org/pdf/2510.11977', + 'benchmark_category': benchmark.category, + 'benchmark_slug': benchmark.slug, }, }, - "eval_library": { - "name": "HAL", - "version": "unknown", + 'eval_library': { + 'name': 'HAL', + 'version': 'unknown', }, - "model_info": { - "name": row.model_raw, - "id": model_id, - "developer": developer if developer != "unknown" else None, - "additional_details": model_additional, + 'model_info': { + 'name': row.model_raw, + 'id': model_id, + 'developer': developer if developer != 'unknown' else None, + 'additional_details': model_additional, }, - "evaluation_results": eval_results, + 'evaluation_results': eval_results, } return record, developer, slugify(model_slug_clean) @@ -712,11 +757,17 @@ def build_eee_record( # I/O # --------------------------------------------------------------------------- -def save_record(record: dict, out_root: Path, developer: str, model_slug: str) -> Path: + +def save_record( + record: dict, out_root: Path, developer: str, model_slug: str +) -> Path: out_dir = out_root / developer / model_slug out_dir.mkdir(parents=True, exist_ok=True) - out_path = out_dir / f"{uuid.uuid4()}.json" - out_path.write_text(json.dumps(record, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + out_path = out_dir / f'{uuid.uuid4()}.json' + out_path.write_text( + json.dumps(record, indent=2, ensure_ascii=False) + '\n', + encoding='utf-8', + ) return out_path @@ -724,29 +775,30 @@ def save_record(record: dict, out_root: Path, developer: str, model_slug: str) - # Main # --------------------------------------------------------------------------- + def process_benchmark( benchmark: BenchmarkDef, out_root: Path, retrieved_timestamp: str, ) -> tuple[int, int]: """Fetch and process one benchmark. Returns (saved, errors).""" - url = f"{HAL_BASE_URL}/{benchmark.slug}" - print(f"\n{'=' * 60}") - print(f"Fetching {benchmark.name} from {url} …") + url = f'{HAL_BASE_URL}/{benchmark.slug}' + print(f'\n{"=" * 60}') + print(f'Fetching {benchmark.name} from {url} …') try: html = _fetch_page(url) except URLError as e: - print(f" ERROR fetching page: {e}") + print(f' ERROR fetching page: {e}') return 0, 1 try: rows = parse_table(html, benchmark) except ValueError as e: - print(f" ERROR parsing table: {e}") + print(f' ERROR parsing table: {e}') return 0, 1 - print(f" Found {len(rows)} leaderboard entries") + print(f' Found {len(rows)} leaderboard entries') benchmark_out_root = out_root / benchmark.output_name saved = 0 @@ -754,14 +806,22 @@ def process_benchmark( for row in rows: try: - record, developer, model_slug = build_eee_record(benchmark, row, retrieved_timestamp) - path = save_record(record, benchmark_out_root, developer, model_slug) - score_pct = f"{row.accuracy * 100:.2f}%" - cost_str = f"${row.cost_usd:.2f}" if row.cost_usd is not None else "N/A" - print(f" [{row.rank:2d}] {row.model_raw:<45s} {score_pct} {cost_str} → {path.relative_to(out_root)}") + record, developer, model_slug = build_eee_record( + benchmark, row, retrieved_timestamp + ) + path = save_record( + record, benchmark_out_root, developer, model_slug + ) + score_pct = f'{row.accuracy * 100:.2f}%' + cost_str = ( + f'${row.cost_usd:.2f}' if row.cost_usd is not None else 'N/A' + ) + print( + f' [{row.rank:2d}] {row.model_raw:<45s} {score_pct} {cost_str} → {path.relative_to(out_root)}' + ) saved += 1 except Exception as e: - print(f" ERROR processing row {row.rank} ({row.model_raw!r}): {e}") + print(f' ERROR processing row {row.rank} ({row.model_raw!r}): {e}') errors += 1 return saved, errors @@ -769,25 +829,25 @@ def process_benchmark( def main() -> None: parser = argparse.ArgumentParser( - description="Fetch HAL leaderboard data and convert to EEE schema." + description='Fetch HAL leaderboard data and convert to EEE schema.' ) parser.add_argument( - "--benchmark", - choices=list(BENCHMARK_BY_SLUG.keys()) + ["all"], - default="all", - help="Which benchmark(s) to fetch (default: all)", + '--benchmark', + choices=list(BENCHMARK_BY_SLUG.keys()) + ['all'], + default='all', + help='Which benchmark(s) to fetch (default: all)', ) parser.add_argument( - "--output-dir", + '--output-dir', type=Path, - default=Path("data"), - help="Root output directory (default: data/)", + default=Path('data'), + help='Root output directory (default: data/)', ) args = parser.parse_args() benchmarks = ( list(BENCHMARK_BY_SLUG.values()) - if args.benchmark == "all" + if args.benchmark == 'all' else [BENCHMARK_BY_SLUG[args.benchmark]] ) @@ -796,13 +856,17 @@ def main() -> None: total_errors = 0 for benchmark in benchmarks: - saved, errors = process_benchmark(benchmark, args.output_dir, retrieved_timestamp) + saved, errors = process_benchmark( + benchmark, args.output_dir, retrieved_timestamp + ) total_saved += saved total_errors += errors - print(f"\n{'=' * 60}") - print(f"Done. Saved {total_saved} files, {total_errors} errors → {args.output_dir}/") + print(f'\n{"=" * 60}') + print( + f'Done. Saved {total_saved} files, {total_errors} errors → {args.output_dir}/' + ) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/utils/helm/adapter.py b/utils/helm/adapter.py index df365f304..5ed78cd1d 100644 --- a/utils/helm/adapter.py +++ b/utils/helm/adapter.py @@ -39,8 +39,9 @@ save_evaluation_log, ) - -HELM_PROJECT_METADATA_URL = "https://crfm.stanford.edu/helm/project_metadata.json" +HELM_PROJECT_METADATA_URL = ( + 'https://crfm.stanford.edu/helm/project_metadata.json' +) def parse_args(): @@ -64,7 +65,7 @@ def parse_args(): parser.add_argument( '--leaderboard_version', type=str, - default="latest", + default='latest', help='Version of the HELM leaderboard to use; defaults to the latest version', ) parser.add_argument( @@ -208,19 +209,19 @@ def convert( # are in the format "dataset_name - metric_name" (e.g. "MMLU - EM") # This boolean indicates whether the special handling is needed. is_helm_air_bench_category_table = ( - leaderboard_name == "helm_air_bench" - and tab_name.startswith("AIR") - and tab_name.endswith("categories") + leaderboard_name == 'helm_air_bench' + and tab_name.startswith('AIR') + and tab_name.endswith('categories') ) full_eval_name = header.get('value') short_name = ( full_eval_name.split()[0] - if '-' in full_eval_name and not is_helm_air_bench_category_table + if '-' in full_eval_name + and not is_helm_air_bench_category_table else full_eval_name ) - is_new_metric = ( tab_name.lower() == 'accuracy' or short_name not in model_results[model_name] @@ -235,7 +236,7 @@ def convert( elif is_helm_air_bench_category_table: dataset_name = full_eval_name evaluation_name = dataset_name - metric_name = "Refusal Rate" + metric_name = 'Refusal Rate' else: dataset_name, metric_name = full_eval_name.split(' - ', 1) evaluation_name = dataset_name @@ -265,7 +266,8 @@ def convert( source_dataset_name = ( leaderboard_name - if leaderboard_name.lower() in ['helm_mmlu', "helm_air_bench"] + if leaderboard_name.lower() + in ['helm_mmlu', 'helm_air_bench'] else dataset_name ) @@ -372,28 +374,36 @@ def convert( def get_leaderboard_versions(leaderboard_id: str) -> List[str]: """Return a list of published versions for the leaderboard""" project_metadata = fetch_json(HELM_PROJECT_METADATA_URL) - project = "" + project = '' for project in project_metadata: - if project["id"] == leaderboard_id: - return project["releases"] - raise ValueError(f"Leaderboard ID {leaderboard_id} not found in HELM project metadata at {HELM_PROJECT_METADATA_URL}") + if project['id'] == leaderboard_id: + return project['releases'] + raise ValueError( + f'Leaderboard ID {leaderboard_id} not found in HELM project metadata at {HELM_PROJECT_METADATA_URL}' + ) def get_source_data_url(leaderboard_id: str, leaderboard_version: str) -> str: """Return the URL of the JSON file containing the results table of the primary group of the leaderboard""" leaderboard_versions = get_leaderboard_versions(leaderboard_id) if not leaderboard_versions: - raise ValueError(f"No versions found for leaderboard {leaderboard_id}") - if leaderboard_version == "latest": + raise ValueError(f'No versions found for leaderboard {leaderboard_id}') + if leaderboard_version == 'latest': leaderboard_version = leaderboard_versions[0] if leaderboard_version not in leaderboard_versions: - raise ValueError(f"Version {leaderboard_version} for leaderboard {leaderboard_id} not found; available versions: {leaderboard_versions}") + raise ValueError( + f'Version {leaderboard_version} for leaderboard {leaderboard_id} not found; available versions: {leaderboard_versions}' + ) - groups_table = fetch_json(f"https://storage.googleapis.com/crfm-helm-public/{leaderboard_id}/benchmark_output/releases/{leaderboard_version}/groups.json") + groups_table = fetch_json( + f'https://storage.googleapis.com/crfm-helm-public/{leaderboard_id}/benchmark_output/releases/{leaderboard_version}/groups.json' + ) # This is un ugly hack to get the first group's ID. # Unfortunately, this is actually how the offical HELM code does it. # See: https://github.com/stanford-crfm/helm/blob/v0.5.14/helm-frontend/src/routes/Leaderboard.tsx#L44-L56 - first_group_name = groups_table[0]["rows"][0][0]["href"].removeprefix("?group=") + first_group_name = groups_table[0]['rows'][0][0]['href'].removeprefix( + '?group=' + ) return f'https://storage.googleapis.com/crfm-helm-public/{leaderboard_id}/benchmark_output/releases/{leaderboard_version}/groups/{first_group_name}.json' @@ -402,12 +412,16 @@ def main(): leaderboard_name = args.leaderboard_name.lower() - if not leaderboard_name.startswith("helm_"): - raise ValueError("leaderboard_name must start with helm_") - leaderboard_id = leaderboard_name.removeprefix("helm_").replace("_", "-") - source_data_url = get_source_data_url(leaderboard_id, args.leaderboard_version) + if not leaderboard_name.startswith('helm_'): + raise ValueError('leaderboard_name must start with helm_') + leaderboard_id = leaderboard_name.removeprefix('helm_').replace('_', '-') + source_data_url = get_source_data_url( + leaderboard_id, args.leaderboard_version + ) - print(f'Fetching {leaderboard_name} {args.leaderboard_version} data from {source_data_url}') + print( + f'Fetching {leaderboard_name} {args.leaderboard_version} data from {source_data_url}' + ) leaderboard_data = fetch_json(source_data_url) convert( diff --git a/utils/multi_swe_bench/adapter.py b/utils/multi_swe_bench/adapter.py index 776870fc8..fbd1c5638 100644 --- a/utils/multi_swe_bench/adapter.py +++ b/utils/multi_swe_bench/adapter.py @@ -40,12 +40,17 @@ SourceDataUrl, SourceMetadata, ) -from every_eval_ever.helpers import SCHEMA_VERSION, get_developer, get_model_id, save_evaluation_log +from every_eval_ever.helpers import ( + SCHEMA_VERSION, + get_developer, + get_model_id, + save_evaluation_log, +) from utils.swe_helpers import parse_date_from_dir, parse_model_from_dir -MULTI_SWE_REPO = "https://github.com/multi-swe-bench/experiments" -LANGUAGES = ["c", "c++", "go", "java", "javascript", "rust", "typescript"] -OUTPUT_BASE = "data/multi-swe-bench-leaderboard" +MULTI_SWE_REPO = 'https://github.com/multi-swe-bench/experiments' +LANGUAGES = ['c', 'c++', 'go', 'java', 'javascript', 'rust', 'typescript'] +OUTPUT_BASE = 'data/multi-swe-bench-leaderboard' def convert_submission( @@ -58,66 +63,70 @@ def convert_submission( import yaml except ImportError as e: raise ImportError( - "pyyaml is required to run this adapter. Install it with: pip install pyyaml" + 'pyyaml is required to run this adapter. Install it with: pip install pyyaml' ) from e dir_name = submission_dir.name - with open(submission_dir / "metadata.yaml") as f: + with open(submission_dir / 'metadata.yaml') as f: metadata = yaml.safe_load(f) - with open(submission_dir / "results" / "results.json") as f: + with open(submission_dir / 'results' / 'results.json') as f: results = json.load(f) - total_instances = results.get("total_instances", 0) + total_instances = results.get('total_instances', 0) if total_instances == 0: - raise ValueError(f"total_instances is 0 for {dir_name}, skipping") + raise ValueError(f'total_instances is 0 for {dir_name}, skipping') - resolved = results.get("resolved", []) + resolved = results.get('resolved', []) score = len(resolved) / total_instances agent, primary_model = parse_model_from_dir(dir_name) developer = get_developer(primary_model) model_id = get_model_id(primary_model, developer) - sanitized_id = re.sub(r"[^a-zA-Z0-9_.-]", "_", model_id.replace("/", "_")) - submission_slug = re.sub(r"[^a-zA-Z0-9_.-]", "_", dir_name) - eval_id = f"multi-swe-bench/{lang}/{sanitized_id}/{submission_slug}/{retrieved_timestamp}" + sanitized_id = re.sub(r'[^a-zA-Z0-9_.-]', '_', model_id.replace('/', '_')) + submission_slug = re.sub(r'[^a-zA-Z0-9_.-]', '_', dir_name) + eval_id = f'multi-swe-bench/{lang}/{sanitized_id}/{submission_slug}/{retrieved_timestamp}' evaluation_timestamp = parse_date_from_dir(dir_name) additional_details: dict[str, str] = { - "submission_name": str(metadata.get("name", "")), - "language": lang, - "oss": str(metadata.get("oss", "")), - "site": str(metadata.get("site", "")), - "verified": str(metadata.get("verified", "")), - "submission_dir": dir_name, - "agent": agent, + 'submission_name': str(metadata.get('name', '')), + 'language': lang, + 'oss': str(metadata.get('oss', '')), + 'site': str(metadata.get('site', '')), + 'verified': str(metadata.get('verified', '')), + 'submission_dir': dir_name, + 'agent': agent, } score_details: dict[str, str] = { - "resolved_count": str(len(resolved)), - "total_instances": str(total_instances), - "submitted_instances": str(results.get("submitted_instances", "")), - "completed_instances": str(results.get("completed_instances", "")), - "unresolved_instances": str(results.get("unresolved_instances", "")), - "empty_error_patch_instances": str(results.get("empty_error_patch_instances", "")), + 'resolved_count': str(len(resolved)), + 'total_instances': str(total_instances), + 'submitted_instances': str(results.get('submitted_instances', '')), + 'completed_instances': str(results.get('completed_instances', '')), + 'unresolved_instances': str(results.get('unresolved_instances', '')), + 'empty_error_patch_instances': str( + results.get('empty_error_patch_instances', '') + ), } - dataset_label = f"Multi-SWE-bench ({lang})" - eval_name = f"Multi-SWE-Bench ({lang})" + dataset_label = f'Multi-SWE-bench ({lang})' + eval_name = f'Multi-SWE-Bench ({lang})' eval_result = EvaluationResult( evaluation_name=eval_name, source_data=SourceDataUrl( dataset_name=dataset_label, - source_type="url", - url=["https://huggingface.co/datasets/ByteDance-Seed/Multi-SWE-bench"], + source_type='url', + url=[ + 'https://huggingface.co/datasets/ByteDance-Seed/Multi-SWE-bench' + ], ), evaluation_timestamp=evaluation_timestamp, metric_config=MetricConfig( - evaluation_description=f"Fraction of {lang} GitHub issues resolved (0.0–1.0)", + evaluation_description=f'Fraction of {lang} GitHub issues resolved (0.0–1.0)', lower_is_better=False, score_type=ScoreType.continuous, min_score=0.0, @@ -130,7 +139,7 @@ def convert_submission( generation_config=GenerationConfig( generation_args=GenerationArgs( agentic_eval_config=AgenticEvalConfig( - available_tools=[AvailableTool(name="bash")], + available_tools=[AvailableTool(name='bash')], ), ), ), @@ -142,17 +151,17 @@ def convert_submission( retrieved_timestamp=retrieved_timestamp, evaluation_timestamp=evaluation_timestamp, source_metadata=SourceMetadata( - source_name="Multi-SWE-Bench Leaderboard", - source_type="documentation", - source_organization_name="ByteDance-Seed", - source_organization_url="https://github.com/multi-swe-bench/experiments", + source_name='Multi-SWE-Bench Leaderboard', + source_type='documentation', + source_organization_name='ByteDance-Seed', + source_organization_url='https://github.com/multi-swe-bench/experiments', evaluator_relationship=EvaluatorRelationship.third_party, ), - eval_library=EvalLibrary(name="multi-swe-bench", version="unknown"), + eval_library=EvalLibrary(name='multi-swe-bench', version='unknown'), model_info=ModelInfo( name=primary_model, id=model_id, - developer=developer if developer != "unknown" else None, + developer=developer if developer != 'unknown' else None, additional_details=additional_details, ), evaluation_results=[eval_result], @@ -165,37 +174,43 @@ def main(): errors = 0 with tempfile.TemporaryDirectory() as tmpdir: - print(f"Cloning {MULTI_SWE_REPO} into {tmpdir} ...") + print(f'Cloning {MULTI_SWE_REPO} into {tmpdir} ...') subprocess.run( - ["git", "clone", "--depth=1", MULTI_SWE_REPO, tmpdir], - env={**os.environ, "GIT_LFS_SKIP_SMUDGE": "1"}, + ['git', 'clone', '--depth=1', MULTI_SWE_REPO, tmpdir], + env={**os.environ, 'GIT_LFS_SKIP_SMUDGE': '1'}, check=True, ) for lang in LANGUAGES: - verified_path = Path(tmpdir) / "evaluation" / lang / "verified" + verified_path = Path(tmpdir) / 'evaluation' / lang / 'verified' if not verified_path.exists(): - print(f" [SKIP] No verified/ dir for language: {lang}") + print(f' [SKIP] No verified/ dir for language: {lang}') continue - submissions = sorted(d for d in verified_path.iterdir() if d.is_dir()) - print(f"\n[{lang}] Found {len(submissions)} submissions") + submissions = sorted( + d for d in verified_path.iterdir() if d.is_dir() + ) + print(f'\n[{lang}] Found {len(submissions)} submissions') for submission_dir in submissions: try: - eval_log = convert_submission(submission_dir, lang, retrieved_timestamp) - dev = eval_log.model_info.developer or "unknown" - model_name = eval_log.model_info.name.split("/")[-1] - filepath = save_evaluation_log(eval_log, OUTPUT_BASE, dev, model_name) + eval_log = convert_submission( + submission_dir, lang, retrieved_timestamp + ) + dev = eval_log.model_info.developer or 'unknown' + model_name = eval_log.model_info.name.split('/')[-1] + filepath = save_evaluation_log( + eval_log, OUTPUT_BASE, dev, model_name + ) score = eval_log.evaluation_results[0].score_details.score - print(f" [{score:.1%}] {submission_dir.name} → {filepath}") + print(f' [{score:.1%}] {submission_dir.name} → {filepath}') count += 1 except Exception as e: - print(f" ERROR {submission_dir.name}: {e}") + print(f' ERROR {submission_dir.name}: {e}') errors += 1 - print(f"\nGenerated {count} files, {errors} errors → {OUTPUT_BASE}/") + print(f'\nGenerated {count} files, {errors} errors → {OUTPUT_BASE}/') -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/utils/swe_bench_verified/adapter.py b/utils/swe_bench_verified/adapter.py index 119ab7d04..fffe5dc9e 100644 --- a/utils/swe_bench_verified/adapter.py +++ b/utils/swe_bench_verified/adapter.py @@ -39,19 +39,24 @@ SourceDataUrl, SourceMetadata, ) -from every_eval_ever.helpers import SCHEMA_VERSION, get_developer, get_model_id, save_evaluation_log +from every_eval_ever.helpers import ( + SCHEMA_VERSION, + get_developer, + get_model_id, + save_evaluation_log, +) from utils.swe_helpers import parse_date_from_dir -SWE_BENCH_REPO = "https://github.com/swe-bench/experiments" -SWE_BENCH_SUBDIR = "evaluation/verified" -OUTPUT_DIR = "data/swe-bench-verified-leaderboard" +SWE_BENCH_REPO = 'https://github.com/swe-bench/experiments' +SWE_BENCH_SUBDIR = 'evaluation/verified' +OUTPUT_DIR = 'data/swe-bench-verified-leaderboard' def normalize_org(org) -> str: """Normalize org field which can be str, list, or None.""" if isinstance(org, list): - return ", ".join(str(o) for o in org if o) - return str(org) if org else "" + return ', '.join(str(o) for o in org if o) + return str(org) if org else '' def normalize_model_name(model) -> str: @@ -62,16 +67,16 @@ def normalize_model_name(model) -> str: - Plain strings returned as-is """ if not model: - return "" + return '' s = str(model) - if s.startswith("https://huggingface.co/"): - s = s[len("https://huggingface.co/"):] + if s.startswith('https://huggingface.co/'): + s = s[len('https://huggingface.co/') :] return s def get_primary_model(tags: dict, info: dict, dir_name: str) -> str: """Extract the primary model name from tags, falling back to submission info.""" - raw = tags.get("model") + raw = tags.get('model') # tags.model can be a list or a plain string if isinstance(raw, list): models = raw @@ -83,10 +88,12 @@ def get_primary_model(tags: dict, info: dict, dir_name: str) -> str: if models: return normalize_model_name(models[0]) # Fallback: use submission name from info - return info.get("name", dir_name) + return info.get('name', dir_name) -def convert_submission(submission_dir: Path, retrieved_timestamp: str, total_instances: int) -> EvaluationLog: +def convert_submission( + submission_dir: Path, retrieved_timestamp: str, total_instances: int +) -> EvaluationLog: """Convert a single SWE-bench submission directory to an EvaluationLog.""" dir_name = submission_dir.name @@ -94,19 +101,19 @@ def convert_submission(submission_dir: Path, retrieved_timestamp: str, total_ins import yaml except ImportError as e: raise ImportError( - "pyyaml is required to run this adapter. Install it with: pip install pyyaml" + 'pyyaml is required to run this adapter. Install it with: pip install pyyaml' ) from e # Read metadata - with open(submission_dir / "metadata.yaml") as f: + with open(submission_dir / 'metadata.yaml') as f: metadata = yaml.safe_load(f) # Read results - with open(submission_dir / "results" / "results.json") as f: + with open(submission_dir / 'results' / 'results.json') as f: results = json.load(f) - tags = metadata.get("tags", {}) or {} - info = metadata.get("info", {}) or {} + tags = metadata.get('tags', {}) or {} + info = metadata.get('info', {}) or {} # Primary model: first element of tags.model (list or string), fallback to submission name primary_model = get_primary_model(tags, info, dir_name) @@ -115,56 +122,54 @@ def convert_submission(submission_dir: Path, retrieved_timestamp: str, total_ins model_id = get_model_id(primary_model, developer) # Score: resolved / total_instances - resolved = results.get("resolved", []) + resolved = results.get('resolved', []) score = len(resolved) / total_instances # Build additional_details (all values must be strings) additional_details: dict[str, str] = { - "submission_name": str(info.get("name", "")), - "agent_organization": normalize_org(tags.get("org", "")), - "open_source_model": str(tags.get("os_model", "")), - "open_source_system": str(tags.get("os_system", "")), - "verified": str(tags.get("checked", "")), - "attempts": str((tags.get("system") or {}).get("attempts", "")), - "submission_dir": dir_name, + 'submission_name': str(info.get('name', '')), + 'agent_organization': normalize_org(tags.get('org', '')), + 'open_source_model': str(tags.get('os_model', '')), + 'open_source_system': str(tags.get('os_system', '')), + 'verified': str(tags.get('checked', '')), + 'attempts': str((tags.get('system') or {}).get('attempts', '')), + 'submission_dir': dir_name, } - site = info.get("site") + site = info.get('site') if site: - additional_details["site"] = str(site) - report = info.get("report") + additional_details['site'] = str(site) + report = info.get('report') if report: - additional_details["report"] = str(report) + additional_details['report'] = str(report) # Score details score_details: dict[str, str] = { - "resolved_count": str(len(resolved)), + 'resolved_count': str(len(resolved)), } - no_generation = results.get("no_generation", []) + no_generation = results.get('no_generation', []) if no_generation: - score_details["no_generation_count"] = str(len(no_generation)) - no_logs = results.get("no_logs", []) + score_details['no_generation_count'] = str(len(no_generation)) + no_logs = results.get('no_logs', []) if no_logs: - score_details["no_logs_count"] = str(len(no_logs)) + score_details['no_logs_count'] = str(len(no_logs)) # Sanitize identifier components for use in evaluation_id - sanitized_id = re.sub(r"[^a-zA-Z0-9_.-]", "_", model_id.replace("/", "_")) - submission_slug = re.sub(r"[^a-zA-Z0-9_.-]", "_", dir_name) - eval_id = ( - f"swe-bench-verified/{sanitized_id}/{submission_slug}/{retrieved_timestamp}" - ) + sanitized_id = re.sub(r'[^a-zA-Z0-9_.-]', '_', model_id.replace('/', '_')) + submission_slug = re.sub(r'[^a-zA-Z0-9_.-]', '_', dir_name) + eval_id = f'swe-bench-verified/{sanitized_id}/{submission_slug}/{retrieved_timestamp}' evaluation_timestamp = parse_date_from_dir(dir_name) eval_result = EvaluationResult( - evaluation_name="SWE-bench Verified", + evaluation_name='SWE-bench Verified', source_data=SourceDataUrl( - dataset_name="SWE-bench Verified", - source_type="url", - url=["https://www.swebench.com"], + dataset_name='SWE-bench Verified', + source_type='url', + url=['https://www.swebench.com'], ), evaluation_timestamp=evaluation_timestamp, metric_config=MetricConfig( evaluation_description=( - "Fraction of 500 verified GitHub issues resolved (0.0–1.0)" + 'Fraction of 500 verified GitHub issues resolved (0.0–1.0)' ), lower_is_better=False, score_type=ScoreType.continuous, @@ -178,7 +183,7 @@ def convert_submission(submission_dir: Path, retrieved_timestamp: str, total_ins generation_config=GenerationConfig( generation_args=GenerationArgs( agentic_eval_config=AgenticEvalConfig( - available_tools=[AvailableTool(name="bash")], + available_tools=[AvailableTool(name='bash')], ), ), ), @@ -190,17 +195,17 @@ def convert_submission(submission_dir: Path, retrieved_timestamp: str, total_ins retrieved_timestamp=retrieved_timestamp, evaluation_timestamp=evaluation_timestamp, source_metadata=SourceMetadata( - source_name="SWE-bench Verified Leaderboard", - source_type="documentation", - source_organization_name="SWE-bench", - source_organization_url="https://www.swebench.com", + source_name='SWE-bench Verified Leaderboard', + source_type='documentation', + source_organization_name='SWE-bench', + source_organization_url='https://www.swebench.com', evaluator_relationship=EvaluatorRelationship.third_party, ), - eval_library=EvalLibrary(name="swe-bench", version="unknown"), + eval_library=EvalLibrary(name='swe-bench', version='unknown'), model_info=ModelInfo( name=primary_model, id=model_id, - developer=developer if developer != "unknown" else None, + developer=developer if developer != 'unknown' else None, additional_details=additional_details, ), evaluation_results=[eval_result], @@ -212,44 +217,50 @@ def main(): from datasets import load_dataset except ImportError as e: raise ImportError( - "datasets is required to run this adapter. Install it with: pip install datasets" + 'datasets is required to run this adapter. Install it with: pip install datasets' ) from e retrieved_timestamp = str(time.time()) count = 0 errors = 0 - ds = load_dataset("SWE-bench/SWE-bench_Verified", split="test") + ds = load_dataset('SWE-bench/SWE-bench_Verified', split='test') total_instances = len(ds) - print(f"Loaded {total_instances} instances from SWE-bench/SWE-bench_Verified\n") + print( + f'Loaded {total_instances} instances from SWE-bench/SWE-bench_Verified\n' + ) with tempfile.TemporaryDirectory() as tmpdir: - print(f"Cloning {SWE_BENCH_REPO} into {tmpdir} ...") + print(f'Cloning {SWE_BENCH_REPO} into {tmpdir} ...') subprocess.run( - ["git", "clone", "--depth=1", SWE_BENCH_REPO, tmpdir], + ['git', 'clone', '--depth=1', SWE_BENCH_REPO, tmpdir], check=True, ) swe_bench_path = Path(tmpdir) / SWE_BENCH_SUBDIR submissions = sorted(d for d in swe_bench_path.iterdir() if d.is_dir()) - print(f"Found {len(submissions)} submission directories\n") + print(f'Found {len(submissions)} submission directories\n') for submission_dir in submissions: try: - eval_log = convert_submission(submission_dir, retrieved_timestamp, total_instances) - dev = eval_log.model_info.developer or "unknown" + eval_log = convert_submission( + submission_dir, retrieved_timestamp, total_instances + ) + dev = eval_log.model_info.developer or 'unknown' # Use model name without developer prefix for the directory - model_name = eval_log.model_info.name.split("/")[-1] - filepath = save_evaluation_log(eval_log, OUTPUT_DIR, dev, model_name) + model_name = eval_log.model_info.name.split('/')[-1] + filepath = save_evaluation_log( + eval_log, OUTPUT_DIR, dev, model_name + ) score = eval_log.evaluation_results[0].score_details.score - print(f" [{score:.1%}] {submission_dir.name} → {filepath}") + print(f' [{score:.1%}] {submission_dir.name} → {filepath}') count += 1 except Exception as e: - print(f" ERROR {submission_dir.name}: {e}") + print(f' ERROR {submission_dir.name}: {e}') errors += 1 - print(f"\nGenerated {count} files, {errors} errors → {OUTPUT_DIR}/") + print(f'\nGenerated {count} files, {errors} errors → {OUTPUT_DIR}/') -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/utils/swe_polybench/adapter.py b/utils/swe_polybench/adapter.py index e54b0620e..506d8d803 100644 --- a/utils/swe_polybench/adapter.py +++ b/utils/swe_polybench/adapter.py @@ -43,20 +43,28 @@ SourceDataHf, SourceMetadata, ) -from every_eval_ever.helpers import SCHEMA_VERSION, get_developer, get_model_id, save_evaluation_log +from every_eval_ever.helpers import ( + SCHEMA_VERSION, + get_developer, + get_model_id, + save_evaluation_log, +) from utils.swe_helpers import parse_date_from_dir, parse_model_from_dir -POLY_REPO = "https://github.com/amazon-science/SWE-PolyBench" -POLY_BRANCH = "submission" +POLY_REPO = 'https://github.com/amazon-science/SWE-PolyBench' +POLY_BRANCH = 'submission' DATASETS = { - "PB": "AmazonScience/SWE-PolyBench", - "PBVerified": "AmazonScience/SWE-PolyBench_Verified", + 'PB': 'AmazonScience/SWE-PolyBench', + 'PBVerified': 'AmazonScience/SWE-PolyBench_Verified', +} +DATASET_LABELS = {'PB': 'pb', 'PBVerified': 'pb-verified'} +DATASET_DISPLAY = { + 'PB': 'SWE-PolyBench', + 'PBVerified': 'SWE-PolyBench Verified', } -DATASET_LABELS = {"PB": "pb", "PBVerified": "pb-verified"} -DATASET_DISPLAY = {"PB": "SWE-PolyBench", "PBVerified": "SWE-PolyBench Verified"} -OUTPUT_BASE = "data/swe-polybench-leaderboard" +OUTPUT_BASE = 'data/swe-polybench-leaderboard' def convert_submission( @@ -79,46 +87,46 @@ def convert_submission( developer = get_developer(primary_model) model_id = get_model_id(primary_model, developer) - sanitized_id = re.sub(r"[^a-zA-Z0-9_.-]", "_", model_id.replace("/", "_")) - submission_slug = re.sub(r"[^a-zA-Z0-9_.-]", "_", dir_name) - eval_id = f"swe-polybench/{ds_label}/{lang}/{sanitized_id}/{submission_slug}/{retrieved_timestamp}" + sanitized_id = re.sub(r'[^a-zA-Z0-9_.-]', '_', model_id.replace('/', '_')) + submission_slug = re.sub(r'[^a-zA-Z0-9_.-]', '_', dir_name) + eval_id = f'swe-polybench/{ds_label}/{lang}/{sanitized_id}/{submission_slug}/{retrieved_timestamp}' evaluation_timestamp = parse_date_from_dir(dir_name) score = resolved_count / total_instances_for_lang additional_details: dict[str, str] = { - "submission_name": str(metadata.get("name", "")), - "language": lang, - "dataset": ds_label, - "oss": str(metadata.get("oss", "")), - "site": str(metadata.get("site", "")), - "pass_rate": str(metadata.get("pass_rate", "")), - "submission_dir": dir_name, - "agent": agent, + 'submission_name': str(metadata.get('name', '')), + 'language': lang, + 'dataset': ds_label, + 'oss': str(metadata.get('oss', '')), + 'site': str(metadata.get('site', '')), + 'pass_rate': str(metadata.get('pass_rate', '')), + 'submission_dir': dir_name, + 'agent': agent, } score_details: dict[str, str] = { - "resolved_count": str(resolved_count), - "total_instances_for_language": str(total_instances_for_lang), - "patch_applied_count": str(patch_applied_count), - "no_p2p_failed_count": str(no_p2p_failed_count), + 'resolved_count': str(resolved_count), + 'total_instances_for_language': str(total_instances_for_lang), + 'patch_applied_count': str(patch_applied_count), + 'no_p2p_failed_count': str(no_p2p_failed_count), } - eval_name = f"{ds_display} ({lang})" - dataset_label = f"{ds_display} ({lang})" + eval_name = f'{ds_display} ({lang})' + dataset_label = f'{ds_display} ({lang})' eval_result = EvaluationResult( evaluation_name=eval_name, source_data=SourceDataHf( dataset_name=dataset_label, - source_type="hf_dataset", + source_type='hf_dataset', hf_repo=hf_repo, - hf_split="test", + hf_split='test', samples_number=total_instances_for_lang, ), evaluation_timestamp=evaluation_timestamp, metric_config=MetricConfig( - evaluation_description=f"Fraction of {lang} GitHub issues resolved (0.0–1.0)", + evaluation_description=f'Fraction of {lang} GitHub issues resolved (0.0–1.0)', lower_is_better=False, score_type=ScoreType.continuous, min_score=0.0, @@ -131,7 +139,7 @@ def convert_submission( generation_config=GenerationConfig( generation_args=GenerationArgs( agentic_eval_config=AgenticEvalConfig( - available_tools=[AvailableTool(name="bash")], + available_tools=[AvailableTool(name='bash')], ), ), ), @@ -143,17 +151,17 @@ def convert_submission( retrieved_timestamp=retrieved_timestamp, evaluation_timestamp=evaluation_timestamp, source_metadata=SourceMetadata( - source_name="SWE-PolyBench Leaderboard", - source_type="documentation", - source_organization_name="AmazonScience", - source_organization_url="https://github.com/amazon-science/SWE-PolyBench", + source_name='SWE-PolyBench Leaderboard', + source_type='documentation', + source_organization_name='AmazonScience', + source_organization_url='https://github.com/amazon-science/SWE-PolyBench', evaluator_relationship=EvaluatorRelationship.third_party, ), - eval_library=EvalLibrary(name="swe-polybench", version="unknown"), + eval_library=EvalLibrary(name='swe-polybench', version='unknown'), model_info=ModelInfo( name=primary_model, id=model_id, - developer=developer if developer != "unknown" else None, + developer=developer if developer != 'unknown' else None, additional_details=additional_details, ), evaluation_results=[eval_result], @@ -166,17 +174,17 @@ def load_hf_instance_maps(ds: str) -> tuple[dict[str, str], Counter]: from datasets import load_dataset except ImportError as e: raise ImportError( - "datasets is required to run this adapter. Install it with: pip install datasets" + 'datasets is required to run this adapter. Install it with: pip install datasets' ) from e hf_repo = DATASETS[ds] - print(f" Loading HF dataset {hf_repo} ...") - dataset = load_dataset(hf_repo, split="test") + print(f' Loading HF dataset {hf_repo} ...') + dataset = load_dataset(hf_repo, split='test') id_to_lang: dict[str, str] = {} lang_counts: Counter = Counter() for row in dataset: - iid = row["instance_id"] - lang = row["language"] + iid = row['instance_id'] + lang = row['language'] id_to_lang[iid] = lang lang_counts[lang] += 1 return id_to_lang, lang_counts @@ -192,20 +200,20 @@ def process_submission( ) -> list[tuple[EvaluationLog, str]]: """Return list of (EvaluationLog, lang) for each language found in this submission.""" dir_name = submission_dir.name - metadata_path = submission_dir / "metadata.yaml" + metadata_path = submission_dir / 'metadata.yaml' if not metadata_path.exists(): - raise FileNotFoundError(f"metadata.yaml not found in {dir_name}") + raise FileNotFoundError(f'metadata.yaml not found in {dir_name}') with open(metadata_path) as f: metadata = yaml.safe_load(f) - logs_dir = submission_dir / "logs" + logs_dir = submission_dir / 'logs' if not logs_dir.exists(): - raise FileNotFoundError(f"logs/ not found in {dir_name}") + raise FileNotFoundError(f'logs/ not found in {dir_name}') - result_files = sorted(logs_dir.glob("*_result.json")) + result_files = sorted(logs_dir.glob('*_result.json')) if not result_files: - raise FileNotFoundError(f"No *_result.json files in {dir_name}/logs/") + raise FileNotFoundError(f'No *_result.json files in {dir_name}/logs/') # Aggregate per language langs_in_submission: set[str] = set() @@ -217,21 +225,23 @@ def process_submission( for result_file in result_files: with open(result_file) as f: data = json.load(f) - iid = data.get("instance_id", "") + iid = data.get('instance_id', '') lang = id_to_lang.get(iid) if lang is None: unknown_ids.append(iid) - lang = "unknown" + lang = 'unknown' langs_in_submission.add(lang) - if data.get("resolved", False): + if data.get('resolved', False): resolved_by_lang[lang] += 1 - if data.get("patch_applied", False): + if data.get('patch_applied', False): patch_applied_by_lang[lang] += 1 - if data.get("no_p2p_failed", False): + if data.get('no_p2p_failed', False): no_p2p_failed_by_lang[lang] += 1 if unknown_ids: - print(f" WARNING: {len(unknown_ids)} instance_ids not in HF dataset, bucketed as 'unknown'") + print( + f" WARNING: {len(unknown_ids)} instance_ids not in HF dataset, bucketed as 'unknown'" + ) # Only emit records for languages actually present in this submission's result # files, to avoid spurious 0-score entries for uncovered languages. @@ -260,7 +270,7 @@ def main(): import yaml except ImportError as e: raise ImportError( - "pyyaml is required to run this adapter. Install it with: pip install pyyaml" + 'pyyaml is required to run this adapter. Install it with: pip install pyyaml' ) from e retrieved_timestamp = str(time.time()) @@ -269,47 +279,67 @@ def main(): # Load HF datasets first hf_maps: dict[str, tuple[dict[str, str], Counter]] = {} - for ds in ("PB", "PBVerified"): + for ds in ('PB', 'PBVerified'): id_to_lang, lang_counts = load_hf_instance_maps(ds) hf_maps[ds] = (id_to_lang, lang_counts) - print(f" [{ds}] {sum(lang_counts.values())} instances: {dict(lang_counts)}") + print( + f' [{ds}] {sum(lang_counts.values())} instances: {dict(lang_counts)}' + ) with tempfile.TemporaryDirectory() as tmpdir: - print(f"\nCloning {POLY_REPO} (branch={POLY_BRANCH}) into {tmpdir} ...") + print(f'\nCloning {POLY_REPO} (branch={POLY_BRANCH}) into {tmpdir} ...') subprocess.run( - ["git", "clone", "--branch", POLY_BRANCH, "--depth=1", POLY_REPO, tmpdir], + [ + 'git', + 'clone', + '--branch', + POLY_BRANCH, + '--depth=1', + POLY_REPO, + tmpdir, + ], check=True, ) - for ds in ("PB", "PBVerified"): - ds_label = DATASET_LABELS[ds] - eval_path = Path(tmpdir) / "evaluation" / ds + for ds in ('PB', 'PBVerified'): + eval_path = Path(tmpdir) / 'evaluation' / ds if not eval_path.exists(): - print(f" [SKIP] No evaluation/{ds} dir") + print(f' [SKIP] No evaluation/{ds} dir') continue id_to_lang, lang_counts = hf_maps[ds] submissions = sorted(d for d in eval_path.iterdir() if d.is_dir()) - print(f"\n[{ds}] Found {len(submissions)} submissions") + print(f'\n[{ds}] Found {len(submissions)} submissions') for submission_dir in submissions: try: logs_results = process_submission( - submission_dir, ds, id_to_lang, lang_counts, retrieved_timestamp, yaml + submission_dir, + ds, + id_to_lang, + lang_counts, + retrieved_timestamp, + yaml, ) for eval_log, lang in logs_results: - dev = eval_log.model_info.developer or "unknown" - model_name = eval_log.model_info.name.split("/")[-1] - filepath = save_evaluation_log(eval_log, OUTPUT_BASE, dev, model_name) - score = eval_log.evaluation_results[0].score_details.score - print(f" [{score:.1%}] {submission_dir.name} [{lang}] → {filepath}") + dev = eval_log.model_info.developer or 'unknown' + model_name = eval_log.model_info.name.split('/')[-1] + filepath = save_evaluation_log( + eval_log, OUTPUT_BASE, dev, model_name + ) + score = eval_log.evaluation_results[ + 0 + ].score_details.score + print( + f' [{score:.1%}] {submission_dir.name} [{lang}] → {filepath}' + ) count += 1 except Exception as e: - print(f" ERROR {submission_dir.name}: {e}") + print(f' ERROR {submission_dir.name}: {e}') errors += 1 - print(f"\nGenerated {count} files, {errors} errors → {OUTPUT_BASE}/") + print(f'\nGenerated {count} files, {errors} errors → {OUTPUT_BASE}/') -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/utils/terminal_bench_2/adapter.py b/utils/terminal_bench_2/adapter.py index 7a1653ae0..d16af019d 100644 --- a/utils/terminal_bench_2/adapter.py +++ b/utils/terminal_bench_2/adapter.py @@ -18,8 +18,6 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) -import json -import uuid from eval_types import ( AgenticEvalConfig, @@ -41,48 +39,48 @@ ) from helpers import save_evaluation_log -LEADERBOARD_URL = "https://www.tbench.ai/leaderboard/terminal-bench/2.0" -OUTPUT_DIR = "data/terminal-bench-2.0" +LEADERBOARD_URL = 'https://www.tbench.ai/leaderboard/terminal-bench/2.0' +OUTPUT_DIR = 'data/terminal-bench-2.0' ORG_SLUG_MAP = { - "Google": "google", - "OpenAI": "openai", - "Anthropic": "anthropic", - "xAI": "xai", - "Moonshot AI": "moonshot-ai", - "Z-AI": "zhipu-ai", - "Z.ai": "zhipu-ai", - "DeepSeek": "deepseek", - "Alibaba": "alibaba", - "MiniMax": "minimax", - "Minimax": "minimax", - "Kimi": "moonshot-ai", - "Multiple": "multiple", - "Block": "block", - "Factory": "factory", - "Forge Code": "forge-code", - "KRAFTON AI": "krafton-ai", - "Coder": "coder", - "OpenBlock Labs": "openblock-labs", - "Bigai": "bigai", - "JetBrains": "jetbrains", - "Feeling AI": "feeling-ai", - "Antigma Labs": "antigma-labs", - "Roam": "roam", - "LangChain": "langchain", - "OpenSage": "opensage", - "Terminal Bench": "terminal-bench", - "Intelligent Internet": "intelligent-internet", - "Warp": "warp", - "Letta": "letta", - "Abacus.AI": "abacus-ai", - "OpenHands": "openhands", - "Anomaly Innovations": "anomaly-innovations", - "CAMEL-AI": "camel-ai", - "ADYA": "adya", - "Princeton": "princeton", - "TUM": "tum", - "iflow": "iflow", + 'Google': 'google', + 'OpenAI': 'openai', + 'Anthropic': 'anthropic', + 'xAI': 'xai', + 'Moonshot AI': 'moonshot-ai', + 'Z-AI': 'zhipu-ai', + 'Z.ai': 'zhipu-ai', + 'DeepSeek': 'deepseek', + 'Alibaba': 'alibaba', + 'MiniMax': 'minimax', + 'Minimax': 'minimax', + 'Kimi': 'moonshot-ai', + 'Multiple': 'multiple', + 'Block': 'block', + 'Factory': 'factory', + 'Forge Code': 'forge-code', + 'KRAFTON AI': 'krafton-ai', + 'Coder': 'coder', + 'OpenBlock Labs': 'openblock-labs', + 'Bigai': 'bigai', + 'JetBrains': 'jetbrains', + 'Feeling AI': 'feeling-ai', + 'Antigma Labs': 'antigma-labs', + 'Roam': 'roam', + 'LangChain': 'langchain', + 'OpenSage': 'opensage', + 'Terminal Bench': 'terminal-bench', + 'Intelligent Internet': 'intelligent-internet', + 'Warp': 'warp', + 'Letta': 'letta', + 'Abacus.AI': 'abacus-ai', + 'OpenHands': 'openhands', + 'Anomaly Innovations': 'anomaly-innovations', + 'CAMEL-AI': 'camel-ai', + 'ADYA': 'adya', + 'Princeton': 'princeton', + 'TUM': 'tum', + 'iflow': 'iflow', } # fmt: off @@ -207,49 +205,53 @@ def get_org_slug(org_name: str) -> str: - return ORG_SLUG_MAP.get(org_name, org_name.lower().replace(" ", "-").replace(".", "-")) + return ORG_SLUG_MAP.get( + org_name, org_name.lower().replace(' ', '-').replace('.', '-') + ) def get_model_slug(model_name: str) -> str: - return model_name.lower().replace(" ", "-") + return model_name.lower().replace(' ', '-') def make_model_id(model_org: str, model_name: str) -> str: - return f"{get_org_slug(model_org)}/{get_model_slug(model_name)}" + return f'{get_org_slug(model_org)}/{get_model_slug(model_name)}' def convert_entry(entry: dict, retrieved_timestamp: str) -> EvaluationLog: """Convert a single leaderboard entry to an EvaluationLog.""" - model_id = make_model_id(entry["model_org"], entry["model"]) - agent_slug = entry["agent"].lower().replace(" ", "-") - model_slug = get_model_slug(entry["model"]) + model_id = make_model_id(entry['model_org'], entry['model']) + agent_slug = entry['agent'].lower().replace(' ', '-') + model_slug = get_model_slug(entry['model']) - eval_id = f"terminal-bench-2.0/{agent_slug}__{model_slug}/{retrieved_timestamp}" + eval_id = ( + f'terminal-bench-2.0/{agent_slug}__{model_slug}/{retrieved_timestamp}' + ) uncertainty = None - if entry["stderr"] is not None: + if entry['stderr'] is not None: uncertainty = Uncertainty( - standard_error=StandardError(value=entry["stderr"]), + standard_error=StandardError(value=entry['stderr']), num_samples=435, ) eval_result = EvaluationResult( - evaluation_name="terminal-bench-2.0", + evaluation_name='terminal-bench-2.0', source_data=SourceDataUrl( - dataset_name="terminal-bench-2.0", - source_type="url", + dataset_name='terminal-bench-2.0', + source_type='url', url=[LEADERBOARD_URL], ), - evaluation_timestamp=entry["date"], + evaluation_timestamp=entry['date'], metric_config=MetricConfig( - evaluation_description="Task resolution accuracy across 87 terminal tasks with 5 trials each", + evaluation_description='Task resolution accuracy across 87 terminal tasks with 5 trials each', lower_is_better=False, score_type=ScoreType.continuous, min_score=0, max_score=100, ), score_details=ScoreDetails( - score=entry["accuracy"], + score=entry['accuracy'], uncertainty=uncertainty, ), generation_config=GenerationConfig( @@ -257,8 +259,8 @@ def convert_entry(entry: dict, retrieved_timestamp: str) -> EvaluationLog: agentic_eval_config=AgenticEvalConfig( available_tools=[ AvailableTool( - name="terminal", - description="Full terminal/shell access", + name='terminal', + description='Full terminal/shell access', ), ], ), @@ -268,25 +270,25 @@ def convert_entry(entry: dict, retrieved_timestamp: str) -> EvaluationLog: ) return EvaluationLog( - schema_version="0.2.2", + schema_version='0.2.2', evaluation_id=eval_id, retrieved_timestamp=retrieved_timestamp, - evaluation_timestamp=entry["date"], + evaluation_timestamp=entry['date'], source_metadata=SourceMetadata( - source_name="Terminal-Bench 2.0", - source_type="documentation", - source_organization_name="Terminal-Bench", - source_organization_url="https://www.tbench.ai", + source_name='Terminal-Bench 2.0', + source_type='documentation', + source_organization_name='Terminal-Bench', + source_organization_url='https://www.tbench.ai', evaluator_relationship=EvaluatorRelationship.third_party, ), - eval_library=EvalLibrary(name="harbor", version="unknown"), + eval_library=EvalLibrary(name='harbor', version='unknown'), model_info=ModelInfo( - name=entry["model"], + name=entry['model'], id=model_id, - developer=entry["model_org"], + developer=entry['model_org'], additional_details={ - "agent_name": entry["agent"], - "agent_organization": entry["agent_org"], + 'agent_name': entry['agent'], + 'agent_organization': entry['agent_org'], }, ), evaluation_results=[eval_result], @@ -300,17 +302,21 @@ def main(): for entry in LEADERBOARD_DATA: try: eval_log = convert_entry(entry, retrieved_timestamp) - org_slug = get_org_slug(entry["model_org"]) - model_slug = get_model_slug(entry["model"]) - filepath = save_evaluation_log(eval_log, OUTPUT_DIR, org_slug, model_slug) - print(f"[{entry['rank']:3d}] {filepath}") + org_slug = get_org_slug(entry['model_org']) + model_slug = get_model_slug(entry['model']) + filepath = save_evaluation_log( + eval_log, OUTPUT_DIR, org_slug, model_slug + ) + print(f'[{entry["rank"]:3d}] {filepath}') count += 1 except Exception as e: - print(f"Error processing rank {entry['rank']} " - f"({entry['agent']} / {entry['model']}): {e}") + print( + f'Error processing rank {entry["rank"]} ' + f'({entry["agent"]} / {entry["model"]}): {e}' + ) - print(f"\nGenerated {count} files in {OUTPUT_DIR}/") + print(f'\nGenerated {count} files in {OUTPUT_DIR}/') -if __name__ == "__main__": +if __name__ == '__main__': main()