diff --git a/tests/test_kaggle_adapter.py b/tests/test_kaggle_adapter.py new file mode 100644 index 000000000..b7ad89a75 --- /dev/null +++ b/tests/test_kaggle_adapter.py @@ -0,0 +1,478 @@ +"""Unit tests for the generalized Kaggle Benchmarks adapter. + +These exercise the pure transformation helpers (no network) plus an +end-to-end ``convert_benchmark`` run that writes to a tmp dir and validates +the saved records against the schema. +""" + +from __future__ import annotations + +import json +import sys +from argparse import Namespace + +import pytest + +from every_eval_ever.eval_types import EvaluationLog, ScoreType +from every_eval_ever.helpers import FetchError +from utils.kaggle import adapter + + +# --------------------------------------------------------------------------- +# Fixture builders mirroring the shape of Kaggle's leaderboard API. +# --------------------------------------------------------------------------- +def numeric_task(name, value, *, ci=None, date=None): + numeric = { + "value": value, + "hasUnevenConfidenceInterval": False, + "confidenceInterval": ci if ci is not None else 0.0, + "hasConfidenceInterval": ci is not None, + } + result = { + "hasNumericResult": True, + "numericResult": numeric, + "hasBooleanResult": False, + "booleanResult": False, + "customAdditionalResults": [], + "resultCase": "numericResult", + "hasEvaluationDate": date is not None, + } + if date is not None: + result["evaluationDate"] = date + return {"benchmarkTaskName": name, "benchmarkTaskSlug": "", "result": result} + + +def boolean_task(name, passed): + return { + "benchmarkTaskName": name, + "result": { + "hasNumericResult": False, + "hasBooleanResult": True, + "booleanResult": passed, + "customAdditionalResults": [], + "resultCase": "booleanResult", + }, + } + + +def empty_task(name): + return { + "benchmarkTaskName": name, + "result": { + "hasNumericResult": False, + "hasBooleanResult": False, + "customAdditionalResults": [], + "resultCase": "none", + }, + } + + +PCT_META = { + "benchmark_id": 42, + "sort_order": "DESCENDING", + "aggregation_type": "PERCENTAGE_PASSED", + "display_type": "PERCENTAGES", +} + + +# --------------------------------------------------------------------------- +# _build_eval_result: result-type handling +# --------------------------------------------------------------------------- +def test_numeric_in_unit_range_is_bounded_continuous(): + r = adapter._build_eval_result(numeric_task("acc", 0.87654), "o", "s") + assert r.score_details.score == 0.8765 # rounded to 4dp + assert r.metric_config.score_type == ScoreType.continuous + assert r.metric_config.min_score == 0.0 + assert r.metric_config.max_score == 1.0 + + +def test_numeric_outside_unit_range_is_left_untyped(): + # Unknown scale (e.g. a count); must NOT fabricate [0, 1] bounds. + r = adapter._build_eval_result(numeric_task("count", 100.0), "o", "s") + assert r.score_details.score == 100.0 + assert r.metric_config.score_type is None + assert r.metric_config.min_score is None + assert r.metric_config.max_score is None + + +def test_boolean_results_become_binary(): + passed = adapter._build_eval_result(boolean_task("solved", True), "o", "s") + failed = adapter._build_eval_result(boolean_task("solved", False), "o", "s") + assert passed.metric_config.score_type == ScoreType.binary + assert passed.score_details.score == 1.0 + assert passed.metric_config.min_score == 0.0 + assert passed.metric_config.max_score == 1.0 + assert failed.score_details.score == 0.0 + + +def test_empty_result_is_skipped(): + assert adapter._build_eval_result(empty_task("unscored"), "o", "s") is None + + +def test_numeric_with_null_value_is_skipped(): + assert adapter._build_eval_result(numeric_task("x", None), "o", "s") is None + + +# --------------------------------------------------------------------------- +# _build_eval_result: enrichment (dates, CI, meta) +# --------------------------------------------------------------------------- +def test_evaluation_date_is_captured_as_timestamp(): + r = adapter._build_eval_result( + numeric_task("acc", 0.5, date="2026-06-20T16:40:52.0Z"), "o", "s" + ) + assert r.evaluation_timestamp == "2026-06-20T16:40:52.0Z" + + +def test_missing_evaluation_date_leaves_timestamp_unset(): + r = adapter._build_eval_result(numeric_task("acc", 0.5), "o", "s") + assert r.evaluation_timestamp is None + + +def test_confidence_interval_brackets_the_score(): + # Kaggle's confidenceInterval is a symmetric half-width around the score, + # so the bounds must bracket the score, not be centered at zero. + r = adapter._build_eval_result(numeric_task("acc", 0.87, ci=0.02), "o", "s") + ci = r.score_details.uncertainty.confidence_interval + assert ci.lower == 0.85 + assert ci.upper == 0.89 + + +def test_non_numeric_confidence_interval_is_ignored_not_fatal(): + task = numeric_task("acc", 0.5) + task["result"]["numericResult"]["hasConfidenceInterval"] = True + task["result"]["numericResult"]["confidenceInterval"] = "oops" + r = adapter._build_eval_result(task, "o", "s") + assert r.score_details.uncertainty is None + + +def test_no_uncertainty_when_ci_absent(): + r = adapter._build_eval_result(numeric_task("acc", 0.5), "o", "s") + assert r.score_details.uncertainty is None + + +def test_meta_sets_direction_unit_and_kind(): + r = adapter._build_eval_result(numeric_task("acc", 0.9), "o", "s", PCT_META) + assert r.metric_config.lower_is_better is False + assert r.metric_config.metric_kind == "pass_rate" + assert r.metric_config.metric_unit == "proportion" + assert r.metric_config.metric_name == "acc" + + +def test_ascending_sort_order_means_lower_is_better(): + meta = {**PCT_META, "sort_order": "ASCENDING"} + r = adapter._build_eval_result(numeric_task("latency", 0.3), "o", "s", meta) + assert r.metric_config.lower_is_better is True + + +def test_boolean_result_does_not_get_aggregation_kind_or_unit(): + # Aggregation-derived kind/unit describe a numeric metric, not a 0/1 result. + r = adapter._build_eval_result(boolean_task("solved", True), "o", "s", PCT_META) + assert r.metric_config.metric_kind is None + assert r.metric_config.metric_unit is None + assert r.metric_config.score_type == ScoreType.binary + + +def test_counts_display_type_maps_to_count_unit(): + meta = {**PCT_META, "display_type": "COUNTS", "aggregation_type": None} + r = adapter._build_eval_result(numeric_task("n", 100.0), "o", "s", meta) + assert r.metric_config.metric_unit == "count" + assert r.metric_config.metric_kind is None + + +def test_without_meta_direction_defaults_false_and_no_unit(): + r = adapter._build_eval_result(numeric_task("acc", 0.9), "o", "s") + assert r.metric_config.lower_is_better is False + assert r.metric_config.metric_unit is None + assert r.metric_config.metric_kind is None + + +def test_task_name_falls_back_to_slug(): + task = numeric_task("", 0.5) + r = adapter._build_eval_result(task, "owner", "myslug") + assert r.evaluation_name == "myslug" + + +# --------------------------------------------------------------------------- +# _benchmark_owner / _benchmark_meta +# --------------------------------------------------------------------------- +def test_owner_prefers_organization_slug(): + bench = { + "organization": {"slug": "cohere-labs"}, + "ownerUser": {"userName": "someuser"}, + } + assert adapter._benchmark_owner(bench) == "cohere-labs" + + +def test_owner_falls_back_to_username(): + bench = {"organization": None, "ownerUser": {"userName": "someuser"}} + assert adapter._benchmark_owner(bench) == "someuser" + + +def test_owner_none_when_unresolvable(): + assert adapter._benchmark_owner({}) is None + + +def test_benchmark_meta_extracts_scoring_config(): + bench = { + "id": 7, + "task": { + "version": { + "sortOrder": "DESCENDING", + "aggregationType": "PERCENTAGE_PASSED", + "displayType": "PERCENTAGES", + } + }, + } + meta = adapter._benchmark_meta(bench) + assert meta == { + "benchmark_id": 7, + "sort_order": "DESCENDING", + "aggregation_type": "PERCENTAGE_PASSED", + "display_type": "PERCENTAGES", + } + + +def test_benchmark_meta_handles_missing_version(): + meta = adapter._benchmark_meta({"id": 1}) + assert meta["sort_order"] is None + assert meta["aggregation_type"] is None + + +# --------------------------------------------------------------------------- +# list_benchmarks: discovery pagination + filtering (network mocked) +# --------------------------------------------------------------------------- +class _FakeResp: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + +class _FakeSession: + """Returns two canned ListBenchmarks pages keyed by pageToken.""" + + def __init__(self, pages): + self._pages = pages + self.calls = [] + + def post(self, url, json=None, headers=None, timeout=None): + token = json["pageToken"] + self.calls.append(token) + return _FakeResp(self._pages[token]) + + +def test_list_benchmarks_paginates_and_filters_unpublished(monkeypatch): + pages = { + "": { + "benchmarks": [ + { + "slug": "global-mmlu-lite", + "name": " Global MMLU Lite ", + "published": True, + "organization": {"slug": "cohere-labs"}, + "ownerUser": {"userName": "creator"}, + "task": {"version": {"sortOrder": "DESCENDING"}}, + }, + # Unpublished -> must be filtered out. + {"slug": "draft", "published": False, "ownerUser": {"userName": "x"}}, + ], + "nextPageToken": "p2", + }, + "p2": { + "benchmarks": [ + { + "slug": "solo", + "name": "Solo", + "published": True, + "organization": None, + "ownerUser": {"userName": "alice"}, + } + ], + "nextPageToken": "", + }, + } + fake = _FakeSession(pages) + monkeypatch.setattr(adapter, "_kaggle_session", lambda: (fake, "xsrf-token")) + + out = list(adapter.list_benchmarks()) + + assert fake.calls == ["", "p2"] # paginated via nextPageToken + assert [(b["owner"], b["slug"]) for b in out] == [ + ("cohere-labs", "global-mmlu-lite"), # org slug preferred over creator + ("alice", "solo"), # falls back to username + ] + assert out[0]["name"] == "Global MMLU Lite" # stripped + assert out[0]["meta"]["sort_order"] == "DESCENDING" + + +# --------------------------------------------------------------------------- +# fetch_leaderboard: failure vs empty contract (network mocked) +# --------------------------------------------------------------------------- +def test_fetch_leaderboard_returns_none_on_fetch_error(monkeypatch): + def boom(url): + raise FetchError("boom") + + monkeypatch.setattr(adapter, "fetch_json", boom) + assert adapter.fetch_leaderboard("o", "s") is None + + +def test_fetch_leaderboard_returns_empty_list_for_no_submissions(monkeypatch): + monkeypatch.setattr(adapter, "fetch_json", lambda url: {"rows": []}) + assert adapter.fetch_leaderboard("o", "s") == [] + + +def test_fetch_leaderboard_returns_none_on_malformed_shape(monkeypatch): + # A 200 with an error envelope / no list-valued rows is a failure, not empty. + monkeypatch.setattr(adapter, "fetch_json", lambda url: {"error": "nope"}) + assert adapter.fetch_leaderboard("o", "s") is None + + +# --------------------------------------------------------------------------- +# _resolve_targets: dedup + discovery-failure flag +# --------------------------------------------------------------------------- +def test_resolve_targets_dedupes_repeated_benchmarks(): + args = Namespace( + benchmark=["cohere-labs/global-mmlu-lite", "cohere-labs/global-mmlu-lite"], + all=False, + limit=None, + ) + targets, discovery_failed = adapter._resolve_targets(args) + assert len(targets) == 1 + assert discovery_failed is False + + +def test_resolve_targets_dedup_prefers_meta_carrying_entry(monkeypatch): + # An explicit (meta-less) --benchmark overlapping an --all discovery entry + # must keep the discovered entry's meta so enrichment isn't lost. + def discover(): + yield { + "owner": "cohere-labs", + "slug": "global-mmlu-lite", + "name": "Global MMLU Lite", + "meta": {"sort_order": "DESCENDING", "display_type": "PERCENTAGES"}, + } + + monkeypatch.setattr(adapter, "list_benchmarks", discover) + args = Namespace( + benchmark=["cohere-labs/global-mmlu-lite"], all=True, limit=None + ) + targets, _ = adapter._resolve_targets(args) + assert len(targets) == 1 + assert targets[0]["meta"]["display_type"] == "PERCENTAGES" + + +def test_resolve_targets_flags_truncated_discovery(monkeypatch): + def partial(): + yield {"owner": "o", "slug": "a", "name": "A", "meta": {}} + raise FetchError("page 2 failed") + + monkeypatch.setattr(adapter, "list_benchmarks", partial) + args = Namespace(benchmark=None, all=True, limit=None) + targets, discovery_failed = adapter._resolve_targets(args) + assert [t["slug"] for t in targets] == ["a"] # partial progress kept + assert discovery_failed is True + + +# --------------------------------------------------------------------------- +# main: non-zero exit when a fetch fails, good benchmark still written +# --------------------------------------------------------------------------- +def test_main_exits_nonzero_when_a_fetch_fails(monkeypatch, tmp_path): + def fake_fetch(owner, slug): + return None if owner == "bad" else sample_rows()[:1] + + monkeypatch.setattr(adapter, "fetch_leaderboard", fake_fetch) + monkeypatch.setattr( + sys, + "argv", + [ + "adapter", + "--benchmark", + "good/one", + "--benchmark", + "bad/two", + "--output-dir", + str(tmp_path), + ], + ) + with pytest.raises(SystemExit) as exc: + adapter.main() + assert exc.value.code == 1 + # The good benchmark was still converted despite the other's failure. + assert list(tmp_path.rglob("*.json")) + + +# --------------------------------------------------------------------------- +# convert_benchmark: end-to-end, schema-valid output +# --------------------------------------------------------------------------- +def sample_rows(): + return [ + { + "modelVersionName": "GPT-4o", + "modelVersionSlug": "gpt-4o-2024-05-13", + "taskResults": [ + numeric_task("Overall", 0.71, date="2026-06-20T16:40:52Z"), + boolean_task("Edge Case", True), + empty_task("Unscored Task"), # dropped, no usable score + ], + }, + # Row with only empty results -> produces no file. + { + "modelVersionName": "Blank", + "modelVersionSlug": "blank-model", + "taskResults": [empty_task("Nothing")], + }, + # Row missing the slug -> skipped entirely. + {"modelVersionName": "No Slug", "taskResults": [numeric_task("x", 0.5)]}, + ] + + +def test_convert_benchmark_writes_only_rows_with_results(tmp_path): + count = adapter.convert_benchmark( + "cohere-labs", + "demo", + "Demo Benchmark", + sample_rows(), + str(tmp_path), + "123.0", + meta=PCT_META, + ) + assert count == 1 # only gpt-4o has usable results + files = list(tmp_path.rglob("*.json")) + assert len(files) == 1 + + +def test_convert_benchmark_output_validates_and_is_enriched(tmp_path): + adapter.convert_benchmark( + "cohere-labs", + "demo", + "Demo Benchmark", + sample_rows(), + str(tmp_path), + "123.0", + meta=PCT_META, + ) + path = next(tmp_path.rglob("*.json")) + log = EvaluationLog.model_validate(json.loads(path.read_text())) + + assert log.schema_version == "0.2.2" + assert log.model_info.id == "openai/gpt-4o-2024-05-13" + assert log.source_metadata.source_organization_name == "cohere-labs" + + details = log.source_metadata.additional_details + assert details["platform"] == "kaggle" + assert details["benchmark_id"] == "42" + assert details["aggregation_type"] == "PERCENTAGE_PASSED" + assert details["display_type"] == "PERCENTAGES" + + # Empty task dropped; numeric + boolean kept. + names = {r.evaluation_name for r in log.evaluation_results} + assert names == {"Overall", "Edge Case"} + + overall = next(r for r in log.evaluation_results if r.evaluation_name == "Overall") + assert overall.evaluation_timestamp == "2026-06-20T16:40:52Z" + assert overall.metric_config.metric_unit == "proportion" + assert overall.metric_config.lower_is_better is False diff --git a/utils/README.md b/utils/README.md index c6e691e45..74a92afb9 100644 --- a/utils/README.md +++ b/utils/README.md @@ -16,6 +16,7 @@ Each adapter is run with `uv run python -m utils..adapter`. | `bfcl` | BFCL leaderboard CSV | Converts BFCL leaderboard data with per-metric evaluation names and bounded continuous scores. | | `sciarena` | SciArena leaderboard API | Converts SciArena leaderboard results. | | `global-mmlu-lite` | Kaggle API | Fetches Global MMLU Lite leaderboard results from Kaggle. | +| `kaggle` | Kaggle Benchmarks API | Generalized Kaggle Community Benchmarks adapter. Converts any benchmark via `--benchmark owner/slug`, or discovers and converts all published benchmarks via `--all` (uses the `ListBenchmarks` RPC). Handles numeric and boolean task results. | | `hfopenllm_v2` | HuggingFace Spaces API | Fetches the Open LLM Leaderboard v2 (4576+ models). | | `helm` | HELM leaderboard | Converts HELM leaderboard data. Supports `--leaderboard_name` for Capabilities/Lite/Classic/Instruct/MMLU. | | `llm_stats` | LLM Stats API | Converts LLM Stats model, benchmark, and score API data into `data/llm-stats/`. | @@ -37,6 +38,42 @@ Each adapter is run with `uv run python -m utils..adapter`. generated artifacts. Prefer temporary output paths for smoke runs unless a data refresh is intentionally part of the change. +### Kaggle Benchmarks + +Convert one or more named benchmarks (smoke run, output outside the repo): + +```bash +uv run python -m utils.kaggle.adapter \ + --benchmark cohere-labs/global-mmlu-lite \ + --output-dir /tmp/eee-kaggle +``` + +Discover and convert all published benchmarks via the `ListBenchmarks` RPC +(slow — 1000+ benchmarks; use `--limit` to cap during testing): + +```bash +uv run python -m utils.kaggle.adapter --all --limit 10 --output-dir /tmp/eee-kaggle +``` + +Notes: +- The `--all` discovery path calls an unauthenticated Kaggle internal RPC + (`benchmarks.BenchmarkService/ListBenchmarks`) that requires the anonymous + XSRF cookie/header handshake; the adapter handles this automatically. +- Numeric results in `[0, 1]` are emitted as bounded `continuous` metrics and + boolean results as `binary`; numeric results outside `[0, 1]` have an unknown + scale (Kaggle does not expose metric bounds), so they are left untyped rather + than given fabricated `[0, 1]` bounds. +- Each result carries its Kaggle `evaluationDate` as `evaluation_timestamp` when + present (~70% of results). +- The `--all` path additionally enriches each record from the benchmark's + scoring config (only available via `ListBenchmarks`): `lower_is_better` from + `sortOrder`, `metric_kind` from `aggregationType`, `metric_unit` from + `displayType`, and `benchmark_id`/`aggregation_type`/`display_type` in + `source_metadata.additional_details`. The targeted `--benchmark` path does not + have this metadata, so those fields fall back to defaults there. +- This supersedes the single-purpose `global-mmlu-lite` adapter, which is kept + for backwards compatibility. + ### Vals.ai Run a live smoke export from the repository root, writing generated output diff --git a/utils/kaggle/__init__.py b/utils/kaggle/__init__.py new file mode 100644 index 000000000..2c3e40b21 --- /dev/null +++ b/utils/kaggle/__init__.py @@ -0,0 +1 @@ +"""Generalized Kaggle Community Benchmarks adapter for the EvalEval schema.""" diff --git a/utils/kaggle/adapter.py b/utils/kaggle/adapter.py new file mode 100644 index 000000000..80c188c76 --- /dev/null +++ b/utils/kaggle/adapter.py @@ -0,0 +1,541 @@ +""" +Generalized adapter for Kaggle Community Benchmarks. + +Kaggle Benchmarks are community-published evaluation suites. Each benchmark is +identified by an ``owner/slug`` pair (e.g. ``cohere-labs/global-mmlu-lite``) and +exposes a public leaderboard. This adapter: + +1. Optionally enumerates ALL published benchmarks via Kaggle's (undocumented but + unauthenticated) ``ListBenchmarks`` RPC, and/or accepts explicit ``owner/slug`` + pairs on the command line. +2. Fetches each benchmark's leaderboard from the public REST endpoint. +3. Converts every model row (and every task result within it) into the EvalEval + schema, handling both numeric and boolean task results. + +Data sources: +- List: POST https://www.kaggle.com/api/i/benchmarks.BenchmarkService/ListBenchmarks +- Leaderboard: GET https://www.kaggle.com/api/v1/benchmarks/{owner}/{slug}/leaderboard + +Usage: + # Convert specific benchmark(s) + uv run python -m utils.kaggle.adapter \ + --benchmark cohere-labs/global-mmlu-lite \ + --output-dir /tmp/eee-kaggle + + # Discover and convert all published benchmarks (slow) + uv run python -m utils.kaggle.adapter --all --output-dir data/kaggle + + # Smoke test: discover but only convert the first 5 + uv run python -m utils.kaggle.adapter --all --limit 5 --output-dir /tmp/eee-kaggle +""" + +import argparse +import time +from typing import Dict, Iterator, List, Optional, Tuple + +import requests + +from every_eval_ever.eval_types import ( + ConfidenceInterval, + EvalLibrary, + EvaluationLog, + EvaluationResult, + EvaluatorRelationship, + MetricConfig, + ScoreDetails, + ScoreType, + SourceDataUrl, + Uncertainty, +) +from every_eval_ever.helpers import ( + SCHEMA_VERSION, + FetchError, + fetch_json, + make_model_info, + make_source_metadata, + save_evaluation_log, +) + +KAGGLE_BASE = "https://www.kaggle.com" +LIST_RPC_URL = f"{KAGGLE_BASE}/api/i/benchmarks.BenchmarkService/ListBenchmarks" +LEADERBOARD_URL = KAGGLE_BASE + "/api/v1/benchmarks/{owner}/{slug}/leaderboard" + +# Server-side cap on ListBenchmarks page size. +LIST_PAGE_SIZE = 200 + + +def _kaggle_session() -> Tuple[requests.Session, str]: + """Open a session with an anonymous XSRF token. + + The ListBenchmarks RPC is unauthenticated but requires the XSRF cookie/header + handshake that Kaggle hands out on any page load. + """ + session = requests.Session() + try: + session.get(f"{KAGGLE_BASE}/benchmarks", timeout=60) + except requests.RequestException as e: + raise FetchError(f"Kaggle XSRF handshake failed: {e}") from e + xsrf = session.cookies.get("XSRF-TOKEN") + if not xsrf: + raise FetchError("Could not obtain XSRF-TOKEN cookie from Kaggle") + return session, xsrf + + +# Kaggle aggregationType -> normalized metric_kind (for safe cross-source aggregation). +# Only map aggregation types that describe the *measured value*; generic reduction +# modes (AVERAGE, SUM) are not metric families and would conflate unrelated metrics, +# so they are intentionally omitted (the raw type is kept in source additional_details). +_AGGREGATION_TO_METRIC_KIND = { + "PERCENTAGE_PASSED": "pass_rate", +} +# Kaggle displayType -> metric_unit. PERCENTAGES are stored as proportions in [0, 1]. +_DISPLAY_TO_UNIT = { + "PERCENTAGES": "proportion", + "COUNTS": "count", +} + + +def _benchmark_owner(bench: dict) -> Optional[str]: + """Resolve the leaderboard URL owner for a benchmark object. + + Org-owned benchmarks are addressed by the organization slug; otherwise the + creating user's username is used. + """ + org = bench.get("organization") + if org and org.get("slug"): + return org["slug"] + return (bench.get("ownerUser") or {}).get("userName") + + +def _benchmark_meta(bench: dict) -> Dict[str, Optional[str]]: + """Extract benchmark-level metadata (only present on the discovery path). + + The leaderboard endpoint omits these, but ``ListBenchmarks`` carries the + scoring config we need to set direction (``lower_is_better``) and metric + unit/kind, which would otherwise be guessed. + """ + version = (bench.get("task") or {}).get("version") or {} + return { + "benchmark_id": bench.get("id"), + "sort_order": version.get("sortOrder"), + "aggregation_type": version.get("aggregationType"), + "display_type": version.get("displayType"), + } + + +def list_benchmarks() -> Iterator[Dict[str, object]]: + """Enumerate published Kaggle benchmarks. + + Yields dicts with ``owner``, ``slug``, ``name`` and ``meta``. The leaderboard + URL owner is the organization slug when the benchmark is org-owned, otherwise + the creating user's username. + """ + session, xsrf = _kaggle_session() + headers = { + "content-type": "application/json", + "accept": "application/json", + "x-xsrf-token": xsrf, + } + page_token = "" + while True: + body = {"filter": {}, "pageSize": LIST_PAGE_SIZE, "pageToken": page_token} + # On a page failure, raise after the benchmarks already yielded: the + # caller keeps that partial progress but is told discovery was truncated, + # so a `--all` run is never reported as a clean success when its tail is + # missing. + try: + resp = session.post(LIST_RPC_URL, json=body, headers=headers, timeout=60) + resp.raise_for_status() + data = resp.json() + except (requests.RequestException, ValueError) as e: + raise FetchError(f"ListBenchmarks page fetch failed: {e}") from e + # A 200 error envelope or changed RPC schema would otherwise look like an + # empty final page and end discovery as a clean success — validate the + # shape so a broken response is a failure, not a silently empty corpus. + if not isinstance(data, dict) or not isinstance(data.get("benchmarks"), list): + raise FetchError( + f"unexpected ListBenchmarks response shape (pageToken={page_token!r})" + ) + for bench in data["benchmarks"]: + if not bench.get("published"): + continue + slug = bench.get("slug") + if not slug: + continue + owner = _benchmark_owner(bench) + if not owner: + continue + yield { + "owner": owner, + "slug": slug, + "name": bench.get("name", slug).strip(), + "meta": _benchmark_meta(bench), + } + page_token = data.get("nextPageToken") or "" + if not page_token: + break + + +def fetch_leaderboard(owner: str, slug: str) -> Optional[List[dict]]: + """Fetch a benchmark leaderboard. + + Returns the list of rows on success (possibly empty for a benchmark with no + submissions), or ``None`` when the fetch itself failed (HTTP/network/parse + error, including the 403 returned for non-existent/private benchmarks). The + caller distinguishes these: an empty list is "no data", ``None`` is a failure + that must be surfaced rather than silently treated as empty. + """ + url = LEADERBOARD_URL.format(owner=owner, slug=slug) + try: + data = fetch_json(url) + except FetchError as e: + print(f" ! could not fetch leaderboard for {owner}/{slug}: {e}") + return None + # A 200 with an error envelope or a changed schema (no list-valued `rows`) is + # a fetch failure, not an empty leaderboard — don't let an upstream API break + # masquerade as "no data". + rows = data.get("rows") if isinstance(data, dict) else None + if not isinstance(rows, list): + print(f" ! unexpected leaderboard response shape for {owner}/{slug}") + return None + return rows + + +def _build_eval_result( + task: dict, owner: str, slug: str, meta: Optional[Dict[str, object]] = None +) -> Optional[EvaluationResult]: + """Convert a single ``taskResult`` entry into an EvaluationResult. + + Handles numeric results (continuous score in [0, 1]) and boolean results + (binary 0/1). Other result cases (e.g. custom tuples) are skipped. + + ``meta`` carries benchmark-level scoring config (from the discovery path) + used to set ``lower_is_better`` and the metric unit/kind; when absent these + fall back to sensible defaults. + """ + meta = meta or {} + task_name = task.get("benchmarkTaskName") or slug + result = task.get("result", {}) + + # Direction: Kaggle sorts leaderboards DESCENDING when higher is better. + lower_is_better = meta.get("sort_order") == "ASCENDING" + + score: Optional[float] = None + score_type: Optional[ScoreType] = None + min_score: Optional[float] = None + max_score: Optional[float] = None + metric_kind: Optional[str] = None + metric_unit: Optional[str] = None + uncertainty: Optional[Uncertainty] = None + + if result.get("hasNumericResult"): + numeric = result.get("numericResult") or {} + value = numeric.get("value") + if value is None: + return None + try: + score = float(value) + except (TypeError, ValueError): + return None + # Aggregation-derived kind/unit describe a numeric metric; they are + # meaningless for a single boolean pass/fail, so only attach them here. + metric_kind = _AGGREGATION_TO_METRIC_KIND.get(meta.get("aggregation_type")) + metric_unit = _DISPLAY_TO_UNIT.get(meta.get("display_type")) + # Kaggle's API does not expose a metric's scale. Most numeric results are + # accuracies / pass-rates in [0, 1]; for those we can safely declare a + # bounded continuous metric. Values outside [0, 1] (counts, percentages, + # latencies, ...) have an unknown scale, so we leave score_type and bounds + # unset rather than fabricate misleading [0, 1] bounds. + # Only declare bounded [0, 1] when the value plausibly is a proportion. + # A COUNTS metric whose value happens to fall in [0, 1] (e.g. a count of + # 0 or 1) must not claim a max possible score of 1, so skip bounding when + # Kaggle marks the metric as counts. + if meta.get("display_type") != "COUNTS" and 0.0 <= score <= 1.0: + score_type = ScoreType.continuous + min_score, max_score = 0.0, 1.0 + # Kaggle's `confidenceInterval` is a symmetric half-width ("confidence + # radius") around the score, so the emitted bounds must bracket the score + # (cf. utils/hle/adapter.py), not be centered at zero. + if numeric.get("hasConfidenceInterval"): + try: + ci = float(numeric.get("confidenceInterval")) + except (TypeError, ValueError): + ci = None + if ci is not None: + uncertainty = Uncertainty( + confidence_interval=ConfidenceInterval( + lower=round(score - ci, 4), + upper=round(score + ci, 4), + method="unknown", + ) + ) + elif result.get("hasBooleanResult"): + score_type = ScoreType.binary + score = 1.0 if result.get("booleanResult") else 0.0 + min_score, max_score = 0.0, 1.0 + else: + return None + + benchmark_url = f"{KAGGLE_BASE}/benchmarks/{owner}/{slug}" + return EvaluationResult( + evaluation_name=task_name, + evaluation_timestamp=result.get("evaluationDate"), + source_data=SourceDataUrl( + dataset_name=slug, + source_type="url", + url=[benchmark_url], + ), + metric_config=MetricConfig( + evaluation_description=f"Kaggle Benchmarks - {task_name}", + metric_name=task_name, + metric_kind=metric_kind, + metric_unit=metric_unit, + lower_is_better=lower_is_better, + score_type=score_type, + min_score=min_score, + max_score=max_score, + ), + score_details=ScoreDetails( + score=round(score, 4), + uncertainty=uncertainty, + ), + ) + + +def convert_benchmark( + owner: str, + slug: str, + name: str, + rows: List[dict], + output_dir: str, + retrieved_timestamp: str, + meta: Optional[Dict[str, object]] = None, +) -> int: + """Convert all leaderboard rows for one benchmark and save them. Returns count.""" + meta = meta or {} + benchmark_url = f"{KAGGLE_BASE}/benchmarks/{owner}/{slug}" + + source_details = { + "platform": "kaggle", + "benchmark_owner": owner, + "benchmark_url": benchmark_url, + } + for key in ("benchmark_id", "aggregation_type", "display_type"): + if meta.get(key) is not None: + source_details[key] = str(meta[key]) + + count = 0 + + for row in rows: + model_slug = row.get("modelVersionSlug") + if not model_slug: + continue + model_display_name = row.get("modelVersionName", "") + + eval_results: List[EvaluationResult] = [] + for task in row.get("taskResults", []): + result = _build_eval_result(task, owner, slug, meta) + if result is not None: + eval_results.append(result) + if not eval_results: + continue + + model_info = make_model_info( + model_name=model_slug, + additional_details={"display_name": model_display_name} + if model_display_name and model_display_name != model_slug + else None, + ) + + # Owner-qualify the id: benchmarks are owner/slug resources, so two + # owners sharing a slug (with the same model and run timestamp) would + # otherwise collide on this logical run identifier. + evaluation_id = ( + f"{owner}/{slug}/{model_info.id.replace('/', '_')}/{retrieved_timestamp}" + ) + eval_log = EvaluationLog( + schema_version=SCHEMA_VERSION, + evaluation_id=evaluation_id, + retrieved_timestamp=retrieved_timestamp, + source_metadata=make_source_metadata( + source_name=f"{name} (Kaggle Benchmarks)", + organization_name=owner, + organization_url="https://www.kaggle.com", + evaluator_relationship=EvaluatorRelationship.third_party, + additional_details=source_details, + ), + eval_library=EvalLibrary( + name="kaggle benchmarks", + version="unknown", + additional_details={"url": benchmark_url}, + ), + model_info=model_info, + evaluation_results=eval_results, + ) + + if "/" in model_info.id: + dev, _ = model_info.id.split("/", 1) + else: + dev = "unknown" + filepath = save_evaluation_log(eval_log, output_dir, dev, model_slug) + print(f" saved {filepath}") + count += 1 + + return count + + +def _resolve_targets(args) -> Tuple[List[Dict[str, object]], bool]: + """Build the (de-duplicated) list of benchmarks to process from CLI args. + + Returns the targets and a flag indicating whether ``--all`` discovery was + truncated by a fetch error, so the caller can exit non-zero instead of + reporting an incomplete corpus as success. + + Note: the ``--benchmark`` path has no benchmark-level ``meta`` (the + leaderboard endpoint omits it), so direction/unit enrichment is only applied + on the ``--all`` discovery path. + """ + targets: List[Dict[str, object]] = [] + for spec in args.benchmark or []: + if "/" not in spec: + raise SystemExit(f"--benchmark expects owner/slug, got: {spec!r}") + owner, slug = spec.split("/", 1) + targets.append({"owner": owner, "slug": slug, "name": slug, "meta": {}}) + + discovery_failed = False + if args.all: + print("Discovering published benchmarks via Kaggle ListBenchmarks RPC...") + try: + for bench in list_benchmarks(): + targets.append(bench) + if args.limit and len(targets) >= args.limit: + break + except FetchError as e: + # Keep what was discovered before the failure, but record that the + # corpus is incomplete so the run does not exit 0. + print(f" ! benchmark discovery truncated: {e}") + discovery_failed = True + + # De-duplicate by (owner, slug), preferring the entry that carries benchmark + # meta (the discovered one) so enrichment is not lost to a bare --benchmark + # duplicate or an explicit/--all overlap. + deduped: Dict[Tuple[str, str], Dict[str, object]] = {} + for t in targets: + key = (t["owner"], t["slug"]) + existing = deduped.get(key) + if existing is None or (not existing.get("meta") and t.get("meta")): + deduped[key] = t + return list(deduped.values()), discovery_failed + + +def main(): + parser = argparse.ArgumentParser( + description="Convert Kaggle Community Benchmarks leaderboards to the EvalEval schema." + ) + parser.add_argument( + "--benchmark", + action="append", + metavar="OWNER/SLUG", + help="Specific benchmark to convert (repeatable), e.g. cohere-labs/global-mmlu-lite", + ) + parser.add_argument( + "--all", + action="store_true", + help="Discover and convert all published benchmarks via the ListBenchmarks RPC.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="When using --all, stop after this many benchmarks (smoke testing).", + ) + parser.add_argument( + "--output-dir", + default="data/kaggle", + help="Base output directory (default: data/kaggle).", + ) + args = parser.parse_args() + + if not args.benchmark and not args.all: + parser.error("provide --benchmark OWNER/SLUG and/or --all") + + targets, discovery_failed = _resolve_targets(args) + retrieved_timestamp = str(time.time()) + + print("=" * 60) + print(f"Converting {len(targets)} Kaggle benchmark(s) -> {args.output_dir}") + print("=" * 60) + + total_models = 0 + total_benchmarks = 0 + failed_fetches = [] + conversion_failures = [] + empty_or_dropped = [] + for bench in targets: + owner, slug, name = bench["owner"], bench["slug"], bench["name"] + print(f"\n[{owner}/{slug}] {name}") + rows = fetch_leaderboard(owner, slug) + if rows is None: + # Fetch failed (transient/HTTP/parse error) — distinct from an empty + # leaderboard, so we don't report it as a clean "no data" success. + failed_fetches.append(f"{owner}/{slug}") + continue + if not rows: + print(" (no rows)") + continue + try: + count = convert_benchmark( + owner, + slug, + name, + rows, + args.output_dir, + retrieved_timestamp, + meta=bench.get("meta"), + ) + except (TypeError, AttributeError, KeyError) as e: + # A single malformed benchmark payload (e.g. taskResults: null) must + # not abort the whole --all run — isolate it as a per-benchmark + # failure, keep going, and exit non-zero at the end. + print(f" ! conversion failed for {owner}/{slug}: {e}") + conversion_failures.append(f"{owner}/{slug}") + continue + print(f" -> {count} model(s)") + if count == 0: + # A non-empty leaderboard that produced nothing means every row was + # unusable (e.g. an unhandled result type) — surface it, don't hide it. + empty_or_dropped.append(f"{owner}/{slug}") + total_models += count + total_benchmarks += 1 + + print("\n" + "=" * 60) + print(f"Done: {total_models} models across {total_benchmarks} benchmarks") + if empty_or_dropped: + print( + f"WARNING: {len(empty_or_dropped)} non-empty leaderboard(s) produced " + f"0 records (all rows unusable): {', '.join(empty_or_dropped)}" + ) + if failed_fetches: + print( + f"WARNING: {len(failed_fetches)} benchmark(s) could not be fetched " + f"and were skipped: {', '.join(failed_fetches)}" + ) + if conversion_failures: + print( + f"WARNING: {len(conversion_failures)} benchmark(s) failed to convert " + f"(malformed payload): {', '.join(conversion_failures)}" + ) + if discovery_failed: + print( + "WARNING: benchmark discovery was truncated by a fetch error; " + "the converted set is incomplete." + ) + print("=" * 60) + # Non-zero exit when any fetch/conversion failed or discovery was truncated, + # so callers and CI don't read a partial run as a clean success. + if failed_fetches or conversion_failures or discovery_failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main()