From e9ba3bfc75b53183e5705005a9937f265b22ceeb Mon Sep 17 00:00:00 2001 From: "mr.Shu" Date: Sun, 21 Jun 2026 22:46:01 +0200 Subject: [PATCH] feat: add generalized Kaggle Community Benchmarks adapter Previously the only Kaggle ingestion was a single hardcoded script (utils/global-mmlu-lite) bound to one benchmark and one metric path. This commit adds a generalized adapter that converts any Kaggle Community Benchmark leaderboard into the EvalEval schema (v0.2.2), either for explicit owner/slug targets or by discovering and converting every published benchmark. A full run converts ~996/1017 benchmarks into ~13.7k schema-valid records, which would make Kaggle the second-largest source in the datastore. Discovery uses Kaggle's unauthenticated internal RPC (benchmarks.BenchmarkService/ListBenchmarks) with the anonymous XSRF cookie/header handshake, because no official package, sitemap, or public endpoint can enumerate community benchmarks. Per-benchmark leaderboards are read from the public REST endpoint (/api/v1/benchmarks/{owner}/{slug}/leaderboard). Both APIs are undocumented for consumers, so the response shapes the adapter relies on are pinned by validation and tests rather than an upstream contract. Schema mapping is deliberately conservative about what Kaggle actually exposes: - Numeric results in [0, 1] become bounded `continuous` metrics; boolean results become `binary`. Numeric values outside [0, 1] have an unknown scale (Kaggle does not publish metric bounds) and are left untyped rather than given fabricated [0, 1] bounds; `COUNTS`-typed metrics are never bounded even when a value happens to fall in [0, 1]. - Confidence intervals are emitted as `[score - ci, score + ci]` because Kaggle's `confidenceInterval` is a symmetric half-width ("confidence radius"), matching the sibling utils/hle adapter. - Each result carries its Kaggle `evaluationDate` as `evaluation_timestamp` when present (~70% of results). - On the discovery path, benchmark scoring config enriches each record: `lower_is_better` from `sortOrder`, `metric_kind` from `aggregationType` (only `PERCENTAGE_PASSED -> pass_rate`; generic reductions like AVERAGE/SUM are intentionally not metric families), `metric_unit` from `displayType`, with `benchmark_id`/`aggregation_type`/`display_type` recorded in `source_metadata.additional_details`. The targeted `--benchmark` path has no access to this metadata, so those fields fall back to defaults there (a documented limitation). Bulk runs are built to fail loudly rather than silently lose data: a fetch failure is distinguished from an empty leaderboard, discovery truncation and per-benchmark conversion errors are isolated and surfaced, and the process exits non-zero when any benchmark could not be fetched, converted, or discovered, so a partial run is never reported as a clean success. - Add `utils/kaggle/adapter.py` with `--benchmark owner/slug` (repeatable), `--all`, `--limit`, and `--output-dir`; targets are de-duplicated by (owner, slug), preferring the metadata-carrying entry - Resolve the leaderboard owner as the organization slug when org-owned, else the creating user's username; owner-qualify `evaluation_id` - Write to the standard data/{benchmark}/{developer}/{model}/{uuid}.json layout via the shared helpers (make_model_info, make_source_metadata, save_evaluation_log) - Add `tests/test_kaggle_adapter.py`: 31 network-free tests covering result-type handling, CI bracketing, metadata enrichment, discovery pagination/filtering, the failure-vs-empty contract, dedup, and the non-zero-exit path - Document usage, the discovery/bounds behavior, and the targeted-path limitation in utils/README.md; retain the global-mmlu-lite adapter for backwards compatibility --- tests/test_kaggle_adapter.py | 478 +++++++++++++++++++++++++++++++ utils/README.md | 37 +++ utils/kaggle/__init__.py | 1 + utils/kaggle/adapter.py | 541 +++++++++++++++++++++++++++++++++++ 4 files changed, 1057 insertions(+) create mode 100644 tests/test_kaggle_adapter.py create mode 100644 utils/kaggle/__init__.py create mode 100644 utils/kaggle/adapter.py diff --git a/tests/test_kaggle_adapter.py b/tests/test_kaggle_adapter.py new file mode 100644 index 000000000..b7ad89a75 --- /dev/null +++ b/tests/test_kaggle_adapter.py @@ -0,0 +1,478 @@ +"""Unit tests for the generalized Kaggle Benchmarks adapter. + +These exercise the pure transformation helpers (no network) plus an +end-to-end ``convert_benchmark`` run that writes to a tmp dir and validates +the saved records against the schema. +""" + +from __future__ import annotations + +import json +import sys +from argparse import Namespace + +import pytest + +from every_eval_ever.eval_types import EvaluationLog, ScoreType +from every_eval_ever.helpers import FetchError +from utils.kaggle import adapter + + +# --------------------------------------------------------------------------- +# Fixture builders mirroring the shape of Kaggle's leaderboard API. +# --------------------------------------------------------------------------- +def numeric_task(name, value, *, ci=None, date=None): + numeric = { + "value": value, + "hasUnevenConfidenceInterval": False, + "confidenceInterval": ci if ci is not None else 0.0, + "hasConfidenceInterval": ci is not None, + } + result = { + "hasNumericResult": True, + "numericResult": numeric, + "hasBooleanResult": False, + "booleanResult": False, + "customAdditionalResults": [], + "resultCase": "numericResult", + "hasEvaluationDate": date is not None, + } + if date is not None: + result["evaluationDate"] = date + return {"benchmarkTaskName": name, "benchmarkTaskSlug": "", "result": result} + + +def boolean_task(name, passed): + return { + "benchmarkTaskName": name, + "result": { + "hasNumericResult": False, + "hasBooleanResult": True, + "booleanResult": passed, + "customAdditionalResults": [], + "resultCase": "booleanResult", + }, + } + + +def empty_task(name): + return { + "benchmarkTaskName": name, + "result": { + "hasNumericResult": False, + "hasBooleanResult": False, + "customAdditionalResults": [], + "resultCase": "none", + }, + } + + +PCT_META = { + "benchmark_id": 42, + "sort_order": "DESCENDING", + "aggregation_type": "PERCENTAGE_PASSED", + "display_type": "PERCENTAGES", +} + + +# --------------------------------------------------------------------------- +# _build_eval_result: result-type handling +# --------------------------------------------------------------------------- +def test_numeric_in_unit_range_is_bounded_continuous(): + r = adapter._build_eval_result(numeric_task("acc", 0.87654), "o", "s") + assert r.score_details.score == 0.8765 # rounded to 4dp + assert r.metric_config.score_type == ScoreType.continuous + assert r.metric_config.min_score == 0.0 + assert r.metric_config.max_score == 1.0 + + +def test_numeric_outside_unit_range_is_left_untyped(): + # Unknown scale (e.g. a count); must NOT fabricate [0, 1] bounds. + r = adapter._build_eval_result(numeric_task("count", 100.0), "o", "s") + assert r.score_details.score == 100.0 + assert r.metric_config.score_type is None + assert r.metric_config.min_score is None + assert r.metric_config.max_score is None + + +def test_boolean_results_become_binary(): + passed = adapter._build_eval_result(boolean_task("solved", True), "o", "s") + failed = adapter._build_eval_result(boolean_task("solved", False), "o", "s") + assert passed.metric_config.score_type == ScoreType.binary + assert passed.score_details.score == 1.0 + assert passed.metric_config.min_score == 0.0 + assert passed.metric_config.max_score == 1.0 + assert failed.score_details.score == 0.0 + + +def test_empty_result_is_skipped(): + assert adapter._build_eval_result(empty_task("unscored"), "o", "s") is None + + +def test_numeric_with_null_value_is_skipped(): + assert adapter._build_eval_result(numeric_task("x", None), "o", "s") is None + + +# --------------------------------------------------------------------------- +# _build_eval_result: enrichment (dates, CI, meta) +# --------------------------------------------------------------------------- +def test_evaluation_date_is_captured_as_timestamp(): + r = adapter._build_eval_result( + numeric_task("acc", 0.5, date="2026-06-20T16:40:52.0Z"), "o", "s" + ) + assert r.evaluation_timestamp == "2026-06-20T16:40:52.0Z" + + +def test_missing_evaluation_date_leaves_timestamp_unset(): + r = adapter._build_eval_result(numeric_task("acc", 0.5), "o", "s") + assert r.evaluation_timestamp is None + + +def test_confidence_interval_brackets_the_score(): + # Kaggle's confidenceInterval is a symmetric half-width around the score, + # so the bounds must bracket the score, not be centered at zero. + r = adapter._build_eval_result(numeric_task("acc", 0.87, ci=0.02), "o", "s") + ci = r.score_details.uncertainty.confidence_interval + assert ci.lower == 0.85 + assert ci.upper == 0.89 + + +def test_non_numeric_confidence_interval_is_ignored_not_fatal(): + task = numeric_task("acc", 0.5) + task["result"]["numericResult"]["hasConfidenceInterval"] = True + task["result"]["numericResult"]["confidenceInterval"] = "oops" + r = adapter._build_eval_result(task, "o", "s") + assert r.score_details.uncertainty is None + + +def test_no_uncertainty_when_ci_absent(): + r = adapter._build_eval_result(numeric_task("acc", 0.5), "o", "s") + assert r.score_details.uncertainty is None + + +def test_meta_sets_direction_unit_and_kind(): + r = adapter._build_eval_result(numeric_task("acc", 0.9), "o", "s", PCT_META) + assert r.metric_config.lower_is_better is False + assert r.metric_config.metric_kind == "pass_rate" + assert r.metric_config.metric_unit == "proportion" + assert r.metric_config.metric_name == "acc" + + +def test_ascending_sort_order_means_lower_is_better(): + meta = {**PCT_META, "sort_order": "ASCENDING"} + r = adapter._build_eval_result(numeric_task("latency", 0.3), "o", "s", meta) + assert r.metric_config.lower_is_better is True + + +def test_boolean_result_does_not_get_aggregation_kind_or_unit(): + # Aggregation-derived kind/unit describe a numeric metric, not a 0/1 result. + r = adapter._build_eval_result(boolean_task("solved", True), "o", "s", PCT_META) + assert r.metric_config.metric_kind is None + assert r.metric_config.metric_unit is None + assert r.metric_config.score_type == ScoreType.binary + + +def test_counts_display_type_maps_to_count_unit(): + meta = {**PCT_META, "display_type": "COUNTS", "aggregation_type": None} + r = adapter._build_eval_result(numeric_task("n", 100.0), "o", "s", meta) + assert r.metric_config.metric_unit == "count" + assert r.metric_config.metric_kind is None + + +def test_without_meta_direction_defaults_false_and_no_unit(): + r = adapter._build_eval_result(numeric_task("acc", 0.9), "o", "s") + assert r.metric_config.lower_is_better is False + assert r.metric_config.metric_unit is None + assert r.metric_config.metric_kind is None + + +def test_task_name_falls_back_to_slug(): + task = numeric_task("", 0.5) + r = adapter._build_eval_result(task, "owner", "myslug") + assert r.evaluation_name == "myslug" + + +# --------------------------------------------------------------------------- +# _benchmark_owner / _benchmark_meta +# --------------------------------------------------------------------------- +def test_owner_prefers_organization_slug(): + bench = { + "organization": {"slug": "cohere-labs"}, + "ownerUser": {"userName": "someuser"}, + } + assert adapter._benchmark_owner(bench) == "cohere-labs" + + +def test_owner_falls_back_to_username(): + bench = {"organization": None, "ownerUser": {"userName": "someuser"}} + assert adapter._benchmark_owner(bench) == "someuser" + + +def test_owner_none_when_unresolvable(): + assert adapter._benchmark_owner({}) is None + + +def test_benchmark_meta_extracts_scoring_config(): + bench = { + "id": 7, + "task": { + "version": { + "sortOrder": "DESCENDING", + "aggregationType": "PERCENTAGE_PASSED", + "displayType": "PERCENTAGES", + } + }, + } + meta = adapter._benchmark_meta(bench) + assert meta == { + "benchmark_id": 7, + "sort_order": "DESCENDING", + "aggregation_type": "PERCENTAGE_PASSED", + "display_type": "PERCENTAGES", + } + + +def test_benchmark_meta_handles_missing_version(): + meta = adapter._benchmark_meta({"id": 1}) + assert meta["sort_order"] is None + assert meta["aggregation_type"] is None + + +# --------------------------------------------------------------------------- +# list_benchmarks: discovery pagination + filtering (network mocked) +# --------------------------------------------------------------------------- +class _FakeResp: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + +class _FakeSession: + """Returns two canned ListBenchmarks pages keyed by pageToken.""" + + def __init__(self, pages): + self._pages = pages + self.calls = [] + + def post(self, url, json=None, headers=None, timeout=None): + token = json["pageToken"] + self.calls.append(token) + return _FakeResp(self._pages[token]) + + +def test_list_benchmarks_paginates_and_filters_unpublished(monkeypatch): + pages = { + "": { + "benchmarks": [ + { + "slug": "global-mmlu-lite", + "name": " Global MMLU Lite ", + "published": True, + "organization": {"slug": "cohere-labs"}, + "ownerUser": {"userName": "creator"}, + "task": {"version": {"sortOrder": "DESCENDING"}}, + }, + # Unpublished -> must be filtered out. + {"slug": "draft", "published": False, "ownerUser": {"userName": "x"}}, + ], + "nextPageToken": "p2", + }, + "p2": { + "benchmarks": [ + { + "slug": "solo", + "name": "Solo", + "published": True, + "organization": None, + "ownerUser": {"userName": "alice"}, + } + ], + "nextPageToken": "", + }, + } + fake = _FakeSession(pages) + monkeypatch.setattr(adapter, "_kaggle_session", lambda: (fake, "xsrf-token")) + + out = list(adapter.list_benchmarks()) + + assert fake.calls == ["", "p2"] # paginated via nextPageToken + assert [(b["owner"], b["slug"]) for b in out] == [ + ("cohere-labs", "global-mmlu-lite"), # org slug preferred over creator + ("alice", "solo"), # falls back to username + ] + assert out[0]["name"] == "Global MMLU Lite" # stripped + assert out[0]["meta"]["sort_order"] == "DESCENDING" + + +# --------------------------------------------------------------------------- +# fetch_leaderboard: failure vs empty contract (network mocked) +# --------------------------------------------------------------------------- +def test_fetch_leaderboard_returns_none_on_fetch_error(monkeypatch): + def boom(url): + raise FetchError("boom") + + monkeypatch.setattr(adapter, "fetch_json", boom) + assert adapter.fetch_leaderboard("o", "s") is None + + +def test_fetch_leaderboard_returns_empty_list_for_no_submissions(monkeypatch): + monkeypatch.setattr(adapter, "fetch_json", lambda url: {"rows": []}) + assert adapter.fetch_leaderboard("o", "s") == [] + + +def test_fetch_leaderboard_returns_none_on_malformed_shape(monkeypatch): + # A 200 with an error envelope / no list-valued rows is a failure, not empty. + monkeypatch.setattr(adapter, "fetch_json", lambda url: {"error": "nope"}) + assert adapter.fetch_leaderboard("o", "s") is None + + +# --------------------------------------------------------------------------- +# _resolve_targets: dedup + discovery-failure flag +# --------------------------------------------------------------------------- +def test_resolve_targets_dedupes_repeated_benchmarks(): + args = Namespace( + benchmark=["cohere-labs/global-mmlu-lite", "cohere-labs/global-mmlu-lite"], + all=False, + limit=None, + ) + targets, discovery_failed = adapter._resolve_targets(args) + assert len(targets) == 1 + assert discovery_failed is False + + +def test_resolve_targets_dedup_prefers_meta_carrying_entry(monkeypatch): + # An explicit (meta-less) --benchmark overlapping an --all discovery entry + # must keep the discovered entry's meta so enrichment isn't lost. + def discover(): + yield { + "owner": "cohere-labs", + "slug": "global-mmlu-lite", + "name": "Global MMLU Lite", + "meta": {"sort_order": "DESCENDING", "display_type": "PERCENTAGES"}, + } + + monkeypatch.setattr(adapter, "list_benchmarks", discover) + args = Namespace( + benchmark=["cohere-labs/global-mmlu-lite"], all=True, limit=None + ) + targets, _ = adapter._resolve_targets(args) + assert len(targets) == 1 + assert targets[0]["meta"]["display_type"] == "PERCENTAGES" + + +def test_resolve_targets_flags_truncated_discovery(monkeypatch): + def partial(): + yield {"owner": "o", "slug": "a", "name": "A", "meta": {}} + raise FetchError("page 2 failed") + + monkeypatch.setattr(adapter, "list_benchmarks", partial) + args = Namespace(benchmark=None, all=True, limit=None) + targets, discovery_failed = adapter._resolve_targets(args) + assert [t["slug"] for t in targets] == ["a"] # partial progress kept + assert discovery_failed is True + + +# --------------------------------------------------------------------------- +# main: non-zero exit when a fetch fails, good benchmark still written +# --------------------------------------------------------------------------- +def test_main_exits_nonzero_when_a_fetch_fails(monkeypatch, tmp_path): + def fake_fetch(owner, slug): + return None if owner == "bad" else sample_rows()[:1] + + monkeypatch.setattr(adapter, "fetch_leaderboard", fake_fetch) + monkeypatch.setattr( + sys, + "argv", + [ + "adapter", + "--benchmark", + "good/one", + "--benchmark", + "bad/two", + "--output-dir", + str(tmp_path), + ], + ) + with pytest.raises(SystemExit) as exc: + adapter.main() + assert exc.value.code == 1 + # The good benchmark was still converted despite the other's failure. + assert list(tmp_path.rglob("*.json")) + + +# --------------------------------------------------------------------------- +# convert_benchmark: end-to-end, schema-valid output +# --------------------------------------------------------------------------- +def sample_rows(): + return [ + { + "modelVersionName": "GPT-4o", + "modelVersionSlug": "gpt-4o-2024-05-13", + "taskResults": [ + numeric_task("Overall", 0.71, date="2026-06-20T16:40:52Z"), + boolean_task("Edge Case", True), + empty_task("Unscored Task"), # dropped, no usable score + ], + }, + # Row with only empty results -> produces no file. + { + "modelVersionName": "Blank", + "modelVersionSlug": "blank-model", + "taskResults": [empty_task("Nothing")], + }, + # Row missing the slug -> skipped entirely. + {"modelVersionName": "No Slug", "taskResults": [numeric_task("x", 0.5)]}, + ] + + +def test_convert_benchmark_writes_only_rows_with_results(tmp_path): + count = adapter.convert_benchmark( + "cohere-labs", + "demo", + "Demo Benchmark", + sample_rows(), + str(tmp_path), + "123.0", + meta=PCT_META, + ) + assert count == 1 # only gpt-4o has usable results + files = list(tmp_path.rglob("*.json")) + assert len(files) == 1 + + +def test_convert_benchmark_output_validates_and_is_enriched(tmp_path): + adapter.convert_benchmark( + "cohere-labs", + "demo", + "Demo Benchmark", + sample_rows(), + str(tmp_path), + "123.0", + meta=PCT_META, + ) + path = next(tmp_path.rglob("*.json")) + log = EvaluationLog.model_validate(json.loads(path.read_text())) + + assert log.schema_version == "0.2.2" + assert log.model_info.id == "openai/gpt-4o-2024-05-13" + assert log.source_metadata.source_organization_name == "cohere-labs" + + details = log.source_metadata.additional_details + assert details["platform"] == "kaggle" + assert details["benchmark_id"] == "42" + assert details["aggregation_type"] == "PERCENTAGE_PASSED" + assert details["display_type"] == "PERCENTAGES" + + # Empty task dropped; numeric + boolean kept. + names = {r.evaluation_name for r in log.evaluation_results} + assert names == {"Overall", "Edge Case"} + + overall = next(r for r in log.evaluation_results if r.evaluation_name == "Overall") + assert overall.evaluation_timestamp == "2026-06-20T16:40:52Z" + assert overall.metric_config.metric_unit == "proportion" + assert overall.metric_config.lower_is_better is False diff --git a/utils/README.md b/utils/README.md index c6e691e45..74a92afb9 100644 --- a/utils/README.md +++ b/utils/README.md @@ -16,6 +16,7 @@ Each adapter is run with `uv run python -m utils..adapter`. | `bfcl` | BFCL leaderboard CSV | Converts BFCL leaderboard data with per-metric evaluation names and bounded continuous scores. | | `sciarena` | SciArena leaderboard API | Converts SciArena leaderboard results. | | `global-mmlu-lite` | Kaggle API | Fetches Global MMLU Lite leaderboard results from Kaggle. | +| `kaggle` | Kaggle Benchmarks API | Generalized Kaggle Community Benchmarks adapter. Converts any benchmark via `--benchmark owner/slug`, or discovers and converts all published benchmarks via `--all` (uses the `ListBenchmarks` RPC). Handles numeric and boolean task results. | | `hfopenllm_v2` | HuggingFace Spaces API | Fetches the Open LLM Leaderboard v2 (4576+ models). | | `helm` | HELM leaderboard | Converts HELM leaderboard data. Supports `--leaderboard_name` for Capabilities/Lite/Classic/Instruct/MMLU. | | `llm_stats` | LLM Stats API | Converts LLM Stats model, benchmark, and score API data into `data/llm-stats/`. | @@ -37,6 +38,42 @@ Each adapter is run with `uv run python -m utils..adapter`. generated artifacts. Prefer temporary output paths for smoke runs unless a data refresh is intentionally part of the change. +### Kaggle Benchmarks + +Convert one or more named benchmarks (smoke run, output outside the repo): + +```bash +uv run python -m utils.kaggle.adapter \ + --benchmark cohere-labs/global-mmlu-lite \ + --output-dir /tmp/eee-kaggle +``` + +Discover and convert all published benchmarks via the `ListBenchmarks` RPC +(slow — 1000+ benchmarks; use `--limit` to cap during testing): + +```bash +uv run python -m utils.kaggle.adapter --all --limit 10 --output-dir /tmp/eee-kaggle +``` + +Notes: +- The `--all` discovery path calls an unauthenticated Kaggle internal RPC + (`benchmarks.BenchmarkService/ListBenchmarks`) that requires the anonymous + XSRF cookie/header handshake; the adapter handles this automatically. +- Numeric results in `[0, 1]` are emitted as bounded `continuous` metrics and + boolean results as `binary`; numeric results outside `[0, 1]` have an unknown + scale (Kaggle does not expose metric bounds), so they are left untyped rather + than given fabricated `[0, 1]` bounds. +- Each result carries its Kaggle `evaluationDate` as `evaluation_timestamp` when + present (~70% of results). +- The `--all` path additionally enriches each record from the benchmark's + scoring config (only available via `ListBenchmarks`): `lower_is_better` from + `sortOrder`, `metric_kind` from `aggregationType`, `metric_unit` from + `displayType`, and `benchmark_id`/`aggregation_type`/`display_type` in + `source_metadata.additional_details`. The targeted `--benchmark` path does not + have this metadata, so those fields fall back to defaults there. +- This supersedes the single-purpose `global-mmlu-lite` adapter, which is kept + for backwards compatibility. + ### Vals.ai Run a live smoke export from the repository root, writing generated output diff --git a/utils/kaggle/__init__.py b/utils/kaggle/__init__.py new file mode 100644 index 000000000..2c3e40b21 --- /dev/null +++ b/utils/kaggle/__init__.py @@ -0,0 +1 @@ +"""Generalized Kaggle Community Benchmarks adapter for the EvalEval schema.""" diff --git a/utils/kaggle/adapter.py b/utils/kaggle/adapter.py new file mode 100644 index 000000000..80c188c76 --- /dev/null +++ b/utils/kaggle/adapter.py @@ -0,0 +1,541 @@ +""" +Generalized adapter for Kaggle Community Benchmarks. + +Kaggle Benchmarks are community-published evaluation suites. Each benchmark is +identified by an ``owner/slug`` pair (e.g. ``cohere-labs/global-mmlu-lite``) and +exposes a public leaderboard. This adapter: + +1. Optionally enumerates ALL published benchmarks via Kaggle's (undocumented but + unauthenticated) ``ListBenchmarks`` RPC, and/or accepts explicit ``owner/slug`` + pairs on the command line. +2. Fetches each benchmark's leaderboard from the public REST endpoint. +3. Converts every model row (and every task result within it) into the EvalEval + schema, handling both numeric and boolean task results. + +Data sources: +- List: POST https://www.kaggle.com/api/i/benchmarks.BenchmarkService/ListBenchmarks +- Leaderboard: GET https://www.kaggle.com/api/v1/benchmarks/{owner}/{slug}/leaderboard + +Usage: + # Convert specific benchmark(s) + uv run python -m utils.kaggle.adapter \ + --benchmark cohere-labs/global-mmlu-lite \ + --output-dir /tmp/eee-kaggle + + # Discover and convert all published benchmarks (slow) + uv run python -m utils.kaggle.adapter --all --output-dir data/kaggle + + # Smoke test: discover but only convert the first 5 + uv run python -m utils.kaggle.adapter --all --limit 5 --output-dir /tmp/eee-kaggle +""" + +import argparse +import time +from typing import Dict, Iterator, List, Optional, Tuple + +import requests + +from every_eval_ever.eval_types import ( + ConfidenceInterval, + EvalLibrary, + EvaluationLog, + EvaluationResult, + EvaluatorRelationship, + MetricConfig, + ScoreDetails, + ScoreType, + SourceDataUrl, + Uncertainty, +) +from every_eval_ever.helpers import ( + SCHEMA_VERSION, + FetchError, + fetch_json, + make_model_info, + make_source_metadata, + save_evaluation_log, +) + +KAGGLE_BASE = "https://www.kaggle.com" +LIST_RPC_URL = f"{KAGGLE_BASE}/api/i/benchmarks.BenchmarkService/ListBenchmarks" +LEADERBOARD_URL = KAGGLE_BASE + "/api/v1/benchmarks/{owner}/{slug}/leaderboard" + +# Server-side cap on ListBenchmarks page size. +LIST_PAGE_SIZE = 200 + + +def _kaggle_session() -> Tuple[requests.Session, str]: + """Open a session with an anonymous XSRF token. + + The ListBenchmarks RPC is unauthenticated but requires the XSRF cookie/header + handshake that Kaggle hands out on any page load. + """ + session = requests.Session() + try: + session.get(f"{KAGGLE_BASE}/benchmarks", timeout=60) + except requests.RequestException as e: + raise FetchError(f"Kaggle XSRF handshake failed: {e}") from e + xsrf = session.cookies.get("XSRF-TOKEN") + if not xsrf: + raise FetchError("Could not obtain XSRF-TOKEN cookie from Kaggle") + return session, xsrf + + +# Kaggle aggregationType -> normalized metric_kind (for safe cross-source aggregation). +# Only map aggregation types that describe the *measured value*; generic reduction +# modes (AVERAGE, SUM) are not metric families and would conflate unrelated metrics, +# so they are intentionally omitted (the raw type is kept in source additional_details). +_AGGREGATION_TO_METRIC_KIND = { + "PERCENTAGE_PASSED": "pass_rate", +} +# Kaggle displayType -> metric_unit. PERCENTAGES are stored as proportions in [0, 1]. +_DISPLAY_TO_UNIT = { + "PERCENTAGES": "proportion", + "COUNTS": "count", +} + + +def _benchmark_owner(bench: dict) -> Optional[str]: + """Resolve the leaderboard URL owner for a benchmark object. + + Org-owned benchmarks are addressed by the organization slug; otherwise the + creating user's username is used. + """ + org = bench.get("organization") + if org and org.get("slug"): + return org["slug"] + return (bench.get("ownerUser") or {}).get("userName") + + +def _benchmark_meta(bench: dict) -> Dict[str, Optional[str]]: + """Extract benchmark-level metadata (only present on the discovery path). + + The leaderboard endpoint omits these, but ``ListBenchmarks`` carries the + scoring config we need to set direction (``lower_is_better``) and metric + unit/kind, which would otherwise be guessed. + """ + version = (bench.get("task") or {}).get("version") or {} + return { + "benchmark_id": bench.get("id"), + "sort_order": version.get("sortOrder"), + "aggregation_type": version.get("aggregationType"), + "display_type": version.get("displayType"), + } + + +def list_benchmarks() -> Iterator[Dict[str, object]]: + """Enumerate published Kaggle benchmarks. + + Yields dicts with ``owner``, ``slug``, ``name`` and ``meta``. The leaderboard + URL owner is the organization slug when the benchmark is org-owned, otherwise + the creating user's username. + """ + session, xsrf = _kaggle_session() + headers = { + "content-type": "application/json", + "accept": "application/json", + "x-xsrf-token": xsrf, + } + page_token = "" + while True: + body = {"filter": {}, "pageSize": LIST_PAGE_SIZE, "pageToken": page_token} + # On a page failure, raise after the benchmarks already yielded: the + # caller keeps that partial progress but is told discovery was truncated, + # so a `--all` run is never reported as a clean success when its tail is + # missing. + try: + resp = session.post(LIST_RPC_URL, json=body, headers=headers, timeout=60) + resp.raise_for_status() + data = resp.json() + except (requests.RequestException, ValueError) as e: + raise FetchError(f"ListBenchmarks page fetch failed: {e}") from e + # A 200 error envelope or changed RPC schema would otherwise look like an + # empty final page and end discovery as a clean success — validate the + # shape so a broken response is a failure, not a silently empty corpus. + if not isinstance(data, dict) or not isinstance(data.get("benchmarks"), list): + raise FetchError( + f"unexpected ListBenchmarks response shape (pageToken={page_token!r})" + ) + for bench in data["benchmarks"]: + if not bench.get("published"): + continue + slug = bench.get("slug") + if not slug: + continue + owner = _benchmark_owner(bench) + if not owner: + continue + yield { + "owner": owner, + "slug": slug, + "name": bench.get("name", slug).strip(), + "meta": _benchmark_meta(bench), + } + page_token = data.get("nextPageToken") or "" + if not page_token: + break + + +def fetch_leaderboard(owner: str, slug: str) -> Optional[List[dict]]: + """Fetch a benchmark leaderboard. + + Returns the list of rows on success (possibly empty for a benchmark with no + submissions), or ``None`` when the fetch itself failed (HTTP/network/parse + error, including the 403 returned for non-existent/private benchmarks). The + caller distinguishes these: an empty list is "no data", ``None`` is a failure + that must be surfaced rather than silently treated as empty. + """ + url = LEADERBOARD_URL.format(owner=owner, slug=slug) + try: + data = fetch_json(url) + except FetchError as e: + print(f" ! could not fetch leaderboard for {owner}/{slug}: {e}") + return None + # A 200 with an error envelope or a changed schema (no list-valued `rows`) is + # a fetch failure, not an empty leaderboard — don't let an upstream API break + # masquerade as "no data". + rows = data.get("rows") if isinstance(data, dict) else None + if not isinstance(rows, list): + print(f" ! unexpected leaderboard response shape for {owner}/{slug}") + return None + return rows + + +def _build_eval_result( + task: dict, owner: str, slug: str, meta: Optional[Dict[str, object]] = None +) -> Optional[EvaluationResult]: + """Convert a single ``taskResult`` entry into an EvaluationResult. + + Handles numeric results (continuous score in [0, 1]) and boolean results + (binary 0/1). Other result cases (e.g. custom tuples) are skipped. + + ``meta`` carries benchmark-level scoring config (from the discovery path) + used to set ``lower_is_better`` and the metric unit/kind; when absent these + fall back to sensible defaults. + """ + meta = meta or {} + task_name = task.get("benchmarkTaskName") or slug + result = task.get("result", {}) + + # Direction: Kaggle sorts leaderboards DESCENDING when higher is better. + lower_is_better = meta.get("sort_order") == "ASCENDING" + + score: Optional[float] = None + score_type: Optional[ScoreType] = None + min_score: Optional[float] = None + max_score: Optional[float] = None + metric_kind: Optional[str] = None + metric_unit: Optional[str] = None + uncertainty: Optional[Uncertainty] = None + + if result.get("hasNumericResult"): + numeric = result.get("numericResult") or {} + value = numeric.get("value") + if value is None: + return None + try: + score = float(value) + except (TypeError, ValueError): + return None + # Aggregation-derived kind/unit describe a numeric metric; they are + # meaningless for a single boolean pass/fail, so only attach them here. + metric_kind = _AGGREGATION_TO_METRIC_KIND.get(meta.get("aggregation_type")) + metric_unit = _DISPLAY_TO_UNIT.get(meta.get("display_type")) + # Kaggle's API does not expose a metric's scale. Most numeric results are + # accuracies / pass-rates in [0, 1]; for those we can safely declare a + # bounded continuous metric. Values outside [0, 1] (counts, percentages, + # latencies, ...) have an unknown scale, so we leave score_type and bounds + # unset rather than fabricate misleading [0, 1] bounds. + # Only declare bounded [0, 1] when the value plausibly is a proportion. + # A COUNTS metric whose value happens to fall in [0, 1] (e.g. a count of + # 0 or 1) must not claim a max possible score of 1, so skip bounding when + # Kaggle marks the metric as counts. + if meta.get("display_type") != "COUNTS" and 0.0 <= score <= 1.0: + score_type = ScoreType.continuous + min_score, max_score = 0.0, 1.0 + # Kaggle's `confidenceInterval` is a symmetric half-width ("confidence + # radius") around the score, so the emitted bounds must bracket the score + # (cf. utils/hle/adapter.py), not be centered at zero. + if numeric.get("hasConfidenceInterval"): + try: + ci = float(numeric.get("confidenceInterval")) + except (TypeError, ValueError): + ci = None + if ci is not None: + uncertainty = Uncertainty( + confidence_interval=ConfidenceInterval( + lower=round(score - ci, 4), + upper=round(score + ci, 4), + method="unknown", + ) + ) + elif result.get("hasBooleanResult"): + score_type = ScoreType.binary + score = 1.0 if result.get("booleanResult") else 0.0 + min_score, max_score = 0.0, 1.0 + else: + return None + + benchmark_url = f"{KAGGLE_BASE}/benchmarks/{owner}/{slug}" + return EvaluationResult( + evaluation_name=task_name, + evaluation_timestamp=result.get("evaluationDate"), + source_data=SourceDataUrl( + dataset_name=slug, + source_type="url", + url=[benchmark_url], + ), + metric_config=MetricConfig( + evaluation_description=f"Kaggle Benchmarks - {task_name}", + metric_name=task_name, + metric_kind=metric_kind, + metric_unit=metric_unit, + lower_is_better=lower_is_better, + score_type=score_type, + min_score=min_score, + max_score=max_score, + ), + score_details=ScoreDetails( + score=round(score, 4), + uncertainty=uncertainty, + ), + ) + + +def convert_benchmark( + owner: str, + slug: str, + name: str, + rows: List[dict], + output_dir: str, + retrieved_timestamp: str, + meta: Optional[Dict[str, object]] = None, +) -> int: + """Convert all leaderboard rows for one benchmark and save them. Returns count.""" + meta = meta or {} + benchmark_url = f"{KAGGLE_BASE}/benchmarks/{owner}/{slug}" + + source_details = { + "platform": "kaggle", + "benchmark_owner": owner, + "benchmark_url": benchmark_url, + } + for key in ("benchmark_id", "aggregation_type", "display_type"): + if meta.get(key) is not None: + source_details[key] = str(meta[key]) + + count = 0 + + for row in rows: + model_slug = row.get("modelVersionSlug") + if not model_slug: + continue + model_display_name = row.get("modelVersionName", "") + + eval_results: List[EvaluationResult] = [] + for task in row.get("taskResults", []): + result = _build_eval_result(task, owner, slug, meta) + if result is not None: + eval_results.append(result) + if not eval_results: + continue + + model_info = make_model_info( + model_name=model_slug, + additional_details={"display_name": model_display_name} + if model_display_name and model_display_name != model_slug + else None, + ) + + # Owner-qualify the id: benchmarks are owner/slug resources, so two + # owners sharing a slug (with the same model and run timestamp) would + # otherwise collide on this logical run identifier. + evaluation_id = ( + f"{owner}/{slug}/{model_info.id.replace('/', '_')}/{retrieved_timestamp}" + ) + eval_log = EvaluationLog( + schema_version=SCHEMA_VERSION, + evaluation_id=evaluation_id, + retrieved_timestamp=retrieved_timestamp, + source_metadata=make_source_metadata( + source_name=f"{name} (Kaggle Benchmarks)", + organization_name=owner, + organization_url="https://www.kaggle.com", + evaluator_relationship=EvaluatorRelationship.third_party, + additional_details=source_details, + ), + eval_library=EvalLibrary( + name="kaggle benchmarks", + version="unknown", + additional_details={"url": benchmark_url}, + ), + model_info=model_info, + evaluation_results=eval_results, + ) + + if "/" in model_info.id: + dev, _ = model_info.id.split("/", 1) + else: + dev = "unknown" + filepath = save_evaluation_log(eval_log, output_dir, dev, model_slug) + print(f" saved {filepath}") + count += 1 + + return count + + +def _resolve_targets(args) -> Tuple[List[Dict[str, object]], bool]: + """Build the (de-duplicated) list of benchmarks to process from CLI args. + + Returns the targets and a flag indicating whether ``--all`` discovery was + truncated by a fetch error, so the caller can exit non-zero instead of + reporting an incomplete corpus as success. + + Note: the ``--benchmark`` path has no benchmark-level ``meta`` (the + leaderboard endpoint omits it), so direction/unit enrichment is only applied + on the ``--all`` discovery path. + """ + targets: List[Dict[str, object]] = [] + for spec in args.benchmark or []: + if "/" not in spec: + raise SystemExit(f"--benchmark expects owner/slug, got: {spec!r}") + owner, slug = spec.split("/", 1) + targets.append({"owner": owner, "slug": slug, "name": slug, "meta": {}}) + + discovery_failed = False + if args.all: + print("Discovering published benchmarks via Kaggle ListBenchmarks RPC...") + try: + for bench in list_benchmarks(): + targets.append(bench) + if args.limit and len(targets) >= args.limit: + break + except FetchError as e: + # Keep what was discovered before the failure, but record that the + # corpus is incomplete so the run does not exit 0. + print(f" ! benchmark discovery truncated: {e}") + discovery_failed = True + + # De-duplicate by (owner, slug), preferring the entry that carries benchmark + # meta (the discovered one) so enrichment is not lost to a bare --benchmark + # duplicate or an explicit/--all overlap. + deduped: Dict[Tuple[str, str], Dict[str, object]] = {} + for t in targets: + key = (t["owner"], t["slug"]) + existing = deduped.get(key) + if existing is None or (not existing.get("meta") and t.get("meta")): + deduped[key] = t + return list(deduped.values()), discovery_failed + + +def main(): + parser = argparse.ArgumentParser( + description="Convert Kaggle Community Benchmarks leaderboards to the EvalEval schema." + ) + parser.add_argument( + "--benchmark", + action="append", + metavar="OWNER/SLUG", + help="Specific benchmark to convert (repeatable), e.g. cohere-labs/global-mmlu-lite", + ) + parser.add_argument( + "--all", + action="store_true", + help="Discover and convert all published benchmarks via the ListBenchmarks RPC.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="When using --all, stop after this many benchmarks (smoke testing).", + ) + parser.add_argument( + "--output-dir", + default="data/kaggle", + help="Base output directory (default: data/kaggle).", + ) + args = parser.parse_args() + + if not args.benchmark and not args.all: + parser.error("provide --benchmark OWNER/SLUG and/or --all") + + targets, discovery_failed = _resolve_targets(args) + retrieved_timestamp = str(time.time()) + + print("=" * 60) + print(f"Converting {len(targets)} Kaggle benchmark(s) -> {args.output_dir}") + print("=" * 60) + + total_models = 0 + total_benchmarks = 0 + failed_fetches = [] + conversion_failures = [] + empty_or_dropped = [] + for bench in targets: + owner, slug, name = bench["owner"], bench["slug"], bench["name"] + print(f"\n[{owner}/{slug}] {name}") + rows = fetch_leaderboard(owner, slug) + if rows is None: + # Fetch failed (transient/HTTP/parse error) — distinct from an empty + # leaderboard, so we don't report it as a clean "no data" success. + failed_fetches.append(f"{owner}/{slug}") + continue + if not rows: + print(" (no rows)") + continue + try: + count = convert_benchmark( + owner, + slug, + name, + rows, + args.output_dir, + retrieved_timestamp, + meta=bench.get("meta"), + ) + except (TypeError, AttributeError, KeyError) as e: + # A single malformed benchmark payload (e.g. taskResults: null) must + # not abort the whole --all run — isolate it as a per-benchmark + # failure, keep going, and exit non-zero at the end. + print(f" ! conversion failed for {owner}/{slug}: {e}") + conversion_failures.append(f"{owner}/{slug}") + continue + print(f" -> {count} model(s)") + if count == 0: + # A non-empty leaderboard that produced nothing means every row was + # unusable (e.g. an unhandled result type) — surface it, don't hide it. + empty_or_dropped.append(f"{owner}/{slug}") + total_models += count + total_benchmarks += 1 + + print("\n" + "=" * 60) + print(f"Done: {total_models} models across {total_benchmarks} benchmarks") + if empty_or_dropped: + print( + f"WARNING: {len(empty_or_dropped)} non-empty leaderboard(s) produced " + f"0 records (all rows unusable): {', '.join(empty_or_dropped)}" + ) + if failed_fetches: + print( + f"WARNING: {len(failed_fetches)} benchmark(s) could not be fetched " + f"and were skipped: {', '.join(failed_fetches)}" + ) + if conversion_failures: + print( + f"WARNING: {len(conversion_failures)} benchmark(s) failed to convert " + f"(malformed payload): {', '.join(conversion_failures)}" + ) + if discovery_failed: + print( + "WARNING: benchmark discovery was truncated by a fetch error; " + "the converted set is incomplete." + ) + print("=" * 60) + # Non-zero exit when any fetch/conversion failed or discovery was truncated, + # so callers and CI don't read a partial run as a clean success. + if failed_fetches or conversion_failures or discovery_failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main()