diff --git a/every_eval_ever/converters/README.md b/every_eval_ever/converters/README.md index 50c94f453..0348beff0 100644 --- a/every_eval_ever/converters/README.md +++ b/every_eval_ever/converters/README.md @@ -122,7 +122,7 @@ options: You can convert HELM evaluation log into unified schema via `every_eval_ever/converters/helm/__main__.py`. For example: ```bash -uv run --extra helm every_eval_ever convert helm --log_path tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0 +uv run --extra helm every_eval_ever convert helm --log_path tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0 ``` The automatic conversion script requires following files generated by HELM to work correctly: diff --git a/every_eval_ever/converters/helm/__main__.py b/every_eval_ever/converters/helm/__main__.py index 33129c341..4a8090ba9 100644 --- a/every_eval_ever/converters/helm/__main__.py +++ b/every_eval_ever/converters/helm/__main__.py @@ -24,7 +24,7 @@ def parse_args(): parser.add_argument( '--log_path', type=str, - default='tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + default='tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', help='Path to directory with single evaluaion or multiple evaluations to convert', ) parser.add_argument('--output_dir', type=str, default='data') diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json b/tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json similarity index 100% rename from tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json rename to tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json b/tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json similarity index 100% rename from tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json rename to tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json b/tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json similarity index 100% rename from tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json rename to tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json b/tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json similarity index 100% rename from tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json rename to tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json b/tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json similarity index 100% rename from tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json rename to tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json b/tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json similarity index 100% rename from tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json rename to tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json b/tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json similarity index 100% rename from tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json rename to tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json b/tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json similarity index 100% rename from tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json rename to tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json b/tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json similarity index 100% rename from tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json rename to tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json b/tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json similarity index 100% rename from tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json rename to tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/per_instance_stats.json b/tests/data/helm/narrative_qa_model=openai_gpt2/per_instance_stats.json similarity index 100% rename from tests/data/helm/narrative_qa:model=openai_gpt2/per_instance_stats.json rename to tests/data/helm/narrative_qa_model=openai_gpt2/per_instance_stats.json diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/run_spec.json b/tests/data/helm/narrative_qa_model=openai_gpt2/run_spec.json similarity index 100% rename from tests/data/helm/narrative_qa:model=openai_gpt2/run_spec.json rename to tests/data/helm/narrative_qa_model=openai_gpt2/run_spec.json diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/scenario.json b/tests/data/helm/narrative_qa_model=openai_gpt2/scenario.json similarity index 100% rename from tests/data/helm/narrative_qa:model=openai_gpt2/scenario.json rename to tests/data/helm/narrative_qa_model=openai_gpt2/scenario.json diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/scenario_state.json b/tests/data/helm/narrative_qa_model=openai_gpt2/scenario_state.json similarity index 100% rename from tests/data/helm/narrative_qa:model=openai_gpt2/scenario_state.json rename to tests/data/helm/narrative_qa_model=openai_gpt2/scenario_state.json diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/stats.json b/tests/data/helm/narrative_qa_model=openai_gpt2/stats.json similarity index 100% rename from tests/data/helm/narrative_qa:model=openai_gpt2/stats.json rename to tests/data/helm/narrative_qa_model=openai_gpt2/stats.json diff --git a/tests/test_helm_adapter.py b/tests/test_helm_adapter.py index 5d2bbb9a0..b1cf53907 100644 --- a/tests/test_helm_adapter.py +++ b/tests/test_helm_adapter.py @@ -66,7 +66,7 @@ def test_mmlu_eval(): converted_eval = _load_eval( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) @@ -110,7 +110,7 @@ def test_hellswag_eval(): converted_eval = _load_eval( adapter, - 'tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0', + 'tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0', metadata_args, ) @@ -152,7 +152,7 @@ def test_narrativeqa_eval(): } converted_eval = _load_eval( - adapter, 'tests/data/helm/narrative_qa:model=openai_gpt2', metadata_args + adapter, 'tests/data/helm/narrative_qa_model=openai_gpt2', metadata_args ) assert converted_eval.evaluation_timestamp is not None @@ -192,7 +192,7 @@ def test_missing_model_deployment_falls_back_to_model(): import json src = Path( 'tests/data/helm/' - 'mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' + 'mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' ) with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/test_helm_instance_level_adapter.py b/tests/test_helm_instance_level_adapter.py index 98e21fcc3..b4a058044 100644 --- a/tests/test_helm_instance_level_adapter.py +++ b/tests/test_helm_instance_level_adapter.py @@ -123,7 +123,7 @@ def test_mmlu_instance_level(): converted_eval, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) @@ -188,7 +188,7 @@ def test_hellaswag_instance_level(): _, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0', + 'tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0', metadata_args, ) @@ -234,7 +234,7 @@ def test_narrativeqa_instance_level(): _, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/narrative_qa:model=openai_gpt2', + 'tests/data/helm/narrative_qa_model=openai_gpt2', metadata_args, ) @@ -282,7 +282,7 @@ def test_per_sample_core_metric_rows_are_emitted(): } _, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) rows_for_id147 = [ @@ -311,7 +311,7 @@ def test_bookkeeping_stats_are_not_emitted_as_metric_rows(): } _, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) @@ -349,7 +349,7 @@ def test_graded_core_metrics_are_not_binary_correctness(): } _, narr_logs = _load_instance_level_data( adapter, - 'tests/data/helm/narrative_qa:model=openai_gpt2', + 'tests/data/helm/narrative_qa_model=openai_gpt2', metadata_args2, ) # Graded generation metrics also should not be coerced into a @@ -381,7 +381,7 @@ def test_is_correct_is_true_for_correct_exact_match_rows(): } _, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) # Correctness metrics are the exception: positive exact-match rows @@ -442,7 +442,7 @@ def test_evaluation_result_id_helper_disambiguates_split_and_perturbation(): def test_total_rows_matches_core_per_instance_stats(): _require_helm() - fixture = 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' + fixture = 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' adapter = HELMAdapter() with tempfile.TemporaryDirectory() as tmpdir: metadata_args = { @@ -478,7 +478,7 @@ def test_instance_evaluation_result_ids_join_to_aggregate_results(): } converted_eval, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) @@ -512,7 +512,7 @@ def test_aggregate_evaluation_result_ids_are_unique_and_non_null(): } converted_eval, _ = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) @@ -588,7 +588,7 @@ def test_reasoning_traces_none_does_not_break_conversion(monkeypatch): } _, instance_logs = _load_instance_level_data( adapter, - 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', + 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2', metadata_args, ) assert instance_logs