fix(helm): remove ':' from test fixture paths so repo is cloneable on Windows

every_eval_ever/converters/README.md

@@ -122,7 +122,7 @@ options:
 You can convert HELM evaluation log into unified schema via `every_eval_ever/converters/helm/__main__.py`. For example:
 
 ```bash
-uv run --extra helm every_eval_ever convert helm --log_path tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0
+uv run --extra helm every_eval_ever convert helm --log_path tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0
 ```
 
 The automatic conversion script requires following files generated by HELM to work correctly:

every_eval_ever/converters/helm/__main__.py

@@ -24,7 +24,7 @@ def parse_args():
     parser.add_argument(
         '--log_path',
         type=str,
-        default='tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+        default='tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
         help='Path to directory with single evaluaion or multiple evaluations to convert',
     )
     parser.add_argument('--output_dir', type=str, default='data')

tests/test_helm_adapter.py

@@ -66,7 +66,7 @@ def test_mmlu_eval():
 
     converted_eval = _load_eval(
         adapter,
-        'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+        'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
         metadata_args,
     )
 
@@ -110,7 +110,7 @@ def test_hellswag_eval():
 
     converted_eval = _load_eval(
         adapter,
-        'tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0',
+        'tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0',
         metadata_args,
     )
 
@@ -152,7 +152,7 @@ def test_narrativeqa_eval():
     }
 
     converted_eval = _load_eval(
-        adapter, 'tests/data/helm/narrative_qa:model=openai_gpt2', metadata_args
+        adapter, 'tests/data/helm/narrative_qa_model=openai_gpt2', metadata_args
     )
 
     assert converted_eval.evaluation_timestamp is not None
@@ -192,7 +192,7 @@ def test_missing_model_deployment_falls_back_to_model():
     import json
     src = Path(
         'tests/data/helm/'
-        'mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2'
+        'mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2'
     )
 
     with tempfile.TemporaryDirectory() as tmpdir:

tests/test_helm_instance_level_adapter.py

@@ -123,7 +123,7 @@ def test_mmlu_instance_level():
 
         converted_eval, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
 
@@ -188,7 +188,7 @@ def test_hellaswag_instance_level():
 
         _, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0',
+            'tests/data/helm/commonsense_dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0',
             metadata_args,
         )
 
@@ -234,7 +234,7 @@ def test_narrativeqa_instance_level():
 
         _, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/narrative_qa:model=openai_gpt2',
+            'tests/data/helm/narrative_qa_model=openai_gpt2',
             metadata_args,
         )
 
@@ -282,7 +282,7 @@ def test_per_sample_core_metric_rows_are_emitted():
         }
         _, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
         rows_for_id147 = [
@@ -311,7 +311,7 @@ def test_bookkeeping_stats_are_not_emitted_as_metric_rows():
         }
         _, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
 
@@ -349,7 +349,7 @@ def test_graded_core_metrics_are_not_binary_correctness():
         }
         _, narr_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/narrative_qa:model=openai_gpt2',
+            'tests/data/helm/narrative_qa_model=openai_gpt2',
             metadata_args2,
         )
         # Graded generation metrics also should not be coerced into a
@@ -381,7 +381,7 @@ def test_is_correct_is_true_for_correct_exact_match_rows():
         }
         _, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
         # Correctness metrics are the exception: positive exact-match rows
@@ -442,7 +442,7 @@ def test_evaluation_result_id_helper_disambiguates_split_and_perturbation():
 
 def test_total_rows_matches_core_per_instance_stats():
     _require_helm()
-    fixture = 'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2'
+    fixture = 'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2'
     adapter = HELMAdapter()
     with tempfile.TemporaryDirectory() as tmpdir:
         metadata_args = {
@@ -478,7 +478,7 @@ def test_instance_evaluation_result_ids_join_to_aggregate_results():
         }
         converted_eval, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
 
@@ -512,7 +512,7 @@ def test_aggregate_evaluation_result_ids_are_unique_and_non_null():
         }
         converted_eval, _ = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
 
@@ -588,7 +588,7 @@ def test_reasoning_traces_none_does_not_break_conversion(monkeypatch):
         }
         _, instance_logs = _load_instance_level_data(
             adapter,
-            'tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
+            'tests/data/helm/mmlu_subject=philosophy,method=multiple_choice_joint,model=openai_gpt2',
             metadata_args,
         )
         assert instance_logs