Evaluate(partially) reasoning enhanced hotpot.

examples/basic_example.ipynb

@@ -180,9 +180,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda env:.mlspace-focus_new]",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "conda-env-.mlspace-focus_new-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -194,7 +194,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.16"
+   "version": "3.11.11"
   }
  },
  "nbformat": 4,

examples/configs/base_processing_hotpot.yaml

@@ -0,0 +1,6 @@
+process_output_fn:
+  path: instruct/output_processing_scripts/hotpot.py
+  fn_name: process_output_cot_hotpot
+process_target_fn:
+  path: instruct/output_processing_scripts/hotpot.py
+  fn_name: process_target_cot_hotpot

examples/configs/estimators/cot_estimators.yaml

@@ -0,0 +1,13 @@
+- name: MaximumSequenceProbability
+- name: Perplexity
+- name: MeanTokenEntropy
+- name: MeanPointwiseMutualInformation
+- name: MeanConditionalPointwiseMutualInformation 
+- name: PTrue
+- name: PTrueSampling
+- name: MonteCarloSequenceEntropy
+- name: MonteCarloNormalizedSequenceEntropy
+- name: EigenScore
+- name: RenyiNeg
+- name: FisherRao
+- name: ProbasMeanWithCoT

examples/configs/estimators/default_estimators.yaml

@@ -82,4 +82,5 @@
     trust_remote_code: True
     idf_seed: 42
     idf_dataset_size: -1
-    spacy_path: "en_core_web_sm"
+    spacy_path: "en_core_web_sm"
+- name: ProbasMeanWithCoT

examples/configs/instruct/output_processing_scripts/hotpot.py

@@ -0,0 +1,15 @@
+import re
+import string
+
+CoT_OUTPUT_IGNORE_REGEX = re.compile(r"(?s).*Final Answer:")
+
+def process_output_cot_hotpot(output: str) -> str:
+    output = CoT_OUTPUT_IGNORE_REGEX.sub("", output).lower().strip()
+    output = output.translate(str.maketrans("", "", string.punctuation))
+    return output
+
+def process_target_cot_hotpot(target: str) -> str:
+    target = target.lower().strip()
+    target = target.translate(str.maketrans("", "", string.punctuation))
+
+    return target

examples/configs/polygraph_eval_cot_hotpot.yaml

@@ -0,0 +1,38 @@
+hydra:
+  run:
+    dir: ${cache_path}/${task}/${model}/${dataset}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+
+defaults:
+  - model: bloomz-560m
+  - estimators: cot_estimators
+  - stat_calculators: default_calculators
+  - base_processing_hotpot
+  - _self_
+
+cache_path: ./workdir/output
+save_path: '${hydra:run.dir}'
+instruct: true
+task: qa
+
+dataset: ['denis1699/hotpot_cot']
+text_column: question
+label_column: answer
+train_split: train
+eval_split: validation
+few_shot_prompt: null
+max_new_tokens: 384
+load_from_disk: false
+trust_remote_code: false
+size: 100
+
+
+subsample_eval_dataset: 20
+
+generation_metrics: null
+
+ignore_exceptions: false
+
+batch_size: 1
+
+seed:
+    - 1