diff --git a/.env.example b/.env.example
new file mode 100644
index 000000000..8e72e11ce
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,19 @@
+# EEE Platform environment variables
+# Copy this to .env and fill in your values
+
+# --- Database ---
+DATABASE_URL=postgresql+asyncpg://eee:eee_password@localhost:5432/eee_platform
+DB_ECHO=false
+
+# --- Google OAuth ---
+# Get these from https://console.cloud.google.com/apis/credentials
+GOOGLE_CLIENT_ID=your-google-client-id
+GOOGLE_CLIENT_SECRET=your-google-client-secret
+GOOGLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/callback
+
+# --- JWT ---
+SECRET_KEY=change-this-to-a-random-string-in-production
+
+# --- General ---
+DEBUG=true
+ALLOWED_ORIGINS=["http://localhost:3000","http://localhost:5173","http://localhost:8000"]
diff --git a/.github/workflows/adapter_cron.yml b/.github/workflows/adapter_cron.yml
new file mode 100644
index 000000000..d3c0d674c
--- /dev/null
+++ b/.github/workflows/adapter_cron.yml
@@ -0,0 +1,50 @@
+name: Run Adapters Cron
+
+on:
+ schedule:
+ - cron: '0 0 * * *' # Daily at midnight UTC
+ workflow_dispatch: # Allow manual triggering
+ inputs:
+ force_all:
+ description: 'Force run all adapters'
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ run-adapters:
+ runs-on: ubuntu-latest
+ environment: adapter_cron
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+ cache-dependency-glob: "uv.lock"
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version-file: "pyproject.toml"
+
+ - name: Install dependencies
+ run: uv sync --locked --all-extras
+
+ - name: Run Adapters Orchestrator
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ MERCOR_EVAL_API_EVALEVAL_KEY: ${{ secrets.MERCOR_EVAL_API_EVALEVAL_KEY }}
+ ARTIFICIAL_ANALYSIS_API_KEY: ${{ secrets.ARTIFICIAL_ANALYSIS_API_KEY }}
+ KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
+ KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
+ LLM_STATS_API_KEY: ${{ secrets.LLM_STATS_API_KEY }}
+ run: |
+ if [ "${{ github.event.inputs.force_all }}" == "true" ]; then
+ uv run python utils/scripts/run_adapters.py --force-all
+ else
+ uv run python utils/scripts/run_adapters.py
+ fi
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 000000000..30cf57ed7
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/.idea/every_eval_ever.iml b/.idea/every_eval_ever.iml
new file mode 100644
index 000000000..37a8d5f71
--- /dev/null
+++ b/.idea/every_eval_ever.iml
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 000000000..84f7d9554
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 000000000..105ce2da2
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 000000000..596eb5ed7
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 000000000..109a5062f
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 000000000..35eb1ddfb
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/aggregate.jsonl b/aggregate.jsonl
new file mode 100644
index 000000000..fb773cfe5
--- /dev/null
+++ b/aggregate.jsonl
@@ -0,0 +1,139 @@
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-3.7-thinking-16k/9b320166-f93a-4db6-9a6a-1e683d485bcc.json", "object_path": "flat/objects/9b/32/9b320166-f93a-4db6-9a6a-1e683d485bcc.json", "object_uuid": "9b320166-f93a-4db6-9a6a-1e683d485bcc", "record_type": "aggregate", "sha256": "00335595ac04b177ef6be164a213dbfe87e2f4aee73ded8a20f4ba0f6a73ad43", "size_bytes": 7804}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-3.7-thinking-1k/b5f0e2bc-ea2b-4d19-9ed6-ac62049a72a4.json", "object_path": "flat/objects/b5/f0/b5f0e2bc-ea2b-4d19-9ed6-ac62049a72a4.json", "object_uuid": "b5f0e2bc-ea2b-4d19-9ed6-ac62049a72a4", "record_type": "aggregate", "sha256": "9b700290c68058f398b54910264aa0cfb7b09c0ca337126ee2252661fc9f8c08", "size_bytes": 7711}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-3.7-thinking-8k/7dabfe8c-45c6-4b49-adca-48f4a7e0a8e1.json", "object_path": "flat/objects/7d/ab/7dabfe8c-45c6-4b49-adca-48f4a7e0a8e1.json", "object_uuid": "7dabfe8c-45c6-4b49-adca-48f4a7e0a8e1", "record_type": "aggregate", "sha256": "24c58aeefaa913e8dd1cd6919d615cc4226e94391a4b2e58d3c6fed3efe252a0", "size_bytes": 7697}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-3.7/b64e3f36-6810-4a96-86c7-6afab7cdf83f.json", "object_path": "flat/objects/b6/4e/b64e3f36-6810-4a96-86c7-6afab7cdf83f.json", "object_uuid": "b64e3f36-6810-4a96-86c7-6afab7cdf83f", "record_type": "aggregate", "sha256": "c12ff1649dbc2c87f155c742d3069b034c1c812c982cbbbdb608755d31adc1a6", "size_bytes": 7487}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-haiku-4-5-20251001-thinking-16k/4d0a1984-4bad-418e-8a71-1776311bf490.json", "object_path": "flat/objects/4d/0a/4d0a1984-4bad-418e-8a71-1776311bf490.json", "object_uuid": "4d0a1984-4bad-418e-8a71-1776311bf490", "record_type": "aggregate", "sha256": "9ea83ba109b1a39d78874691aa61f6d9c1a47bf2899eb9e4d47efe62cc8e0c2c", "size_bytes": 10495}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-haiku-4-5-20251001-thinking-1k/c0764c65-0b35-4234-8d0a-eaff8fc32981.json", "object_path": "flat/objects/c0/76/c0764c65-0b35-4234-8d0a-eaff8fc32981.json", "object_uuid": "c0764c65-0b35-4234-8d0a-eaff8fc32981", "record_type": "aggregate", "sha256": "a5635af88333385db44d0dccea600e348e9809340d0586bf8befd1d2df65d961", "size_bytes": 10388}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-haiku-4-5-20251001-thinking-32k/26616376-2ee6-48dc-ae36-0e4bea532598.json", "object_path": "flat/objects/26/61/26616376-2ee6-48dc-ae36-0e4bea532598.json", "object_uuid": "26616376-2ee6-48dc-ae36-0e4bea532598", "record_type": "aggregate", "sha256": "a9fa33ef635dc999e244918cea5d94adbb304322794d25db3f782b1b349f44bc", "size_bytes": 10705}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-haiku-4-5-20251001-thinking-8k/a3c7a6cb-f059-43ed-8e36-5c149c68ef14.json", "object_path": "flat/objects/a3/c7/a3c7a6cb-f059-43ed-8e36-5c149c68ef14.json", "object_uuid": "a3c7a6cb-f059-43ed-8e36-5c149c68ef14", "record_type": "aggregate", "sha256": "33f7e523cc81212197e06f26311388d2e04b4c467186803e7ba01c3a03ece492", "size_bytes": 10390}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-haiku-4-5-20251001/8c972c09-f84a-47e3-adc8-6236391537c5.json", "object_path": "flat/objects/8c/97/8c972c09-f84a-47e3-adc8-6236391537c5.json", "object_uuid": "8c972c09-f84a-47e3-adc8-6236391537c5", "record_type": "aggregate", "sha256": "59b7af86e3b8f2511522002fa25b4399b63de95a9330462986ff08831ed0b00c", "size_bytes": 10040}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-20250514-thinking-16k/c2eb70e4-fd46-486a-bcce-6f28e14abebe.json", "object_path": "flat/objects/c2/eb/c2eb70e4-fd46-486a-bcce-6f28e14abebe.json", "object_uuid": "c2eb70e4-fd46-486a-bcce-6f28e14abebe", "record_type": "aggregate", "sha256": "bfa28b80ed4d7c81f8c9ec844a82fa2aa3a4929acd6d6bad68b8ff794368cd6b", "size_bytes": 10338}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-20250514-thinking-1k/d1aeb640-2024-47df-bc5c-73b60233e3ee.json", "object_path": "flat/objects/d1/ae/d1aeb640-2024-47df-bc5c-73b60233e3ee.json", "object_uuid": "d1aeb640-2024-47df-bc5c-73b60233e3ee", "record_type": "aggregate", "sha256": "48acb416a6ae7e88bfd26a04d5b52355d4811ee096a1f0b84ba29435825f54a3", "size_bytes": 10369}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-20250514-thinking-8k/79946c01-e237-4d8d-984e-d9dda5c543e2.json", "object_path": "flat/objects/79/94/79946c01-e237-4d8d-984e-d9dda5c543e2.json", "object_uuid": "79946c01-e237-4d8d-984e-d9dda5c543e2", "record_type": "aggregate", "sha256": "6836718990a8e689fb9fce0c092b89938abfac64347a6b34c6af2a43ccb3eb2d", "size_bytes": 10307}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-20250514/832d1006-41b8-43f8-be1f-045723bd2d6d.json", "object_path": "flat/objects/83/2d/832d1006-41b8-43f8-be1f-045723bd2d6d.json", "object_uuid": "832d1006-41b8-43f8-be1f-045723bd2d6d", "record_type": "aggregate", "sha256": "bf48caa15cfa2411d111562b45ee28e3f95cf3f9850c48c11b15ea7ff9d12dd8", "size_bytes": 9957}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-5-20251101-thinking-16k/df05c4b7-7ae8-4208-920a-d2873adb9013.json", "object_path": "flat/objects/df/05/df05c4b7-7ae8-4208-920a-d2873adb9013.json", "object_uuid": "df05c4b7-7ae8-4208-920a-d2873adb9013", "record_type": "aggregate", "sha256": "6dfaff554cb6944c08716c512152f0c28ba89b8f22e3138af20a32d2ff7e48b8", "size_bytes": 10386}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-5-20251101-thinking-32k/780846c7-4220-4ab3-b0da-523f07ffe893.json", "object_path": "flat/objects/78/08/780846c7-4220-4ab3-b0da-523f07ffe893.json", "object_uuid": "780846c7-4220-4ab3-b0da-523f07ffe893", "record_type": "aggregate", "sha256": "06f3b54ad785a27860a311bcb4a256ea34867943dd6bf25f27a8b8b053d6f138", "size_bytes": 8048}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-5-20251101-thinking-64k/2a47fdfd-c075-4017-b92a-fdc05f2058ac.json", "object_path": "flat/objects/2a/47/2a47fdfd-c075-4017-b92a-fdc05f2058ac.json", "object_uuid": "2a47fdfd-c075-4017-b92a-fdc05f2058ac", "record_type": "aggregate", "sha256": "d0b5ed54165f80442cf342c25f059db2a61cfb0e7480aeb4a18471424312ebf2", "size_bytes": 5726}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-5-20251101-thinking-8k/d6c5bbca-7346-4b9e-aa61-33fe5eca678c.json", "object_path": "flat/objects/d6/c5/d6c5bbca-7346-4b9e-aa61-33fe5eca678c.json", "object_uuid": "d6c5bbca-7346-4b9e-aa61-33fe5eca678c", "record_type": "aggregate", "sha256": "8291461e1b7a912022756f9f32de7b00736abbc653df28b14ad1382b649590d6", "size_bytes": 10349}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-5-20251101-thinking-none/b9386ce1-c126-4056-a89b-ead4bbff245b.json", "object_path": "flat/objects/b9/38/b9386ce1-c126-4056-a89b-ead4bbff245b.json", "object_uuid": "b9386ce1-c126-4056-a89b-ead4bbff245b", "record_type": "aggregate", "sha256": "2932a95e32f8b75f2240ceba04d92df626c30bc5b8f217d33d460d802b02fe87", "size_bytes": 10403}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-6-thinking-120k-high/244305c6-a830-4882-9b9f-70f9b6a7927b.json", "object_path": "flat/objects/24/43/244305c6-a830-4882-9b9f-70f9b6a7927b.json", "object_uuid": "244305c6-a830-4882-9b9f-70f9b6a7927b", "record_type": "aggregate", "sha256": "39428ca0623c1bd3a21dd7042e012c8cb438e7f6c6ab67e7bddbda68b6d4f7af", "size_bytes": 10375}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-6-thinking-120k-low/a267ea0e-3dc0-4705-8352-8f8d0d9453fc.json", "object_path": "flat/objects/a2/67/a267ea0e-3dc0-4705-8352-8f8d0d9453fc.json", "object_uuid": "a267ea0e-3dc0-4705-8352-8f8d0d9453fc", "record_type": "aggregate", "sha256": "58256aaeeae931fa7160d637c1d564caf21301524815f9e66605db6354c88bb5", "size_bytes": 10282}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-6-thinking-120k-max/a6c39cd1-7a69-41f1-8e00-c65ff1602906.json", "object_path": "flat/objects/a6/c3/a6c39cd1-7a69-41f1-8e00-c65ff1602906.json", "object_uuid": "a6c39cd1-7a69-41f1-8e00-c65ff1602906", "record_type": "aggregate", "sha256": "4fa8bda8ce38e989414a25eb93dfe5c524c0aaa690a080590662dc9714090eee", "size_bytes": 10474}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-opus-4-6-thinking-120k-medium/b44228f1-f5ea-4390-b0ac-89f9a4d3039c.json", "object_path": "flat/objects/b4/42/b44228f1-f5ea-4390-b0ac-89f9a4d3039c.json", "object_uuid": "b44228f1-f5ea-4390-b0ac-89f9a4d3039c", "record_type": "aggregate", "sha256": "5d18e0c334660d884bd338db12b5c91aea0efe92bb45508d30772ba5a34bf295", "size_bytes": 10373}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-20250514-thinking-16k-bedrock/4d461977-2f3e-47ae-acc1-08815f821da3.json", "object_path": "flat/objects/4d/46/4d461977-2f3e-47ae-acc1-08815f821da3.json", "object_uuid": "4d461977-2f3e-47ae-acc1-08815f821da3", "record_type": "aggregate", "sha256": "3e0d450f2eedb1353cc2f6deff5f27be7a9af7f8d6f15e49dbd48217e966cc74", "size_bytes": 10630}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-20250514-thinking-1k/b60e2c5c-80fb-456a-8586-546c7c8b0c1c.json", "object_path": "flat/objects/b6/0e/b60e2c5c-80fb-456a-8586-546c7c8b0c1c.json", "object_uuid": "b60e2c5c-80fb-456a-8586-546c7c8b0c1c", "record_type": "aggregate", "sha256": "07af19cccda87748633d9b5775df26c0615f9c0cdc16a33ad4cfdf947a3dc398", "size_bytes": 10371}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-20250514-thinking-8k-bedrock/4de2519b-b77e-44b0-b456-826d6c955de0.json", "object_path": "flat/objects/4d/e2/4de2519b-b77e-44b0-b456-826d6c955de0.json", "object_uuid": "4de2519b-b77e-44b0-b456-826d6c955de0", "record_type": "aggregate", "sha256": "4790a52f0a6b09a78e43824754d79df3b7fdf8b96f2834af65b8940c76257a41", "size_bytes": 10595}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-20250514/6e3cce3e-d30a-44b5-befe-cb841d249827.json", "object_path": "flat/objects/6e/3c/6e3cce3e-d30a-44b5-befe-cb841d249827.json", "object_uuid": "6e3cce3e-d30a-44b5-befe-cb841d249827", "record_type": "aggregate", "sha256": "5272dcb83560a3827f840ad4f6fd2e7aa06b7c9ca52ba727988c46b914ef0806", "size_bytes": 9995}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-5-20250929-thinking-16k/91613e3b-eb44-47e2-8d45-f6f1924b7f70.json", "object_path": "flat/objects/91/61/91613e3b-eb44-47e2-8d45-f6f1924b7f70.json", "object_uuid": "91613e3b-eb44-47e2-8d45-f6f1924b7f70", "record_type": "aggregate", "sha256": "ad896ee39307c69f6e04bca2751206ae4d21d94cfc346a7c3324066e9ac56ce3", "size_bytes": 10524}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-5-20250929-thinking-1k/96bd08c4-b8b5-4129-87e4-07f8d1ac5b28.json", "object_path": "flat/objects/96/bd/96bd08c4-b8b5-4129-87e4-07f8d1ac5b28.json", "object_uuid": "96bd08c4-b8b5-4129-87e4-07f8d1ac5b28", "record_type": "aggregate", "sha256": "88121a8d515d0882c230e745ec687929711c891c0696974d82792e16d1dbc917", "size_bytes": 10421}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-5-20250929-thinking-32k/4f93a9bf-568f-4404-95fe-8ffea13d00e2.json", "object_path": "flat/objects/4f/93/4f93a9bf-568f-4404-95fe-8ffea13d00e2.json", "object_uuid": "4f93a9bf-568f-4404-95fe-8ffea13d00e2", "record_type": "aggregate", "sha256": "5167200cba8d72acb0bbf31ac02669d3e44579b6dc23ba7aa498d8425021056c", "size_bytes": 10524}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-5-20250929-thinking-8k/a2c1d4cc-9b27-4ff9-b44e-8f448c216791.json", "object_path": "flat/objects/a2/c1/a2c1d4cc-9b27-4ff9-b44e-8f448c216791.json", "object_uuid": "a2c1d4cc-9b27-4ff9-b44e-8f448c216791", "record_type": "aggregate", "sha256": "bfc8282c1ae267f82d655fb7fff17ff0c9642ac31fef0ce1cbc557b0bcfc82fc", "size_bytes": 10411}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-5-20250929/1a56b654-596a-42a3-a234-0e3482ec2df8.json", "object_path": "flat/objects/1a/56/1a56b654-596a-42a3-a234-0e3482ec2df8.json", "object_uuid": "1a56b654-596a-42a3-a234-0e3482ec2df8", "record_type": "aggregate", "sha256": "756293a739616a136ca65c400eefd1afa8ef9e9c92fc90ba729bbfa7f91147e3", "size_bytes": 10075}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-6-high/f4860c78-7671-4536-81c4-7810dad00491.json", "object_path": "flat/objects/f4/86/f4860c78-7671-4536-81c4-7810dad00491.json", "object_uuid": "f4860c78-7671-4536-81c4-7810dad00491", "record_type": "aggregate", "sha256": "35fd0ff7670aaa164dbca1991f6c0c5be6b26c7e28e7eb20f79b55a6d9498cd6", "size_bytes": 9957}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/claude-sonnet-4-6-max/2d11550d-eac6-4ae9-aea0-63a144a0dc7e.json", "object_path": "flat/objects/2d/11/2d11550d-eac6-4ae9-aea0-63a144a0dc7e.json", "object_uuid": "2d11550d-eac6-4ae9-aea0-63a144a0dc7e", "record_type": "aggregate", "sha256": "8cdb354638e2886eafbf03918e2cd8e01b48bad5a0c2f4451d612383a88b6d27", "size_bytes": 9936}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/anthropic/opus-4-6-max-effort/9ccc355f-a48a-4acd-8dc6-c3c9c4683b3c.json", "object_path": "flat/objects/9c/cc/9ccc355f-a48a-4acd-8dc6-c3c9c4683b3c.json", "object_uuid": "9ccc355f-a48a-4acd-8dc6-c3c9c4683b3c", "record_type": "aggregate", "sha256": "1c36c133e2ea0125904e89c10baec0115d5583a0fc184ad49db25fdec38f793b", "size_bytes": 3360}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/arcprize/2025-human-panel/70bee9a0-6a64-4efe-931c-e2d84d3daedc.json", "object_path": "flat/objects/70/be/70bee9a0-6a64-4efe-931c-e2d84d3daedc.json", "object_uuid": "70bee9a0-6a64-4efe-931c-e2d84d3daedc", "record_type": "aggregate", "sha256": "4f7aa5532d88d4b21fad79ef7924fc8634d7c85a5ffad8aa559a054397645824", "size_bytes": 9730}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/architects/11e1a241-eedb-48d9-b487-c1c9567e7b5b.json", "object_path": "flat/objects/11/e1/11e1a241-eedb-48d9-b487-c1c9567e7b5b.json", "object_uuid": "11e1a241-eedb-48d9-b487-c1c9567e7b5b", "record_type": "aggregate", "sha256": "61fc2d6781eeb20208fa6046d15396d297f081ab849602c33a8aa0de8b9774dd", "size_bytes": 9585}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/icecuber/0dce9214-0320-4909-8b8d-45fdc9a17252.json", "object_path": "flat/objects/0d/ce/0dce9214-0320-4909-8b8d-45fdc9a17252.json", "object_uuid": "0dce9214-0320-4909-8b8d-45fdc9a17252", "record_type": "aggregate", "sha256": "32446965bea69f24562596d09e05c662333ba2247a5b52c27216c41711065941", "size_bytes": 9533}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/jeremy-sept-2025/1e76c71d-b6dc-4afe-8385-d5993bf67f7c.json", "object_path": "flat/objects/1e/76/1e76c71d-b6dc-4afe-8385-d5993bf67f7c.json", "object_uuid": "1e76c71d-b6dc-4afe-8385-d5993bf67f7c", "record_type": "aggregate", "sha256": "f090debf887301010020e03ef202ced422166f6d2c567786441fc92b77531c8a", "size_bytes": 5369}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/johan-land-gpt-5-2-refine/57731052-3c99-4517-8e06-ceb7a25428e5.json", "object_path": "flat/objects/57/73/57731052-3c99-4517-8e06-ceb7a25428e5.json", "object_uuid": "57731052-3c99-4517-8e06-ceb7a25428e5", "record_type": "aggregate", "sha256": "de5220c0dbff5abd48c7637bdf93d47890afe6d284a1421f4397fa5352897318", "size_bytes": 5528}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/nvarc-2025/74678790-7f16-4f6e-aaa5-ffded7ad1a71.json", "object_path": "flat/objects/74/67/74678790-7f16-4f6e-aaa5-ffded7ad1a71.json", "object_uuid": "74678790-7f16-4f6e-aaa5-ffded7ad1a71", "record_type": "aggregate", "sha256": "9b8ffd83434a566972c39ce79aadbb4eb34e9f91a1d38f3cded8b425fe0da6bf", "size_bytes": 3097}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/poetiq-arc-agi-gemini-3-d-v1/ca0ac43b-4831-4fb3-b7e8-96b62f96f3f1.json", "object_path": "flat/objects/ca/0a/ca0ac43b-4831-4fb3-b7e8-96b62f96f3f1.json", "object_uuid": "ca0ac43b-4831-4fb3-b7e8-96b62f96f3f1", "record_type": "aggregate", "sha256": "4ae798c7f1a3f00729263cb4afbc84805afd7ad95ec8c2c5175add38c7ccff5f", "size_bytes": 3359}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/stem-grad/19f7653c-256b-4420-9d95-f0c280b267b4.json", "object_path": "flat/objects/19/f7/19f7653c-256b-4420-9d95-f0c280b267b4.json", "object_uuid": "19f7653c-256b-4420-9d95-f0c280b267b4", "record_type": "aggregate", "sha256": "44009960e1680b01d1d971aeb6cdd4202d83f8ab1b758f1ff6b4e43438ec8f54", "size_bytes": 3082}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/community/trm-2025-10-07/808efe0f-ea5e-40ac-9ef0-93cf8d221a79.json", "object_path": "flat/objects/80/8e/808efe0f-ea5e-40ac-9ef0-93cf8d221a79.json", "object_uuid": "808efe0f-ea5e-40ac-9ef0-93cf8d221a79", "record_type": "aggregate", "sha256": "9c2b3aabc1a02c3bb27b4c8eba747f556f1ba2580c1b7cf948c9a2553cc95142", "size_bytes": 9705}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/deepseek/r1-0528-openrouter/2dc14d78-543b-491a-8ff9-33f056ac9ed6.json", "object_path": "flat/objects/2d/c1/2dc14d78-543b-491a-8ff9-33f056ac9ed6.json", "object_uuid": "2dc14d78-543b-491a-8ff9-33f056ac9ed6", "record_type": "aggregate", "sha256": "63d7f8b2396a187e4f3752faa7e29cd38c43b5d92352d9e9452dd2fbd9a17126", "size_bytes": 10003}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/deepseek/r1/b12e7cf4-f1c2-48f6-afef-2a6352358748.json", "object_path": "flat/objects/b1/2e/b12e7cf4-f1c2-48f6-afef-2a6352358748.json", "object_uuid": "b12e7cf4-f1c2-48f6-afef-2a6352358748", "record_type": "aggregate", "sha256": "0f6a45dfe228eeca3c3e67364ef744dbf3fcf4c7ef775b130c950b444b5f2ae0", "size_bytes": 7234}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/deepseek/v3.2/88d65dec-f757-40bc-93fd-e6e9d6432137.json", "object_path": "flat/objects/88/d6/88d65dec-f757-40bc-93fd-e6e9d6432137.json", "object_uuid": "88d65dec-f757-40bc-93fd-e6e9d6432137", "record_type": "aggregate", "sha256": "d7e67cbe464bf73e1f43bc2bb27a5cdf4b8e21bbac3865f48ef89cb08aa5477e", "size_bytes": 9579}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-1.5-pro/e2aa63d9-773f-4aae-9adf-bdfe029c1523.json", "object_path": "flat/objects/e2/aa/e2aa63d9-773f-4aae-9adf-bdfe029c1523.json", "object_uuid": "e2aa63d9-773f-4aae-9adf-bdfe029c1523", "record_type": "aggregate", "sha256": "acd0cc977b6b2810107f463412e7ad58882ba6952b2aad06ff7dd9247c51ed6c", "size_bytes": 5300}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-flash-preview-05-20-thinking-16k/33ccfb73-0fb4-48d0-8383-d6acb368417f.json", "object_path": "flat/objects/33/cc/33ccfb73-0fb4-48d0-8383-d6acb368417f.json", "object_uuid": "33ccfb73-0fb4-48d0-8383-d6acb368417f", "record_type": "aggregate", "sha256": "5cc8b98b1746936a4e6adada06448def35a21fa60032b5d2d76fd3e9a2b303f2", "size_bytes": 10541}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-flash-preview-05-20-thinking-1k/60280f77-5bba-47d3-afca-b38f118aeb17.json", "object_path": "flat/objects/60/28/60280f77-5bba-47d3-afca-b38f118aeb17.json", "object_uuid": "60280f77-5bba-47d3-afca-b38f118aeb17", "record_type": "aggregate", "sha256": "7fdf7a1ec5c306258c1d887a0310188a2d7b8438a7b9d74bf5dcc7843938a4ef", "size_bytes": 10488}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-flash-preview-05-20-thinking-24k/590cabd0-a948-4b6c-91f4-b4c4344a65ac.json", "object_path": "flat/objects/59/0c/590cabd0-a948-4b6c-91f4-b4c4344a65ac.json", "object_uuid": "590cabd0-a948-4b6c-91f4-b4c4344a65ac", "record_type": "aggregate", "sha256": "43d588520cc72fe904acc742f05e62fd757264c4606859a6f6b76dd42d4682d5", "size_bytes": 10547}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-flash-preview-05-20-thinking-8k/1989f5cb-a4f6-44ae-af36-3b4d3981087a.json", "object_path": "flat/objects/19/89/1989f5cb-a4f6-44ae-af36-3b4d3981087a.json", "object_uuid": "1989f5cb-a4f6-44ae-af36-3b4d3981087a", "record_type": "aggregate", "sha256": "be43058afc012af8de547b4cbb81e4c199698ec502b2ab07359425ee9b5a23cc", "size_bytes": 10518}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-flash-preview-05-20/b30accb2-c84e-4620-a914-7f999bca5d0b.json", "object_path": "flat/objects/b3/0a/b30accb2-c84e-4620-a914-7f999bca5d0b.json", "object_uuid": "b30accb2-c84e-4620-a914-7f999bca5d0b", "record_type": "aggregate", "sha256": "99788073605645b74e0bf563d31917a3f1dc91942d2710a88997eb160826d8c7", "size_bytes": 10164}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-pro-2025-06-17-thinking-16k/69d9e262-348d-4a88-a576-b373c63d9f07.json", "object_path": "flat/objects/69/d9/69d9e262-348d-4a88-a576-b373c63d9f07.json", "object_uuid": "69d9e262-348d-4a88-a576-b373c63d9f07", "record_type": "aggregate", "sha256": "47f80fc795e5fe673c3dfb296e91c3bf00fb7744b33e0b4de2bbc78240e69894", "size_bytes": 10398}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-pro-2025-06-17-thinking-1k/236488c0-b7ef-4252-afbe-ba5cf9a62f33.json", "object_path": "flat/objects/23/64/236488c0-b7ef-4252-afbe-ba5cf9a62f33.json", "object_uuid": "236488c0-b7ef-4252-afbe-ba5cf9a62f33", "record_type": "aggregate", "sha256": "1bf0ee249e95c07902eb034b969284799717b2ffe0a464b2b2ec388300ad2ee9", "size_bytes": 10357}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-pro-2025-06-17-thinking-32k/a85938e2-a1ae-4bb2-8a07-12809994b99c.json", "object_path": "flat/objects/a8/59/a85938e2-a1ae-4bb2-8a07-12809994b99c.json", "object_uuid": "a85938e2-a1ae-4bb2-8a07-12809994b99c", "record_type": "aggregate", "sha256": "341e27cc9d09bf5ac084f0a503021079701e5c11c4dc3cf0f572040f852c84a6", "size_bytes": 10386}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2-5-pro-2025-06-17-thinking-8k/24784ed0-2200-4f75-8c86-43ffa8438c05.json", "object_path": "flat/objects/24/78/24784ed0-2200-4f75-8c86-43ffa8438c05.json", "object_uuid": "24784ed0-2200-4f75-8c86-43ffa8438c05", "record_type": "aggregate", "sha256": "3721b9b9647daba0899ee7babc342faaf6699460b216758108d22ac5f365657f", "size_bytes": 10371}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-2.0-flash/0dee38a8-dcca-4fb2-8e45-b9759b185310.json", "object_path": "flat/objects/0d/ee/0dee38a8-dcca-4fb2-8e45-b9759b185310.json", "object_uuid": "0dee38a8-dcca-4fb2-8e45-b9759b185310", "record_type": "aggregate", "sha256": "92aab38fcc14195bce3242e003bcc0bf29d30cab2de9107a12279bea370e398b", "size_bytes": 3150}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-1-pro-preview/cffc6028-5a65-4cc8-a342-848154ecd7cc.json", "object_path": "flat/objects/cf/fc/cffc6028-5a65-4cc8-a342-848154ecd7cc.json", "object_uuid": "cffc6028-5a65-4cc8-a342-848154ecd7cc", "record_type": "aggregate", "sha256": "570726470a9a94f1818c7377de55572679fcdc2cdcf8b90bc5ac9799bf4137fb", "size_bytes": 12329}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-deep-think-022026/0978ca63-ec46-4b97-a6ce-21a7e74d2f78.json", "object_path": "flat/objects/09/78/0978ca63-ec46-4b97-a6ce-21a7e74d2f78.json", "object_uuid": "0978ca63-ec46-4b97-a6ce-21a7e74d2f78", "record_type": "aggregate", "sha256": "84623bbff33ecc1fdd6cfc272b803df2d213854e08c2a9f11b4f59de3bd4915d", "size_bytes": 5652}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-deep-think-preview/b49b33fb-df1f-43ab-a9a2-e4be9ab97818.json", "object_path": "flat/objects/b4/9b/b49b33fb-df1f-43ab-a9a2-e4be9ab97818.json", "object_uuid": "b49b33fb-df1f-43ab-a9a2-e4be9ab97818", "record_type": "aggregate", "sha256": "7e90b153aa36a56b9cc95aed83a8149595ecdd4ebb942c6e11764038a838573a", "size_bytes": 5633}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-flash-preview-thinking-high/12198531-4006-492b-8c57-ec69e5ffa9eb.json", "object_path": "flat/objects/12/19/12198531-4006-492b-8c57-ec69e5ffa9eb.json", "object_uuid": "12198531-4006-492b-8c57-ec69e5ffa9eb", "record_type": "aggregate", "sha256": "de7de49fcb675059a180c7c2971a6500042ea87019ff1998be521a30be7a93f1", "size_bytes": 10334}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-flash-preview-thinking-low/0abbd5f9-49fb-420c-9dd0-07bb0d24728e.json", "object_path": "flat/objects/0a/bb/0abbd5f9-49fb-420c-9dd0-07bb0d24728e.json", "object_uuid": "0abbd5f9-49fb-420c-9dd0-07bb0d24728e", "record_type": "aggregate", "sha256": "9b42b04b86de266702d062aaec2d46297798c413787687e527673891c175e718", "size_bytes": 10301}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-flash-preview-thinking-medium/a2c2c16a-1003-454a-a91d-8b9d439498d2.json", "object_path": "flat/objects/a2/c2/a2c2c16a-1003-454a-a91d-8b9d439498d2.json", "object_uuid": "a2c2c16a-1003-454a-a91d-8b9d439498d2", "record_type": "aggregate", "sha256": "76bf451bf3075bdc0d839076abc8fc35213233f244451e98db416ec66297de54", "size_bytes": 10402}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-flash-preview-thinking-minimal/fb3fcdcc-6690-475e-8b87-bf2652035390.json", "object_path": "flat/objects/fb/3f/fb3fcdcc-6690-475e-8b87-bf2652035390.json", "object_uuid": "fb3fcdcc-6690-475e-8b87-bf2652035390", "record_type": "aggregate", "sha256": "e3900113fa19e44fbe0cef90c797015a913508492354b57fc2ba578cdbe455f7", "size_bytes": 10417}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/google/gemini-3-pro-preview/e13453ef-f08a-42f6-993d-60534f5b337a.json", "object_path": "flat/objects/e1/34/e13453ef-f08a-42f6-993d-60534f5b337a.json", "object_uuid": "e13453ef-f08a-42f6-993d-60534f5b337a", "record_type": "aggregate", "sha256": "2aa990e70fe301ff02f19d65de6074348dd0565ffb946945b6fc4e980bb6427e", "size_bytes": 5422}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/meta/llama-4-maverick-17b-128e-instruct-fp8-together/67bbe731-ec6d-4f75-9e54-d61b45e6ffdf.json", "object_path": "flat/objects/67/bb/67bbe731-ec6d-4f75-9e54-d61b45e6ffdf.json", "object_uuid": "67bbe731-ec6d-4f75-9e54-d61b45e6ffdf", "record_type": "aggregate", "sha256": "c4e90a3fedc607b54703ae66ef981472bd3f477e34525021ebcba90385f0ce49", "size_bytes": 10615}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/meta/llama-4-scout-17b-16e-instruct-together/45788b3f-054c-4ab9-ab34-aed0dff8db7b.json", "object_path": "flat/objects/45/78/45788b3f-054c-4ab9-ab34-aed0dff8db7b.json", "object_uuid": "45788b3f-054c-4ab9-ab34-aed0dff8db7b", "record_type": "aggregate", "sha256": "c5d940fcc84605c1df8819191c15bdf1df4838552bb03575d7501a2a986245f7", "size_bytes": 10389}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/minimax/m2.5/8ac026e7-68e4-408f-a2c7-e782d82362ac.json", "object_path": "flat/objects/8a/c0/8ac026e7-68e4-408f-a2c7-e782d82362ac.json", "object_uuid": "8ac026e7-68e4-408f-a2c7-e782d82362ac", "record_type": "aggregate", "sha256": "274c66fb79efeb667fdb5c93f7de73b2b9d659816e95e0a132295bbdd010174e", "size_bytes": 9563}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/mistralai/magistral-medium-2506-thinking/862e45c7-8e42-464a-bd20-b297af0a77fd.json", "object_path": "flat/objects/86/2e/862e45c7-8e42-464a-bd20-b297af0a77fd.json", "object_uuid": "862e45c7-8e42-464a-bd20-b297af0a77fd", "record_type": "aggregate", "sha256": "c070a92861af1f937f088e7196ba086d497a2bfb31e5c71b6c1605e081362bed", "size_bytes": 10185}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/mistralai/magistral-medium-2506/1bcd3970-cb4a-4933-9b35-5752b8c577b7.json", "object_path": "flat/objects/1b/cd/1bcd3970-cb4a-4933-9b35-5752b8c577b7.json", "object_uuid": "1bcd3970-cb4a-4933-9b35-5752b8c577b7", "record_type": "aggregate", "sha256": "92d88d7177b96f237a2ea9d6e15d7d286d7a37a156e03618a38e242974cf790f", "size_bytes": 9914}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/mistralai/magistral-small-2506/221a0973-87a6-446b-b40e-f386e98e710b.json", "object_path": "flat/objects/22/1a/221a0973-87a6-446b-b40e-f386e98e710b.json", "object_uuid": "221a0973-87a6-446b-b40e-f386e98e710b", "record_type": "aggregate", "sha256": "1e51ee91006b7480fbadf0464ab822ac4d4ba68d3170a28b75d08c8090987712", "size_bytes": 9883}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/moonshotai/kimi-k2.5/852caa17-762f-4c27-b27e-2b2101ef2df6.json", "object_path": "flat/objects/85/2c/852caa17-762f-4c27-b27e-2b2101ef2df6.json", "object_uuid": "852caa17-762f-4c27-b27e-2b2101ef2df6", "record_type": "aggregate", "sha256": "f3eac85b4a70ed8932a160386c7b1c14bcdab99882648f2fb0ba33a898aece78", "size_bytes": 9577}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/codex-mini-latest/f98bbf3c-68a3-420b-8f81-846b5396fa0c.json", "object_path": "flat/objects/f9/8b/f98bbf3c-68a3-420b-8f81-846b5396fa0c.json", "object_uuid": "f98bbf3c-68a3-420b-8f81-846b5396fa0c", "record_type": "aggregate", "sha256": "195dea21ffee036013b841b5bed7f009010a82e48466b45e8cd956874946db12", "size_bytes": 9773}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-4-1-2025-04-14/6153a395-cf97-4056-83bd-0f5b2d4118f8.json", "object_path": "flat/objects/61/53/6153a395-cf97-4056-83bd-0f5b2d4118f8.json", "object_uuid": "6153a395-cf97-4056-83bd-0f5b2d4118f8", "record_type": "aggregate", "sha256": "eeada04f56fad10ab8d243511e90452cff315f078c0f6a484519c575927b6b58", "size_bytes": 9796}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-4-1-mini-2025-04-14/569fd2e8-6cb8-4251-a2c4-3d5c52813c06.json", "object_path": "flat/objects/56/9f/569fd2e8-6cb8-4251-a2c4-3d5c52813c06.json", "object_uuid": "569fd2e8-6cb8-4251-a2c4-3d5c52813c06", "record_type": "aggregate", "sha256": "7817c3f197842dad834a9db45d579ff8422e79bbe91077cb08fad333003d8f2a", "size_bytes": 9939}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-4-1-nano-2025-04-14/c3d740ff-0930-4e8b-a4a1-d3c4ad7bcd77.json", "object_path": "flat/objects/c3/d7/c3d740ff-0930-4e8b-a4a1-d3c4ad7bcd77.json", "object_uuid": "c3d740ff-0930-4e8b-a4a1-d3c4ad7bcd77", "record_type": "aggregate", "sha256": "31855692a6ff00883f1fa8b7d70b156cfe79c7d172e509ebbe90127f7c3b5c2f", "size_bytes": 9933}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-4-5-2025-02-27/32b43574-4074-439f-b163-7a40442a5607.json", "object_path": "flat/objects/32/b4/32b43574-4074-439f-b163-7a40442a5607.json", "object_uuid": "32b43574-4074-439f-b163-7a40442a5607", "record_type": "aggregate", "sha256": "69273051d0c6281b22549dba1b1fd5f12536ae6077975472f7a1450699ff86db", "size_bytes": 7576}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-4o-2024-11-20/efecfcc0-1b7a-4e85-a710-9bf297e9072c.json", "object_path": "flat/objects/ef/ec/efecfcc0-1b7a-4e85-a710-9bf297e9072c.json", "object_uuid": "efecfcc0-1b7a-4e85-a710-9bf297e9072c", "record_type": "aggregate", "sha256": "401ea34e288e3d35a8133288d235a441261c9237f97b1df5efd74b1e32669513", "size_bytes": 7549}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-4o-mini-2024-07-18/c2db0630-9aa8-467b-a4f6-b5d19771bc01.json", "object_path": "flat/objects/c2/db/c2db0630-9aa8-467b-a4f6-b5d19771bc01.json", "object_uuid": "c2db0630-9aa8-467b-a4f6-b5d19771bc01", "record_type": "aggregate", "sha256": "2ea63ecf732453691ae5699bc6b46c0371a618b840aa3657a8528b49fe2abcf1", "size_bytes": 5430}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-1-2025-11-13-thinking-high/8962bdac-bc2d-411e-ae86-a8ef247edf86.json", "object_path": "flat/objects/89/62/8962bdac-bc2d-411e-ae86-a8ef247edf86.json", "object_uuid": "8962bdac-bc2d-411e-ae86-a8ef247edf86", "record_type": "aggregate", "sha256": "4043402efc6f15d8518f44e1d93d32530c4f3aa432fcd6db89ff2da8ed7ab86b", "size_bytes": 10224}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-1-2025-11-13-thinking-low/df3fa2f5-1b58-4423-a32f-5cb42db7fcbf.json", "object_path": "flat/objects/df/3f/df3fa2f5-1b58-4423-a32f-5cb42db7fcbf.json", "object_uuid": "df3fa2f5-1b58-4423-a32f-5cb42db7fcbf", "record_type": "aggregate", "sha256": "9736c692728116f2e97feb908134ee718b41d37e032bb461927e5fcdd537a1fa", "size_bytes": 10187}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-1-2025-11-13-thinking-medium/3dec3d5d-766e-4ee8-b1b5-a8f8a79c7d03.json", "object_path": "flat/objects/3d/ec/3dec3d5d-766e-4ee8-b1b5-a8f8a79c7d03.json", "object_uuid": "3dec3d5d-766e-4ee8-b1b5-a8f8a79c7d03", "record_type": "aggregate", "sha256": "4fdf12d716d0303b28b0862486c84c0acb0fad6d0ec11f9bb55468ef0ea8b4cc", "size_bytes": 10274}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-1-2025-11-13-thinking-none/b184c615-5321-417c-851c-77f6c491f484.json", "object_path": "flat/objects/b1/84/b184c615-5321-417c-851c-77f6c491f484.json", "object_uuid": "b184c615-5321-417c-851c-77f6c491f484", "record_type": "aggregate", "sha256": "0066dedc05fc158023bb479e1ce604dc129f36453e3ca308cc24e68a53dd3666", "size_bytes": 10220}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-2025-12-11-thinking-high/5f9eaceb-43e6-43cc-af61-a039de7b8f2c.json", "object_path": "flat/objects/5f/9e/5f9eaceb-43e6-43cc-af61-a039de7b8f2c.json", "object_uuid": "5f9eaceb-43e6-43cc-af61-a039de7b8f2c", "record_type": "aggregate", "sha256": "f1d8229e296ad8a895170d838f237233f601de54ee14df92f7bb712c33500e94", "size_bytes": 10214}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-2025-12-11-thinking-low/4f80fe00-b5b3-42d4-aa69-94bd1fc7eb1b.json", "object_path": "flat/objects/4f/80/4f80fe00-b5b3-42d4-aa69-94bd1fc7eb1b.json", "object_uuid": "4f80fe00-b5b3-42d4-aa69-94bd1fc7eb1b", "record_type": "aggregate", "sha256": "ccbabea766138a8a703e9ef00ab7fd3a8ff606ddae655780cb9494d18d827607", "size_bytes": 10189}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-2025-12-11-thinking-medium/8145ba1b-7766-4eb0-8c4a-16d2f3094bc3.json", "object_path": "flat/objects/81/45/8145ba1b-7766-4eb0-8c4a-16d2f3094bc3.json", "object_uuid": "8145ba1b-7766-4eb0-8c4a-16d2f3094bc3", "record_type": "aggregate", "sha256": "8ffa52e58e6757cff9611b8c0b82a818810d9fa332a8423d238774e76dab6d4b", "size_bytes": 10276}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-2025-12-11-thinking-none/5b38d83a-f1a0-49ff-bc19-efb27c335a1f.json", "object_path": "flat/objects/5b/38/5b38d83a-f1a0-49ff-bc19-efb27c335a1f.json", "object_uuid": "5b38d83a-f1a0-49ff-bc19-efb27c335a1f", "record_type": "aggregate", "sha256": "61409fdee2df56164bed598237a38a96391301eccb5a7a0fb7be0885d7aa33c8", "size_bytes": 10218}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-2025-12-11-thinking-xhigh/c73b2a3e-2243-4a5a-a03f-2617855d0458.json", "object_path": "flat/objects/c7/3b/c73b2a3e-2243-4a5a-a03f-2617855d0458.json", "object_uuid": "c73b2a3e-2243-4a5a-a03f-2617855d0458", "record_type": "aggregate", "sha256": "6390e92343ca8f409417725c4a1db07bdc2c94700214edf4df8fc107f5a6df8f", "size_bytes": 10233}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-pro-2025-12-11-high/bb937691-5c4f-411b-bebd-26b436ac1200.json", "object_path": "flat/objects/bb/93/bb937691-5c4f-411b-bebd-26b436ac1200.json", "object_uuid": "bb937691-5c4f-411b-bebd-26b436ac1200", "record_type": "aggregate", "sha256": "157a9527307c72e93b48f2da1d548e0f8eaafa566bc45090eaeb4875e60624c1", "size_bytes": 10193}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-pro-2025-12-11-medium/4991e254-8cae-4728-abfe-de24ca189451.json", "object_path": "flat/objects/49/91/4991e254-8cae-4728-abfe-de24ca189451.json", "object_uuid": "4991e254-8cae-4728-abfe-de24ca189451", "record_type": "aggregate", "sha256": "d85969af03c5c4dd7020293e7b25c2e15577e4d4cd6b44167afa8b4e14cd1208", "size_bytes": 10131}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2-pro-2025-12-11-xhigh/bf730ec9-4aa1-4ee8-a9c4-168553db1c8d.json", "object_path": "flat/objects/bf/73/bf730ec9-4aa1-4ee8-a9c4-168553db1c8d.json", "object_uuid": "bf730ec9-4aa1-4ee8-a9c4-168553db1c8d", "record_type": "aggregate", "sha256": "11c322fc738ddf586e06b926eadb80914aa08a3b4886e45663f3ab4e0a07439d", "size_bytes": 5562}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2025-08-07-high/0f86b39c-c913-420e-b056-507958f1ceac.json", "object_path": "flat/objects/0f/86/0f86b39c-c913-420e-b056-507958f1ceac.json", "object_uuid": "0f86b39c-c913-420e-b056-507958f1ceac", "record_type": "aggregate", "sha256": "9a19f7c4db7d1b44f8d8176711152a29d2911cebd00d9d74e8271176dbf3e44e", "size_bytes": 9897}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2025-08-07-low/a94237a0-6fce-4245-9dbe-4834f65604ab.json", "object_path": "flat/objects/a9/42/a94237a0-6fce-4245-9dbe-4834f65604ab.json", "object_uuid": "a94237a0-6fce-4245-9dbe-4834f65604ab", "record_type": "aggregate", "sha256": "718b1dae99237b7e3712d3e6cdcbec1ed5e380896ed6efdf6e268603a1b4417f", "size_bytes": 9864}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2025-08-07-medium/c215742c-a7a7-4e36-9fd1-b4a7ea88592c.json", "object_path": "flat/objects/c2/15/c215742c-a7a7-4e36-9fd1-b4a7ea88592c.json", "object_uuid": "c215742c-a7a7-4e36-9fd1-b4a7ea88592c", "record_type": "aggregate", "sha256": "12b87b702c27469876bb86adb1e7b69c742ced3ea2b5bc39990ad5becb438f89", "size_bytes": 9965}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-2025-08-07-minimal/0fe0c039-e609-4db8-8739-30ec8ea40ab7.json", "object_path": "flat/objects/0f/e0/0fe0c039-e609-4db8-8739-30ec8ea40ab7.json", "object_uuid": "0fe0c039-e609-4db8-8739-30ec8ea40ab7", "record_type": "aggregate", "sha256": "6220a547adfce2e715591edeb594a78760246395ff7bcf455ba1039067f07e48", "size_bytes": 9956}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-2026-03-05-high/dcb9e927-86df-432d-8186-19badd59c3ce.json", "object_path": "flat/objects/dc/b9/dcb9e927-86df-432d-8186-19badd59c3ce.json", "object_uuid": "dcb9e927-86df-432d-8186-19badd59c3ce", "record_type": "aggregate", "sha256": "cb3d96c9d13b91004c9551e4edf22740733f492a7c3a55a83e34858b54319be5", "size_bytes": 3374}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-high/51daef9d-ce8a-4d65-99ad-ef83e6e5d72f.json", "object_path": "flat/objects/51/da/51daef9d-ce8a-4d65-99ad-ef83e6e5d72f.json", "object_uuid": "51daef9d-ce8a-4d65-99ad-ef83e6e5d72f", "record_type": "aggregate", "sha256": "95b9a175337764df70153d6f3fb7926668e1ccee68435e060e3d01237465027f", "size_bytes": 9620}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-low/32795adc-1fee-471f-8947-b0e4dcc53e47.json", "object_path": "flat/objects/32/79/32795adc-1fee-471f-8947-b0e4dcc53e47.json", "object_uuid": "32795adc-1fee-471f-8947-b0e4dcc53e47", "record_type": "aggregate", "sha256": "319764d06da7a4eb213a6a10100819f0fc5a579065df36ce741795e111510f70", "size_bytes": 9597}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-medium/e63595d6-e7a3-4455-9122-0eaa923e7c9b.json", "object_path": "flat/objects/e6/35/e63595d6-e7a3-4455-9122-0eaa923e7c9b.json", "object_uuid": "e63595d6-e7a3-4455-9122-0eaa923e7c9b", "record_type": "aggregate", "sha256": "1f07f13f84428dbc9002739222bcb3a28cb624a116f316256ff818c6d3bdd5cf", "size_bytes": 9686}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-mini-high/ff2db4c2-9d9a-41e9-80b9-bc0044a85c88.json", "object_path": "flat/objects/ff/2d/ff2db4c2-9d9a-41e9-80b9-bc0044a85c88.json", "object_uuid": "ff2db4c2-9d9a-41e9-80b9-bc0044a85c88", "record_type": "aggregate", "sha256": "08bb7f52943e7fa8a38ee8cb7c88d9bbe47e1c9da90180103e47c6e5abe1b9e0", "size_bytes": 9763}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-mini-low/df2814fe-8d0d-4804-910a-bde483fd9895.json", "object_path": "flat/objects/df/28/df2814fe-8d0d-4804-910a-bde483fd9895.json", "object_uuid": "df2814fe-8d0d-4804-910a-bde483fd9895", "record_type": "aggregate", "sha256": "bbcd9b8d81918c7485a5d673b97cd0a92c5058516dae806aab2cd4621003f37e", "size_bytes": 9734}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-mini-medium/76d15cc6-c90d-4563-8f8e-11526547242e.json", "object_path": "flat/objects/76/d1/76d15cc6-c90d-4563-8f8e-11526547242e.json", "object_uuid": "76d15cc6-c90d-4563-8f8e-11526547242e", "record_type": "aggregate", "sha256": "f98d5b4b0d74542345f31ebde26592b7284643352cf5f7adf294fa5acfc35d99", "size_bytes": 9825}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-mini-xhigh/dc4234dd-ff41-476b-b73b-93432261b24c.json", "object_path": "flat/objects/dc/42/dc4234dd-ff41-476b-b73b-93432261b24c.json", "object_uuid": "dc4234dd-ff41-476b-b73b-93432261b24c", "record_type": "aggregate", "sha256": "f79e5b004432a01e5fdec8db3c9ea10257cb1cc6e23a5de9c798b1ce293ee9a5", "size_bytes": 9794}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-nano-high/a855ae7b-064e-470f-b5b2-661f13fd41ed.json", "object_path": "flat/objects/a8/55/a855ae7b-064e-470f-b5b2-661f13fd41ed.json", "object_uuid": "a855ae7b-064e-470f-b5b2-661f13fd41ed", "record_type": "aggregate", "sha256": "441174dc314cb850a9ecebaa5082db36be6bd49637f0a8b9342171debedbab4b", "size_bytes": 9767}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-nano-low/c129a837-4fc6-4300-b0e8-0216bedfca8c.json", "object_path": "flat/objects/c1/29/c129a837-4fc6-4300-b0e8-0216bedfca8c.json", "object_uuid": "c129a837-4fc6-4300-b0e8-0216bedfca8c", "record_type": "aggregate", "sha256": "7fb6c909f8040c6c95d0e21c356461774a07b6be1c29aebb794010face369a2b", "size_bytes": 9732}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-nano-medium/3cc47452-50b3-479a-8b75-bee6192c11bb.json", "object_path": "flat/objects/3c/c4/3cc47452-50b3-479a-8b75-bee6192c11bb.json", "object_uuid": "3cc47452-50b3-479a-8b75-bee6192c11bb", "record_type": "aggregate", "sha256": "eb635ae0861602c7ac6bbe0000704b81d0c3db2175e62aa52a6adbf670c85b6a", "size_bytes": 9821}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-nano-xhigh/d0451a9c-4ea3-4c56-87a0-f3242f157f63.json", "object_path": "flat/objects/d0/45/d0451a9c-4ea3-4c56-87a0-f3242f157f63.json", "object_uuid": "d0451a9c-4ea3-4c56-87a0-f3242f157f63", "record_type": "aggregate", "sha256": "b199425f82e9f0702b15582b9410740612f0dc6f3c8c5b603c8181e136dbe8ce", "size_bytes": 9802}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-pro-xhigh/76c59754-c6d3-44ab-ae7c-d209acf2cd20.json", "object_path": "flat/objects/76/c5/76c59754-c6d3-44ab-ae7c-d209acf2cd20.json", "object_uuid": "76c59754-c6d3-44ab-ae7c-d209acf2cd20", "record_type": "aggregate", "sha256": "63867e6ae56c20f996aca22b3dd8c60dd51f579e058ea0abb998402169770c35", "size_bytes": 9767}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-4-xhigh/fe7c3e43-84f7-4159-8145-fb53bde09930.json", "object_path": "flat/objects/fe/7c/fe7c3e43-84f7-4159-8145-fb53bde09930.json", "object_uuid": "fe7c3e43-84f7-4159-8145-fb53bde09930", "record_type": "aggregate", "sha256": "d5cf60d222e61500a0de45cbb25c780746d934a593090677f42ecd98185cb150", "size_bytes": 9661}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-mini-2025-08-07-high/e2aadafe-496b-4a27-8e2e-0ed9afa16f2d.json", "object_path": "flat/objects/e2/aa/e2aadafe-496b-4a27-8e2e-0ed9afa16f2d.json", "object_uuid": "e2aadafe-496b-4a27-8e2e-0ed9afa16f2d", "record_type": "aggregate", "sha256": "497d54b3ceeeb474e7e511008fa9110f7f6260e4694f2ca2cff7e9ada6975b81", "size_bytes": 10030}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-mini-2025-08-07-low/2ecfc8ed-cfa8-4e7d-b493-e5de2eb3c35f.json", "object_path": "flat/objects/2e/cf/2ecfc8ed-cfa8-4e7d-b493-e5de2eb3c35f.json", "object_uuid": "2ecfc8ed-cfa8-4e7d-b493-e5de2eb3c35f", "record_type": "aggregate", "sha256": "c31172514067d3d354499a544c1bb166a357a331013b68dd9453c49dc942bd13", "size_bytes": 10025}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-mini-2025-08-07-medium/b4617018-4859-4c0a-bbbd-4928b35a6b4a.json", "object_path": "flat/objects/b4/61/b4617018-4859-4c0a-bbbd-4928b35a6b4a.json", "object_uuid": "b4617018-4859-4c0a-bbbd-4928b35a6b4a", "record_type": "aggregate", "sha256": "823c9f419101b0af48bd97a5b947a9b0524945d2284f718f351cd6468edef66d", "size_bytes": 10110}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-mini-2025-08-07-minimal/f3c3573a-6ca4-4e40-88ab-5c1588a16dcf.json", "object_path": "flat/objects/f3/c3/f3c3573a-6ca4-4e40-88ab-5c1588a16dcf.json", "object_uuid": "f3c3573a-6ca4-4e40-88ab-5c1588a16dcf", "record_type": "aggregate", "sha256": "690f900af433b15fadcc07b0ab43ffaf2af234ebf946ae688aefb4536cb225bc", "size_bytes": 10123}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-nano-2025-08-07-high/9777d9b5-e89f-4fc9-b621-fbef22e76e56.json", "object_path": "flat/objects/97/77/9777d9b5-e89f-4fc9-b621-fbef22e76e56.json", "object_uuid": "9777d9b5-e89f-4fc9-b621-fbef22e76e56", "record_type": "aggregate", "sha256": "b7051d2e00b4b6a484a81935c23dbc9e694c0e8b0afc28241bab7c7888e96d64", "size_bytes": 10042}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-nano-2025-08-07-low/d7cedccf-3026-4b48-8c6d-8c74cfee683a.json", "object_path": "flat/objects/d7/ce/d7cedccf-3026-4b48-8c6d-8c74cfee683a.json", "object_uuid": "d7cedccf-3026-4b48-8c6d-8c74cfee683a", "record_type": "aggregate", "sha256": "cf8dcc135c8304b525ea5dba4fae54d2141f0ce8190f6df9dc015353332c3153", "size_bytes": 9999}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-nano-2025-08-07-medium/a67ae2be-ef5a-4e13-9757-7c70d4645a3e.json", "object_path": "flat/objects/a6/7a/a67ae2be-ef5a-4e13-9757-7c70d4645a3e.json", "object_uuid": "a67ae2be-ef5a-4e13-9757-7c70d4645a3e", "record_type": "aggregate", "sha256": "00276b58b62567f2e477ed58f1025ee510bb158322a0bc2a3fc64bf5c0476722", "size_bytes": 10104}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-nano-2025-08-07-minimal/a9e53599-5f48-411c-9476-5320ea49c32b.json", "object_path": "flat/objects/a9/e5/a9e53599-5f48-411c-9476-5320ea49c32b.json", "object_uuid": "a9e53599-5f48-411c-9476-5320ea49c32b", "record_type": "aggregate", "sha256": "455a878da0b59fedc8aa9f66b9c2b98a6d96a11c21e92b431e7acb00a0590e82", "size_bytes": 10113}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/gpt-5-pro-2025-10-06/15e6df2c-027f-4b84-8beb-6f4fe1adcd1d.json", "object_path": "flat/objects/15/e6/15e6df2c-027f-4b84-8beb-6f4fe1adcd1d.json", "object_uuid": "15e6df2c-027f-4b84-8beb-6f4fe1adcd1d", "record_type": "aggregate", "sha256": "0ee398b7db165a428e383da32868ec00e09e55fc8ccc5682af091b277f67040a", "size_bytes": 9876}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o1-mini-2024-09-12/72a56e6a-25c0-41dc-83ad-dc46e11a85a6.json", "object_path": "flat/objects/72/a5/72a56e6a-25c0-41dc-83ad-dc46e11a85a6.json", "object_uuid": "72a56e6a-25c0-41dc-83ad-dc46e11a85a6", "record_type": "aggregate", "sha256": "41219a8c28e6e585b4dc679713dcd97511868e38261826a5a2457d78f2bf9264", "size_bytes": 9810}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-2025-04-16-high/a666d9bd-9f0a-4e36-9a90-819ca1f31945.json", "object_path": "flat/objects/a6/66/a666d9bd-9f0a-4e36-9a90-819ca1f31945.json", "object_uuid": "a666d9bd-9f0a-4e36-9a90-819ca1f31945", "record_type": "aggregate", "sha256": "9001c4b4a06436f69b464d5b5fde881da771c9f02626e6af1afa7098a7d39da8", "size_bytes": 9812}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-2025-04-16-low/06db9e6a-fd0c-4615-9339-ba21c28184f7.json", "object_path": "flat/objects/06/db/06db9e6a-fd0c-4615-9339-ba21c28184f7.json", "object_uuid": "06db9e6a-fd0c-4615-9339-ba21c28184f7", "record_type": "aggregate", "sha256": "01bfeec9401aa0fe61d8852c0fec78dde71ce4c063503ff3614da88d5133167e", "size_bytes": 9779}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-2025-04-16-medium/e6347367-dd90-4bb9-9a7e-8a604e2d04ae.json", "object_path": "flat/objects/e6/34/e6347367-dd90-4bb9-9a7e-8a604e2d04ae.json", "object_uuid": "e6347367-dd90-4bb9-9a7e-8a604e2d04ae", "record_type": "aggregate", "sha256": "f39b01890d55182eb6460e6886d7774b2fafd90287ecb33d37b23905fdae0297", "size_bytes": 9880}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-mini-2025-01-31-high/81493e61-fbf0-4346-839a-74df3ebf9b5b.json", "object_path": "flat/objects/81/49/81493e61-fbf0-4346-839a-74df3ebf9b5b.json", "object_uuid": "81493e61-fbf0-4346-839a-74df3ebf9b5b", "record_type": "aggregate", "sha256": "523df6e30d661ddfe9e5555cfea81f0e9c4a0b282622bcf9ced8e3e3a415602e", "size_bytes": 9949}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-mini-2025-01-31-low/d71f7512-c1f5-4ff8-81a9-af5723c54882.json", "object_path": "flat/objects/d7/1f/d71f7512-c1f5-4ff8-81a9-af5723c54882.json", "object_uuid": "d71f7512-c1f5-4ff8-81a9-af5723c54882", "record_type": "aggregate", "sha256": "86dd13c4dec9aa898a11a1c1d40454710230046449c08b30637f6d491c510416", "size_bytes": 9924}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-mini-2025-01-31-medium/d50c0ec4-3694-43e3-abd2-9bef55812973.json", "object_path": "flat/objects/d5/0c/d50c0ec4-3694-43e3-abd2-9bef55812973.json", "object_uuid": "d50c0ec4-3694-43e3-abd2-9bef55812973", "record_type": "aggregate", "sha256": "6c77005a96e9a90856e240f4e7b05385d5fbb0fbcdcdebaf1989e96f4d4707ac", "size_bytes": 10017}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-pro-2025-06-10-high/978812b0-48fd-4588-b5c1-91f15feb01a2.json", "object_path": "flat/objects/97/88/978812b0-48fd-4588-b5c1-91f15feb01a2.json", "object_uuid": "978812b0-48fd-4588-b5c1-91f15feb01a2", "record_type": "aggregate", "sha256": "58a39b6c1bf9adb2c388c9367af4dba3d52946064612a5490869d73e6cb8eb0b", "size_bytes": 9924}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-pro-2025-06-10-low/205def2a-7497-4a5f-a55c-46e2dad4adb7.json", "object_path": "flat/objects/20/5d/205def2a-7497-4a5f-a55c-46e2dad4adb7.json", "object_uuid": "205def2a-7497-4a5f-a55c-46e2dad4adb7", "record_type": "aggregate", "sha256": "2246f71b90cb73a8f657fd60f3e8c01ab6637b3ba0c68aaacc2af38e1d40838c", "size_bytes": 9897}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o3-pro-2025-06-10-medium/27b462e9-e1c9-4da2-94ed-fa75e66bc57b.json", "object_path": "flat/objects/27/b4/27b462e9-e1c9-4da2-94ed-fa75e66bc57b.json", "object_uuid": "27b462e9-e1c9-4da2-94ed-fa75e66bc57b", "record_type": "aggregate", "sha256": "938ea92ffe693a7024d6b78fb210016ba1e0f9a51b1612af21f7e9adede3fb00", "size_bytes": 9980}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o4-mini-2025-04-16-high/4fcb1c71-bba2-4754-b35f-ec5b54e3f692.json", "object_path": "flat/objects/4f/cb/4fcb1c71-bba2-4754-b35f-ec5b54e3f692.json", "object_uuid": "4fcb1c71-bba2-4754-b35f-ec5b54e3f692", "record_type": "aggregate", "sha256": "5fd2f2457e812aca22b2f685e096c6f3e9559721590883a69f48d37784f7a2d6", "size_bytes": 9935}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o4-mini-2025-04-16-low/213b6c9b-eb95-4857-877b-534e94a4bb8e.json", "object_path": "flat/objects/21/3b/213b6c9b-eb95-4857-877b-534e94a4bb8e.json", "object_uuid": "213b6c9b-eb95-4857-877b-534e94a4bb8e", "record_type": "aggregate", "sha256": "285fcb09836d0c560c14775f56b361b0d936e509167eda09a25db44dff5979e6", "size_bytes": 9934}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/openai/o4-mini-2025-04-16-medium/04db58da-e9ed-493f-8c65-f88a572abf2d.json", "object_path": "flat/objects/04/db/04db58da-e9ed-493f-8c65-f88a572abf2d.json", "object_uuid": "04db58da-e9ed-493f-8c65-f88a572abf2d", "record_type": "aggregate", "sha256": "d1f0af71807d646b2e4594ac467fd3f634dc627ef6bdca071159a28a2867bff9", "size_bytes": 10007}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/qwen/qwen3-235b-a22b-instruct-2507/929438a0-89d5-44fa-ae38-4cf583f4b778.json", "object_path": "flat/objects/92/94/929438a0-89d5-44fa-ae38-4cf583f4b778.json", "object_uuid": "929438a0-89d5-44fa-ae38-4cf583f4b778", "record_type": "aggregate", "sha256": "0228a1c88f9e526a774426b9846f770b01d7f7f88a33b4b74f3d096a8c3a5866", "size_bytes": 10111}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/xai/grok-3-mini-beta-low-openrouter/d6299979-fa98-4b4b-934c-c7d2dde2d0d0.json", "object_path": "flat/objects/d6/29/d6299979-fa98-4b4b-934c-c7d2dde2d0d0.json", "object_uuid": "d6299979-fa98-4b4b-934c-c7d2dde2d0d0", "record_type": "aggregate", "sha256": "ff914b57ee49cffb65486d151aaa0874ec8b2b6d0ada6c00f3a24da7ee1e85dc", "size_bytes": 10146}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/xai/grok-3-openrouter/0cbb4662-77ae-4245-b53a-4f3af687decb.json", "object_path": "flat/objects/0c/bb/0cbb4662-77ae-4245-b53a-4f3af687decb.json", "object_uuid": "0cbb4662-77ae-4245-b53a-4f3af687decb", "record_type": "aggregate", "sha256": "cb11a8b0201eba4ad2c1f3e5e073b95a132b5a6078e3b0b7859edf0585c19c1a", "size_bytes": 9736}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/xai/grok-4-0709/47a87599-00ae-42be-863d-8380c5437107.json", "object_path": "flat/objects/47/a8/47a87599-00ae-42be-863d-8380c5437107.json", "object_uuid": "47a87599-00ae-42be-863d-8380c5437107", "record_type": "aggregate", "sha256": "5ec41bfa5737e0a4168a298baefa29047c209e0e9b0466c3eaa62f206ddf72fe", "size_bytes": 9588}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/xai/grok-4-20-beta-0309-reasoning/48dfb7b5-c0d1-49f2-88e6-c98fe14a08c3.json", "object_path": "flat/objects/48/df/48dfb7b5-c0d1-49f2-88e6-c98fe14a08c3.json", "object_uuid": "48dfb7b5-c0d1-49f2-88e6-c98fe14a08c3", "record_type": "aggregate", "sha256": "547218a939d1a10264bda7530c813feafff994fa43afd5aaae254a1d4fce5ffa", "size_bytes": 3390}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/xai/grok-4-fast-reasoning/b415752b-9557-43cf-8a52-9b98d42ff053.json", "object_path": "flat/objects/b4/15/b415752b-9557-43cf-8a52-9b98d42ff053.json", "object_uuid": "b415752b-9557-43cf-8a52-9b98d42ff053", "record_type": "aggregate", "sha256": "66095daaf484d4de5f1e3c6c5ca88eebf9d54f5bdf418acbabb65562fafeffa5", "size_bytes": 9872}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/xai/grok-4.20-beta-0309b-reasoning/c3c2d5a8-bf1c-46d5-ab3d-4c22528e34f8.json", "object_path": "flat/objects/c3/c2/c3c2d5a8-bf1c-46d5-ab3d-4c22528e34f8.json", "object_uuid": "c3c2d5a8-bf1c-46d5-ab3d-4c22528e34f8", "record_type": "aggregate", "sha256": "5c144bf6cef1c6c3589c160155673487067cc6f7f8d3c52775209b07099623c6", "size_bytes": 10107}
+{"benchmark": "arc-agi", "eval_schema_version": "0.2.2", "legacy_path": "data/arc-agi/zhipu/glm-5/18f23454-de5c-446a-a00b-849a02f81282.json", "object_path": "flat/objects/18/f2/18f23454-de5c-446a-a00b-849a02f81282.json", "object_uuid": "18f23454-de5c-446a-a00b-849a02f81282", "record_type": "aggregate", "sha256": "70b261f515077d62ea06ee1f80e5adf63a4579fb62181592f0b96d40878abb98", "size_bytes": 9408}
diff --git a/eval.schema.json b/eval.schema.json
deleted file mode 120000
index c1943c02f..000000000
--- a/eval.schema.json
+++ /dev/null
@@ -1 +0,0 @@
-every_eval_ever/schemas/eval.schema.json
\ No newline at end of file
diff --git a/eval.schema.json b/eval.schema.json
new file mode 100644
index 000000000..c1943c02f
--- /dev/null
+++ b/eval.schema.json
@@ -0,0 +1 @@
+every_eval_ever/schemas/eval.schema.json
\ No newline at end of file
diff --git a/every_eval_ever-main.zip b/every_eval_ever-main.zip
new file mode 100644
index 000000000..6061ea0a1
Binary files /dev/null and b/every_eval_ever-main.zip differ
diff --git a/every_eval_ever/helpers/io.py b/every_eval_ever/helpers/io.py
index 88ac19730..8e2c1d2fa 100644
--- a/every_eval_ever/helpers/io.py
+++ b/every_eval_ever/helpers/io.py
@@ -21,7 +21,12 @@ def sanitize_filename(name: str) -> str:
Sanitized string safe for filesystem use
"""
# Replace characters invalid on Windows/Unix filesystems
- return re.sub(r'[<>:"/\\|?*]', '_', name)
+ name = re.sub(r'[<>:"/\\|?*]', '_', name)
+ # Hugging Face Hub blocks paths containing "/x00". If a name starts with x00,
+ # it will create a /x00 in the path when joined.
+ if name.startswith("x00"):
+ name = "x_00" + name[3:]
+ return name
def generate_output_path(
diff --git a/every_eval_ever/helpers/schema.py b/every_eval_ever/helpers/schema.py
index 12b7eb190..8c8704542 100644
--- a/every_eval_ever/helpers/schema.py
+++ b/every_eval_ever/helpers/schema.py
@@ -7,7 +7,7 @@
def _load_schema_version() -> str:
- schema_path = Path(__file__).parent.parent.parent / 'eval.schema.json'
+ schema_path = Path(__file__).parent.parent / 'schemas' / 'eval.schema.json'
with schema_path.open() as f:
return json.load(f)['version']
diff --git a/instance_level.jsonl b/instance_level.jsonl
new file mode 100644
index 000000000..e69de29bb
diff --git a/instance_level_eval.schema.json b/instance_level_eval.schema.json
deleted file mode 120000
index 7e10082d4..000000000
--- a/instance_level_eval.schema.json
+++ /dev/null
@@ -1 +0,0 @@
-every_eval_ever/schemas/instance_level_eval.schema.json
\ No newline at end of file
diff --git a/instance_level_eval.schema.json b/instance_level_eval.schema.json
new file mode 100644
index 000000000..7e10082d4
--- /dev/null
+++ b/instance_level_eval.schema.json
@@ -0,0 +1 @@
+every_eval_ever/schemas/instance_level_eval.schema.json
\ No newline at end of file
diff --git a/pr_corrector.py b/pr_corrector.py
new file mode 100644
index 000000000..fdc3a0eac
--- /dev/null
+++ b/pr_corrector.py
@@ -0,0 +1,500 @@
+import os
+import re
+import json
+import shutil
+import stat
+import urllib.request
+from pathlib import Path
+from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_download
+
+from every_eval_ever.validate import validate_file
+
+# --- CONFIGURATION ---
+REPO_ID = "evaleval/EEE_datastore"
+REPO_TYPE = "dataset"
+WORKSPACE = "EEE_Targeted_Workspace"
+SCHEMA_PATH = "eval.schema.json"
+
+
+def remove_readonly(func, path, excinfo):
+ """Clear the read-only bit on Windows so shutil can delete files."""
+ os.chmod(path, stat.S_IWRITE)
+ func(path)
+
+
+def load_schema():
+ with open(SCHEMA_PATH, "r", encoding="utf-8") as f:
+ return json.load(f)
+
+
+def resolve_ref(schema, ref):
+ parts = ref.replace("#/", "").split("/")
+ current = schema
+ for part in parts:
+ current = current.get(part, {})
+ return current
+
+
+def get_schema_info(loc_string, root_schema):
+ """Parses loc string like 'evaluation_results -> [0] -> metric_config' and returns its schema."""
+ parts = loc_string.split(" -> ")
+ current = root_schema
+
+ for part in parts:
+ if "$ref" in current:
+ current = resolve_ref(root_schema, current["$ref"])
+
+ if part.startswith("[") and part.endswith("]"):
+ current = current.get("items", {})
+ else:
+ if "properties" in current and part in current["properties"]:
+ current = current["properties"][part]
+ elif "oneOf" in current:
+ found = False
+ for option in current["oneOf"]:
+ opt = option
+ if "$ref" in opt:
+ opt = resolve_ref(root_schema, opt["$ref"])
+ if "properties" in opt and part in opt["properties"]:
+ current = opt["properties"][part]
+ found = True
+ break
+ if not found:
+ current = {}
+ else:
+ current = {}
+
+ if "$ref" in current:
+ current = resolve_ref(root_schema, current["$ref"])
+
+ return current
+
+
+def convert_value(user_input, schema_info):
+ if not schema_info:
+ return user_input
+
+ expected_type = schema_info.get("type")
+
+ if expected_type == "boolean":
+ return user_input.lower() in ["true", "1", "yes", "y", "t"]
+ elif expected_type == "integer":
+ try:
+ return int(user_input)
+ except ValueError:
+ return user_input
+ elif expected_type == "number":
+ try:
+ return float(user_input)
+ except ValueError:
+ return user_input
+ elif expected_type == "array":
+ try:
+ parsed = json.loads(user_input)
+ if isinstance(parsed, list):
+ return parsed
+ except json.JSONDecodeError:
+ pass
+ # Simple fallback for arrays
+ return [s.strip() for s in user_input.split(",") if s.strip()]
+
+ # If type is an array of options e.g., ["null", "number"]
+ if isinstance(expected_type, list):
+ if "number" in expected_type:
+ try:
+ return float(user_input)
+ except ValueError:
+ pass
+ if "integer" in expected_type:
+ try:
+ return int(user_input)
+ except ValueError:
+ pass
+
+ return user_input
+
+
+def apply_fix_exact(data, loc_string, value):
+ """Dynamically traverses a JSON object and injects the value at the exact location."""
+ parts = loc_string.split(" -> ")
+ if not parts:
+ return
+
+ current = data
+ for i in range(len(parts) - 1):
+ part = parts[i]
+ if part.startswith("[") and part.endswith("]"):
+ index = int(part[1:-1])
+ current = current[index]
+ else:
+ if part not in current:
+ current[part] = {}
+ current = current[part]
+
+ last_part = parts[-1]
+ if last_part.startswith("[") and last_part.endswith("]"):
+ index = int(last_part[1:-1])
+ current[index] = value
+ else:
+ current[last_part] = value
+
+
+def apply_fix_fuzzy(data, path_str, key, value):
+ """Dynamically traverses a JSON object using dot notation and injects the key/value pair."""
+ parts = path_str.split('.') if path_str else []
+
+ def traverse(current_obj, path_parts):
+ if not path_parts:
+ if isinstance(current_obj, dict):
+ current_obj[key] = value
+ return
+
+ part = path_parts[0]
+ if isinstance(current_obj, dict):
+ if part not in current_obj:
+ current_obj[part] = {}
+ if len(path_parts) == 1:
+ if isinstance(current_obj[part], list):
+ for item in current_obj[part]:
+ if isinstance(item, dict):
+ item[key] = value
+ else:
+ current_obj[part][key] = value
+ else:
+ traverse(current_obj[part], path_parts[1:])
+ elif isinstance(current_obj, list):
+ for item in current_obj:
+ traverse(item, path_parts)
+
+ traverse(data, parts)
+
+
+def get_pr_files(repo_id, pr_num, token):
+ """Fetches the list of modified .json files in the PR using diffUrl or HTML scraping as a fallback."""
+ url = f"https://huggingface.co/api/datasets/{repo_id}/discussions/{pr_num}"
+ req = urllib.request.Request(url)
+ if token:
+ req.add_header("Authorization", f"Bearer {token}")
+
+ try:
+ with urllib.request.urlopen(req) as res:
+ data = json.loads(res.read().decode('utf-8'))
+
+ diff_url = data.get('diffUrl')
+ if diff_url:
+ if diff_url.startswith('/'):
+ diff_url = "https://huggingface.co" + diff_url
+ diff_req = urllib.request.Request(diff_url)
+ if token:
+ diff_req.add_header("Authorization", f"Bearer {token}")
+ with urllib.request.urlopen(diff_req) as diff_res:
+ diff_text = diff_res.read().decode('utf-8')
+ files = re.findall(r'^\+\+\+ b/(data/.*\.json)$', diff_text, flags=re.MULTILINE)
+ if files:
+ return list(set(files))
+ except Exception as e:
+ pass
+
+ # Fallback to HTML scraping
+ html_url = f"https://huggingface.co/datasets/{repo_id}/discussions/{pr_num}/files"
+ html_req = urllib.request.Request(html_url)
+ if token:
+ html_req.add_header("Authorization", f"Bearer {token}")
+ try:
+ with urllib.request.urlopen(html_req) as res:
+ html = res.read().decode('utf-8')
+ files = re.findall(r'(data/[\w\-\./]+\.json)', html)
+ return list(set(files))
+ except Exception as e:
+ print(f"❌ Failed to parse files from PR: {e}")
+
+ return []
+
+
+def custom_validate(data):
+ """Custom heuristic validator for warnings not strictly in the schema."""
+ bot_warnings = []
+
+ # 1. Check for deployment_type and dependent fields
+ model_info = data.get('model_info', {})
+ additional = model_info.get('additional_details', {})
+
+ deployment_type = additional.get('deployment_type')
+ if not deployment_type:
+ bot_warnings.append(('model_info.additional_details', 'deployment_type', 'Expected one of: api, local, unknown'))
+ else:
+ if deployment_type == 'api' and 'model_availability' not in additional:
+ bot_warnings.append(('model_info.additional_details', 'model_availability', "Expected one of: closed_source, open_weights_deployment, other"))
+
+ return bot_warnings
+
+
+def get_all_json_paths(data, parent_path=""):
+ paths = []
+ if isinstance(data, dict):
+ for k, v in data.items():
+ current_path = f"{parent_path} -> {k}" if parent_path else k
+ if isinstance(v, (dict, list)) and v:
+ paths.extend(get_all_json_paths(v, current_path))
+ else:
+ paths.append((current_path, v))
+ elif isinstance(data, list):
+ for i, item in enumerate(data):
+ current_path = f"{parent_path} -> [{i}]"
+ if isinstance(item, (dict, list)) and item:
+ paths.extend(get_all_json_paths(item, current_path))
+ else:
+ paths.append((current_path, item))
+ else:
+ paths.append((parent_path, data))
+ return paths
+
+
+def print_json_summary(data):
+ print("\n--- Current JSON Fields & Values ---")
+ paths = get_all_json_paths(data)
+ for path, val in paths:
+ val_str = str(val)
+ if len(val_str) > 80:
+ val_str = val_str[:77] + "..."
+ print(f" {path} : {val_str}")
+ print("------------------------------------\n")
+
+
+def main():
+ print("🤖 EEE Datastore Targeted PR Fixer 🤖\n")
+
+ hf_token = input("Enter your Hugging Face Access Token: ").strip()
+ pr_input = input("Enter the PR number or URL (e.g., 136 or https://.../136): ").strip()
+
+ pr_num = re.search(r'(\d+)$', pr_input).group(1)
+ revision = f"refs/pr/{pr_num}"
+ api = HfApi(token=hf_token)
+
+ try:
+ schema = load_schema()
+ except Exception as e:
+ print(f"❌ Failed to load {SCHEMA_PATH}: {e}")
+ return
+
+ print("\n📂 Fetching file list for the PR...")
+ json_files = get_pr_files(REPO_ID, pr_num, hf_token)
+
+ if not json_files:
+ print("❌ No aggregate JSON files found in this PR to fix. Is the PR number correct?")
+ return
+
+ print(f"Found {len(json_files)} file(s) in PR #{pr_num}.")
+
+ if os.path.exists(WORKSPACE):
+ shutil.rmtree(WORKSPACE, onexc=remove_readonly)
+ os.makedirs(WORKSPACE)
+
+ operations = []
+ apply_to_all_answers = {}
+ print(f"Downloading {len(json_files)} JSON file(s) locally...\n")
+
+ for file_path in json_files:
+ try:
+ local_path = hf_hub_download(
+ repo_id=REPO_ID,
+ repo_type=REPO_TYPE,
+ filename=file_path,
+ revision=revision,
+ token=hf_token,
+ local_dir=WORKSPACE,
+ local_dir_use_symlinks=False
+ )
+ except Exception as e:
+ print(f"❌ Failed to download {file_path}: {e}")
+ continue
+
+ modified = False
+
+ while True:
+ # Validate the downloaded file
+ report = validate_file(Path(local_path))
+
+ with open(local_path, "r", encoding="utf-8") as f:
+ data = json.load(f)
+
+ bot_warnings = custom_validate(data)
+ missing_errors = [err for err in report.errors if "missing" in err.get("type", "")]
+
+ # 1. If there are missing fields or bot warnings, process them automatically first.
+ if missing_errors or bot_warnings:
+ print(f"\n📄 Auto-fixing {file_path.split('/')[-1]} ({len(missing_errors)} missing fields, {len(bot_warnings)} bot warnings)")
+ made_changes_this_round = False
+
+ # Process standard validation missing errors
+ for err in missing_errors:
+ loc_str = err['loc']
+ last_key = loc_str.split(" -> ")[-1]
+
+ if last_key in apply_to_all_answers:
+ apply_fix_exact(data, loc_str, apply_to_all_answers[last_key])
+ made_changes_this_round = True
+ continue
+
+ schema_info = get_schema_info(loc_str, schema)
+ desc = schema_info.get("description", "No description available")
+ expected_type = schema_info.get("type", "unknown")
+
+ print(f"\n Missing: '{last_key}' at '{loc_str}'")
+ print(f" Type: {expected_type} | Description: {desc}")
+
+ user_input = input(f" Enter value (or 'skip', or 'all:your_value'): ").strip()
+
+ if user_input.lower() == 'skip' or user_input == '':
+ print(" ⏭️ Skipped.")
+ continue
+
+ if user_input.lower().startswith("all:"):
+ raw_value = user_input[4:].strip()
+ converted = convert_value(raw_value, schema_info)
+ apply_to_all_answers[last_key] = converted
+ apply_fix_exact(data, loc_str, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}' to this and all future occurrences of '{last_key}'.")
+ else:
+ converted = convert_value(user_input, schema_info)
+ apply_fix_exact(data, loc_str, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}'.")
+
+ # Process bot warnings
+ for path_str, key, desc in bot_warnings:
+ if key in apply_to_all_answers:
+ apply_fix_fuzzy(data, path_str, key, apply_to_all_answers[key])
+ made_changes_this_round = True
+ continue
+
+ print(f"\n Bot Warning Missing: '{key}' inside '{path_str}'")
+ print(f" Description: {desc}")
+ user_input = input(f" Enter value (or 'skip', or 'all:your_value'): ").strip()
+
+ if user_input.lower() == 'skip' or user_input == '':
+ print(" ⏭️ Skipped.")
+ continue
+
+ if user_input.lower().startswith("all:"):
+ raw_value = user_input[4:].strip()
+ converted = convert_value(raw_value, {})
+ apply_to_all_answers[key] = converted
+ apply_fix_fuzzy(data, path_str, key, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}' to this and all future files.")
+ else:
+ converted = convert_value(user_input, {})
+ apply_fix_fuzzy(data, path_str, key, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}'.")
+
+ if made_changes_this_round:
+ modified = True
+ with open(local_path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=4, ensure_ascii=False)
+ f.write("\n")
+
+ # Re-validate after auto-fixes
+ continue
+
+ # 2. Re-evaluate errors after processing auto-fixes (or if there weren't any)
+ if report.valid and not bot_warnings:
+ print(f"✅ {file_path.split('/')[-1]} is valid and has no bot warnings!")
+ else:
+ print(f"⚠️ {file_path.split('/')[-1]} has validation issues:")
+ for err in report.errors:
+ print(f" - {err['loc']}: {err['msg']}")
+
+ # 3. Prompt for manual editing
+ manual_choice = input("\nDo you want to manually edit any fields in this file? (y/n) [n]: ").strip().lower()
+ if manual_choice in ['y', 'yes']:
+ while True:
+ # Print summary of current JSON keys and values
+ print_json_summary(data)
+ loc_str = input("Enter the exact path to edit (e.g., model_info -> name), or 'q' to finish: ").strip()
+ if not loc_str or loc_str.lower() == 'q':
+ break
+
+ schema_info = get_schema_info(loc_str, schema)
+ expected_type = schema_info.get("type", "unknown")
+ desc = schema_info.get("description", "No description available")
+
+ # Try to retrieve the current value for display
+ current_val = "Not Set"
+ try:
+ parts = loc_str.split(" -> ")
+ temp = data
+ for part in parts:
+ if part.startswith("[") and part.endswith("]"):
+ temp = temp[int(part[1:-1])]
+ else:
+ temp = temp[part]
+ current_val = temp
+ except Exception:
+ pass
+
+ print(f"\nEditing: '{loc_str}'")
+ print(f"Type: {expected_type} | Description: {desc}")
+ print(f"Current Value: {current_val}")
+ new_val_raw = input("Enter new value (or press Enter to cancel): ").strip()
+ if new_val_raw == "":
+ print("⏭️ Cancelled.")
+ continue
+
+ converted = convert_value(new_val_raw, schema_info)
+
+ try:
+ apply_fix_exact(data, loc_str, converted)
+ with open(local_path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=4, ensure_ascii=False)
+ f.write("\n")
+ modified = True
+ print(f"✅ Successfully updated '{loc_str}' to '{converted}'.")
+ except Exception as e:
+ print(f"❌ Failed to apply fix: {e}")
+
+ # Re-validate within editing loop to show updated errors
+ report = validate_file(Path(local_path))
+ bot_warnings = custom_validate(data)
+ if report.valid and not bot_warnings:
+ print("✅ Validation passes successfully now!")
+ else:
+ print(f"⚠️ Validation issues remaining: {len(report.errors)} errors, {len(bot_warnings)} warnings")
+
+ # 4. Check if we should break or stay in the loop for the file
+ report = validate_file(Path(local_path))
+ bot_warnings = custom_validate(data)
+ if report.valid and not bot_warnings:
+ break
+ else:
+ proceed = input("\nFile still has validation issues. Proceed anyway, or continue editing/fixing? (p = proceed/skip, f = edit/fix) [p]: ").strip().lower()
+ if proceed == 'f' or proceed == 'fix' or proceed == 'edit':
+ continue
+ else:
+ break
+
+ if modified:
+ operations.append(CommitOperationAdd(path_in_repo=file_path, path_or_fileobj=local_path))
+
+ if operations:
+ print(f"\n📤 Committing {len(operations)} updated file(s) to PR #{pr_num}...")
+ api.create_commit(
+ repo_id=REPO_ID,
+ repo_type=REPO_TYPE,
+ revision=revision,
+ operations=operations,
+ commit_message="Fix schema warnings via interactive script"
+ )
+ print("🎉 Success! Your PR has been updated.")
+ else:
+ print("\n⚠️ No changes were made that require a commit.")
+
+ print("🧹 Cleaning up local files...")
+ shutil.rmtree(WORKSPACE, onexc=remove_readonly)
+ print("✨ Cleanup complete!")
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 450f30f30..a9bc6ec7a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,10 +12,12 @@ authors = [
]
requires-python = ">=3.12"
dependencies = [
+ "aiohttp>=3.13.3",
"datamodel-code-generator[ruff]>=0.52.2",
"duckdb>=1.5.2",
"huggingface-hub>=0.36.0,<1.0.0",
"jsonschema>=4.26.0,<5.0.0",
+ "locust==2.44.5.dev5",
"matplotlib>=3.10.8",
"numpy>=2.4.1",
"pandas>=2.3.3",
diff --git a/start_dev.ps1 b/start_dev.ps1
new file mode 100644
index 000000000..1dc5d47e4
--- /dev/null
+++ b/start_dev.ps1
@@ -0,0 +1,6 @@
+# Start FastAPI backend in the background
+Start-Process -NoNewWindow -FilePath "python" -ArgumentList "-m", "uvicorn", "backend.api.main:app", "--reload", "--port", "8000"
+
+# Start Next.js frontend in the foreground
+cd web
+npm run dev
diff --git a/test_eval.json b/test_eval.json
new file mode 100644
index 000000000..b0a014079
--- /dev/null
+++ b/test_eval.json
@@ -0,0 +1 @@
+{"schema_version": "0.2.2", "model_info": {"name": "test"}}
diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json b/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json
deleted file mode 100644
index b05d3ab41..000000000
--- a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/per_instance_stats.json
+++ /dev/null
@@ -1,3742 +0,0 @@
-[
- {
- "instance_id": "id44874",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 718,
- "sum_squared": 515524,
- "min": 718,
- "max": 718,
- "mean": 718.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 18.749841690063477,
- "sum_squared": 351.5565634024424,
- "min": 18.749841690063477,
- "max": 18.749841690063477,
- "mean": 18.749841690063477,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id47299",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 684,
- "sum_squared": 467856,
- "min": 684,
- "max": 684,
- "mean": 684.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 17.995312690734863,
- "sum_squared": 323.8312788373232,
- "min": 17.995312690734863,
- "max": 17.995312690734863,
- "mean": 17.995312690734863,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id45277",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 664,
- "sum_squared": 440896,
- "min": 664,
- "max": 664,
- "mean": 664.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 16.95018243789673,
- "sum_squared": 287.3086846779827,
- "min": 16.95018243789673,
- "max": 16.95018243789673,
- "mean": 16.95018243789673,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id41992",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 523,
- "sum_squared": 273529,
- "min": 523,
- "max": 523,
- "mean": 523.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 13.807421445846558,
- "sum_squared": 190.64488698322344,
- "min": 13.807421445846558,
- "max": 13.807421445846558,
- "mean": 13.807421445846558,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id49438",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 636,
- "sum_squared": 404496,
- "min": 636,
- "max": 636,
- "mean": 636.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 16.23285961151123,
- "sum_squared": 263.50573116703254,
- "min": 16.23285961151123,
- "max": 16.23285961151123,
- "mean": 16.23285961151123,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id42841",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 496,
- "sum_squared": 246016,
- "min": 496,
- "max": 496,
- "mean": 496.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 12.17577576637268,
- "sum_squared": 148.24951551298824,
- "min": 12.17577576637268,
- "max": 12.17577576637268,
- "mean": 12.17577576637268,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id46128",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 681,
- "sum_squared": 463761,
- "min": 681,
- "max": 681,
- "mean": 681.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 16.331517696380615,
- "sum_squared": 266.7184702671932,
- "min": 16.331517696380615,
- "max": 16.331517696380615,
- "mean": 16.331517696380615,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id47975",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 700,
- "sum_squared": 490000,
- "min": 700,
- "max": 700,
- "mean": 700.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 16.919055700302124,
- "sum_squared": 286.2544457899258,
- "min": 16.919055700302124,
- "max": 16.919055700302124,
- "mean": 16.919055700302124,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id41468",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 534,
- "sum_squared": 285156,
- "min": 534,
- "max": 534,
- "mean": 534.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 9.699313402175903,
- "sum_squared": 94.0766804736291,
- "min": 9.699313402175903,
- "max": 9.699313402175903,
- "mean": 9.699313402175903,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id44284",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 678,
- "sum_squared": 459684,
- "min": 678,
- "max": 678,
- "mean": 678.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 9.90688705444336,
- "sum_squared": 98.14641110949742,
- "min": 9.90688705444336,
- "max": 9.90688705444336,
- "mean": 9.90688705444336,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- }
-]
\ No newline at end of file
diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json b/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json
deleted file mode 100644
index d1f3d5666..000000000
--- a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/run_spec.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
- "name": "commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0",
- "scenario_spec": {
- "class_name": "helm.benchmark.scenarios.commonsense_scenario.HellaSwagScenario",
- "args": {}
- },
- "adapter_spec": {
- "method": "multiple_choice_joint",
- "global_prefix": "",
- "global_suffix": "",
- "instructions": "The following are multiple choice questions (with answers) about common sense.\n",
- "input_prefix": "Question: ",
- "input_suffix": "\n",
- "reference_prefix": "A. ",
- "reference_suffix": "\n",
- "chain_of_thought_prefix": "",
- "chain_of_thought_suffix": "\n",
- "output_prefix": "Answer: ",
- "output_suffix": "\n",
- "instance_prefix": "\n",
- "substitutions": [],
- "max_train_instances": 5,
- "max_eval_instances": 10,
- "num_outputs": 5,
- "num_train_trials": 1,
- "num_trials": 1,
- "sample_train": true,
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "temperature": 0.0,
- "max_tokens": 1,
- "stop_sequences": [
- "\n"
- ],
- "multi_label": false
- },
- "metric_specs": [
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
- "args": {
- "names": [
- "exact_match",
- "quasi_exact_match",
- "prefix_exact_match",
- "quasi_prefix_exact_match"
- ]
- }
- },
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
- "args": {}
- },
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
- "args": {}
- }
- ],
- "data_augmenter_spec": {
- "perturbation_specs": [],
- "should_augment_train_instances": false,
- "should_include_original_train": false,
- "should_skip_unchanged_train": false,
- "should_augment_eval_instances": false,
- "should_include_original_eval": false,
- "should_skip_unchanged_eval": false,
- "seeds_per_instance": 1
- },
- "groups": [
- "hellaswag"
- ]
-}
\ No newline at end of file
diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json b/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json
deleted file mode 100644
index 449f227bd..000000000
--- a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "name": "hellaswag",
- "description": "Benchmark from https://arxiv.org/pdf/1905.07830.pdf.",
- "tags": [
- "knowledge",
- "multiple_choice"
- ],
- "definition_path": "https://github.com/stanford-crfm/helm/blob/main/src/helm/benchmark/scenarios/commonsense_scenario.py"
-}
\ No newline at end of file
diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json b/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json
deleted file mode 100644
index 39d847e96..000000000
--- a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/scenario_state.json
+++ /dev/null
@@ -1,844 +0,0 @@
-{
- "adapter_spec": {
- "method": "multiple_choice_joint",
- "global_prefix": "",
- "global_suffix": "",
- "instructions": "The following are multiple choice questions (with answers) about common sense.\n",
- "input_prefix": "Question: ",
- "input_suffix": "\n",
- "reference_prefix": "A. ",
- "reference_suffix": "\n",
- "chain_of_thought_prefix": "",
- "chain_of_thought_suffix": "\n",
- "output_prefix": "Answer: ",
- "output_suffix": "\n",
- "instance_prefix": "\n",
- "substitutions": [],
- "max_train_instances": 5,
- "max_eval_instances": 10,
- "num_outputs": 5,
- "num_train_trials": 1,
- "num_trials": 1,
- "sample_train": true,
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "temperature": 0.0,
- "max_tokens": 1,
- "stop_sequences": [
- "\n"
- ],
- "multi_label": false
- },
- "request_states": [
- {
- "instance": {
- "input": {
- "text": "Personal Care and Style: [header] How to dye your hair with semi permanent hair dye [title] Find the color you want. [step] There are many popular brands and hundreds of different colors to choose from. Semi-permanent dyes can be found in a variety of places, ranging from grocery stores to specialized fashion shops, with the biggest selection at beauty supply stores. "
- },
- "references": [
- {
- "output": {
- "text": "It is important to select the color that represents your hair type when you register your hair color. [substeps] Traditional semi-permanent dyes will generally not be available for hair color, like blow-dryers, curling irons, and appliances."
- },
- "tags": []
- },
- {
- "output": {
- "text": "If you're not planning on dying your hair, there are other coloration measures you can take to dye your hair. [step] Photoshop hd darkers work well, but don't lack the style that can be coupled with it."
- },
- "tags": []
- },
- {
- "output": {
- "text": "Pick the color that's your favorite, matches your wardrobe best, and/or is most flattering for your eye color and skin tone. Semi-permanent dyes work on all hair colors, but show up brightest on light hair."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "However, you can also take your color, added color, and texture into account when deciding what to dye, and what you will use it for. [substeps] Consider adding your hair dye to your hair if you have it long or curly."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id44874"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "It is important to select the color that represents your hair type when you register your hair color. [substeps] Traditional semi-permanent dyes will generally not be available for hair color, like blow-dryers, curling irons, and appliances.",
- "B": "If you're not planning on dying your hair, there are other coloration measures you can take to dye your hair. [step] Photoshop hd darkers work well, but don't lack the style that can be coupled with it.",
- "C": "Pick the color that's your favorite, matches your wardrobe best, and/or is most flattering for your eye color and skin tone. Semi-permanent dyes work on all hair colors, but show up brightest on light hair.",
- "D": "However, you can also take your color, added color, and texture into account when deciding what to dye, and what you will use it for. [substeps] Consider adding your hair dye to your hair if you have it long or curly."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Personal Care and Style: [header] How to dye your hair with semi permanent hair dye [title] Find the color you want. [step] There are many popular brands and hundreds of different colors to choose from. Semi-permanent dyes can be found in a variety of places, ranging from grocery stores to specialized fashion shops, with the biggest selection at beauty supply stores. \nA. It is important to select the color that represents your hair type when you register your hair color. [substeps] Traditional semi-permanent dyes will generally not be available for hair color, like blow-dryers, curling irons, and appliances.\nB. If you're not planning on dying your hair, there are other coloration measures you can take to dye your hair. [step] Photoshop hd darkers work well, but don't lack the style that can be coupled with it.\nC. Pick the color that's your favorite, matches your wardrobe best, and/or is most flattering for your eye color and skin tone. Semi-permanent dyes work on all hair colors, but show up brightest on light hair.\nD. However, you can also take your color, added color, and texture into account when deciding what to dye, and what you will use it for. [substeps] Consider adding your hair dye to your hair if you have it long or curly.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " B",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " B",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 18.749841690063477,
- "request_datetime": 1751729998
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Home and Garden: [header] How to grow grape vines [title] Choose a type of grape. [step] As with any plant, certain types of grapes grow better in different areas and offer up different flavors and appearances. There are three general types of grapes: american, european, and muscadine grapes. "
- },
- "references": [
- {
- "output": {
- "text": "The' bat' variety is quite dark, with a hectic shape and a bit of texture. Popular grapes grow quickly for the same years and are often planted with white grapes and a rose color."
- },
- "tags": []
- },
- {
- "output": {
- "text": "[substeps] Traditional grape grapes are made of rich grapes and have light yellow and orange coloring. If the grapes are not from the wild and you are looking for a more bitter flavor, look for grapes grown in known regions that are not based on the wild grape."
- },
- "tags": []
- },
- {
- "output": {
- "text": "[substeps] American grapes are naturally sweet and plump, with skins that are slightly crisped. European grapes grow best in warm, dry conditions with ripened fruits on a stalk."
- },
- "tags": []
- },
- {
- "output": {
- "text": "American grapes grow best in warm, sunny climates like that of central california. European grapes are common in europe and northern parts of the us, and muscadine grapes are commonly found in the southern us."
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "valid",
- "id": "id47299"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "The' bat' variety is quite dark, with a hectic shape and a bit of texture. Popular grapes grow quickly for the same years and are often planted with white grapes and a rose color.",
- "B": "[substeps] Traditional grape grapes are made of rich grapes and have light yellow and orange coloring. If the grapes are not from the wild and you are looking for a more bitter flavor, look for grapes grown in known regions that are not based on the wild grape.",
- "C": "[substeps] American grapes are naturally sweet and plump, with skins that are slightly crisped. European grapes grow best in warm, dry conditions with ripened fruits on a stalk.",
- "D": "American grapes grow best in warm, sunny climates like that of central california. European grapes are common in europe and northern parts of the us, and muscadine grapes are commonly found in the southern us."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Home and Garden: [header] How to grow grape vines [title] Choose a type of grape. [step] As with any plant, certain types of grapes grow better in different areas and offer up different flavors and appearances. There are three general types of grapes: american, european, and muscadine grapes. \nA. The' bat' variety is quite dark, with a hectic shape and a bit of texture. Popular grapes grow quickly for the same years and are often planted with white grapes and a rose color.\nB. [substeps] Traditional grape grapes are made of rich grapes and have light yellow and orange coloring. If the grapes are not from the wild and you are looking for a more bitter flavor, look for grapes grown in known regions that are not based on the wild grape.\nC. [substeps] American grapes are naturally sweet and plump, with skins that are slightly crisped. European grapes grow best in warm, dry conditions with ripened fruits on a stalk.\nD. American grapes grow best in warm, sunny climates like that of central california. European grapes are common in europe and northern parts of the us, and muscadine grapes are commonly found in the southern us.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " B",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " B",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 17.995312690734863,
- "request_datetime": 1751729998
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Personal Care and Style: [header] How to do edwardian hairstyles [title] Backcomb or tease your hair to create volume. [step] Scrunch some volumizing powder or mousse into your roots, then grab a teasing brush or boar bristle flat brush. Starting at your hairline, take a section of hair about the size of your brush and hold the ends up in the air. "
- },
- "references": [
- {
- "output": {
- "text": "Gently pull the ends up towards the roots. [title] Flatten your hairstyle to create natural streaks, and then repeat the process for extra volume."
- },
- "tags": []
- },
- {
- "output": {
- "text": "Lightly brush the hair back towards the root for 2-3 strokes. Work towards the back of your hair, repeating the process until all of your hair is teased."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "Sweep the styling iron from your ear (teasing the ends slightly in the direction toward your root) to create a teasing mist. Move around so the entire section is teased."
- },
- "tags": []
- },
- {
- "output": {
- "text": "Pin the sections above or below your roots so your hair lays flat. Lift a section of hair up and pull it down."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id45277"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "Gently pull the ends up towards the roots. [title] Flatten your hairstyle to create natural streaks, and then repeat the process for extra volume.",
- "B": "Lightly brush the hair back towards the root for 2-3 strokes. Work towards the back of your hair, repeating the process until all of your hair is teased.",
- "C": "Sweep the styling iron from your ear (teasing the ends slightly in the direction toward your root) to create a teasing mist. Move around so the entire section is teased.",
- "D": "Pin the sections above or below your roots so your hair lays flat. Lift a section of hair up and pull it down."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Personal Care and Style: [header] How to do edwardian hairstyles [title] Backcomb or tease your hair to create volume. [step] Scrunch some volumizing powder or mousse into your roots, then grab a teasing brush or boar bristle flat brush. Starting at your hairline, take a section of hair about the size of your brush and hold the ends up in the air. \nA. Gently pull the ends up towards the roots. [title] Flatten your hairstyle to create natural streaks, and then repeat the process for extra volume.\nB. Lightly brush the hair back towards the root for 2-3 strokes. Work towards the back of your hair, repeating the process until all of your hair is teased.\nC. Sweep the styling iron from your ear (teasing the ends slightly in the direction toward your root) to create a teasing mist. Move around so the entire section is teased.\nD. Pin the sections above or below your roots so your hair lays flat. Lift a section of hair up and pull it down.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " B",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " B",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 16.95018243789673,
- "request_datetime": 1751729998
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Having an ice cream: A young child is seen holding an ice cream cone and speaking to the camera while smiling. She"
- },
- "references": [
- {
- "output": {
- "text": "continues speaking while using her mouth and pointing to the camera."
- },
- "tags": []
- },
- {
- "output": {
- "text": "continues speaking more and picking up ice cream and taking a chunk."
- },
- "tags": []
- },
- {
- "output": {
- "text": "licks the ice cream cone and continues eating around her toy."
- },
- "tags": []
- },
- {
- "output": {
- "text": "speaks to the camera man holding a doll in the other hand, then sits down on the ground."
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "valid",
- "id": "id41992"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "continues speaking while using her mouth and pointing to the camera.",
- "B": "continues speaking more and picking up ice cream and taking a chunk.",
- "C": "licks the ice cream cone and continues eating around her toy.",
- "D": "speaks to the camera man holding a doll in the other hand, then sits down on the ground."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Having an ice cream: A young child is seen holding an ice cream cone and speaking to the camera while smiling. She\nA. continues speaking while using her mouth and pointing to the camera.\nB. continues speaking more and picking up ice cream and taking a chunk.\nC. licks the ice cream cone and continues eating around her toy.\nD. speaks to the camera man holding a doll in the other hand, then sits down on the ground.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 13.807421445846558,
- "request_datetime": 1751729998
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Personal Care and Style: [header] How to use a credit card system [title] Open a merchant account. [step] A merchant account is a bank account, but it is different from a business checking account. A merchant account communicates with your customers' credit card issuers. "
- },
- "references": [
- {
- "output": {
- "text": "You fill out credits and other credit forms into your merchant account. You can do any transactions you'd like on a merchant account."
- },
- "tags": []
- },
- {
- "output": {
- "text": "It takes the customer's credit card information and verifies and approves the sale. Your merchant account communicates with the card issuer to authenticate the user, confirming that the card is not stolen and that the customer has enough of a balance to pay for the sale."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "Credit cards handle the transactions linked to your account. [title] Determine your business's credit score."
- },
- "tags": []
- },
- {
- "output": {
- "text": "Credit card companies offer a variety of services : [substeps] Subscription. Credit card issuers charge a fee for a card and fill out forms to use."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id49438"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "You fill out credits and other credit forms into your merchant account. You can do any transactions you'd like on a merchant account.",
- "B": "It takes the customer's credit card information and verifies and approves the sale. Your merchant account communicates with the card issuer to authenticate the user, confirming that the card is not stolen and that the customer has enough of a balance to pay for the sale.",
- "C": "Credit cards handle the transactions linked to your account. [title] Determine your business's credit score.",
- "D": "Credit card companies offer a variety of services : [substeps] Subscription. Credit card issuers charge a fee for a card and fill out forms to use."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Personal Care and Style: [header] How to use a credit card system [title] Open a merchant account. [step] A merchant account is a bank account, but it is different from a business checking account. A merchant account communicates with your customers' credit card issuers. \nA. You fill out credits and other credit forms into your merchant account. You can do any transactions you'd like on a merchant account.\nB. It takes the customer's credit card information and verifies and approves the sale. Your merchant account communicates with the card issuer to authenticate the user, confirming that the card is not stolen and that the customer has enough of a balance to pay for the sale.\nC. Credit cards handle the transactions linked to your account. [title] Determine your business's credit score.\nD. Credit card companies offer a variety of services : [substeps] Subscription. Credit card issuers charge a fee for a card and fill out forms to use.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 16.23285961151123,
- "request_datetime": 1751730012
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Playing accordion: The fingers are pressing the black keys. One finger is pressing one key. The fingers"
- },
- "references": [
- {
- "output": {
- "text": "are moving to different keys and pressing them."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "are pumping the hand."
- },
- "tags": []
- },
- {
- "output": {
- "text": "stopped playing and stop at the bottom."
- },
- "tags": []
- },
- {
- "output": {
- "text": "flay the black keys in the middle of the keyboard."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id42841"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "are moving to different keys and pressing them.",
- "B": "are pumping the hand.",
- "C": "stopped playing and stop at the bottom.",
- "D": "flay the black keys in the middle of the keyboard."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Playing accordion: The fingers are pressing the black keys. One finger is pressing one key. The fingers\nA. are moving to different keys and pressing them.\nB. are pumping the hand.\nC. stopped playing and stop at the bottom.\nD. flay the black keys in the middle of the keyboard.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 12.17577576637268,
- "request_datetime": 1751730015
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Personal Care and Style: [header] How to wear a denim dress [title] Consider the wash of the dress. [step] Denim comes in a variety of colors, but most denim dresses are usually some shade of blue. However, you still have options when it comes to the depth of the blue. "
- },
- "references": [
- {
- "output": {
- "text": "Some denim washes work best for more polished looks, while others are ideal for a relaxed outfit. [substeps] Dark wash denim tends to have a more dressed up look, so they work well for work or an evening out."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "Depending on your dress's style, some of the colors will show up darker, and others won't. [substeps] In general, the darker the shade of the denim, the darker the dress."
- },
- "tags": []
- },
- {
- "output": {
- "text": "[substeps] Jeans tend to look better when washed and ironed on both sides. Likewise, jeans can also be washed and ironed on both sides of the dress, but with less work involved."
- },
- "tags": []
- },
- {
- "output": {
- "text": "[substeps] Look for dresses made of the same material or a lighter blue. Choose washes made of the same material, usually denim colors such as fuchsia, navy, and tan."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id46128"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "Some denim washes work best for more polished looks, while others are ideal for a relaxed outfit. [substeps] Dark wash denim tends to have a more dressed up look, so they work well for work or an evening out.",
- "B": "Depending on your dress's style, some of the colors will show up darker, and others won't. [substeps] In general, the darker the shade of the denim, the darker the dress.",
- "C": "[substeps] Jeans tend to look better when washed and ironed on both sides. Likewise, jeans can also be washed and ironed on both sides of the dress, but with less work involved.",
- "D": "[substeps] Look for dresses made of the same material or a lighter blue. Choose washes made of the same material, usually denim colors such as fuchsia, navy, and tan."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Personal Care and Style: [header] How to wear a denim dress [title] Consider the wash of the dress. [step] Denim comes in a variety of colors, but most denim dresses are usually some shade of blue. However, you still have options when it comes to the depth of the blue. \nA. Some denim washes work best for more polished looks, while others are ideal for a relaxed outfit. [substeps] Dark wash denim tends to have a more dressed up look, so they work well for work or an evening out.\nB. Depending on your dress's style, some of the colors will show up darker, and others won't. [substeps] In general, the darker the shade of the denim, the darker the dress.\nC. [substeps] Jeans tend to look better when washed and ironed on both sides. Likewise, jeans can also be washed and ironed on both sides of the dress, but with less work involved.\nD. [substeps] Look for dresses made of the same material or a lighter blue. Choose washes made of the same material, usually denim colors such as fuchsia, navy, and tan.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " B",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " B",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 16.331517696380615,
- "request_datetime": 1751730016
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Family Life: [header] How to adopt a russian baby [title] Fill out form i-600a, application for advance processing of orphan petition. [step] This form is available through the u.s. citizenship and immigration services, and it is the necessary first step in obtaining an immigrant visa for your adoptive child. You do not need to have a specific child in mind to complete this form. "
- },
- "references": [
- {
- "output": {
- "text": "[substeps] The form needs to include information about the child and the country they are from. Don't forget to include all the information that you know: \" parent, child, next of kin."
- },
- "tags": []
- },
- {
- "output": {
- "text": "[title] Contact the uscis or a state-licensed adoption agency to request a home study. [step] Home studies must be performed in order to determine the fitness of you and the home environment you intend to bring a child into."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "[title] Complete section ii of form ii of visa application, application for advance processing of a russian baby. [step] This form is available through the u.s."
- },
- "tags": []
- },
- {
- "output": {
- "text": "The form is brief to provide basic information about yourself and your adoptive child, as well as what you are looking for and need to bring with you. [title] Submit the completed form via the mail or by fax."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id47975"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "[substeps] The form needs to include information about the child and the country they are from. Don't forget to include all the information that you know: \" parent, child, next of kin.",
- "B": "[title] Contact the uscis or a state-licensed adoption agency to request a home study. [step] Home studies must be performed in order to determine the fitness of you and the home environment you intend to bring a child into.",
- "C": "[title] Complete section ii of form ii of visa application, application for advance processing of a russian baby. [step] This form is available through the u.s.",
- "D": "The form is brief to provide basic information about yourself and your adoptive child, as well as what you are looking for and need to bring with you. [title] Submit the completed form via the mail or by fax."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Family Life: [header] How to adopt a russian baby [title] Fill out form i-600a, application for advance processing of orphan petition. [step] This form is available through the u.s. citizenship and immigration services, and it is the necessary first step in obtaining an immigrant visa for your adoptive child. You do not need to have a specific child in mind to complete this form. \nA. [substeps] The form needs to include information about the child and the country they are from. Don't forget to include all the information that you know: \" parent, child, next of kin.\nB. [title] Contact the uscis or a state-licensed adoption agency to request a home study. [step] Home studies must be performed in order to determine the fitness of you and the home environment you intend to bring a child into.\nC. [title] Complete section ii of form ii of visa application, application for advance processing of a russian baby. [step] This form is available through the u.s.\nD. The form is brief to provide basic information about yourself and your adoptive child, as well as what you are looking for and need to bring with you. [title] Submit the completed form via the mail or by fax.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " A",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " A",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 16.919055700302124,
- "request_datetime": 1751730017
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Running a marathon: There are pictures of male participants shown dominating the scene where she was the only female participant. She talks about her experience as she shows more pictures of her participation against all odds. She"
- },
- "references": [
- {
- "output": {
- "text": "eventually walks off the set."
- },
- "tags": []
- },
- {
- "output": {
- "text": "also shares information on how things changed later when she was finally allowed to participate freely."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "talks about her experience again in waking and reading."
- },
- "tags": []
- },
- {
- "output": {
- "text": "challenges her opponents with her stance and high kick, doing incredible falls on the mat in the process."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id41468"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "eventually walks off the set.",
- "B": "also shares information on how things changed later when she was finally allowed to participate freely.",
- "C": "talks about her experience again in waking and reading.",
- "D": "challenges her opponents with her stance and high kick, doing incredible falls on the mat in the process."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Running a marathon: There are pictures of male participants shown dominating the scene where she was the only female participant. She talks about her experience as she shows more pictures of her participation against all odds. She\nA. eventually walks off the set.\nB. also shares information on how things changed later when she was finally allowed to participate freely.\nC. talks about her experience again in waking and reading.\nD. challenges her opponents with her stance and high kick, doing incredible falls on the mat in the process.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 9.699313402175903,
- "request_datetime": 1751730027
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Personal Care and Style: [header] How to keep hair from curling with humidity [title] Lock in moisture with conditioner. [step] Dry hair, especially curly hair, needs lots of moisture. This is because curly hair tends to be on the drier end of the spectrum compared to straight hair. "
- },
- "references": [
- {
- "output": {
- "text": "Instead of styling damp hair, work in conditioner to restore moisture to your hair. [substeps] In severe weather, it's a good idea to mist your hair twice a day, once in the morning and once at night."
- },
- "tags": []
- },
- {
- "output": {
- "text": "If you have curly hair or textured hair, moisturizing is the key to preventing frizz when it's humid. [substeps] Choose a shampoo and conditioner that is made for curly hair."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "As such, this may make your hair release moisture into the air instead of in wet curls. [substeps] Use a steam wand or mini-cooler to draw moisture from your hair, such as a hair dryer or a head dryer."
- },
- "tags": []
- },
- {
- "output": {
- "text": "Set your hair loose with a spritz of water as needed. If it tends to curl less, wipe away the excess moisture with a silk scarf."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id44284"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "Instead of styling damp hair, work in conditioner to restore moisture to your hair. [substeps] In severe weather, it's a good idea to mist your hair twice a day, once in the morning and once at night.",
- "B": "If you have curly hair or textured hair, moisturizing is the key to preventing frizz when it's humid. [substeps] Choose a shampoo and conditioner that is made for curly hair.",
- "C": "As such, this may make your hair release moisture into the air instead of in wet curls. [substeps] Use a steam wand or mini-cooler to draw moisture from your hair, such as a hair dryer or a head dryer.",
- "D": "Set your hair loose with a spritz of water as needed. If it tends to curl less, wipe away the excess moisture with a silk scarf."
- },
- "request": {
- "model_deployment": "huggingface/pythia-1b-v0",
- "model": "eleutherai/pythia-1b-v0",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about common sense.\n\nQuestion: Putting on makeup: The words how to apply mascara appear. A woman with long black hair\nA. presents a bow and ribbon, followed by painted eye shadow, and a gift wrap.\nB. appears on a black screen with white lines, soap, and information at the bottom.\nC. appears in the large window.\nD. is talking to the camera.\nAnswer: D\n\nQuestion: Polishing shoes: The man uses the polish on his shoes. The man brushes his shoes with occasional blowing. The man shines his shoes. The man\nA. replaces the frame and arm strap on the shoes.\nB. shines balls of polish onto his shoes.\nC. puts the shoes on the stand to style them.\nD. talks to the camera.\nAnswer: D\n\nQuestion: Laying tile: The floor is swept, cleaned, and prepared for the process. The men begin to lay the vinyl flooring across the floor one piece at a time. The final result\nA. is shown on the screen.\nB. of the carpet is shown.\nC. is displayed in the male success rate.\nD. is seen with one ran through the floor.\nAnswer: A\n\nQuestion: Waterskiing: A person is water skiing behind a boat. They are going back and forth behind the boat. Words\nA. are on the screen.\nB. come onto the screen at the end.\nC. appear on the screen.\nD. are shown on the screen.\nAnswer: D\n\nQuestion: Ping-pong: We see an instructional title screen. The man demonstrates hitting a ball and we see him in play hitting the ball. We\nA. see the man beating a bag over a net.\nB. see the ending title screen.\nC. see a disc fly from 2 people.\nD. see the ending title screen again.\nAnswer: B\n\nQuestion: Personal Care and Style: [header] How to keep hair from curling with humidity [title] Lock in moisture with conditioner. [step] Dry hair, especially curly hair, needs lots of moisture. This is because curly hair tends to be on the drier end of the spectrum compared to straight hair. \nA. Instead of styling damp hair, work in conditioner to restore moisture to your hair. [substeps] In severe weather, it's a good idea to mist your hair twice a day, once in the morning and once at night.\nB. If you have curly hair or textured hair, moisturizing is the key to preventing frizz when it's humid. [substeps] Choose a shampoo and conditioner that is made for curly hair.\nC. As such, this may make your hair release moisture into the air instead of in wet curls. [substeps] Use a steam wand or mini-cooler to draw moisture from your hair, such as a hair dryer or a head dryer.\nD. Set your hair loose with a spritz of water as needed. If it tends to curl less, wipe away the excess moisture with a silk scarf.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " B",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " B",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 9.90688705444336,
- "request_datetime": 1751730028
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- }
- ]
-}
\ No newline at end of file
diff --git a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json b/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json
deleted file mode 100644
index 8cbc8363c..000000000
--- a/tests/data/helm/commonsense:dataset=hellaswag,method=multiple_choice_joint,model=eleutherai_pythia-1b-v0/stats.json
+++ /dev/null
@@ -1,1426 +0,0 @@
-[
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 631.4,
- "sum_squared": 398665.95999999996,
- "min": 631.4,
- "max": 631.4,
- "mean": 631.4,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 14.876816749572754,
- "sum_squared": 221.31967660036844,
- "min": 14.876816749572754,
- "max": 14.876816749572754,
- "mean": 14.876816749572754,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "perplexity",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bits_per_byte",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob_per_byte",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 631.4,
- "sum_squared": 398665.95999999996,
- "min": 631.4,
- "max": 631.4,
- "mean": 631.4,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 631.4,
- "sum_squared": 398665.95999999996,
- "min": 631.4,
- "max": 631.4,
- "mean": 631.4,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 14.876816749572754,
- "sum_squared": 221.31967660036844,
- "min": 14.876816749572754,
- "max": 14.876816749572754,
- "mean": 14.876816749572754,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 14.876816749572754,
- "sum_squared": 221.31967660036844,
- "min": 14.876816749572754,
- "max": 14.876816749572754,
- "mean": 14.876816749572754,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.3,
- "sum_squared": 0.09,
- "min": 0.3,
- "max": 0.3,
- "mean": 0.3,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 10.0,
- "sum_squared": 100.0,
- "min": 10.0,
- "max": 10.0,
- "mean": 10.0,
- "variance": 0.0,
- "stddev": 0.0
- }
-]
\ No newline at end of file
diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json b/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json
deleted file mode 100644
index bebce63b7..000000000
--- a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/per_instance_stats.json
+++ /dev/null
@@ -1,3742 +0,0 @@
-[
- {
- "instance_id": "id147",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 333,
- "sum_squared": 110889,
- "min": 333,
- "max": 333,
- "mean": 333.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.6803672313690186,
- "sum_squared": 0.4628995695207436,
- "min": 0.6803672313690186,
- "max": 0.6803672313690186,
- "mean": 0.6803672313690186,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id11",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 420,
- "sum_squared": 176400,
- "min": 420,
- "max": 420,
- "mean": 420.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.6785750389099121,
- "sum_squared": 0.46046408343158873,
- "min": 0.6785750389099121,
- "max": 0.6785750389099121,
- "mean": 0.6785750389099121,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id65",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 424,
- "sum_squared": 179776,
- "min": 424,
- "max": 424,
- "mean": 424.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.6800699234008789,
- "sum_squared": 0.4624951007144773,
- "min": 0.6800699234008789,
- "max": 0.6800699234008789,
- "mean": 0.6800699234008789,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id344",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 359,
- "sum_squared": 128881,
- "min": 359,
- "max": 359,
- "mean": 359.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.6793179512023926,
- "sum_squared": 0.4614728788258162,
- "min": 0.6793179512023926,
- "max": 0.6793179512023926,
- "mean": 0.6793179512023926,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id59",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 336,
- "sum_squared": 112896,
- "min": 336,
- "max": 336,
- "mean": 336.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.10651206970214844,
- "sum_squared": 0.011344820992235327,
- "min": 0.10651206970214844,
- "max": 0.10651206970214844,
- "mean": 0.10651206970214844,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id291",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 360,
- "sum_squared": 129600,
- "min": 360,
- "max": 360,
- "mean": 360.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.11817407608032227,
- "sum_squared": 0.013965112257437795,
- "min": 0.11817407608032227,
- "max": 0.11817407608032227,
- "mean": 0.11817407608032227,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id131",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 336,
- "sum_squared": 112896,
- "min": 336,
- "max": 336,
- "mean": 336.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.11505007743835449,
- "sum_squared": 0.013236520318571365,
- "min": 0.11505007743835449,
- "max": 0.11505007743835449,
- "mean": 0.11505007743835449,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id222",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 342,
- "sum_squared": 116964,
- "min": 342,
- "max": 342,
- "mean": 342.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.11363410949707031,
- "sum_squared": 0.012912710841192165,
- "min": 0.11363410949707031,
- "max": 0.11363410949707031,
- "mean": 0.11363410949707031,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id259",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 337,
- "sum_squared": 113569,
- "min": 337,
- "max": 337,
- "mean": 337.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0825653076171875,
- "sum_squared": 0.0068170300219208,
- "min": 0.0825653076171875,
- "max": 0.0825653076171875,
- "mean": 0.0825653076171875,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id105",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4,
- "sum_squared": 16,
- "min": 4,
- "max": 4,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 327,
- "sum_squared": 106929,
- "min": 327,
- "max": 327,
- "mean": 327.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.07625007629394531,
- "sum_squared": 0.005814074134832481,
- "min": 0.07625007629394531,
- "max": 0.07625007629394531,
- "mean": 0.07625007629394531,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- }
-]
\ No newline at end of file
diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json b/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json
deleted file mode 100644
index 84b0d0527..000000000
--- a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/run_spec.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "name": "mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2",
- "scenario_spec": {
- "class_name": "helm.benchmark.scenarios.mmlu_scenario.MMLUScenario",
- "args": {
- "subject": "philosophy"
- }
- },
- "adapter_spec": {
- "method": "multiple_choice_joint",
- "global_prefix": "",
- "global_suffix": "",
- "instructions": "The following are multiple choice questions (with answers) about philosophy.\n",
- "input_prefix": "Question: ",
- "input_suffix": "\n",
- "reference_prefix": "A. ",
- "reference_suffix": "\n",
- "chain_of_thought_prefix": "",
- "chain_of_thought_suffix": "\n",
- "output_prefix": "Answer: ",
- "output_suffix": "\n",
- "instance_prefix": "\n",
- "substitutions": [],
- "max_train_instances": 5,
- "max_eval_instances": 10,
- "num_outputs": 5,
- "num_train_trials": 1,
- "num_trials": 1,
- "sample_train": true,
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "temperature": 0.0,
- "max_tokens": 1,
- "stop_sequences": [
- "\n"
- ],
- "multi_label": false
- },
- "metric_specs": [
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
- "args": {
- "names": [
- "exact_match",
- "quasi_exact_match",
- "prefix_exact_match",
- "quasi_prefix_exact_match"
- ]
- }
- },
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
- "args": {}
- },
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
- "args": {}
- }
- ],
- "data_augmenter_spec": {
- "perturbation_specs": [],
- "should_augment_train_instances": false,
- "should_include_original_train": false,
- "should_skip_unchanged_train": false,
- "should_augment_eval_instances": false,
- "should_include_original_eval": false,
- "should_skip_unchanged_eval": false,
- "seeds_per_instance": 1
- },
- "groups": [
- "mmlu"
- ]
-}
\ No newline at end of file
diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json b/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json
deleted file mode 100644
index ad14658ca..000000000
--- a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "name": "mmlu",
- "description": "Massive Multitask Language Understanding",
- "tags": [
- "knowledge",
- "multiple_choice"
- ],
- "definition_path": "https://github.com/stanford-crfm/helm/blob/main/src/helm/benchmark/scenarios/mmlu_scenario.py"
-}
\ No newline at end of file
diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json b/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json
deleted file mode 100644
index 81d679e9a..000000000
--- a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/scenario_state.json
+++ /dev/null
@@ -1,844 +0,0 @@
-{
- "adapter_spec": {
- "method": "multiple_choice_joint",
- "global_prefix": "",
- "global_suffix": "",
- "instructions": "The following are multiple choice questions (with answers) about philosophy.\n",
- "input_prefix": "Question: ",
- "input_suffix": "\n",
- "reference_prefix": "A. ",
- "reference_suffix": "\n",
- "chain_of_thought_prefix": "",
- "chain_of_thought_suffix": "\n",
- "output_prefix": "Answer: ",
- "output_suffix": "\n",
- "instance_prefix": "\n",
- "substitutions": [],
- "max_train_instances": 5,
- "max_eval_instances": 10,
- "num_outputs": 5,
- "num_train_trials": 1,
- "num_trials": 1,
- "sample_train": true,
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "temperature": 0.0,
- "max_tokens": 1,
- "stop_sequences": [
- "\n"
- ],
- "multi_label": false
- },
- "request_states": [
- {
- "instance": {
- "input": {
- "text": "What does the notion of \u201cmeaning in life\u201d refer to?"
- },
- "references": [
- {
- "output": {
- "text": "external meaning"
- },
- "tags": []
- },
- {
- "output": {
- "text": "god's plan"
- },
- "tags": []
- },
- {
- "output": {
- "text": "internalmeaning"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "meaning in an afterlife"
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id147"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "external meaning",
- "B": "god's plan",
- "C": "internalmeaning",
- "D": "meaning in an afterlife"
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: What does the notion of \u201cmeaning in life\u201d refer to?\nA. external meaning\nB. god's plan\nC. internalmeaning\nD. meaning in an afterlife\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.6803672313690186,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "In Plato's idealism, the unchanging Ideas or \"Forms\" in terms of which sensible objects both exist and are known must transcend (that is, exist beyond) the changing realm of appearances; because if Forms changed, then:"
- },
- "references": [
- {
- "output": {
- "text": "the only things in the sensible world that we could ever experience would be concepts."
- },
- "tags": []
- },
- {
- "output": {
- "text": "the sensible realm (in contrast to the intelligible realm) would consist only of copies of real things."
- },
- "tags": []
- },
- {
- "output": {
- "text": "nothing in the experienced world could be or be identified as one determinate thing or another."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "the sensible world would consist of unchanging Forms."
- },
- "tags": []
- }
- ],
- "split": "valid",
- "id": "id11"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "the only things in the sensible world that we could ever experience would be concepts.",
- "B": "the sensible realm (in contrast to the intelligible realm) would consist only of copies of real things.",
- "C": "nothing in the experienced world could be or be identified as one determinate thing or another.",
- "D": "the sensible world would consist of unchanging Forms."
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: In Plato's idealism, the unchanging Ideas or \"Forms\" in terms of which sensible objects both exist and are known must transcend (that is, exist beyond) the changing realm of appearances; because if Forms changed, then:\nA. the only things in the sensible world that we could ever experience would be concepts.\nB. the sensible realm (in contrast to the intelligible realm) would consist only of copies of real things.\nC. nothing in the experienced world could be or be identified as one determinate thing or another.\nD. the sensible world would consist of unchanging Forms.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.6785750389099121,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Aristotle says that what makes things be what they are--their essence--does not exist apart from individ-uals that exist in the world. So if all the members of a species were destroyed, then their essence or form:"
- },
- "references": [
- {
- "output": {
- "text": "would likewise be destroyed."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "would be destroyed only if there were no one around to remember the species."
- },
- "tags": []
- },
- {
- "output": {
- "text": "would continue existing (as with Plato's Forms) in some other realm of being."
- },
- "tags": []
- },
- {
- "output": {
- "text": "would not be destroyed because there was no essence or form originally to be destroyed; there are only individuals, not universal essences or natures of things."
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id65"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "would likewise be destroyed.",
- "B": "would be destroyed only if there were no one around to remember the species.",
- "C": "would continue existing (as with Plato's Forms) in some other realm of being.",
- "D": "would not be destroyed because there was no essence or form originally to be destroyed; there are only individuals, not universal essences or natures of things."
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Aristotle says that what makes things be what they are--their essence--does not exist apart from individ-uals that exist in the world. So if all the members of a species were destroyed, then their essence or form:\nA. would likewise be destroyed.\nB. would be destroyed only if there were no one around to remember the species.\nC. would continue existing (as with Plato's Forms) in some other realm of being.\nD. would not be destroyed because there was no essence or form originally to be destroyed; there are only individuals, not universal essences or natures of things.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.6800699234008789,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Aesthetic values have to do with _______"
- },
- "references": [
- {
- "output": {
- "text": "the moral value of works of art and other objects that could be judged beautiful"
- },
- "tags": []
- },
- {
- "output": {
- "text": "good and bad works of art and other objects that could be judged beautiful"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "the moral values of artists and critics"
- },
- "tags": []
- },
- {
- "output": {
- "text": "pragmatic decisions regarding the display of art"
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id344"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "the moral value of works of art and other objects that could be judged beautiful",
- "B": "good and bad works of art and other objects that could be judged beautiful",
- "C": "the moral values of artists and critics",
- "D": "pragmatic decisions regarding the display of art"
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Aesthetic values have to do with _______\nA. the moral value of works of art and other objects that could be judged beautiful\nB. good and bad works of art and other objects that could be judged beautiful\nC. the moral values of artists and critics\nD. pragmatic decisions regarding the display of art\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.6793179512023926,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "According to Moore, we are thinking about good whenever we think about:"
- },
- "references": [
- {
- "output": {
- "text": "pleasure."
- },
- "tags": []
- },
- {
- "output": {
- "text": "things that we desire."
- },
- "tags": []
- },
- {
- "output": {
- "text": "intrinsic value."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "none of the above."
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id59"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "pleasure.",
- "B": "things that we desire.",
- "C": "intrinsic value.",
- "D": "none of the above."
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: According to Moore, we are thinking about good whenever we think about:\nA. pleasure.\nB. things that we desire.\nC. intrinsic value.\nD. none of the above.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.10651206970214844,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Craig argues that the series of events in time cannot be actually infinite, so we know that _____."
- },
- "references": [
- {
- "output": {
- "text": "the universe is finite in the past and began to exist"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "the universe is infinite in the past"
- },
- "tags": []
- },
- {
- "output": {
- "text": "the universe never is uncaused"
- },
- "tags": []
- },
- {
- "output": {
- "text": "the universe exists only in the mind"
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id291"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "the universe is finite in the past and began to exist",
- "B": "the universe is infinite in the past",
- "C": "the universe never is uncaused",
- "D": "the universe exists only in the mind"
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Craig argues that the series of events in time cannot be actually infinite, so we know that _____.\nA. the universe is finite in the past and began to exist\nB. the universe is infinite in the past\nC. the universe never is uncaused\nD. the universe exists only in the mind\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.11817407608032227,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Baier claims that ethical skepticism is often due to:"
- },
- "references": [
- {
- "output": {
- "text": "confused metaphysical theories."
- },
- "tags": []
- },
- {
- "output": {
- "text": "confused epistemological theories."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "confused religious views."
- },
- "tags": []
- },
- {
- "output": {
- "text": "confused scientific theories."
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id131"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "confused metaphysical theories.",
- "B": "confused epistemological theories.",
- "C": "confused religious views.",
- "D": "confused scientific theories."
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Baier claims that ethical skepticism is often due to:\nA. confused metaphysical theories.\nB. confused epistemological theories.\nC. confused religious views.\nD. confused scientific theories.\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.11505007743835449,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Mill says that the ultimate end of utilitarianism is an existence as free of pain as possible and as rich as possible in _____."
- },
- "references": [
- {
- "output": {
- "text": "lower pleasures"
- },
- "tags": []
- },
- {
- "output": {
- "text": "spiritual attainment"
- },
- "tags": []
- },
- {
- "output": {
- "text": "social achievement"
- },
- "tags": []
- },
- {
- "output": {
- "text": "enjoyments"
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "test",
- "id": "id222"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "lower pleasures",
- "B": "spiritual attainment",
- "C": "social achievement",
- "D": "enjoyments"
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Mill says that the ultimate end of utilitarianism is an existence as free of pain as possible and as rich as possible in _____.\nA. lower pleasures\nB. spiritual attainment\nC. social achievement\nD. enjoyments\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.11363410949707031,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Philo says the analogy that Cleanthes uses to make his case is _____."
- },
- "references": [
- {
- "output": {
- "text": "too complicated"
- },
- "tags": []
- },
- {
- "output": {
- "text": "weak"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "strong"
- },
- "tags": []
- },
- {
- "output": {
- "text": "not based on a legitimate method of reasoning"
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id259"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "too complicated",
- "B": "weak",
- "C": "strong",
- "D": "not based on a legitimate method of reasoning"
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Philo says the analogy that Cleanthes uses to make his case is _____.\nA. too complicated\nB. weak\nC. strong\nD. not based on a legitimate method of reasoning\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.0825653076171875,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Baggini accuses pessimists of mixing up the two senses of _____."
- },
- "references": [
- {
- "output": {
- "text": "philosophy"
- },
- "tags": []
- },
- {
- "output": {
- "text": "religion"
- },
- "tags": []
- },
- {
- "output": {
- "text": "meaning"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "time"
- },
- "tags": []
- }
- ],
- "split": "test",
- "id": "id105"
- },
- "train_trial_index": 0,
- "output_mapping": {
- "A": "philosophy",
- "B": "religion",
- "C": "meaning",
- "D": "time"
- },
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "The following are multiple choice questions (with answers) about philosophy.\n\nQuestion: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\nA. metaphysics\nB. epistemology\nC. quantum physics\nD. axiology\nAnswer: A\n\nQuestion: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\nA. pleasure.\nB. happiness.\nC. good.\nD. virtue.\nAnswer: C\n\nQuestion: Psychological egoism is:\nA. an ethical theory about how we ought to behave.\nB. a generalization concerning the way people tend to behave.\nC. a claim about human nature and the ways people are capable of behaving.\nD. none of the above.\nAnswer: C\n\nQuestion: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\nA. optimist\nB. satisfied\nC. nominally religious\nD. pessimist\nAnswer: D\n\nQuestion: According to d'Holbach, people always act according to _____.\nA. free choices\nB. dictates of the soul\nC. necessary natural laws\nD. undetermined will\nAnswer: C\n\nQuestion: Baggini accuses pessimists of mixing up the two senses of _____.\nA. philosophy\nB. religion\nC. meaning\nD. time\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 5,
- "max_tokens": 1,
- "stop_sequences": [],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " D",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " D",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.07625007629394531,
- "request_datetime": 1762354922
- },
- "num_train_instances": 5,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- }
- ]
-}
\ No newline at end of file
diff --git a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json b/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json
deleted file mode 100644
index 25374cbb6..000000000
--- a/tests/data/helm/mmlu:subject=philosophy,method=multiple_choice_joint,model=openai_gpt2/stats.json
+++ /dev/null
@@ -1,2850 +0,0 @@
-[
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 350.44444444444446,
- "sum_squared": 122811.30864197532,
- "min": 350.44444444444446,
- "max": 350.44444444444446,
- "mean": 350.44444444444446,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.2946600914001465,
- "sum_squared": 0.08682456946394268,
- "min": 0.2946600914001465,
- "max": 0.2946600914001465,
- "mean": 0.2946600914001465,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test"
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 420.0,
- "sum_squared": 176400.0,
- "min": 420.0,
- "max": 420.0,
- "mean": 420.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.6785750389099121,
- "sum_squared": 0.46046408343158873,
- "min": 0.6785750389099121,
- "max": 0.6785750389099121,
- "mean": 0.6785750389099121,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "perplexity",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bits_per_byte",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob_per_byte",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "perplexity",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bits_per_byte",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob_per_byte",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 350.44444444444446,
- "sum_squared": 122811.30864197532,
- "min": 350.44444444444446,
- "max": 350.44444444444446,
- "mean": 350.44444444444446,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 350.44444444444446,
- "sum_squared": 122811.30864197532,
- "min": 350.44444444444446,
- "max": 350.44444444444446,
- "mean": 350.44444444444446,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.2946600914001465,
- "sum_squared": 0.08682456946394268,
- "min": 0.2946600914001465,
- "max": 0.2946600914001465,
- "mean": 0.2946600914001465,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.2946600914001465,
- "sum_squared": 0.08682456946394268,
- "min": 0.2946600914001465,
- "max": 0.2946600914001465,
- "mean": 0.2946600914001465,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.1111111111111111,
- "sum_squared": 0.012345679012345678,
- "min": 0.1111111111111111,
- "max": 0.1111111111111111,
- "mean": 0.1111111111111111,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 420.0,
- "sum_squared": 176400.0,
- "min": 420.0,
- "max": 420.0,
- "mean": 420.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 420.0,
- "sum_squared": 176400.0,
- "min": 420.0,
- "max": 420.0,
- "mean": 420.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.6785750389099121,
- "sum_squared": 0.46046408343158873,
- "min": 0.6785750389099121,
- "max": 0.6785750389099121,
- "mean": 0.6785750389099121,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.6785750389099121,
- "sum_squared": 0.46046408343158873,
- "min": 0.6785750389099121,
- "max": 0.6785750389099121,
- "mean": 0.6785750389099121,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 5.0,
- "sum_squared": 25.0,
- "min": 5.0,
- "max": 5.0,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_prefix_exact_match@5",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 9.0,
- "sum_squared": 81.0,
- "min": 9.0,
- "max": 9.0,
- "mean": 9.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- }
-]
\ No newline at end of file
diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/per_instance_stats.json b/tests/data/helm/narrative_qa:model=openai_gpt2/per_instance_stats.json
deleted file mode 100644
index 2a7de0caa..000000000
--- a/tests/data/helm/narrative_qa:model=openai_gpt2/per_instance_stats.json
+++ /dev/null
@@ -1,1732 +0,0 @@
-[
- {
- "instance_id": "id1413",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 790,
- "sum_squared": 624100,
- "min": 790,
- "max": 790,
- "mean": 790.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.8313617706298828,
- "sum_squared": 0.6911623936648539,
- "min": 0.8313617706298828,
- "max": 0.8313617706298828,
- "mean": 0.8313617706298828,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 7,
- "sum_squared": 49,
- "min": 7,
- "max": 7,
- "mean": 7.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id1332",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 653,
- "sum_squared": 426409,
- "min": 653,
- "max": 653,
- "mean": 653.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 45,
- "sum_squared": 2025,
- "min": 45,
- "max": 45,
- "mean": 45.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 45,
- "sum_squared": 2025,
- "min": 45,
- "max": 45,
- "mean": 45.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 1.622730016708374,
- "sum_squared": 2.63325270712636,
- "min": 1.622730016708374,
- "max": 1.622730016708374,
- "mean": 1.622730016708374,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test"
- },
- "count": 1,
- "sum": 0.3333333333333333,
- "sum_squared": 0.1111111111111111,
- "min": 0.3333333333333333,
- "max": 0.3333333333333333,
- "mean": 0.3333333333333333,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test"
- },
- "count": 1,
- "sum": 0.09090909090909091,
- "sum_squared": 0.008264462809917356,
- "min": 0.09090909090909091,
- "max": 0.09090909090909091,
- "mean": 0.09090909090909091,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test"
- },
- "count": 1,
- "sum": 0.06666666666666667,
- "sum_squared": 0.0044444444444444444,
- "min": 0.06666666666666667,
- "max": 0.06666666666666667,
- "mean": 0.06666666666666667,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test"
- },
- "count": 1,
- "sum": 2.2250738585072626e-308,
- "sum_squared": 0.0,
- "min": 2.2250738585072626e-308,
- "max": 2.2250738585072626e-308,
- "mean": 2.2250738585072626e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 45,
- "sum_squared": 2025,
- "min": 45,
- "max": 45,
- "mean": 45.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 213,
- "sum_squared": 45369,
- "min": 213,
- "max": 213,
- "mean": 213.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id1123",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 686,
- "sum_squared": 470596,
- "min": 686,
- "max": 686,
- "mean": 686.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 57,
- "sum_squared": 3249,
- "min": 57,
- "max": 57,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 57,
- "sum_squared": 3249,
- "min": 57,
- "max": 57,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.743454933166504,
- "sum_squared": 3.0396351039826186,
- "min": 1.743454933166504,
- "max": 1.743454933166504,
- "mean": 1.743454933166504,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "valid"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.03636363636363636,
- "sum_squared": 0.0013223140495867767,
- "min": 0.03636363636363636,
- "max": 0.03636363636363636,
- "mean": 0.03636363636363636,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.03508771929824561,
- "sum_squared": 0.0012311480455524776,
- "min": 0.03508771929824561,
- "max": 0.03508771929824561,
- "mean": 0.03508771929824561,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "valid"
- },
- "count": 1,
- "sum": 2.2250738585072626e-308,
- "sum_squared": 0.0,
- "min": 2.2250738585072626e-308,
- "max": 2.2250738585072626e-308,
- "mean": 2.2250738585072626e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 57,
- "sum_squared": 3249,
- "min": 57,
- "max": 57,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 270,
- "sum_squared": 72900,
- "min": 270,
- "max": 270,
- "mean": 270.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id1514",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 772,
- "sum_squared": 595984,
- "min": 772,
- "max": 772,
- "mean": 772.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 0.8950591087341309,
- "sum_squared": 0.8011308081279367,
- "min": 0.8950591087341309,
- "max": 0.8950591087341309,
- "mean": 0.8950591087341309,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 5,
- "sum_squared": 25,
- "min": 5,
- "max": 5,
- "mean": 5.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 13,
- "sum_squared": 169,
- "min": 13,
- "max": 13,
- "mean": 13.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- },
- {
- "instance_id": "id1340",
- "train_trial_index": 0,
- "stats": [
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 2,
- "sum_squared": 4,
- "min": 2,
- "max": 2,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 635,
- "sum_squared": 403225,
- "min": 635,
- "max": 635,
- "mean": 635.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 100,
- "sum_squared": 10000,
- "min": 100,
- "max": 100,
- "mean": 100.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 100,
- "sum_squared": 10000,
- "min": 100,
- "max": 100,
- "mean": 100.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 1.3460521697998047,
- "sum_squared": 1.8118564438227622,
- "min": 1.3460521697998047,
- "max": 1.3460521697998047,
- "mean": 1.3460521697998047,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 1,
- "sum_squared": 1,
- "min": 1,
- "max": 1,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0,
- "sum_squared": 0,
- "min": 0,
- "max": 0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test"
- },
- "count": 1,
- "sum": 0.36363636363636365,
- "sum_squared": 0.1322314049586777,
- "min": 0.36363636363636365,
- "max": 0.36363636363636365,
- "mean": 0.36363636363636365,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test"
- },
- "count": 1,
- "sum": 0.12371134020618557,
- "sum_squared": 0.015304495695610586,
- "min": 0.12371134020618557,
- "max": 0.12371134020618557,
- "mean": 0.12371134020618557,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test"
- },
- "count": 1,
- "sum": 0.07608695652173914,
- "sum_squared": 0.005789224952741021,
- "min": 0.07608695652173914,
- "max": 0.07608695652173914,
- "mean": 0.07608695652173914,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test"
- },
- "count": 1,
- "sum": 2.2250738585072626e-308,
- "sum_squared": 0.0,
- "min": 2.2250738585072626e-308,
- "max": 2.2250738585072626e-308,
- "mean": 2.2250738585072626e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 100,
- "sum_squared": 10000,
- "min": 100,
- "max": 100,
- "mean": 100.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 379,
- "sum_squared": 143641,
- "min": 379,
- "max": 379,
- "mean": 379.0,
- "variance": 0.0,
- "stddev": 0.0
- }
- ]
- }
-]
\ No newline at end of file
diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/run_spec.json b/tests/data/helm/narrative_qa:model=openai_gpt2/run_spec.json
deleted file mode 100644
index ff005c9fc..000000000
--- a/tests/data/helm/narrative_qa:model=openai_gpt2/run_spec.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "name": "narrative_qa:model=openai_gpt2",
- "scenario_spec": {
- "class_name": "helm.benchmark.scenarios.narrativeqa_scenario.NarrativeQAScenario",
- "args": {}
- },
- "adapter_spec": {
- "method": "generation",
- "global_prefix": "",
- "global_suffix": "",
- "instructions": "",
- "input_prefix": "Passage: ",
- "input_suffix": "\n",
- "reference_prefix": "A. ",
- "reference_suffix": "\n",
- "chain_of_thought_prefix": "",
- "chain_of_thought_suffix": "\n",
- "output_prefix": "Answer: ",
- "output_suffix": "\n",
- "instance_prefix": "\n",
- "substitutions": [],
- "max_train_instances": 5,
- "max_eval_instances": 5,
- "num_outputs": 1,
- "num_train_trials": 1,
- "num_trials": 1,
- "sample_train": true,
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "temperature": 0.0,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "multi_label": false
- },
- "metric_specs": [
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
- "args": {
- "names": [
- "exact_match",
- "quasi_exact_match",
- "f1_score",
- "rouge_l",
- "bleu_1",
- "bleu_4"
- ]
- }
- },
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
- "args": {}
- },
- {
- "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
- "args": {}
- }
- ],
- "data_augmenter_spec": {
- "perturbation_specs": [],
- "should_augment_train_instances": false,
- "should_include_original_train": false,
- "should_skip_unchanged_train": false,
- "should_augment_eval_instances": false,
- "should_include_original_eval": false,
- "should_skip_unchanged_eval": false,
- "seeds_per_instance": 1
- },
- "groups": [
- "narrative_qa"
- ]
-}
\ No newline at end of file
diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/scenario.json b/tests/data/helm/narrative_qa:model=openai_gpt2/scenario.json
deleted file mode 100644
index 8d4345fc5..000000000
--- a/tests/data/helm/narrative_qa:model=openai_gpt2/scenario.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "name": "narrativeqa",
- "description": "Question answering using summaries of books/movie scripts.",
- "tags": [
- "question_answering"
- ],
- "definition_path": "https://github.com/stanford-crfm/helm/blob/main/src/helm/benchmark/scenarios/narrativeqa_scenario.py"
-}
\ No newline at end of file
diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/scenario_state.json b/tests/data/helm/narrative_qa:model=openai_gpt2/scenario_state.json
deleted file mode 100644
index 936e68472..000000000
--- a/tests/data/helm/narrative_qa:model=openai_gpt2/scenario_state.json
+++ /dev/null
@@ -1,1185 +0,0 @@
-{
- "adapter_spec": {
- "method": "generation",
- "global_prefix": "",
- "global_suffix": "",
- "instructions": "",
- "input_prefix": "Passage: ",
- "input_suffix": "\n",
- "reference_prefix": "A. ",
- "reference_suffix": "\n",
- "chain_of_thought_prefix": "",
- "chain_of_thought_suffix": "\n",
- "output_prefix": "Answer: ",
- "output_suffix": "\n",
- "instance_prefix": "\n",
- "substitutions": [],
- "max_train_instances": 5,
- "max_eval_instances": 5,
- "num_outputs": 1,
- "num_train_trials": 1,
- "num_trials": 1,
- "sample_train": true,
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "temperature": 0.0,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "multi_label": false
- },
- "request_states": [
- {
- "instance": {
- "input": {
- "text": "Olive Penderghast, a 17-year-old girl living in Ojai, California lies to her best friend Rhiannon Abernathy about going on a date in order to get out of camping with Rhiannon's hippie parents. Instead, she hangs around the house all weekend listening to Natasha Bedingfield's \"Pocketful of Sunshine\", which is played by a greeting card she was sent. The following Monday, pressed by Rhiannon, Olive lies about losing her virginity to a college guy. Marianne Bryant, a prissy and strictly religious Christian at their school, overhears her telling the lie and soon it spreads like wildfire. The school's conservative church group run by Marianne decides Olive will be their next project. Olive confides the truth to her friend Brandon, and he explains how others bully him because of his homosexuality. He later asks Olive to pretend to sleep with him so that he will be accepted by everyone as a 'straight stud'.\nBrandon convinces Olive to help him and they pretend to have sex at a party. After having a fight with Rhiannon over Olive's new identity as a \"dirty skank\", Olive decides to counteract the harassment by embracing her new image as the school tramp. She begins to wear more provocative clothing and stitches a red \"A\" to everything she wears. Boys who usually have had no luck with girls in the past beg Olive to say they have had sex with her in order to increase their own popularity, in exchange for gift cards to various stores, in turn increasing her reputation. Things get worse when Micah, Marianne's 22-year-old boyfriend, contracts chlamydia from sleeping with Mrs. Griffith, the school guidance counsellor, and blames it all on Olive. Olive agrees to lie to cover up the affair so that the marriage of her favorite teacher, Mr. Griffith, would be spared.\nMarianne's religious clique, which now includes Rhiannon, begins harassing Olive in order to get her to leave school. After an ill-fated date with Anson, a boy who wants to pay her to actually sleep with him and not just pretend she did, Olive reconnects with Todd, her old crush, who is also the school's mascot. Todd then tells her that he does not believe the rumors because he remembers when she lied for him when he was not ready for his first kiss years ago. Olive then begins to ask everyone she lied for to help her out by telling the truth, but Brandon and Micah have abruptly left town and everyone else is enjoying their newfound popularity and do not want the truth to get out. Mrs. Griffith also refuses to tell the truth and when Olive threatens to expose her, Mrs. Griffith rebuffs her, saying no one would believe her.\nOlive, out of spite, then immediately tells Mr. Griffith, who believes her and separates from Mrs. Griffith. After a friendly talk with her eccentric, open-minded mother Rosemary, Olive comes up with a plan to get everything finally out in the open. She then does a song and dance number at a school pep rally to get people's attention to watch her via web cam, where she confesses what she has done (the web cam is the framing device of the film). The various boys whose reputations Olive helped improve are also shown watching. Later, Olive texts Rhiannon, apologizing for lying to her. When she is finishing up her web cast, Todd comes by riding a lawnmower and tells her to come outside. She signs off by saying she may lose her virginity to Todd, and proudly declares it's nobody's business (much to Marianne's disgrace). She goes outside to meet him, they kiss and the two are shown riding off on the lawnmower.\nQuestion: Who is Todd besides Olive's old crush?"
- },
- "references": [
- {
- "output": {
- "text": "The school Mascot"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "the schools mascot"
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "test",
- "id": "id1413"
- },
- "train_trial_index": 0,
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "Passage: Olive Penderghast, a 17-year-old girl living in Ojai, California lies to her best friend Rhiannon Abernathy about going on a date in order to get out of camping with Rhiannon's hippie parents. Instead, she hangs around the house all weekend listening to Natasha Bedingfield's \"Pocketful of Sunshine\", which is played by a greeting card she was sent. The following Monday, pressed by Rhiannon, Olive lies about losing her virginity to a college guy. Marianne Bryant, a prissy and strictly religious Christian at their school, overhears her telling the lie and soon it spreads like wildfire. The school's conservative church group run by Marianne decides Olive will be their next project. Olive confides the truth to her friend Brandon, and he explains how others bully him because of his homosexuality. He later asks Olive to pretend to sleep with him so that he will be accepted by everyone as a 'straight stud'.\nBrandon convinces Olive to help him and they pretend to have sex at a party. After having a fight with Rhiannon over Olive's new identity as a \"dirty skank\", Olive decides to counteract the harassment by embracing her new image as the school tramp. She begins to wear more provocative clothing and stitches a red \"A\" to everything she wears. Boys who usually have had no luck with girls in the past beg Olive to say they have had sex with her in order to increase their own popularity, in exchange for gift cards to various stores, in turn increasing her reputation. Things get worse when Micah, Marianne's 22-year-old boyfriend, contracts chlamydia from sleeping with Mrs. Griffith, the school guidance counsellor, and blames it all on Olive. Olive agrees to lie to cover up the affair so that the marriage of her favorite teacher, Mr. Griffith, would be spared.\nMarianne's religious clique, which now includes Rhiannon, begins harassing Olive in order to get her to leave school. After an ill-fated date with Anson, a boy who wants to pay her to actually sleep with him and not just pretend she did, Olive reconnects with Todd, her old crush, who is also the school's mascot. Todd then tells her that he does not believe the rumors because he remembers when she lied for him when he was not ready for his first kiss years ago. Olive then begins to ask everyone she lied for to help her out by telling the truth, but Brandon and Micah have abruptly left town and everyone else is enjoying their newfound popularity and do not want the truth to get out. Mrs. Griffith also refuses to tell the truth and when Olive threatens to expose her, Mrs. Griffith rebuffs her, saying no one would believe her.\nOlive, out of spite, then immediately tells Mr. Griffith, who believes her and separates from Mrs. Griffith. After a friendly talk with her eccentric, open-minded mother Rosemary, Olive comes up with a plan to get everything finally out in the open. She then does a song and dance number at a school pep rally to get people's attention to watch her via web cam, where she confesses what she has done (the web cam is the framing device of the film). The various boys whose reputations Olive helped improve are also shown watching. Later, Olive texts Rhiannon, apologizing for lying to her. When she is finishing up her web cast, Todd comes by riding a lawnmower and tells her to come outside. She signs off by saying she may lose her virginity to Todd, and proudly declares it's nobody's business (much to Marianne's disgrace). She goes outside to meet him, they kiss and the two are shown riding off on the lawnmower.\nQuestion: Who is Todd besides Olive's old crush?\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 1,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " Olive.",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " Olive",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.8313617706298828,
- "request_datetime": 1763479296
- },
- "num_train_instances": 0,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Indefer Jones is the aged squire, between seventy and eighty years of age, of a large manor, Llanfeare, in Carmarthen, Wales. His niece, Isabel Brodrick, has lived with him for years after the remarriage of her father, and endeared herself to everyone. However, according to his strong traditional beliefs, the estate should be bequeathed to a male heir.\nHis sole male blood relative is his nephew Henry Jones, a London clerk. Henry has, in the past, incurred debts that the squire had paid off, been \"sent away from Oxford\", and generally made a poor impression on his occasional visits to Llanfeare. Nevertheless, Henry is told of his uncle's intention to make him the heir to the estate and is invited to pay a visit. Isabel rejects her uncle's suggestion that she solve his dilemma by marrying Henry, as she cannot stand her cousin. Indefer Jones finds his nephew to be just as detestable as ever. As a result, he overcomes his prejudice and changes his will one final time, in Isabel's favour. Unfortunately, he dies before he can tell anyone.\nFinding the document hidden in a book of sermons by accident, Henry vacillates between keeping silent and revealing its location. He is neither good enough to give up the estate nor evil enough to burn the document, fearing disgrace, a long jail sentence and, not least, eternal damnation. Instead, he comforts himself by reasoning that doing nothing cannot be a crime.\nIndefer Jones had had his last will witnessed by two of his tenants, but since the will cannot be found despite a thorough search of the house, Henry inherits the estate. However, already extant suspicions are only strengthened by his guilty manner. He endures abuse from everyone; his own servants either quit or treat him with disrespect. He takes to spending hours in the library, where the will is hidden.\nThe local newspaper begins to publish accounts of the affair that are insulting and seemingly libelous to Henry. It accuses him of destroying the will and usurping the estate from Isabel, whom everybody knows and respects. The old squire's lawyer, Mr Apjohn, himself suspecting that Henry knows more than he lets on, approaches the new squire about the articles, pressuring the unwilling young man into taking legal action against the editor. Henry finds that this only makes things worse. The prospect of being cross examined in the witness box fills him with dread. He realises the truth would be dragged out of him in court.\nMr Apjohn, by clever questioning, gets a good idea about where the will is. Henry knows that time is running out, but once again procrastinates. Mr Apjohn and Mr Brodrick, Isabel's father, visit Henry at home and find the document, despite Henry's ineffectual efforts to stop them. Because he did not destroy the will, Henry is permitted to return to his job in London with his reputation intact and \u00c2\u01414000, the amount Isabel was bequeathed in the other will.\nQuestion: How is Isabel Brodrick related to Indefer Jones?"
- },
- "references": [
- {
- "output": {
- "text": "She is his neice."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "His niece."
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "test",
- "id": "id1332"
- },
- "train_trial_index": 0,
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "Passage: Indefer Jones is the aged squire, between seventy and eighty years of age, of a large manor, Llanfeare, in Carmarthen, Wales. His niece, Isabel Brodrick, has lived with him for years after the remarriage of her father, and endeared herself to everyone. However, according to his strong traditional beliefs, the estate should be bequeathed to a male heir.\nHis sole male blood relative is his nephew Henry Jones, a London clerk. Henry has, in the past, incurred debts that the squire had paid off, been \"sent away from Oxford\", and generally made a poor impression on his occasional visits to Llanfeare. Nevertheless, Henry is told of his uncle's intention to make him the heir to the estate and is invited to pay a visit. Isabel rejects her uncle's suggestion that she solve his dilemma by marrying Henry, as she cannot stand her cousin. Indefer Jones finds his nephew to be just as detestable as ever. As a result, he overcomes his prejudice and changes his will one final time, in Isabel's favour. Unfortunately, he dies before he can tell anyone.\nFinding the document hidden in a book of sermons by accident, Henry vacillates between keeping silent and revealing its location. He is neither good enough to give up the estate nor evil enough to burn the document, fearing disgrace, a long jail sentence and, not least, eternal damnation. Instead, he comforts himself by reasoning that doing nothing cannot be a crime.\nIndefer Jones had had his last will witnessed by two of his tenants, but since the will cannot be found despite a thorough search of the house, Henry inherits the estate. However, already extant suspicions are only strengthened by his guilty manner. He endures abuse from everyone; his own servants either quit or treat him with disrespect. He takes to spending hours in the library, where the will is hidden.\nThe local newspaper begins to publish accounts of the affair that are insulting and seemingly libelous to Henry. It accuses him of destroying the will and usurping the estate from Isabel, whom everybody knows and respects. The old squire's lawyer, Mr Apjohn, himself suspecting that Henry knows more than he lets on, approaches the new squire about the articles, pressuring the unwilling young man into taking legal action against the editor. Henry finds that this only makes things worse. The prospect of being cross examined in the witness box fills him with dread. He realises the truth would be dragged out of him in court.\nMr Apjohn, by clever questioning, gets a good idea about where the will is. Henry knows that time is running out, but once again procrastinates. Mr Apjohn and Mr Brodrick, Isabel's father, visit Henry at home and find the document, despite Henry's ineffectual efforts to stop them. Because he did not destroy the will, Henry is permitted to return to his job in London with his reputation intact and \u00c2\u01414000, the amount Isabel was bequeathed in the other will.\nQuestion: How is Isabel Brodrick related to Indefer Jones?\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 1,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " Isabel is a very good and intelligent woman. She is a very good and intelligent woman. She is a very good and intelligent woman. She is a very good and intelligent woman. She is a very good and intelligent woman.",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " Isabel",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " very",
- "logprob": 0.0
- },
- {
- "text": " good",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " intelligent",
- "logprob": 0.0
- },
- {
- "text": " woman",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " very",
- "logprob": 0.0
- },
- {
- "text": " good",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " intelligent",
- "logprob": 0.0
- },
- {
- "text": " woman",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " very",
- "logprob": 0.0
- },
- {
- "text": " good",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " intelligent",
- "logprob": 0.0
- },
- {
- "text": " woman",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " very",
- "logprob": 0.0
- },
- {
- "text": " good",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " intelligent",
- "logprob": 0.0
- },
- {
- "text": " woman",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " very",
- "logprob": 0.0
- },
- {
- "text": " good",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " intelligent",
- "logprob": 0.0
- },
- {
- "text": " woman",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 1.622730016708374,
- "request_datetime": 1763479296
- },
- "num_train_instances": 0,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "The subject of Cratylus is the correctness of names (\u03c0\u03b5\u03c1\u1f76 \u1f40\u03bd\u03bf\u03bc\u03ac\u03c4\u03c9\u03bd \u1f40\u03c1\u03b8\u03cc\u03c4\u03b7\u03c4\u03bf\u03c2), in other words, it is a critique on the subject of naming (Baxter).\nWhen discussing a \u1f44\u03bd\u03bf\u03bc\u03b1 (onoma ) and how it would relate to its subject, Socrates compares the original creation of a word to the work of an artist. An artist uses color to express the essence of his subject in a painting. In much the same way, the creator of words uses letters containing certain sounds to express the essence of a word's subject. There is a letter that is best for soft things, one for liquid things, and so on. He comments;\nthe best possible way to speak consists in using names all (or most) of which are like the things they name (that is, are appropriate to them), while the worst is to use the opposite kind of names.\nOne countering position, held by Hermogenes, is that names have come about due to custom and convention. They do not express the essence of their subject, so they can be swapped with something unrelated by the individuals or communities who use them.\nThe line between the two perspectives is often blurred. During more than half of the dialogue, Socrates makes guesses at Hermogenes' request as to where names and words have come from. These include the names of the Olympian gods, personified deities, and many words that describe abstract concepts. He examines whether, for example, giving names of \"streams\" to Cronus and Rhea (\u03a1\u03bf\u03ae \u2013 flow or space) are purely accidental.\nDon't you think he who gave to the ancestors of the other gods the names \u201cRhea\u201d and \u201cCronus\u201d had the same thought as Heracleitus? Do you think he gave both of them the names of streams (\u1fe5\u03b5\u03c5\u03bc\u03ac\u03c4\u03c9\u03bd \u1f40\u03bd\u03cc\u03bc\u03b1\u03c4\u03b1) merely by chance?\nThe Greek term \"\u1fe5\u03b5\u1fe6\u03bc\u03b1\" may refer to the flow of any medium and is not restricted to the flow of water or liquids. Many of the words which Socrates uses as examples may have come from an idea originally linked to the name, but have changed over time. Those of which he cannot find a link, he often assumes have come from foreign origins or have changed so much as to lose all resemblance to the original word. He states, \"names have been so twisted in all manner of ways, that I should not be surprised if the old language when compared with that now in use would appear to us to be a barbarous tongue.\"\nThe final theory of relations between name and object named is posited by Cratylus, a disciple of Heraclitus, who believes that names arrive from divine origins, making them necessarily correct. Socrates rebukes this theory by reminding Cratylus of the imperfection of certain names in capturing the objects they seek to signify. From this point, Socrates ultimately rejects the study of language, believing it to be philosophically inferior to a study of things themselves.\nQuestion: What does the old language sound compared with the new language?"
- },
- "references": [
- {
- "output": {
- "text": "like a barbaric tongue."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "barbarous tongue"
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "valid",
- "id": "id1123"
- },
- "train_trial_index": 0,
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "Passage: The subject of Cratylus is the correctness of names (\u03c0\u03b5\u03c1\u1f76 \u1f40\u03bd\u03bf\u03bc\u03ac\u03c4\u03c9\u03bd \u1f40\u03c1\u03b8\u03cc\u03c4\u03b7\u03c4\u03bf\u03c2), in other words, it is a critique on the subject of naming (Baxter).\nWhen discussing a \u1f44\u03bd\u03bf\u03bc\u03b1 (onoma ) and how it would relate to its subject, Socrates compares the original creation of a word to the work of an artist. An artist uses color to express the essence of his subject in a painting. In much the same way, the creator of words uses letters containing certain sounds to express the essence of a word's subject. There is a letter that is best for soft things, one for liquid things, and so on. He comments;\nthe best possible way to speak consists in using names all (or most) of which are like the things they name (that is, are appropriate to them), while the worst is to use the opposite kind of names.\nOne countering position, held by Hermogenes, is that names have come about due to custom and convention. They do not express the essence of their subject, so they can be swapped with something unrelated by the individuals or communities who use them.\nThe line between the two perspectives is often blurred. During more than half of the dialogue, Socrates makes guesses at Hermogenes' request as to where names and words have come from. These include the names of the Olympian gods, personified deities, and many words that describe abstract concepts. He examines whether, for example, giving names of \"streams\" to Cronus and Rhea (\u03a1\u03bf\u03ae \u2013 flow or space) are purely accidental.\nDon't you think he who gave to the ancestors of the other gods the names \u201cRhea\u201d and \u201cCronus\u201d had the same thought as Heracleitus? Do you think he gave both of them the names of streams (\u1fe5\u03b5\u03c5\u03bc\u03ac\u03c4\u03c9\u03bd \u1f40\u03bd\u03cc\u03bc\u03b1\u03c4\u03b1) merely by chance?\nThe Greek term \"\u1fe5\u03b5\u1fe6\u03bc\u03b1\" may refer to the flow of any medium and is not restricted to the flow of water or liquids. Many of the words which Socrates uses as examples may have come from an idea originally linked to the name, but have changed over time. Those of which he cannot find a link, he often assumes have come from foreign origins or have changed so much as to lose all resemblance to the original word. He states, \"names have been so twisted in all manner of ways, that I should not be surprised if the old language when compared with that now in use would appear to us to be a barbarous tongue.\"\nThe final theory of relations between name and object named is posited by Cratylus, a disciple of Heraclitus, who believes that names arrive from divine origins, making them necessarily correct. Socrates rebukes this theory by reminding Cratylus of the imperfection of certain names in capturing the objects they seek to signify. From this point, Socrates ultimately rejects the study of language, believing it to be philosophically inferior to a study of things themselves.\nQuestion: What does the old language sound compared with the new language?\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 1,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " The old language is a dialect of the Greek language, which is the language of the Greeks. The old language is a dialect of the Greek language, which is the language of the Greeks. The old language is a dialect of the Greek language, which is the language of the Greeks.",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " The",
- "logprob": 0.0
- },
- {
- "text": " old",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " dialect",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " Greek",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": ",",
- "logprob": 0.0
- },
- {
- "text": " which",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " Greeks",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " The",
- "logprob": 0.0
- },
- {
- "text": " old",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " dialect",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " Greek",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": ",",
- "logprob": 0.0
- },
- {
- "text": " which",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " Greeks",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " The",
- "logprob": 0.0
- },
- {
- "text": " old",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " dialect",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " Greek",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": ",",
- "logprob": 0.0
- },
- {
- "text": " which",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " language",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " Greeks",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 1.743454933166504,
- "request_datetime": 1763479296
- },
- "num_train_instances": 0,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "The novel begins in Manchester, where we are introduced to the Bartons and the Wilsons, two working-class families. John Barton is a questioner of the distribution of wealth and the relations between rich and poor. Soon his wife dies\u00e2\u0080\u0094he blames it on her grief over the disappearance of her sister Esther. Having already lost his son Tom at a young age, Barton is left to raise his daughter, Mary, alone and now falls into depression and begins to involve himself in the Chartist, trade-union movement.\nChapter 1 takes place in countryside where Moss Side is now.\nMary takes up work at a dressmaker's (her father having objected to her working in a factory) and becomes subject to the affections of hard-working Jem Wilson and Harry Carson, son of a wealthy mill owner. She fondly hopes, by marrying Carson, to secure a comfortable life for herself and her father, but immediately after refusing Jem's offer of marriage she realises that she truly loves him. She therefore decides to evade Carson, planning to show her feelings to Jem in the course of time. Jem believes her decision to be final, though this does not change his feelings for her.\nMeanwhile, Esther, a \"street-walker,\" returns to warn John Barton that he must save Mary from becoming like her. He simply pushes her away, however, and she's sent to jail for a month on the charge of vagrancy. Upon her release she talks to Jem with the same purpose. He promises that he will protect Mary and confronts Carson, eventually entering into a fight with him, which is witnessed by a policeman passing by.\nNot long afterwards, Carson is shot dead, and Jem is arrested for the crime, his gun having been found at the scene. Esther decides to investigate the matter further and discovers that the wadding for the gun was a piece of paper on which is written Mary's name.\nShe visits her niece to warn her to save the one she loves, and after she leaves Mary realises that the murderer is not Jem but her father. She is now faced with having to save her lover without giving away her father. With the help of Job Legh (the intelligent grandfather of her blind friend Margaret), Mary travels to Liverpool to find the only person who could provide an alibi for Jem \u00e2\u0080\u0093 Will Wilson, Jem's cousin and a sailor, who was with him on the night of the murder. Unfortunately, Will's ship is already departing, so that, after Mary chases after the ship in a small boat, the only thing Will can do is promise to return in the pilot ship and testify the next day.\nDuring the trial, Jem learns of Mary's great love for him. Will arrives in court to testify, and Jem is found \"not guilty\". Mary has fallen ill during the trial and is nursed by Mr Sturgis, an old sailor, and his wife. When she finally returns to Manchester she has to face her father, who is crushed by his remorse. He summons John Carson, Harry's father, to confess to him that he is the murderer. Carson is still set on justice, but after turning to the Bible he forgives Barton, who dies soon afterwards in Carson's arms. Not long after this Esther comes back to Mary's home, where she, too, soon dies.\nJem decides to leave England, where, his reputation damaged, it would be difficult for him to find a new job. The novel ends with the wedded Mary and Jem, their little child, and Mrs Wilson living happily in Canada. News comes that Margaret has regained her sight and that she and Will, soon to be married, will visit.\nQuestion: Who actually killed Harry Carson?"
- },
- "references": [
- {
- "output": {
- "text": "John Barton"
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "John Barton"
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "test",
- "id": "id1514"
- },
- "train_trial_index": 0,
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "Passage: The novel begins in Manchester, where we are introduced to the Bartons and the Wilsons, two working-class families. John Barton is a questioner of the distribution of wealth and the relations between rich and poor. Soon his wife dies\u00e2\u0080\u0094he blames it on her grief over the disappearance of her sister Esther. Having already lost his son Tom at a young age, Barton is left to raise his daughter, Mary, alone and now falls into depression and begins to involve himself in the Chartist, trade-union movement.\nChapter 1 takes place in countryside where Moss Side is now.\nMary takes up work at a dressmaker's (her father having objected to her working in a factory) and becomes subject to the affections of hard-working Jem Wilson and Harry Carson, son of a wealthy mill owner. She fondly hopes, by marrying Carson, to secure a comfortable life for herself and her father, but immediately after refusing Jem's offer of marriage she realises that she truly loves him. She therefore decides to evade Carson, planning to show her feelings to Jem in the course of time. Jem believes her decision to be final, though this does not change his feelings for her.\nMeanwhile, Esther, a \"street-walker,\" returns to warn John Barton that he must save Mary from becoming like her. He simply pushes her away, however, and she's sent to jail for a month on the charge of vagrancy. Upon her release she talks to Jem with the same purpose. He promises that he will protect Mary and confronts Carson, eventually entering into a fight with him, which is witnessed by a policeman passing by.\nNot long afterwards, Carson is shot dead, and Jem is arrested for the crime, his gun having been found at the scene. Esther decides to investigate the matter further and discovers that the wadding for the gun was a piece of paper on which is written Mary's name.\nShe visits her niece to warn her to save the one she loves, and after she leaves Mary realises that the murderer is not Jem but her father. She is now faced with having to save her lover without giving away her father. With the help of Job Legh (the intelligent grandfather of her blind friend Margaret), Mary travels to Liverpool to find the only person who could provide an alibi for Jem \u00e2\u0080\u0093 Will Wilson, Jem's cousin and a sailor, who was with him on the night of the murder. Unfortunately, Will's ship is already departing, so that, after Mary chases after the ship in a small boat, the only thing Will can do is promise to return in the pilot ship and testify the next day.\nDuring the trial, Jem learns of Mary's great love for him. Will arrives in court to testify, and Jem is found \"not guilty\". Mary has fallen ill during the trial and is nursed by Mr Sturgis, an old sailor, and his wife. When she finally returns to Manchester she has to face her father, who is crushed by his remorse. He summons John Carson, Harry's father, to confess to him that he is the murderer. Carson is still set on justice, but after turning to the Bible he forgives Barton, who dies soon afterwards in Carson's arms. Not long after this Esther comes back to Mary's home, where she, too, soon dies.\nJem decides to leave England, where, his reputation damaged, it would be difficult for him to find a new job. The novel ends with the wedded Mary and Jem, their little child, and Mrs Wilson living happily in Canada. News comes that Margaret has regained her sight and that she and Will, soon to be married, will visit.\nQuestion: Who actually killed Harry Carson?\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 1,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " The Wilsons.",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " The",
- "logprob": 0.0
- },
- {
- "text": " W",
- "logprob": 0.0
- },
- {
- "text": "ils",
- "logprob": 0.0
- },
- {
- "text": "ons",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 0.8950591087341309,
- "request_datetime": 1763479296
- },
- "num_train_instances": 0,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- },
- {
- "instance": {
- "input": {
- "text": "Tarzan returns to Opar, the source of the gold where a lost colony of fabled Atlantis is located, in order to make good on some financial reverses he has recently suffered. While Atlantis itself sank beneath the waves thousands of years ago, the workers of Opar continued to mine all of the gold, which means there is a rather huge stockpile but which is now lost to the memory of the Oparians and only Tarzan knows its secret location.\nA greedy, outlawed Belgian army officer, Albert Werper, in the employ of a criminal Arab, secretly follows Tarzan to Opar. There, Tarzan loses his memory after being struck on the head by a falling rock in the treasure room during an earthquake. On encountering La, the high priestess who is the servant of the Flaming God of Opar, and who is also very beautiful, Tarzan once again rejects her love which enrages her and she tries to have him killed; she had fallen in love with the apeman during their first encounter and La and her high priests are not going to allow Tarzan to escape their sacrificial knives this time.\nIn the meanwhile, Jane has been kidnapped by the Arab and wonders what is keeping her husband from once again coming to her rescue. A now amnesiac Tarzan and the Werper escape from Opar, bearing away the sacrificial knife of Opar which La and some retainers set out to recover. There is intrigue and counter intrigue the rest of the way.\nQuestion: Who is La?"
- },
- "references": [
- {
- "output": {
- "text": "La is the high priestess that serves the Flaming God of Opar."
- },
- "tags": [
- "correct"
- ]
- },
- {
- "output": {
- "text": "A high priestess."
- },
- "tags": [
- "correct"
- ]
- }
- ],
- "split": "test",
- "id": "id1340"
- },
- "train_trial_index": 0,
- "request": {
- "model_deployment": "huggingface/gpt2",
- "model": "openai/gpt2",
- "embedding": false,
- "prompt": "Passage: The Little White Bird is a series of short episodes, including both accounts of the narrator's day-to-day activities in contemporary London and fanciful tales set in Kensington Gardens and elsewhere.The story is set in several locations; the earlier chapters are set in the town of London, contemporaneous to the time of Barrie's writing, and involving some time travel of a few years, and other fantasy elements, while remaining within the London setting. The middle chapters that later became Peter Pan in Kensington Gardens are set in London's famous Kensington Gardens, introduced by the statement that \"All perambulators lead to Kensington Gardens\". The Kensington Gardens chapters include detailed descriptions of the features of the Gardens, along with fantasy names given to the locations by the story's characters, especially after \"Lock-Out Time\", described by Barrie as the time at the end of the day when the park gates are closed to the public, and the fairies and other magical inhabitants of the park can move about more freely than during the daylight, when they must hide from ordinary people. The third section of the book, following the Kensington Gardens chapters, are again set generally in London, though there are some short returns to the Gardens that are not part of the Peter Pan stories. In a two-page diversion in chapter 24, Barrie brings the story to Patagonia, and a journey by ship returning to England at the \"white cliffs of Albion\".\nQuestion: Where was the ship in Pentagonia returned to?\nAnswer: England\n\nPassage: Tarzan returns to Opar, the source of the gold where a lost colony of fabled Atlantis is located, in order to make good on some financial reverses he has recently suffered. While Atlantis itself sank beneath the waves thousands of years ago, the workers of Opar continued to mine all of the gold, which means there is a rather huge stockpile but which is now lost to the memory of the Oparians and only Tarzan knows its secret location.\nA greedy, outlawed Belgian army officer, Albert Werper, in the employ of a criminal Arab, secretly follows Tarzan to Opar. There, Tarzan loses his memory after being struck on the head by a falling rock in the treasure room during an earthquake. On encountering La, the high priestess who is the servant of the Flaming God of Opar, and who is also very beautiful, Tarzan once again rejects her love which enrages her and she tries to have him killed; she had fallen in love with the apeman during their first encounter and La and her high priests are not going to allow Tarzan to escape their sacrificial knives this time.\nIn the meanwhile, Jane has been kidnapped by the Arab and wonders what is keeping her husband from once again coming to her rescue. A now amnesiac Tarzan and the Werper escape from Opar, bearing away the sacrificial knife of Opar which La and some retainers set out to recover. There is intrigue and counter intrigue the rest of the way.\nQuestion: Who is La?\nAnswer:",
- "temperature": 0.0,
- "num_completions": 1,
- "top_k_per_token": 1,
- "max_tokens": 100,
- "stop_sequences": [
- "\n"
- ],
- "echo_prompt": false,
- "top_p": 1,
- "presence_penalty": 0,
- "frequency_penalty": 0
- },
- "result": {
- "success": true,
- "embedding": [],
- "completions": [
- {
- "text": " La is a young woman who is a member of the royal family of Opar. She is a member of the royal family of Opar, and is a member of the royal family of Opar. She is a member of the royal family of Opar, and is a member of the royal family of Opar. She is a member of the royal family of Opar, and is a member of the royal family of Opar. She is a member of the royal family of Opar",
- "logprob": 0.0,
- "tokens": [
- {
- "text": " La",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " young",
- "logprob": 0.0
- },
- {
- "text": " woman",
- "logprob": 0.0
- },
- {
- "text": " who",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ",",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ",",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ",",
- "logprob": 0.0
- },
- {
- "text": " and",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- },
- {
- "text": ".",
- "logprob": 0.0
- },
- {
- "text": " She",
- "logprob": 0.0
- },
- {
- "text": " is",
- "logprob": 0.0
- },
- {
- "text": " a",
- "logprob": 0.0
- },
- {
- "text": " member",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " the",
- "logprob": 0.0
- },
- {
- "text": " royal",
- "logprob": 0.0
- },
- {
- "text": " family",
- "logprob": 0.0
- },
- {
- "text": " of",
- "logprob": 0.0
- },
- {
- "text": " O",
- "logprob": 0.0
- },
- {
- "text": "par",
- "logprob": 0.0
- }
- ]
- }
- ],
- "cached": false,
- "request_time": 1.3460521697998047,
- "request_datetime": 1763479297
- },
- "num_train_instances": 1,
- "prompt_truncated": false,
- "num_conditioning_tokens": 0
- }
- ]
-}
\ No newline at end of file
diff --git a/tests/data/helm/narrative_qa:model=openai_gpt2/stats.json b/tests/data/helm/narrative_qa:model=openai_gpt2/stats.json
deleted file mode 100644
index d6d6444aa..000000000
--- a/tests/data/helm/narrative_qa:model=openai_gpt2/stats.json
+++ /dev/null
@@ -1,2634 +0,0 @@
-[
- {
- "name": {
- "name": "num_references",
- "split": "test"
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 712.5,
- "sum_squared": 507656.25,
- "min": 712.5,
- "max": 712.5,
- "mean": 712.5,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "test"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test"
- },
- "count": 1,
- "sum": 1.173800766468048,
- "sum_squared": 1.3778082393609772,
- "min": 1.173800766468048,
- "max": 1.173800766468048,
- "mean": 1.173800766468048,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 0.25,
- "sum_squared": 0.0625,
- "min": 0.25,
- "max": 0.25,
- "mean": 0.25,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test"
- },
- "count": 1,
- "sum": 0.17424242424242425,
- "sum_squared": 0.030360422405876955,
- "min": 0.17424242424242425,
- "max": 0.17424242424242425,
- "mean": 0.17424242424242425,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test"
- },
- "count": 1,
- "sum": 0.05365510777881912,
- "sum_squared": 0.002878870590756696,
- "min": 0.05365510777881912,
- "max": 0.05365510777881912,
- "mean": 0.05365510777881912,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test"
- },
- "count": 1,
- "sum": 0.03568840579710145,
- "sum_squared": 0.001273662308338584,
- "min": 0.03568840579710145,
- "max": 0.03568840579710145,
- "mean": 0.03568840579710145,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test"
- },
- "count": 1,
- "sum": 1.1125369292536313e-308,
- "sum_squared": 0.0,
- "min": 1.1125369292536313e-308,
- "max": 1.1125369292536313e-308,
- "mean": 1.1125369292536313e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test"
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test"
- },
- "count": 1,
- "sum": 153.0,
- "sum_squared": 23409.0,
- "min": 153.0,
- "max": 153.0,
- "mean": 153.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid"
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 686.0,
- "sum_squared": 470596.0,
- "min": 686.0,
- "max": 686.0,
- "mean": 686.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "training_co2_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "training_energy_cost",
- "split": "valid"
- },
- "count": 0,
- "sum": 0,
- "sum_squared": 0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.743454933166504,
- "sum_squared": 3.0396351039826186,
- "min": 1.743454933166504,
- "max": 1.743454933166504,
- "mean": 1.743454933166504,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.03636363636363636,
- "sum_squared": 0.0013223140495867767,
- "min": 0.03636363636363636,
- "max": 0.03636363636363636,
- "mean": 0.03636363636363636,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.03508771929824561,
- "sum_squared": 0.0012311480455524776,
- "min": 0.03508771929824561,
- "max": 0.03508771929824561,
- "mean": 0.03508771929824561,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "valid"
- },
- "count": 1,
- "sum": 2.2250738585072626e-308,
- "sum_squared": 0.0,
- "min": 2.2250738585072626e-308,
- "max": 2.2250738585072626e-308,
- "mean": 2.2250738585072626e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid"
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid"
- },
- "count": 1,
- "sum": 270.0,
- "sum_squared": 72900.0,
- "min": 270.0,
- "max": 270.0,
- "mean": 270.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "perplexity",
- "split": "test"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bits_per_byte",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob_per_byte",
- "split": "test"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "perplexity",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bits_per_byte",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob_per_byte",
- "split": "valid"
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 712.5,
- "sum_squared": 507656.25,
- "min": 712.5,
- "max": 712.5,
- "mean": 712.5,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 712.5,
- "sum_squared": 507656.25,
- "min": 712.5,
- "max": 712.5,
- "mean": 712.5,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.173800766468048,
- "sum_squared": 1.3778082393609772,
- "min": 1.173800766468048,
- "max": 1.173800766468048,
- "mean": 1.173800766468048,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.173800766468048,
- "sum_squared": 1.3778082393609772,
- "min": 1.173800766468048,
- "max": 1.173800766468048,
- "mean": 1.173800766468048,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.25,
- "sum_squared": 0.0625,
- "min": 0.25,
- "max": 0.25,
- "mean": 0.25,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.25,
- "sum_squared": 0.0625,
- "min": 0.25,
- "max": 0.25,
- "mean": 0.25,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.17424242424242425,
- "sum_squared": 0.030360422405876955,
- "min": 0.17424242424242425,
- "max": 0.17424242424242425,
- "mean": 0.17424242424242425,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.17424242424242425,
- "sum_squared": 0.030360422405876955,
- "min": 0.17424242424242425,
- "max": 0.17424242424242425,
- "mean": 0.17424242424242425,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.05365510777881912,
- "sum_squared": 0.002878870590756696,
- "min": 0.05365510777881912,
- "max": 0.05365510777881912,
- "mean": 0.05365510777881912,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.05365510777881912,
- "sum_squared": 0.002878870590756696,
- "min": 0.05365510777881912,
- "max": 0.05365510777881912,
- "mean": 0.05365510777881912,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.03568840579710145,
- "sum_squared": 0.001273662308338584,
- "min": 0.03568840579710145,
- "max": 0.03568840579710145,
- "mean": 0.03568840579710145,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.03568840579710145,
- "sum_squared": 0.001273662308338584,
- "min": 0.03568840579710145,
- "max": 0.03568840579710145,
- "mean": 0.03568840579710145,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.1125369292536313e-308,
- "sum_squared": 0.0,
- "min": 1.1125369292536313e-308,
- "max": 1.1125369292536313e-308,
- "mean": 1.1125369292536313e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.1125369292536313e-308,
- "sum_squared": 0.0,
- "min": 1.1125369292536313e-308,
- "max": 1.1125369292536313e-308,
- "mean": 1.1125369292536313e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 38.0,
- "sum_squared": 1444.0,
- "min": 38.0,
- "max": 38.0,
- "mean": 38.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 153.0,
- "sum_squared": 23409.0,
- "min": 153.0,
- "max": 153.0,
- "mean": 153.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "test",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 153.0,
- "sum_squared": 23409.0,
- "min": 153.0,
- "max": 153.0,
- "mean": 153.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_references",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.0,
- "sum_squared": 4.0,
- "min": 2.0,
- "max": 2.0,
- "mean": 2.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_trials",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 686.0,
- "sum_squared": 470596.0,
- "min": 686.0,
- "max": 686.0,
- "mean": 686.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_prompt_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 686.0,
- "sum_squared": 470596.0,
- "min": 686.0,
- "max": 686.0,
- "mean": 686.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_completion_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_output_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.743454933166504,
- "sum_squared": 3.0396351039826186,
- "min": 1.743454933166504,
- "max": 1.743454933166504,
- "mean": 1.743454933166504,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "inference_runtime",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.743454933166504,
- "sum_squared": 3.0396351039826186,
- "min": 1.743454933166504,
- "max": 1.743454933166504,
- "mean": 1.743454933166504,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "batch_size",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_length",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_stop",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_endoftext",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "finish_reason_unknown",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_train_instances",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "prompt_truncated",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "max_prob",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "quasi_exact_match",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "f1_score",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.03636363636363636,
- "sum_squared": 0.0013223140495867767,
- "min": 0.03636363636363636,
- "max": 0.03636363636363636,
- "mean": 0.03636363636363636,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "rouge_l",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.03636363636363636,
- "sum_squared": 0.0013223140495867767,
- "min": 0.03636363636363636,
- "max": 0.03636363636363636,
- "mean": 0.03636363636363636,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.03508771929824561,
- "sum_squared": 0.0012311480455524776,
- "min": 0.03508771929824561,
- "max": 0.03508771929824561,
- "mean": 0.03508771929824561,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_1",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.03508771929824561,
- "sum_squared": 0.0012311480455524776,
- "min": 0.03508771929824561,
- "max": 0.03508771929824561,
- "mean": 0.03508771929824561,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.2250738585072626e-308,
- "sum_squared": 0.0,
- "min": 2.2250738585072626e-308,
- "max": 2.2250738585072626e-308,
- "mean": 2.2250738585072626e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "bleu_4",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 2.2250738585072626e-308,
- "sum_squared": 0.0,
- "min": 2.2250738585072626e-308,
- "max": 2.2250738585072626e-308,
- "mean": 2.2250738585072626e-308,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "logprob",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 0.0,
- "sum_squared": 0.0,
- "min": 0.0,
- "max": 0.0,
- "mean": 0.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_perplexity_tokens",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 57.0,
- "sum_squared": 3249.0,
- "min": 57.0,
- "max": 57.0,
- "mean": 57.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid",
- "perturbation": {
- "name": "robustness",
- "robustness": true,
- "fairness": false,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 270.0,
- "sum_squared": 72900.0,
- "min": 270.0,
- "max": 270.0,
- "mean": 270.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_bytes",
- "split": "valid",
- "perturbation": {
- "name": "fairness",
- "robustness": false,
- "fairness": true,
- "computed_on": "worst"
- }
- },
- "count": 1,
- "sum": 270.0,
- "sum_squared": 72900.0,
- "min": 270.0,
- "max": 270.0,
- "mean": 270.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_instances",
- "split": "test"
- },
- "count": 1,
- "sum": 4.0,
- "sum_squared": 16.0,
- "min": 4.0,
- "max": 4.0,
- "mean": 4.0,
- "variance": 0.0,
- "stddev": 0.0
- },
- {
- "name": {
- "name": "num_instances",
- "split": "valid"
- },
- "count": 1,
- "sum": 1.0,
- "sum_squared": 1.0,
- "min": 1.0,
- "max": 1.0,
- "mean": 1.0,
- "variance": 0.0,
- "stddev": 0.0
- }
-]
\ No newline at end of file
diff --git a/utils/__init__.py b/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/utils/arc_agi/adapter.py b/utils/arc_agi/adapter.py
index c209569a3..b9c4ff43e 100644
--- a/utils/arc_agi/adapter.py
+++ b/utils/arc_agi/adapter.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
+import hashlib
import json
import re
import time
@@ -307,10 +308,61 @@ def make_log(
def write_log(log: dict, out_root: Path, developer: str, model: str) -> Path:
- out_dir = out_root / "arc-agi" / developer / model
+ filename = str(uuid.uuid4())
+ out_dir = out_root / filename[:2] / filename[2:4]
out_dir.mkdir(parents=True, exist_ok=True)
- out_path = out_dir / f"{uuid.uuid4()}.json"
- out_path.write_text(json.dumps(log, indent=2) + "\n", encoding="utf-8")
+ out_path = out_dir / f"{filename}.json"
+
+ content_str = json.dumps(log, indent=2) + "\n"
+ content_bytes = content_str.encode("utf-8")
+ out_path.write_bytes(content_bytes)
+
+ size_bytes = len(content_bytes)
+ sha256 = hashlib.sha256(content_bytes).hexdigest()
+
+ try:
+ legacy_path = out_path.relative_to(out_root.parent).as_posix()
+ except ValueError:
+ legacy_path = f"data/arc-agi/{developer}/{model}/{filename}.json"
+
+ object_path = f"flat/objects/{filename[:2]}/{filename[2:4]}/{filename}.json"
+
+ aggregate_record = {
+ "benchmark": "arc-agi",
+ "eval_schema_version": SCHEMA_VERSION,
+ "legacy_path": legacy_path,
+ "object_path": object_path,
+ "object_uuid": filename,
+ "record_type": "aggregate",
+ "sha256": sha256,
+ "size_bytes": size_bytes
+ }
+
+ instance_record = {
+ "benchmark": "arc-agi",
+ "eval_schema_version": SCHEMA_VERSION,
+ "instance_object_path": None,
+ "instance_sha256": None,
+ "instance_size_bytes": None,
+ "legacy_path": legacy_path,
+ "object_path": object_path,
+ "object_uuid": filename,
+ "record_type": "aggregate",
+ "sha256": sha256,
+ "size_bytes": size_bytes
+ }
+
+ index_dir = out_root / "indexes" / "by_collection" / "arc-agi"
+ index_dir.mkdir(parents=True, exist_ok=True)
+
+ aggregate_index_path = index_dir / "aggregate.jsonl"
+ with open(aggregate_index_path, "a", encoding="utf-8") as f:
+ f.write(json.dumps(aggregate_record) + "\n")
+
+ instance_index_path = index_dir / "instance_level.jsonl"
+ with open(instance_index_path, "a", encoding="utf-8") as f:
+ f.write(json.dumps(instance_record) + "\n")
+
return out_path
diff --git a/utils/arc_agi/adapter_flat.py b/utils/arc_agi/adapter_flat.py
new file mode 100644
index 000000000..62dfc6cc4
--- /dev/null
+++ b/utils/arc_agi/adapter_flat.py
@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import time
+import uuid
+from collections import defaultdict
+from pathlib import Path
+
+from every_eval_ever.helpers import SCHEMA_VERSION
+
+SOURCE_URL = "https://github.com/fchollet/ARC-AGI/tree/master/data"
+
+
+def make_source_data() -> dict:
+ return {
+ "source_type": "url",
+ "dataset_name": "ARC Prize evaluations leaderboard JSON",
+ "url": [SOURCE_URL],
+ }
+
+
+def load_rows(input_json: Path) -> list[dict]:
+ return json.loads(input_json.read_text(encoding="utf-8"))
+
+
+def compute_metric_bounds(rows: list[dict]) -> dict[str, dict[str, float]]:
+ cost_per_task_values = [
+ float(row["costPerTask"])
+ for row in rows
+ if row.get("costPerTask") is not None
+ ]
+ cost_values = [
+ float(row["cost"]) for row in rows if row.get("cost") is not None
+ ]
+
+ bounds = {
+ "score": {
+ "min_score": 0.0,
+ "max_score": 1.0,
+ }
+ }
+
+ if cost_per_task_values:
+ bounds["cost_per_task"] = {
+ "min_score": 0.0,
+ "max_score": max(cost_per_task_values),
+ }
+
+ if cost_values:
+ bounds["cost"] = {
+ "min_score": 0.0,
+ "max_score": max(cost_values),
+ }
+
+ return bounds
+
+
+def infer_developer(raw_model_id: str) -> str:
+ s = raw_model_id.strip().lower().replace("_", "-")
+
+ if s.startswith(("openai-", "gpt-", "o1-", "o3-", "o4-", "codex")):
+ return "openai"
+ if s.startswith(("anthropic-", "claude")):
+ return "anthropic"
+ if s.startswith(("google-", "gemini")):
+ return "google"
+ if s.startswith(("xai-", "grok")):
+ return "xai"
+ if s.startswith(("qwen", "qwq")):
+ return "qwen"
+ if s.startswith("deepseek") or s == "r1":
+ return "deepseek"
+ if s.startswith("glm"):
+ return "zhipu"
+ if s.startswith("kimi"):
+ return "moonshotai"
+ if s.startswith(("mistral", "magistral")):
+ return "mistralai"
+ if s.startswith("llama"):
+ return "meta"
+ if s.startswith("olmo"):
+ return "allenai"
+ if s.startswith("minimax"):
+ return "minimax"
+
+ if raw_model_id in {"2025_human_panel"}:
+ return "arcprize"
+
+ return "community"
+
+
+def slugify_model_name(raw_model_id: str, developer_name: str) -> str:
+ s = raw_model_id.strip().lower()
+ s = s.replace("_", "-")
+ s = re.sub(r"\s+", "-", s)
+ s = re.sub(r"[^a-z0-9.\-]+", "-", s)
+ s = re.sub(r"-{2,}", "-", s).strip("-")
+
+ prefix = developer_name + "-"
+ if s.startswith(prefix):
+ s = s[len(prefix):]
+
+ return s
+
+
+def normalize_model(raw_model_id: str) -> tuple[str, str]:
+ developer_name = infer_developer(raw_model_id)
+ model_name = slugify_model_name(raw_model_id, developer_name)
+ return developer_name, model_name
+
+
+def stringify_details(row: dict, exclude_keys: set[str]) -> dict[str, str]:
+ details = {}
+ for k, v in row.items():
+ if k in exclude_keys:
+ continue
+ details[k] = str(v)
+ return details
+
+
+def choose_primary_raw_model_id(rows_for_canonical: list[dict], developer_name: str) -> str:
+ aliases = sorted({row["modelId"] for row in rows_for_canonical})
+ prefix = developer_name + "-"
+ aliases.sort(
+ key=lambda raw: (
+ raw.lower().replace("_", "-").startswith(prefix),
+ len(raw),
+ raw.lower(),
+ )
+ )
+ return aliases[0]
+
+
+def choose_best_row(rows: list[dict], developer_name: str) -> dict:
+ prefix = developer_name + "-"
+ return sorted(
+ rows,
+ key=lambda row: (
+ row["modelId"].lower().replace("_", "-").startswith(prefix),
+ len(row["modelId"]),
+ row["modelId"].lower(),
+ ),
+ )[0]
+
+
+def make_results(
+ rows_for_canonical: list[dict],
+ developer_name: str,
+ metric_bounds: dict[str, dict[str, float]],
+) -> list[dict]:
+ results = []
+ by_dataset = defaultdict(list)
+
+ for row in rows_for_canonical:
+ by_dataset[row["datasetId"]].append(row)
+
+ for dataset_id in sorted(by_dataset):
+ row = choose_best_row(by_dataset[dataset_id], developer_name)
+ aliases_for_dataset = sorted({r["modelId"] for r in by_dataset[dataset_id]})
+
+ results.append(
+ {
+ "evaluation_result_id": f"{dataset_id}::score",
+ "evaluation_name": dataset_id,
+ "source_data": make_source_data(),
+ "metric_config": {
+ "metric_id": "score",
+ "metric_name": "ARC score",
+ "metric_kind": "accuracy",
+ "metric_unit": "proportion",
+ "lower_is_better": False,
+ "score_type": "continuous",
+ **metric_bounds["score"],
+ "additional_details": {
+ "raw_metric_field": "score",
+ },
+ },
+ "score_details": {
+ "score": float(row["score"]),
+ "details": {
+ **stringify_details(
+ row,
+ exclude_keys={"score", "modelId"},
+ ),
+ "raw_model_id": row["modelId"],
+ "raw_model_aliases_json": json.dumps(aliases_for_dataset),
+ },
+ },
+ }
+ )
+
+ if "costPerTask" in row and row["costPerTask"] is not None:
+ results.append(
+ {
+ "evaluation_result_id": f"{dataset_id}::cost_per_task",
+ "evaluation_name": dataset_id,
+ "source_data": make_source_data(),
+ "metric_config": {
+ "metric_id": "cost_per_task",
+ "metric_name": "Cost per task",
+ "metric_kind": "cost",
+ "metric_unit": "usd",
+ "lower_is_better": True,
+ "score_type": "continuous",
+ **metric_bounds["cost_per_task"],
+ "additional_details": {
+ "raw_metric_field": "costPerTask",
+ },
+ },
+ "score_details": {
+ "score": float(row["costPerTask"]),
+ "details": {
+ **stringify_details(
+ row,
+ exclude_keys={"costPerTask", "modelId"},
+ ),
+ "raw_model_id": row["modelId"],
+ "raw_model_aliases_json": json.dumps(aliases_for_dataset),
+ },
+ },
+ }
+ )
+ elif "cost" in row and row["cost"] is not None:
+ results.append(
+ {
+ "evaluation_result_id": f"{dataset_id}::cost",
+ "evaluation_name": dataset_id,
+ "source_data": make_source_data(),
+ "metric_config": {
+ "metric_id": "cost",
+ "metric_name": "Cost",
+ "metric_kind": "cost",
+ "metric_unit": "usd",
+ "lower_is_better": True,
+ "score_type": "continuous",
+ **metric_bounds["cost"],
+ "additional_details": {
+ "raw_metric_field": "cost",
+ },
+ },
+ "score_details": {
+ "score": float(row["cost"]),
+ "details": {
+ **stringify_details(
+ row,
+ exclude_keys={"cost", "modelId"},
+ ),
+ "raw_model_id": row["modelId"],
+ "raw_model_aliases_json": json.dumps(aliases_for_dataset),
+ },
+ },
+ }
+ )
+
+ return results
+
+
+def make_log(
+ rows_for_canonical: list[dict],
+ developer_name: str,
+ model_name: str,
+ metric_bounds: dict[str, dict[str, float]],
+ retrieved_timestamp: str,
+) -> tuple[dict, str, str]:
+ primary_raw_model_id = choose_primary_raw_model_id(rows_for_canonical, developer_name)
+ all_aliases = sorted({row["modelId"] for row in rows_for_canonical})
+
+ log = {
+ "schema_version": SCHEMA_VERSION,
+ "evaluation_id": (
+ f"arc-agi/{developer_name}/{model_name}/{retrieved_timestamp}"
+ ),
+ "retrieved_timestamp": retrieved_timestamp,
+ "source_metadata": {
+ "source_name": "ARC Prize leaderboard JSON",
+ "source_type": "documentation",
+ "source_organization_name": "ARC Prize",
+ "source_organization_url": "https://arcprize.org/leaderboard",
+ "evaluator_relationship": "third_party",
+ "additional_details": {
+ "api_endpoint": SOURCE_URL,
+ "filtered_to_display_true": "True",
+ },
+ },
+ "eval_library": {
+ "name": "ARC Prize leaderboard",
+ "version": "unknown",
+ },
+ "model_info": {
+ "name": primary_raw_model_id,
+ "id": f"{developer_name}/{model_name}",
+ "developer": developer_name,
+ "additional_details": {
+ "raw_model_id": primary_raw_model_id,
+ "raw_model_aliases_json": json.dumps(all_aliases),
+ },
+ },
+ "evaluation_results": make_results(
+ rows_for_canonical, developer_name, metric_bounds
+ ),
+ }
+
+ return log, developer_name, model_name
+
+
+def write_log(log: dict, out_root: Path, developer: str, model: str) -> Path:
+ object_uuid = str(uuid.uuid4())
+ d1 = object_uuid[:2]
+ d2 = object_uuid[2:4]
+
+ out_dir = out_root / "objects" / d1 / d2
+ out_dir.mkdir(parents=True, exist_ok=True)
+ out_path = out_dir / f"{object_uuid}.json"
+
+ content = json.dumps(log, indent=2) + "\n"
+ content_bytes = content.encode("utf-8")
+ out_path.write_bytes(content_bytes)
+
+ sha256 = hashlib.sha256(content_bytes).hexdigest()
+ size_bytes = len(content_bytes)
+
+ index_dir = out_root / "indexes" / "by_benchmark" / "arc-agi"
+ index_dir.mkdir(parents=True, exist_ok=True)
+
+ instance_index_path = index_dir / "instance_level.jsonl"
+ instance_index_path.touch(exist_ok=True)
+
+ aggregate_index_path = index_dir / "aggregate.jsonl"
+ legacy_path = f"data/arc-agi/{developer}/{model}/{object_uuid}.json"
+ object_path = f"flat/objects/{d1}/{d2}/{object_uuid}.json"
+
+ index_entry = {
+ "benchmark": "arc-agi",
+ "eval_schema_version": SCHEMA_VERSION,
+ "legacy_path": legacy_path,
+ "object_path": object_path,
+ "object_uuid": object_uuid,
+ "record_type": "aggregate",
+ "sha256": sha256,
+ "size_bytes": size_bytes
+ }
+
+ with open(aggregate_index_path, "a", encoding="utf-8") as f:
+ f.write(json.dumps(index_entry) + "\n")
+
+ return out_path
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--input-json", type=Path, required=True)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ args = parser.parse_args()
+
+ rows = load_rows(args.input_json)
+ rows = [r for r in rows if r.get("display") is True]
+ metric_bounds = compute_metric_bounds(rows)
+ retrieved_timestamp = str(time.time())
+
+ by_canonical = defaultdict(list)
+ for row in rows:
+ developer_name, model_name = normalize_model(row["modelId"])
+ by_canonical[(developer_name, model_name)].append(row)
+
+ exported = 0
+ for (developer_name, model_name), rows_for_canonical in sorted(by_canonical.items()):
+ log, developer, model = make_log(
+ rows_for_canonical,
+ developer_name,
+ model_name,
+ metric_bounds,
+ retrieved_timestamp,
+ )
+ out_path = write_log(log, args.output_dir, developer, model)
+ print(out_path)
+ exported += 1
+
+ print(f"Exported {exported} model(s).")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/utils/arc_agi/config.json b/utils/arc_agi/config.json
new file mode 100644
index 000000000..38a01ff2f
--- /dev/null
+++ b/utils/arc_agi/config.json
@@ -0,0 +1,6 @@
+{
+ "requires_json": true,
+ "requires_csv": false,
+ "url": "https://arcprize.org/media/data/leaderboard/evaluations.json",
+ "arg_name": "--input-json"
+}
diff --git a/utils/artificial_analysis/config.json b/utils/artificial_analysis/config.json
new file mode 100644
index 000000000..8ee9bf2ca
--- /dev/null
+++ b/utils/artificial_analysis/config.json
@@ -0,0 +1,6 @@
+{
+ "requires_json": false,
+ "requires_csv": false,
+ "url": "https://artificialanalysis.ai/api/v2/data/llms/models",
+ "arg_name": null
+}
diff --git a/utils/bfcl/config.json b/utils/bfcl/config.json
new file mode 100644
index 000000000..29ab57420
--- /dev/null
+++ b/utils/bfcl/config.json
@@ -0,0 +1,6 @@
+{
+ "requires_json": false,
+ "requires_csv": true,
+ "url": "https://gorilla.cs.berkeley.edu/data_overall.csv",
+ "arg_name": "--input-csv"
+}
diff --git a/utils/exgentic/adapter.py b/utils/exgentic/adapter.py
index b7a4c02b5..653f42ea1 100644
--- a/utils/exgentic/adapter.py
+++ b/utils/exgentic/adapter.py
@@ -11,7 +11,7 @@
Data source:
- Exgentic experiments output: results.json files produced by `exgentic batch aggregate`
-- HuggingFace dataset: https://huggingface.co/datasets/Exgentic/open-agent-leaderboard-results
+- HuggingFace dataset: https://huggingface.co/datasets/Exgentic/results
Usage:
# From local experiment results
@@ -31,7 +31,7 @@
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
sys.path.insert(0, str(Path(__file__).parent.parent))
-from eval_types import (
+from every_eval_ever.eval_types import (
AgenticEvalConfig,
EvalLibrary,
EvaluationLog,
@@ -51,7 +51,7 @@
SCHEMA_VERSION = "0.2.2"
OUTPUT_DIR = "data/exgentic"
-HF_DATASET = "Exgentic/open-agent-leaderboard-results"
+HF_DATASET = "Exgentic/results"
# Map model name prefixes to developer organizations
MODEL_DEVELOPER_MAP = {
diff --git a/utils/helm/adapter.py b/utils/helm/adapter.py
index df365f304..bd4d484fa 100644
--- a/utils/helm/adapter.py
+++ b/utils/helm/adapter.py
@@ -79,6 +79,12 @@ def parse_args():
default='unknown',
help='Version of the evaluation library',
)
+ parser.add_argument(
+ '--output-dir',
+ type=str,
+ default=None,
+ help='Output directory',
+ )
return parser.parse_args()
@@ -176,6 +182,7 @@ def convert(
eval_library_name: str = 'helm',
eval_library_version: str = 'unknown',
source_data_url: str = 'unknown',
+ output_dir: str | None = None,
):
"""Convert HELM leaderboard data into unified evaluation logs."""
retrieved_timestamp = str(time.time())
@@ -360,9 +367,10 @@ def convert(
developer = model_info.developer
model = model_id
+ out_path = output_dir if output_dir else f'data/{leaderboard_name}'
filepath = save_evaluation_log(
eval_log,
- f'data/{leaderboard_name}',
+ out_path,
developer,
model,
)
@@ -416,6 +424,7 @@ def main():
eval_library_name=args.eval_library_name,
eval_library_version=args.eval_library_version,
source_data_url=source_data_url,
+ output_dir=args.output_dir,
)
print('Done!')
diff --git a/utils/hfopenllm_v2/config.json b/utils/hfopenllm_v2/config.json
new file mode 100644
index 000000000..538b0bd79
--- /dev/null
+++ b/utils/hfopenllm_v2/config.json
@@ -0,0 +1,6 @@
+{
+ "requires_json": false,
+ "requires_csv": false,
+ "url": "https://open-llm-leaderboard-open-llm-leaderboard.hf.space/api/leaderboard/formatted",
+ "arg_name": null
+}
diff --git a/utils/mmlu_pro/config.json b/utils/mmlu_pro/config.json
new file mode 100644
index 000000000..343d38e64
--- /dev/null
+++ b/utils/mmlu_pro/config.json
@@ -0,0 +1,6 @@
+{
+ "requires_json": false,
+ "requires_csv": false,
+ "url": "https://huggingface.co/datasets/TIGER-Lab/mmlu_pro_leaderboard_submission/resolve/main/results.csv",
+ "arg_name": null
+}
diff --git a/utils/multi_swe_bench/adapter.py b/utils/multi_swe_bench/adapter.py
index 776870fc8..f1196d2b8 100644
--- a/utils/multi_swe_bench/adapter.py
+++ b/utils/multi_swe_bench/adapter.py
@@ -160,6 +160,11 @@ def convert_submission(
def main():
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--output-dir", type=str, default=OUTPUT_BASE)
+ args = parser.parse_args()
+
retrieved_timestamp = str(time.time())
count = 0
errors = 0
@@ -186,7 +191,7 @@ def main():
eval_log = convert_submission(submission_dir, lang, retrieved_timestamp)
dev = eval_log.model_info.developer or "unknown"
model_name = eval_log.model_info.name.split("/")[-1]
- filepath = save_evaluation_log(eval_log, OUTPUT_BASE, dev, model_name)
+ filepath = save_evaluation_log(eval_log, args.output_dir, dev, model_name)
score = eval_log.evaluation_results[0].score_details.score
print(f" [{score:.1%}] {submission_dir.name} → {filepath}")
count += 1
@@ -194,7 +199,7 @@ def main():
print(f" ERROR {submission_dir.name}: {e}")
errors += 1
- print(f"\nGenerated {count} files, {errors} errors → {OUTPUT_BASE}/")
+ print(f"\nGenerated {count} files, {errors} errors → {args.output_dir}/")
if __name__ == "__main__":
diff --git a/utils/rewardbench/adapter.py b/utils/rewardbench/adapter.py
index 0784ed239..56b1845c2 100644
--- a/utils/rewardbench/adapter.py
+++ b/utils/rewardbench/adapter.py
@@ -134,10 +134,10 @@ def _make_model_info(
)
-def _save_eval_log(eval_log: EvaluationLog, developer: str, model: str) -> Path:
+def _save_eval_log(eval_log: EvaluationLog, developer: str, model: str, output_dir: Path = OUTPUT_DIR) -> Path:
"""Save an evaluation log to the standard directory structure."""
dir_path = (
- OUTPUT_DIR / sanitize_filename(developer) / sanitize_filename(model)
+ output_dir / sanitize_filename(developer) / sanitize_filename(model)
)
dir_path.mkdir(parents=True, exist_ok=True)
@@ -172,7 +172,7 @@ def parse_score(value: str) -> Optional[float]:
return None
-def fetch_rewardbench_v1(retrieved_timestamp: str) -> int:
+def fetch_rewardbench_v1(retrieved_timestamp: str, output_dir: Path = OUTPUT_DIR) -> int:
"""Fetch and process RewardBench v1 results from the CSV file."""
print('Fetching RewardBench v1 CSV...')
@@ -233,14 +233,14 @@ def fetch_rewardbench_v1(retrieved_timestamp: str) -> int:
else:
dev, model = 'unknown', model_info.id
- filepath = _save_eval_log(eval_log, dev, model)
+ filepath = _save_eval_log(eval_log, dev, model, output_dir)
print(f'Saved: {filepath}')
count += 1
return count
-def fetch_rewardbench_v2(retrieved_timestamp: str) -> int:
+def fetch_rewardbench_v2(retrieved_timestamp: str, output_dir: Path = OUTPUT_DIR) -> int:
"""Fetch and process RewardBench v2 results from the HuggingFace dataset."""
print('Fetching RewardBench v2 model list...')
@@ -348,7 +348,7 @@ def fetch_rewardbench_v2(retrieved_timestamp: str) -> int:
else:
dev, model = 'unknown', model_info.id
- filepath = _save_eval_log(eval_log, dev, model)
+ filepath = _save_eval_log(eval_log, dev, model, output_dir)
print(f' Saved: {filepath}')
count += 1
@@ -357,6 +357,11 @@ def fetch_rewardbench_v2(retrieved_timestamp: str) -> int:
def main():
"""Main function to fetch and process RewardBench results."""
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--output-dir', type=Path, default=OUTPUT_DIR)
+ args = parser.parse_args()
+
retrieved_timestamp = str(time.time())
print('=' * 60)
@@ -364,7 +369,7 @@ def main():
print('=' * 60)
try:
- v1_count = fetch_rewardbench_v1(retrieved_timestamp)
+ v1_count = fetch_rewardbench_v1(retrieved_timestamp, args.output_dir)
print(f'\nProcessed {v1_count} models from RewardBench v1')
except Exception as e:
print(f'Error processing RewardBench v1: {e}')
@@ -377,7 +382,7 @@ def main():
print('=' * 60)
try:
- v2_count = fetch_rewardbench_v2(retrieved_timestamp)
+ v2_count = fetch_rewardbench_v2(retrieved_timestamp, args.output_dir)
print(f'\nProcessed {v2_count} models from RewardBench v2')
except Exception as e:
print(f'Error processing RewardBench v2: {e}')
diff --git a/utils/sciarena/config.json b/utils/sciarena/config.json
new file mode 100644
index 000000000..60c0c25d8
--- /dev/null
+++ b/utils/sciarena/config.json
@@ -0,0 +1,6 @@
+{
+ "requires_json": true,
+ "requires_csv": false,
+ "url": "https://sciarena.allen.ai/api/leaderboard",
+ "arg_name": "--input-json"
+}
diff --git a/utils/scripts/pr_corrector.py b/utils/scripts/pr_corrector.py
new file mode 100644
index 000000000..7e6f4e8df
--- /dev/null
+++ b/utils/scripts/pr_corrector.py
@@ -0,0 +1,404 @@
+import os
+import re
+import json
+import shutil
+import stat
+import urllib.request
+from pathlib import Path
+from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_download
+
+from every_eval_ever.validate import validate_file
+
+# --- CONFIGURATION ---
+REPO_ID = "evaleval/EEE_datastore"
+REPO_TYPE = "dataset"
+WORKSPACE = "EEE_Targeted_Workspace"
+SCHEMA_PATH = "eval.schema.json"
+
+
+def remove_readonly(func, path, excinfo):
+ """Clear the read-only bit on Windows so shutil can delete files."""
+ os.chmod(path, stat.S_IWRITE)
+ func(path)
+
+
+def load_schema():
+ with open(SCHEMA_PATH, "r", encoding="utf-8") as f:
+ return json.load(f)
+
+
+def resolve_ref(schema, ref):
+ parts = ref.replace("#/", "").split("/")
+ current = schema
+ for part in parts:
+ current = current.get(part, {})
+ return current
+
+
+def get_schema_info(loc_string, root_schema):
+ """Parses loc string like 'evaluation_results -> [0] -> metric_config' and returns its schema."""
+ parts = loc_string.split(" -> ")
+ current = root_schema
+
+ for part in parts:
+ if "$ref" in current:
+ current = resolve_ref(root_schema, current["$ref"])
+
+ if part.startswith("[") and part.endswith("]"):
+ current = current.get("items", {})
+ else:
+ if "properties" in current and part in current["properties"]:
+ current = current["properties"][part]
+ elif "oneOf" in current:
+ found = False
+ for option in current["oneOf"]:
+ opt = option
+ if "$ref" in opt:
+ opt = resolve_ref(root_schema, opt["$ref"])
+ if "properties" in opt and part in opt["properties"]:
+ current = opt["properties"][part]
+ found = True
+ break
+ if not found:
+ current = {}
+ else:
+ current = {}
+
+ if "$ref" in current:
+ current = resolve_ref(root_schema, current["$ref"])
+
+ return current
+
+
+def convert_value(user_input, schema_info):
+ if not schema_info:
+ return user_input
+
+ expected_type = schema_info.get("type")
+
+ if expected_type == "boolean":
+ return user_input.lower() in ["true", "1", "yes", "y", "t"]
+ elif expected_type == "integer":
+ try:
+ return int(user_input)
+ except ValueError:
+ return user_input
+ elif expected_type == "number":
+ try:
+ return float(user_input)
+ except ValueError:
+ return user_input
+ elif expected_type == "array":
+ try:
+ parsed = json.loads(user_input)
+ if isinstance(parsed, list):
+ return parsed
+ except json.JSONDecodeError:
+ pass
+ # Simple fallback for arrays
+ return [s.strip() for s in user_input.split(",") if s.strip()]
+
+ # If type is an array of options e.g., ["null", "number"]
+ if isinstance(expected_type, list):
+ if "number" in expected_type:
+ try:
+ return float(user_input)
+ except ValueError:
+ pass
+ if "integer" in expected_type:
+ try:
+ return int(user_input)
+ except ValueError:
+ pass
+
+ return user_input
+
+
+def apply_fix_exact(data, loc_string, value):
+ """Dynamically traverses a JSON object and injects the value at the exact location."""
+ parts = loc_string.split(" -> ")
+ if not parts:
+ return
+
+ current = data
+ for i in range(len(parts) - 1):
+ part = parts[i]
+ if part.startswith("[") and part.endswith("]"):
+ index = int(part[1:-1])
+ current = current[index]
+ else:
+ if part not in current:
+ current[part] = {}
+ current = current[part]
+
+ last_part = parts[-1]
+ if last_part.startswith("[") and last_part.endswith("]"):
+ index = int(last_part[1:-1])
+ current[index] = value
+ else:
+ current[last_part] = value
+
+
+def apply_fix_fuzzy(data, path_str, key, value):
+ """Dynamically traverses a JSON object using dot notation and injects the key/value pair."""
+ parts = path_str.split('.') if path_str else []
+
+ def traverse(current_obj, path_parts):
+ if not path_parts:
+ if isinstance(current_obj, dict):
+ current_obj[key] = value
+ return
+
+ part = path_parts[0]
+ if isinstance(current_obj, dict):
+ if part not in current_obj:
+ current_obj[part] = {}
+ if len(path_parts) == 1:
+ if isinstance(current_obj[part], list):
+ for item in current_obj[part]:
+ if isinstance(item, dict):
+ item[key] = value
+ else:
+ current_obj[part][key] = value
+ else:
+ traverse(current_obj[part], path_parts[1:])
+ elif isinstance(current_obj, list):
+ for item in current_obj:
+ traverse(item, path_parts)
+
+ traverse(data, parts)
+
+
+def get_pr_files(repo_id, pr_num, token):
+ """Fetches the list of modified .json files in the PR using diffUrl or HTML scraping as a fallback."""
+ url = f"https://huggingface.co/api/datasets/{repo_id}/discussions/{pr_num}"
+ req = urllib.request.Request(url)
+ if token:
+ req.add_header("Authorization", f"Bearer {token}")
+
+ try:
+ with urllib.request.urlopen(req) as res:
+ data = json.loads(res.read().decode('utf-8'))
+
+ diff_url = data.get('diffUrl')
+ if diff_url:
+ if diff_url.startswith('/'):
+ diff_url = "https://huggingface.co" + diff_url
+ diff_req = urllib.request.Request(diff_url)
+ if token:
+ diff_req.add_header("Authorization", f"Bearer {token}")
+ with urllib.request.urlopen(diff_req) as diff_res:
+ diff_text = diff_res.read().decode('utf-8')
+ files = re.findall(r'^\+\+\+ b/(data/.*\.json)$', diff_text, flags=re.MULTILINE)
+ if files:
+ return list(set(files))
+ except Exception as e:
+ pass
+
+ # Fallback to HTML scraping
+ html_url = f"https://huggingface.co/datasets/{repo_id}/discussions/{pr_num}/files"
+ html_req = urllib.request.Request(html_url)
+ if token:
+ html_req.add_header("Authorization", f"Bearer {token}")
+ try:
+ with urllib.request.urlopen(html_req) as res:
+ html = res.read().decode('utf-8')
+ files = re.findall(r'(data/[\w\-\./]+\.json)', html)
+ return list(set(files))
+ except Exception as e:
+ print(f"❌ Failed to parse files from PR: {e}")
+
+ return []
+
+
+def custom_validate(data):
+ """Custom heuristic validator for warnings not strictly in the schema."""
+ bot_warnings = []
+
+ # 1. Check for deployment_type and dependent fields
+ model_info = data.get('model_info', {})
+ additional = model_info.get('additional_details', {})
+
+ deployment_type = additional.get('deployment_type')
+ if not deployment_type:
+ bot_warnings.append(('model_info.additional_details', 'deployment_type', 'Expected one of: api, local, unknown'))
+ else:
+ if deployment_type == 'api' and 'model_availability' not in additional:
+ bot_warnings.append(('model_info.additional_details', 'model_availability', "Expected one of: closed_source, open_weights_deployment, other"))
+
+ return bot_warnings
+
+
+def main():
+ print("🤖 EEE Datastore Targeted PR Fixer 🤖\n")
+
+ hf_token = input("Enter your Hugging Face Access Token: ").strip()
+ pr_input = input("Enter the PR number or URL (e.g., 136 or https://.../136): ").strip()
+
+ pr_num = re.search(r'(\d+)$', pr_input).group(1)
+ revision = f"refs/pr/{pr_num}"
+ api = HfApi(token=hf_token)
+
+ try:
+ schema = load_schema()
+ except Exception as e:
+ print(f"❌ Failed to load {SCHEMA_PATH}: {e}")
+ return
+
+ print("\n📂 Fetching file list for the PR...")
+ json_files = get_pr_files(REPO_ID, pr_num, hf_token)
+
+ if not json_files:
+ print("❌ No aggregate JSON files found in this PR to fix. Is the PR number correct?")
+ return
+
+ print(f"Found {len(json_files)} file(s) in PR #{pr_num}.")
+
+ if os.path.exists(WORKSPACE):
+ shutil.rmtree(WORKSPACE, onexc=remove_readonly)
+ os.makedirs(WORKSPACE)
+
+ operations = []
+ apply_to_all_answers = {}
+ print(f"Downloading {len(json_files)} JSON file(s) locally...\n")
+
+ for file_path in json_files:
+ try:
+ local_path = hf_hub_download(
+ repo_id=REPO_ID,
+ repo_type=REPO_TYPE,
+ filename=file_path,
+ revision=revision,
+ token=hf_token,
+ local_dir=WORKSPACE,
+ local_dir_use_symlinks=False
+ )
+ except Exception as e:
+ print(f"❌ Failed to download {file_path}: {e}")
+ continue
+
+ modified = False
+
+ while True:
+ # Validate the downloaded file
+ report = validate_file(Path(local_path))
+
+ with open(local_path, "r", encoding="utf-8") as f:
+ data = json.load(f)
+
+ bot_warnings = custom_validate(data)
+ missing_errors = [err for err in report.errors if "missing" in err.get("type", "")]
+
+ if report.valid and not bot_warnings and not missing_errors:
+ if not modified:
+ print(f"✅ {file_path.split('/')[-1]} passed validation and has no bot warnings! No fixes needed.")
+ break
+
+ if not missing_errors and not bot_warnings:
+ print(f"⚠️ {file_path.split('/')[-1]} has non-missing validation errors:")
+ for err in report.errors:
+ print(f" - {err['loc']}: {err['msg']}")
+ break
+
+ print(f"\n📄 Fixing {file_path.split('/')[-1]} ({len(missing_errors)} missing fields, {len(bot_warnings)} bot warnings)")
+ made_changes_this_round = False
+
+ # Process standard validation missing errors
+ for err in missing_errors:
+ loc_str = err['loc']
+ last_key = loc_str.split(" -> ")[-1]
+
+ # Check for "apply all" override based on the key name
+ if last_key in apply_to_all_answers:
+ apply_fix_exact(data, loc_str, apply_to_all_answers[last_key])
+ made_changes_this_round = True
+ continue
+
+ schema_info = get_schema_info(loc_str, schema)
+ desc = schema_info.get("description", "No description available")
+ expected_type = schema_info.get("type", "unknown")
+
+ print(f"\n Missing: '{last_key}' at '{loc_str}'")
+ print(f" Type: {expected_type} | Description: {desc}")
+
+ user_input = input(f" Enter value (or 'skip', or 'all:your_value'): ").strip()
+
+ if user_input.lower() == 'skip' or user_input == '':
+ print(" ⏭️ Skipped.")
+ continue
+
+ if user_input.lower().startswith("all:"):
+ raw_value = user_input[4:].strip()
+ converted = convert_value(raw_value, schema_info)
+ apply_to_all_answers[last_key] = converted
+ apply_fix_exact(data, loc_str, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}' to this and all future occurrences of '{last_key}'.")
+ else:
+ converted = convert_value(user_input, schema_info)
+ apply_fix_exact(data, loc_str, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}'.")
+
+ # Process bot warnings
+ for path_str, key, desc in bot_warnings:
+ if key in apply_to_all_answers:
+ apply_fix_fuzzy(data, path_str, key, apply_to_all_answers[key])
+ made_changes_this_round = True
+ continue
+
+ print(f"\n Bot Warning Missing: '{key}' inside '{path_str}'")
+ print(f" Description: {desc}")
+ user_input = input(f" Enter value (or 'skip', or 'all:your_value'): ").strip()
+
+ if user_input.lower() == 'skip' or user_input == '':
+ print(" ⏭️ Skipped.")
+ continue
+
+ # Fallback for bot warning schema lookup is harder without exact array indices
+ if user_input.lower().startswith("all:"):
+ raw_value = user_input[4:].strip()
+ # for bot warnings we just inject string directly or attempt basic inference
+ converted = convert_value(raw_value, {})
+ apply_to_all_answers[key] = converted
+ apply_fix_fuzzy(data, path_str, key, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}' to this and all future files.")
+ else:
+ converted = convert_value(user_input, {})
+ apply_fix_fuzzy(data, path_str, key, converted)
+ made_changes_this_round = True
+ print(f" ✅ Applied '{converted}'.")
+
+ if made_changes_this_round:
+ modified = True
+ with open(local_path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=4, ensure_ascii=False)
+ f.write("\n")
+ else:
+ # User skipped everything, break to avoid infinite loop
+ break
+
+ if modified:
+ operations.append(CommitOperationAdd(path_in_repo=file_path, path_or_fileobj=local_path))
+
+ if operations:
+ print(f"\n📤 Committing {len(operations)} updated file(s) to PR #{pr_num}...")
+ api.create_commit(
+ repo_id=REPO_ID,
+ repo_type=REPO_TYPE,
+ revision=revision,
+ operations=operations,
+ commit_message="Fix schema warnings via interactive script"
+ )
+ print("🎉 Success! Your PR has been updated.")
+ else:
+ print("\n⚠️ No changes were made that require a commit.")
+
+ print("🧹 Cleaning up local files...")
+ shutil.rmtree(WORKSPACE, onexc=remove_readonly)
+ print("✨ Cleanup complete!")
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/utils/scripts/run_adapters.py b/utils/scripts/run_adapters.py
new file mode 100644
index 000000000..4b8d23ae3
--- /dev/null
+++ b/utils/scripts/run_adapters.py
@@ -0,0 +1,810 @@
+#!/usr/bin/env python3
+"""Orchestrator for running Every Eval Ever adapters.
+
+Handles intelligent scheduling via adapter_stats, duplicate detection using
+content-aware fingerprinting, schema validation, and upload to HuggingFace.
+Designed to run daily via GitHub Actions.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import traceback
+import urllib.request
+import urllib.error
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.utils import EntryNotFoundError
+
+from every_eval_ever.check_duplicate_entries import normalized_hash
+
+# ── Configuration ────────────────────────────────────────────────────────────
+
+REPO_ID = "deeplumiere/EEE_datastore"
+REPO_TYPE = "dataset"
+DATA_DIR = Path("data")
+STATS_FILE = DATA_DIR / "adapter_stats.json"
+REPORT_FILE = DATA_DIR / "run_report.json"
+UTILS_DIR = Path("utils")
+
+# Adapters exceeding these thresholds are classified as "heavy" and only run
+# during the monthly window (first 7 days) or when their source is stale.
+HEAVY_TIME_S = 900
+HEAVY_SIZE_MB = 250
+
+
+# ── File & Network Utilities ─────────────────────────────────────────────────
+
+
+def get_dir_size_mb(path: Path) -> float:
+ """Calculate total size of all files in a directory, in megabytes."""
+ if not path.exists():
+ return 0.0
+ return sum(
+ f.stat().st_size for f in path.rglob("*") if f.is_file()
+ ) / (1024 * 1024)
+
+
+def download_hf_json(
+ filename: str,
+ default: dict | list,
+ revision: str = "main",
+) -> dict | list:
+ """Download and parse a JSON file from HuggingFace Hub.
+
+ Returns *default* when the file doesn't exist on the given revision.
+ """
+ try:
+ path = hf_hub_download(
+ repo_id=REPO_ID,
+ filename=filename,
+ repo_type=REPO_TYPE,
+ revision=revision,
+ )
+ with open(path, "r", encoding="utf-8") as f:
+ return json.load(f)
+ except EntryNotFoundError:
+ return default
+ except Exception as e:
+ print(f"Warning: Could not download {filename} from HF ({e})")
+ return default
+
+
+def download_file(url: str, output: Path) -> None:
+ """Download a remote file to a local path."""
+ headers = {"User-Agent": "every-eval-ever adapter runner"}
+ req = urllib.request.Request(url, headers=headers)
+ try:
+ with urllib.request.urlopen(req, timeout=60) as resp:
+ with open(output, "wb") as f:
+ shutil.copyfileobj(resp, f)
+ except urllib.error.HTTPError as e:
+ print(f"HTTP Error {e.code} while downloading {url}: {e.reason}")
+ raise ValueError(f"Download failed with HTTP {e.code}: {e.reason}") from e
+
+
+def check_url_headers(url: str | None) -> dict[str, str | None]:
+ """Fetch HTTP HEAD headers from a URL to detect source-data changes.
+ Falls back to GET if HEAD is not allowed, and injects known API keys.
+ """
+ if not url:
+ return {}
+
+ headers = {"User-Agent": "every-eval-ever adapter runner"}
+
+ if "artificialanalysis.ai" in url and os.environ.get("ARTIFICIAL_ANALYSIS_API_KEY"):
+ headers["x-api-key"] = os.environ.get("ARTIFICIAL_ANALYSIS_API_KEY")
+ if "llm-stats.com" in url and os.environ.get("LLM_STATS_API_KEY"):
+ headers["Authorization"] = f'Bearer {os.environ.get("LLM_STATS_API_KEY")}'
+ headers["x-api-key"] = os.environ.get("LLM_STATS_API_KEY")
+ if ("huggingface.co" in url or "hf.space" in url) and os.environ.get("HF_TOKEN"):
+ headers["Authorization"] = f'Bearer {os.environ.get("HF_TOKEN")}'
+
+ def _fetch(method: str) -> dict[str, str | None]:
+ req = urllib.request.Request(url, method=method, headers=headers)
+ with urllib.request.urlopen(req, timeout=10) as resp:
+ return {
+ "url_etag": resp.headers.get("ETag"),
+ "url_last_modified": resp.headers.get("Last-Modified"),
+ "url_content_length": resp.headers.get("Content-Length"),
+ }
+
+ try:
+ return _fetch("HEAD")
+ except urllib.error.HTTPError as e:
+ if e.code == 405:
+ try:
+ return _fetch("GET")
+ except Exception as e2:
+ print(f"Warning: GET fallback failed for {url}: {e2}")
+ else:
+ print(f"Warning: HEAD request failed for {url}: {e}")
+ except Exception as e:
+ print(f"Warning: HEAD request failed for {url}: {e}")
+
+ return {}
+
+
+def save_json(path: Path, data: Any) -> None:
+ """Write *data* to a JSON file with readable formatting."""
+ with open(path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=2, sort_keys=True)
+
+
+# ── Staleness Detection ─────────────────────────────────────────────────────
+
+_HEADER_CHECKS: list[tuple[str, str]] = [
+ ("url_etag", "ETag changed"),
+ ("url_last_modified", "Last-Modified changed"),
+ ("url_content_length", "Content-Length changed"),
+]
+
+
+def get_assigned_days_of_week(adapter_name: str) -> set[int]:
+ """Consistently assign an adapter to 3 days of the week (0-6) based on its name."""
+ hash_val = int(hashlib.md5(adapter_name.encode('utf-8')).hexdigest(), 16)
+ return {(hash_val + (i * 2)) % 7 for i in range(3)}
+
+
+def is_stale(
+ adapter: str,
+ stats: dict,
+ current_headers: dict[str, str | None],
+) -> tuple[bool, str]:
+ """Decide whether an adapter's data is stale and needs re-running.
+
+ Priority order:
+ 1. Assigned days of week have not arrived → not stale (deferred).
+ 2. Previous run failed → stale (retry immediately).
+ 3. Source HTTP headers changed → stale.
+ 4. New headers available that weren't previously tracked → stale.
+ 5. 7+ days since last *data change* → stale (fallback).
+ """
+ assigned_dows = get_assigned_days_of_week(adapter)
+ current_dow = datetime.datetime.now().weekday() # Monday is 0, Sunday is 6
+
+ stat = stats.get(adapter, {})
+ last_check_ts = stat.get("last_check_ts", 0)
+
+ if current_dow not in assigned_dows:
+ return False, f"assigned to days {sorted(list(assigned_dows))}, today is {current_dow}"
+
+ if stat.get("last_failed"):
+ return True, "last run failed"
+
+ # Compare current source headers against stored values.
+ if current_headers:
+ has_new_header = False
+ for key, label in _HEADER_CHECKS:
+ current = current_headers.get(key)
+ stored = stat.get(key)
+ if current and stored and current != stored:
+ return True, label
+ if current and not stored:
+ has_new_header = True
+ if has_new_header:
+ return True, "new header available"
+ else:
+ # If we failed to get headers (e.g. 405, 401, 404, or no URL)
+ # we can't reliably know if it's stale, so default to running.
+ return True, "headers unavailable"
+
+ # Fallback: time since last data change (or last success if never tracked).
+ last_change = stat.get(
+ "last_data_change_ts", stat.get("last_success_ts", 0)
+ )
+ if (time.time() - last_change) / 86400 >= 7:
+ return True, "7-day fallback"
+
+ return False, "not stale"
+
+
+# ── Adapter Discovery ───────────────────────────────────────────────────────
+
+
+def load_adapter_config(adapter_dir: Path) -> dict:
+ """Read adapter configuration from ``config.json``."""
+ config_path = adapter_dir / "config.json"
+ if not config_path.exists():
+ return {}
+ try:
+ with open(config_path, "r", encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as e:
+ print(f"Warning: Failed to parse {config_path}: {e}")
+ return {}
+
+
+def discover_adapters(stats: dict) -> list[dict]:
+ """Scan *UTILS_DIR* for adapters and gather scheduling metadata.
+
+ Returns a list of info dicts, sorted stale-first then alphabetically.
+ """
+ infos: list[dict] = []
+ for adapter_dir in UTILS_DIR.iterdir():
+ adapter_path = adapter_dir / "adapter.py"
+ if not adapter_dir.is_dir() or not adapter_path.exists():
+ continue
+
+ config = load_adapter_config(adapter_dir)
+ url = config.get("url")
+ headers = check_url_headers(url)
+ stale, reason = is_stale(adapter_dir.name, stats, headers)
+
+ infos.append({
+ "name": adapter_dir.name,
+ "adapter_path": adapter_path,
+ "stale": stale,
+ "reason": reason,
+ "url": url,
+ "headers": headers,
+ "requires_json": config.get("requires_json", False),
+ "requires_csv": config.get("requires_csv", False),
+ "arg_name": config.get("arg_name"),
+ })
+
+ infos.sort(key=lambda x: (not x["stale"], x["name"]))
+ return infos
+
+
+# ── Duplicate Detection ─────────────────────────────────────────────────────
+
+
+def compute_data_fingerprint(data_dir: Path, known_hashes: set[str] | None = None) -> tuple[str, list[str]]:
+ """Compute a stable fingerprint for all JSON outputs in a directory.
+
+ Uses ``normalized_hash`` from ``check_duplicate_entries`` which strips
+ scrape-specific fields (``retrieved_timestamp``, ``evaluation_id``)
+ before hashing. This means identical evaluation data always produces
+ the same fingerprint regardless of when it was scraped.
+
+ Additionally, deletes any duplicate JSON files found within the directory
+ (or matching `known_hashes`) to ensure the datastore only receives NEW entries.
+
+ Returns (fingerprint, new_hashes_list).
+ """
+ if known_hashes is None:
+ known_hashes = set()
+
+ file_hashes: list[str] = []
+ seen_hashes: set[str] = set(known_hashes)
+ for json_file in sorted(data_dir.rglob("*.json")):
+ try:
+ with open(json_file, "r", encoding="utf-8") as f:
+ payload = json.load(f)
+ h = normalized_hash(payload)
+ if h in seen_hashes:
+ print(f" Warning: Deleting repetitive entry {json_file.name}")
+ json_file.unlink()
+ continue
+ seen_hashes.add(h)
+ file_hashes.append(h)
+ except (json.JSONDecodeError, OSError) as e:
+ print(f" Warning: Could not hash {json_file.name}: {e}")
+ continue
+
+ if not file_hashes:
+ return "", []
+
+ combined = "\n".join(sorted(file_hashes))
+ return hashlib.sha256(combined.encode("utf-8")).hexdigest(), file_hashes
+
+
+# ── Adapter Execution ────────────────────────────────────────────────────────
+
+
+def build_adapter_env() -> dict[str, str]:
+ """Build the environment dict for adapter subprocesses.
+
+ Adds ``every_eval_ever`` to ``PYTHONPATH`` so adapters can import it.
+ """
+ env = os.environ.copy()
+ eee_path = str(Path("every_eval_ever").absolute())
+ if "PYTHONPATH" in env:
+ env["PYTHONPATH"] = f"{eee_path}{os.pathsep}{env['PYTHONPATH']}"
+ else:
+ env["PYTHONPATH"] = eee_path
+ return env
+
+
+def prepare_adapter_command(
+ info: dict,
+ adapter_data_dir: Path,
+) -> tuple[list[str], Path | None]:
+ """Build the CLI command for an adapter and download any required input.
+
+ Returns ``(command, tmp_input_file)``. *tmp_input_file* is ``None``
+ when no download was needed.
+
+ Raises ``ValueError`` when the adapter requires input data but no URL
+ is configured.
+ """
+ adapter = info["name"]
+ adapter_path: Path = info["adapter_path"]
+ content = adapter_path.read_text(encoding="utf-8")
+
+ cmd = ["uv", "run", "python", "-m", f"utils.{adapter}.adapter"]
+ if "--output-dir" in content:
+ cmd.extend(["--output-dir", str(adapter_data_dir)])
+ if "--from-hf" in content:
+ cmd.append("--from-hf")
+
+ requires_json: bool = info["requires_json"]
+ requires_csv: bool = info["requires_csv"]
+ url: str | None = info["url"]
+ arg_name: str | None = info["arg_name"]
+
+ if (requires_json or requires_csv) and not url:
+ raise ValueError(
+ f"requires {'JSON' if requires_json else 'CSV'} input "
+ f"({arg_name}) but no URL configured"
+ )
+
+ tmp_file: Path | None = None
+ if requires_json and url:
+ tmp_file = DATA_DIR / f"{adapter}_input.json"
+ print(f"[{adapter}] Downloading JSON input from {url}")
+ download_file(url, tmp_file)
+ cmd.extend([arg_name, str(tmp_file)])
+ elif requires_csv and url:
+ tmp_file = DATA_DIR / f"{adapter}_input.csv"
+ print(f"[{adapter}] Downloading CSV input from {url}")
+ download_file(url, tmp_file)
+ cmd.extend([arg_name, str(tmp_file)])
+
+ return cmd, tmp_file
+
+
+def validate_adapter_outputs(
+ adapter_data_dir: Path,
+ env: dict[str, str],
+) -> tuple[int, int, list]:
+ """Run schema validation on adapter outputs, deleting invalid files.
+
+ Returns ``(valid_count, failed_count, error_list)``.
+ """
+ cmd = [
+ "uv", "run", "python", "-m", "every_eval_ever",
+ "validate", "--format", "json", str(adapter_data_dir),
+ ]
+ result = subprocess.run(cmd, capture_output=True, text=True, env=env)
+
+ try:
+ stdout = result.stdout.strip()
+ idx = stdout.find("[")
+ val_data = json.loads(stdout[idx:]) if idx != -1 else []
+ except json.JSONDecodeError as e:
+ print(f"[{adapter_data_dir.name}] Failed to parse validation output: {e}")
+ val_data = []
+
+ valid_count = 0
+ failed_count = 0
+ errors: list = []
+
+ for report in val_data:
+ if report.get("valid"):
+ valid_count += 1
+ else:
+ failed_count += 1
+ errors.extend(report.get("errors", []))
+ invalid_path = Path(report.get("file", ""))
+ if invalid_path.exists():
+ if invalid_path.is_file():
+ invalid_path.unlink()
+ else:
+ print(f"[{adapter_data_dir.name}] Warning: Validation reported a directory as a failed file: {invalid_path}")
+
+ return valid_count, failed_count, errors
+
+
+# ── PR Management ────────────────────────────────────────────────────────────
+
+
+def find_existing_pr(api: HfApi) -> Any | None:
+ """Find the most recent open PR by the current user on the HF repo."""
+ try:
+ current_user = api.whoami().get("name")
+ except Exception as e:
+ print(f"Warning: Could not identify current HF user: {e}")
+ current_user = None
+
+ try:
+ discussions = api.get_repo_discussions(
+ repo_id=REPO_ID, repo_type=REPO_TYPE
+ )
+ open_prs = [
+ d
+ for d in discussions
+ if getattr(d, "is_pull_request", False)
+ and d.status in ("open", "draft")
+ and (d.author == current_user if current_user else True)
+ ]
+ return max(open_prs, key=lambda x: x.num) if open_prs else None
+ except Exception as e:
+ print(f"Warning: Could not fetch PRs: {e}")
+ traceback.print_exc()
+ return None
+
+
+def load_remote_state(revision: str) -> tuple[dict, dict]:
+ """Download adapter_stats and run_report from HuggingFace.
+
+ Tries the ``data/`` prefix first, then falls back to root-level paths
+ to handle both repository layouts.
+ """
+ stats = download_hf_json("data/adapter_stats.json", {}, revision=revision)
+ if not stats:
+ stats = download_hf_json("adapter_stats.json", {}, revision=revision)
+
+ report = download_hf_json("data/run_report.json", {}, revision=revision)
+ if not report:
+ report = download_hf_json("run_report.json", {}, revision=revision)
+
+ return stats, report
+
+
+def create_new_pr(api: HfApi) -> int:
+ """Create a new PR on the HF dataset repo and return its number."""
+ pr = api.create_pull_request(
+ repo_id=REPO_ID,
+ title="Automated Adapter Data Update",
+ description="Data update from GitHub Actions",
+ repo_type=REPO_TYPE,
+ )
+ print(f" Created new PR #{pr.num} ({pr.url})")
+ return pr.num
+
+
+# ── Upload ───────────────────────────────────────────────────────────────────
+
+
+def upload_to_hf(
+ api: HfApi,
+ existing_pr: Any | None,
+) -> bool:
+ """Upload data directory to HuggingFace via a PR.
+
+ Reuses an existing open PR when available, otherwise creates a new one.
+ Returns ``True`` on success.
+ """
+ if not os.environ.get("HF_TOKEN"):
+ print("ERROR: HF_TOKEN not set, skipping upload.")
+ return False
+
+ try:
+ if existing_pr:
+ pr_num = existing_pr.num
+ print(f" Uploading to existing PR #{pr_num}")
+ else:
+ pr_num = create_new_pr(api)
+
+ revision = f"refs/pr/{pr_num}"
+ today = datetime.datetime.now().strftime("%Y-%m-%d")
+
+ api.upload_folder(
+ repo_id=REPO_ID,
+ folder_path=str(DATA_DIR),
+ path_in_repo="data",
+ repo_type=REPO_TYPE,
+ revision=revision,
+ commit_message=f"Automated data update ({today})",
+ )
+ print(f" Upload complete to PR #{pr_num}")
+ return True
+ except Exception as e:
+ print(f" Upload failed: {e}")
+ traceback.print_exc()
+ return False
+
+
+# ── Single-Adapter Processing ────────────────────────────────────────────────
+
+
+def process_adapter(
+ info: dict,
+ stats: dict,
+ today: datetime.datetime,
+ env: dict[str, str],
+) -> dict:
+ """Run a single adapter through the full pipeline.
+
+ Steps: schedule check → execute → validate → fingerprint → update stats.
+
+ Returns a result dict with keys: ``status``, ``report_entry``, ``failed``.
+ """
+ adapter = info["name"]
+ stat = stats.get(adapter, {})
+ is_heavy = (
+ stat.get("time_s", 0) > HEAVY_TIME_S
+ or stat.get("size_mb", 0) > HEAVY_SIZE_MB
+ )
+
+ # ── Scheduling gate ──────────────────────────────────────────────────
+ # Non-stale adapters are skipped.
+ if not info["stale"]:
+ label = "heavy, deferred" if is_heavy else "not stale"
+ print(f"[{adapter}] Skipping ({label})")
+ return _skip_result(label)
+
+ # ── Prepare workspace ────────────────────────────────────────────────
+ adapter_data_dir = DATA_DIR / adapter
+ if adapter_data_dir.exists():
+ shutil.rmtree(adapter_data_dir)
+ adapter_data_dir.mkdir(parents=True, exist_ok=True)
+
+ tmp_file: Path | None = None
+ try:
+ cmd, tmp_file = prepare_adapter_command(info, adapter_data_dir)
+
+ # ── Execute ──────────────────────────────────────────────────────
+ start = time.time()
+ result = subprocess.run(cmd, capture_output=True, text=True, env=env)
+ elapsed = time.time() - start
+
+ if result.returncode != 0:
+ print(f"[{adapter}] FAILED ({elapsed:.1f}s):\n{result.stderr[-500:]}")
+ stats.setdefault(adapter, {})["last_failed"] = True
+ return {
+ "status": "exec_failed",
+ "report_entry": {
+ "execution_failed": True,
+ "error": result.stderr[-500:],
+ },
+ "failed": True,
+ }
+
+ # ── Validate ─────────────────────────────────────────────────────
+ print(f"[{adapter}] Ran in {elapsed:.1f}s, validating...")
+ valid, failed, errors = validate_adapter_outputs(adapter_data_dir, env)
+ print(f"[{adapter}] Validation: {valid} valid, {failed} failed")
+
+ if valid == 0 or failed > 0:
+ stats.setdefault(adapter, {})["last_failed"] = True
+ return {
+ "status": "validation_failed",
+ "report_entry": {
+ "execution_failed": False,
+ "valid_files": valid,
+ "failed_files": failed,
+ "error": f"Validation: {valid} valid, {failed} failed",
+ "errors": errors[:50],
+ },
+ "failed": True,
+ }
+
+ # ── Duplicate detection ──────────────────────────────────────────
+ known_hashes = set(stat.get("entry_hashes", []))
+ fingerprint, new_hashes = compute_data_fingerprint(adapter_data_dir, known_hashes)
+
+ # If we have no entry_hashes yet but we DO have a data_fingerprint,
+ # we are migrating. To avoid duplicating everything, we can just treat
+ # the current local output as "known" if the fingerprint matches perfectly.
+ stored_fingerprint = stat.get("data_fingerprint", "")
+ if not known_hashes and stored_fingerprint and fingerprint == stored_fingerprint:
+ data_changed = False
+ # Backfill the entry_hashes so we know them for next time!
+ stat["entry_hashes"] = new_hashes
+ else:
+ data_changed = bool(new_hashes)
+
+ if data_changed:
+ print(f"[{adapter}] New data detected ({len(new_hashes)} new entries)")
+ else:
+ print(f"[{adapter}] Data unchanged (all entries already known). Discarding outputs.")
+ shutil.rmtree(adapter_data_dir)
+
+ # ── Update stats ─────────────────────────────────────────────────
+ update: dict[str, Any] = {
+ "time_s": elapsed,
+ "size_mb": get_dir_size_mb(adapter_data_dir) if adapter_data_dir.exists() else 0,
+ "last_success_ts": time.time(),
+ "last_check_ts": time.time(),
+ "last_failed": False,
+ "data_fingerprint": fingerprint,
+ }
+ if data_changed:
+ update["last_data_change_ts"] = time.time()
+ update["entry_hashes"] = list(known_hashes.union(new_hashes))
+
+ # Persist source headers for future staleness comparisons.
+ for key, value in info["headers"].items():
+ if value:
+ update[key] = value
+
+ stats.setdefault(adapter, {}).update(update)
+
+ return {
+ "status": "success",
+ "report_entry": {
+ "execution_failed": False,
+ "valid_files": valid,
+ "failed_files": failed,
+ "data_changed": data_changed,
+ "errors": errors[:50],
+ },
+ "failed": False,
+ }
+
+ except ValueError as e:
+ # Raised by prepare_adapter_command when input URL is missing.
+ print(f"[{adapter}] Skipping: {e}")
+ return _skip_result(str(e))
+
+ except Exception as e:
+ print(f"[{adapter}] Exception: {e}")
+ traceback.print_exc()
+ stats.setdefault(adapter, {})["last_failed"] = True
+ return {
+ "status": "exception",
+ "report_entry": {
+ "execution_failed": True,
+ "error": str(e),
+ },
+ "failed": True,
+ }
+
+ finally:
+ if tmp_file and tmp_file.exists():
+ tmp_file.unlink()
+
+
+def _skip_result(reason: str) -> dict:
+ """Build a result dict for a skipped adapter."""
+ return {
+ "status": "skipped",
+ "report_entry": None,
+ "failed": False,
+ "skip_reason": reason,
+ }
+
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+
+
+def parse_args() -> argparse.Namespace:
+ """Parse command-line arguments."""
+ parser = argparse.ArgumentParser(
+ description="Run all Every Eval Ever adapters with intelligent scheduling.",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Run adapters locally but do not upload to HuggingFace",
+ )
+ parser.add_argument(
+ "--force-all",
+ action="store_true",
+ help="Force run all adapters regardless of staleness",
+ )
+ return parser.parse_args()
+
+
+def main() -> int:
+ """Entry point: discover, schedule, run, and upload adapters."""
+ args = parse_args()
+ DATA_DIR.mkdir(exist_ok=True)
+
+ # ── PR discovery & remote state ──────────────────────────────────────
+ api = HfApi()
+ print("Checking for existing PRs...")
+ existing_pr = find_existing_pr(api)
+
+ revision = f"refs/pr/{existing_pr.num}" if existing_pr else "main"
+ if existing_pr:
+ print(
+ f"Found PR #{existing_pr.num} ({existing_pr.url}), "
+ f"loading state from {revision}"
+ )
+ else:
+ print("No open PR found, loading state from main")
+
+ stats, _ = load_remote_state(revision)
+
+ # ── Discover & analyse adapters ──────────────────────────────────────
+ print("\nAnalysing adapters...")
+ adapter_infos = discover_adapters(stats)
+
+ if getattr(args, "force_all", False):
+ print("Force run requested! Marking all adapters as stale.")
+ for info in adapter_infos:
+ info["stale"] = True
+ info["reason"] = "forced by --force-all flag"
+
+ print(
+ f"Found {len(adapter_infos)} adapter(s): "
+ f"{sum(1 for a in adapter_infos if a['stale'])} stale, "
+ f"{sum(1 for a in adapter_infos if not a['stale'])} current"
+ )
+
+ today = datetime.datetime.now()
+ env = build_adapter_env()
+ report: dict[str, Any] = {
+ "date": today.strftime("%Y-%m-%d"),
+ "adapters": {},
+ }
+ any_failures = False
+ summary: dict[str, list[str]] = {
+ "ran": [],
+ "skipped": [],
+ "failed": [],
+ }
+
+ # ── Process each adapter ─────────────────────────────────────────────
+ import concurrent.futures
+
+ print(f"\n{'─' * 60}")
+ for info in adapter_infos:
+ print(f"Adapter: {info['name']} | Stale: {info['stale']} ({info['reason']})")
+
+ print("\nRunning adapters in parallel...")
+ with concurrent.futures.ThreadPoolExecutor() as executor:
+ futures = {
+ executor.submit(process_adapter, info, stats, today, env): info["name"]
+ for info in adapter_infos
+ }
+
+ for future in concurrent.futures.as_completed(futures):
+ adapter = futures[future]
+ try:
+ result = future.result()
+ if result["report_entry"]:
+ report["adapters"][adapter] = result["report_entry"]
+
+ if result["failed"]:
+ any_failures = True
+ summary["failed"].append(adapter)
+ elif result["status"] == "success":
+ summary["ran"].append(adapter)
+ else:
+ summary["skipped"].append(adapter)
+ except Exception as e:
+ print(f"[{adapter}] Unhandled exception in thread: {e}")
+ traceback.print_exc()
+ any_failures = True
+ summary["failed"].append(adapter)
+
+ # ── Summary ──────────────────────────────────────────────────────────
+ print(f"\n{'═' * 60}")
+ print(f"Run summary ({today.strftime('%Y-%m-%d')})")
+ print(f" Ran: {len(summary['ran'])} {summary['ran']}")
+ print(f" Skipped: {len(summary['skipped'])} {summary['skipped']}")
+ print(f" Failed: {len(summary['failed'])} {summary['failed']}")
+ print(f"{'═' * 60}")
+
+ # ── Save state ───────────────────────────────────────────────────────
+ save_json(STATS_FILE, stats)
+ save_json(REPORT_FILE, report)
+
+ # ── Upload ───────────────────────────────────────────────────────────
+ has_new_data = any(
+ entry.get("data_changed", False)
+ for entry in report["adapters"].values()
+ if isinstance(entry, dict)
+ )
+ if not args.dry_run and has_new_data:
+ print("\nUploading data directory to HuggingFace (only changed files will be uploaded)...")
+ if not upload_to_hf(api, existing_pr):
+ any_failures = True
+ elif not has_new_data:
+ print("\nNo new data to upload. Skipping HuggingFace PR creation.")
+
+ if any_failures:
+ print("\nFinished with failures.")
+ return 1
+
+ print("\nAll done!")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/utils/swe_bench_verified/adapter.py b/utils/swe_bench_verified/adapter.py
index 119ab7d04..df3a60bbd 100644
--- a/utils/swe_bench_verified/adapter.py
+++ b/utils/swe_bench_verified/adapter.py
@@ -215,6 +215,11 @@ def main():
"datasets is required to run this adapter. Install it with: pip install datasets"
) from e
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--output-dir", type=str, default=OUTPUT_DIR)
+ args = parser.parse_args()
+
retrieved_timestamp = str(time.time())
count = 0
errors = 0
@@ -240,7 +245,7 @@ def main():
dev = eval_log.model_info.developer or "unknown"
# Use model name without developer prefix for the directory
model_name = eval_log.model_info.name.split("/")[-1]
- filepath = save_evaluation_log(eval_log, OUTPUT_DIR, dev, model_name)
+ filepath = save_evaluation_log(eval_log, args.output_dir, dev, model_name)
score = eval_log.evaluation_results[0].score_details.score
print(f" [{score:.1%}] {submission_dir.name} → {filepath}")
count += 1
@@ -248,7 +253,7 @@ def main():
print(f" ERROR {submission_dir.name}: {e}")
errors += 1
- print(f"\nGenerated {count} files, {errors} errors → {OUTPUT_DIR}/")
+ print(f"\nGenerated {count} files, {errors} errors → {args.output_dir}/")
if __name__ == "__main__":
diff --git a/utils/swe_polybench/adapter.py b/utils/swe_polybench/adapter.py
index e54b0620e..b35a3f7b2 100644
--- a/utils/swe_polybench/adapter.py
+++ b/utils/swe_polybench/adapter.py
@@ -263,6 +263,11 @@ def main():
"pyyaml is required to run this adapter. Install it with: pip install pyyaml"
) from e
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--output-dir", type=str, default=OUTPUT_BASE)
+ args = parser.parse_args()
+
retrieved_timestamp = str(time.time())
count = 0
errors = 0
@@ -300,7 +305,7 @@ def main():
for eval_log, lang in logs_results:
dev = eval_log.model_info.developer or "unknown"
model_name = eval_log.model_info.name.split("/")[-1]
- filepath = save_evaluation_log(eval_log, OUTPUT_BASE, dev, model_name)
+ filepath = save_evaluation_log(eval_log, args.output_dir, dev, model_name)
score = eval_log.evaluation_results[0].score_details.score
print(f" [{score:.1%}] {submission_dir.name} [{lang}] → {filepath}")
count += 1
@@ -308,7 +313,7 @@ def main():
print(f" ERROR {submission_dir.name}: {e}")
errors += 1
- print(f"\nGenerated {count} files, {errors} errors → {OUTPUT_BASE}/")
+ print(f"\nGenerated {count} files, {errors} errors → {args.output_dir}/")
if __name__ == "__main__":
diff --git a/utils/terminal_bench_2/adapter.py b/utils/terminal_bench_2/adapter.py
index 7a1653ae0..f3a6f6082 100644
--- a/utils/terminal_bench_2/adapter.py
+++ b/utils/terminal_bench_2/adapter.py
@@ -18,10 +18,11 @@
sys.path.insert(0, str(Path(__file__).parent.parent))
+import argparse
import json
import uuid
-from eval_types import (
+from every_eval_ever.eval_types import (
AgenticEvalConfig,
AvailableTool,
EvalLibrary,
@@ -294,6 +295,10 @@ def convert_entry(entry: dict, retrieved_timestamp: str) -> EvaluationLog:
def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--output-dir", type=Path, default=Path(OUTPUT_DIR))
+ args = parser.parse_args()
+
retrieved_timestamp = str(time.time())
count = 0
@@ -302,14 +307,14 @@ def main():
eval_log = convert_entry(entry, retrieved_timestamp)
org_slug = get_org_slug(entry["model_org"])
model_slug = get_model_slug(entry["model"])
- filepath = save_evaluation_log(eval_log, OUTPUT_DIR, org_slug, model_slug)
+ filepath = save_evaluation_log(eval_log, args.output_dir, org_slug, model_slug)
print(f"[{entry['rank']:3d}] {filepath}")
count += 1
except Exception as e:
print(f"Error processing rank {entry['rank']} "
f"({entry['agent']} / {entry['model']}): {e}")
- print(f"\nGenerated {count} files in {OUTPUT_DIR}/")
+ print(f"\nGenerated {count} files in {args.output_dir}/")
if __name__ == "__main__":
diff --git a/uv.lock b/uv.lock
index 01f0357dc..ad173d2d8 100644
--- a/uv.lock
+++ b/uv.lock
@@ -234,6 +234,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" },
]
+[[package]]
+name = "bidict"
+version = "0.23.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" },
+]
+
[[package]]
name = "black"
version = "25.12.0"
@@ -266,6 +275,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/68/11/21331aed19145a952ad28fca2756a1433ee9308079bd03bd898e903a2e53/black-25.12.0-py3-none-any.whl", hash = "sha256:48ceb36c16dbc84062740049eef990bb2ce07598272e673c17d1a7720c71c828", size = 206191, upload-time = "2025-12-08T01:40:50.963Z" },
]
+[[package]]
+name = "blinker"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" },
+]
+
[[package]]
name = "blis"
version = "1.3.3"
@@ -335,6 +353,44 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bb/1f/5977ea88c6a3df6199db97d320e5da816d415d1eb75a987a1f6823d5cc9d/bottle-0.12.25-py3-none-any.whl", hash = "sha256:d6f15f9d422670b7c073d63bd8d287b135388da187a0f3e3c19293626ce034ea", size = 90181, upload-time = "2023-03-04T15:34:16.243Z" },
]
+[[package]]
+name = "brotli"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f7/16/c92ca344d646e71a43b8bb353f0a6490d7f6e06210f8554c8f874e454285/brotli-1.2.0.tar.gz", hash = "sha256:e310f77e41941c13340a95976fe66a8a95b01e783d430eeaf7a2f87e0a57dd0a", size = 7388632, upload-time = "2025-11-05T18:39:42.86Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/11/ee/b0a11ab2315c69bb9b45a2aaed022499c9c24a205c3a49c3513b541a7967/brotli-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:35d382625778834a7f3061b15423919aa03e4f5da34ac8e02c074e4b75ab4f84", size = 861543, upload-time = "2025-11-05T18:38:24.183Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/2f/29c1459513cd35828e25531ebfcbf3e92a5e49f560b1777a9af7203eb46e/brotli-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a61c06b334bd99bc5ae84f1eeb36bfe01400264b3c352f968c6e30a10f9d08b", size = 444288, upload-time = "2025-11-05T18:38:25.139Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/6f/feba03130d5fceadfa3a1bb102cb14650798c848b1df2a808356f939bb16/brotli-1.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acec55bb7c90f1dfc476126f9711a8e81c9af7fb617409a9ee2953115343f08d", size = 1528071, upload-time = "2025-11-05T18:38:26.081Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/38/f3abb554eee089bd15471057ba85f47e53a44a462cfce265d9bf7088eb09/brotli-1.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:260d3692396e1895c5034f204f0db022c056f9e2ac841593a4cf9426e2a3faca", size = 1626913, upload-time = "2025-11-05T18:38:27.284Z" },
+ { url = "https://files.pythonhosted.org/packages/03/a7/03aa61fbc3c5cbf99b44d158665f9b0dd3d8059be16c460208d9e385c837/brotli-1.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:072e7624b1fc4d601036ab3f4f27942ef772887e876beff0301d261210bca97f", size = 1419762, upload-time = "2025-11-05T18:38:28.295Z" },
+ { url = "https://files.pythonhosted.org/packages/21/1b/0374a89ee27d152a5069c356c96b93afd1b94eae83f1e004b57eb6ce2f10/brotli-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adedc4a67e15327dfdd04884873c6d5a01d3e3b6f61406f99b1ed4865a2f6d28", size = 1484494, upload-time = "2025-11-05T18:38:29.29Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/57/69d4fe84a67aef4f524dcd075c6eee868d7850e85bf01d778a857d8dbe0a/brotli-1.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7a47ce5c2288702e09dc22a44d0ee6152f2c7eda97b3c8482d826a1f3cfc7da7", size = 1593302, upload-time = "2025-11-05T18:38:30.639Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/3b/39e13ce78a8e9a621c5df3aeb5fd181fcc8caba8c48a194cd629771f6828/brotli-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af43b8711a8264bb4e7d6d9a6d004c3a2019c04c01127a868709ec29962b6036", size = 1487913, upload-time = "2025-11-05T18:38:31.618Z" },
+ { url = "https://files.pythonhosted.org/packages/62/28/4d00cb9bd76a6357a66fcd54b4b6d70288385584063f4b07884c1e7286ac/brotli-1.2.0-cp312-cp312-win32.whl", hash = "sha256:e99befa0b48f3cd293dafeacdd0d191804d105d279e0b387a32054c1180f3161", size = 334362, upload-time = "2025-11-05T18:38:32.939Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/4e/bc1dcac9498859d5e353c9b153627a3752868a9d5f05ce8dedd81a2354ab/brotli-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:b35c13ce241abdd44cb8ca70683f20c0c079728a36a996297adb5334adfc1c44", size = 369115, upload-time = "2025-11-05T18:38:33.765Z" },
+ { url = "https://files.pythonhosted.org/packages/6c/d4/4ad5432ac98c73096159d9ce7ffeb82d151c2ac84adcc6168e476bb54674/brotli-1.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e5825ba2c9998375530504578fd4d5d1059d09621a02065d1b6bfc41a8e05ab", size = 861523, upload-time = "2025-11-05T18:38:34.67Z" },
+ { url = "https://files.pythonhosted.org/packages/91/9f/9cc5bd03ee68a85dc4bc89114f7067c056a3c14b3d95f171918c088bf88d/brotli-1.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0cf8c3b8ba93d496b2fae778039e2f5ecc7cff99df84df337ca31d8f2252896c", size = 444289, upload-time = "2025-11-05T18:38:35.6Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/b6/fe84227c56a865d16a6614e2c4722864b380cb14b13f3e6bef441e73a85a/brotli-1.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8565e3cdc1808b1a34714b553b262c5de5fbda202285782173ec137fd13709f", size = 1528076, upload-time = "2025-11-05T18:38:36.639Z" },
+ { url = "https://files.pythonhosted.org/packages/55/de/de4ae0aaca06c790371cf6e7ee93a024f6b4bb0568727da8c3de112e726c/brotli-1.2.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26e8d3ecb0ee458a9804f47f21b74845cc823fd1bb19f02272be70774f56e2a6", size = 1626880, upload-time = "2025-11-05T18:38:37.623Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/16/a1b22cbea436642e071adcaf8d4b350a2ad02f5e0ad0da879a1be16188a0/brotli-1.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67a91c5187e1eec76a61625c77a6c8c785650f5b576ca732bd33ef58b0dff49c", size = 1419737, upload-time = "2025-11-05T18:38:38.729Z" },
+ { url = "https://files.pythonhosted.org/packages/46/63/c968a97cbb3bdbf7f974ef5a6ab467a2879b82afbc5ffb65b8acbb744f95/brotli-1.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4ecdb3b6dc36e6d6e14d3a1bdc6c1057c8cbf80db04031d566eb6080ce283a48", size = 1484440, upload-time = "2025-11-05T18:38:39.916Z" },
+ { url = "https://files.pythonhosted.org/packages/06/9d/102c67ea5c9fc171f423e8399e585dabea29b5bc79b05572891e70013cdd/brotli-1.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3e1b35d56856f3ed326b140d3c6d9db91740f22e14b06e840fe4bb1923439a18", size = 1593313, upload-time = "2025-11-05T18:38:41.24Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/4a/9526d14fa6b87bc827ba1755a8440e214ff90de03095cacd78a64abe2b7d/brotli-1.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54a50a9dad16b32136b2241ddea9e4df159b41247b2ce6aac0b3276a66a8f1e5", size = 1487945, upload-time = "2025-11-05T18:38:42.277Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/e8/3fe1ffed70cbef83c5236166acaed7bb9c766509b157854c80e2f766b38c/brotli-1.2.0-cp313-cp313-win32.whl", hash = "sha256:1b1d6a4efedd53671c793be6dd760fcf2107da3a52331ad9ea429edf0902f27a", size = 334368, upload-time = "2025-11-05T18:38:43.345Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/91/e739587be970a113b37b821eae8097aac5a48e5f0eca438c22e4c7dd8648/brotli-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:b63daa43d82f0cdabf98dee215b375b4058cce72871fd07934f179885aad16e8", size = 369116, upload-time = "2025-11-05T18:38:44.609Z" },
+ { url = "https://files.pythonhosted.org/packages/17/e1/298c2ddf786bb7347a1cd71d63a347a79e5712a7c0cba9e3c3458ebd976f/brotli-1.2.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6c12dad5cd04530323e723787ff762bac749a7b256a5bece32b2243dd5c27b21", size = 863080, upload-time = "2025-11-05T18:38:45.503Z" },
+ { url = "https://files.pythonhosted.org/packages/84/0c/aac98e286ba66868b2b3b50338ffbd85a35c7122e9531a73a37a29763d38/brotli-1.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3219bd9e69868e57183316ee19c84e03e8f8b5a1d1f2667e1aa8c2f91cb061ac", size = 445453, upload-time = "2025-11-05T18:38:46.433Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/f1/0ca1f3f99ae300372635ab3fe2f7a79fa335fee3d874fa7f9e68575e0e62/brotli-1.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:963a08f3bebd8b75ac57661045402da15991468a621f014be54e50f53a58d19e", size = 1528168, upload-time = "2025-11-05T18:38:47.371Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/a6/2ebfc8f766d46df8d3e65b880a2e220732395e6d7dc312c1e1244b0f074a/brotli-1.2.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9322b9f8656782414b37e6af884146869d46ab85158201d82bab9abbcb971dc7", size = 1627098, upload-time = "2025-11-05T18:38:48.385Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/2f/0976d5b097ff8a22163b10617f76b2557f15f0f39d6a0fe1f02b1a53e92b/brotli-1.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf9cba6f5b78a2071ec6fb1e7bd39acf35071d90a81231d67e92d637776a6a63", size = 1419861, upload-time = "2025-11-05T18:38:49.372Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/97/d76df7176a2ce7616ff94c1fb72d307c9a30d2189fe877f3dd99af00ea5a/brotli-1.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7547369c4392b47d30a3467fe8c3330b4f2e0f7730e45e3103d7d636678a808b", size = 1484594, upload-time = "2025-11-05T18:38:50.655Z" },
+ { url = "https://files.pythonhosted.org/packages/d3/93/14cf0b1216f43df5609f5b272050b0abd219e0b54ea80b47cef9867b45e7/brotli-1.2.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1530af5c3c275b8524f2e24841cbe2599d74462455e9bae5109e9ff42e9361", size = 1593455, upload-time = "2025-11-05T18:38:51.624Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/73/3183c9e41ca755713bdf2cc1d0810df742c09484e2e1ddd693bee53877c1/brotli-1.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d2d085ded05278d1c7f65560aae97b3160aeb2ea2c0b3e26204856beccb60888", size = 1488164, upload-time = "2025-11-05T18:38:53.079Z" },
+ { url = "https://files.pythonhosted.org/packages/64/6a/0c78d8f3a582859236482fd9fa86a65a60328a00983006bcf6d83b7b2253/brotli-1.2.0-cp314-cp314-win32.whl", hash = "sha256:832c115a020e463c2f67664560449a7bea26b0c1fdd690352addad6d0a08714d", size = 339280, upload-time = "2025-11-05T18:38:54.02Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/10/56978295c14794b2c12007b07f3e41ba26acda9257457d7085b0bb3bb90c/brotli-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e7c0af964e0b4e3412a0ebf341ea26ec767fa0b4cf81abb5e897c9338b5ad6a3", size = 375639, upload-time = "2025-11-05T18:38:55.67Z" },
+]
+
[[package]]
name = "catalogue"
version = "2.0.10"
@@ -534,6 +590,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0c/00/3106b1854b45bd0474ced037dfe6b73b90fe68a68968cef47c23de3d43d2/confection-0.1.5-py3-none-any.whl", hash = "sha256:e29d3c3f8eac06b3f77eb9dfb4bf2fc6bcc9622a98ca00a698e3d019c6430b14", size = 35451, upload-time = "2024-05-31T16:16:59.075Z" },
]
+[[package]]
+name = "configargparse"
+version = "1.7.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/30328302903c55218ffc5199646d0e9d28348ff26c02ba77b2ffc58d294a/configargparse-1.7.5.tar.gz", hash = "sha256:e3f9a7bb6be34d66b2e3c4a2f58e3045f8dfae47b0dc039f87bcfaa0f193fb0f", size = 53548, upload-time = "2026-03-11T02:19:38.144Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fe/19/3ba5e1b0bcc7b91aeab6c258afd70e4907d220fed3972febe38feb40db30/configargparse-1.7.5-py3-none-any.whl", hash = "sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592", size = 27692, upload-time = "2026-03-11T02:19:36.442Z" },
+]
+
[[package]]
name = "contourpy"
version = "1.3.3"
@@ -847,10 +912,12 @@ name = "every-eval-ever"
version = "0.2.3rc1"
source = { editable = "." }
dependencies = [
+ { name = "aiohttp" },
{ name = "datamodel-code-generator", extra = ["ruff"] },
{ name = "duckdb" },
{ name = "huggingface-hub" },
{ name = "jsonschema" },
+ { name = "locust" },
{ name = "matplotlib" },
{ name = "numpy" },
{ name = "pandas" },
@@ -884,6 +951,7 @@ dev = [
[package.metadata]
requires-dist = [
+ { name = "aiohttp", specifier = ">=3.13.3" },
{ name = "crfm-helm", marker = "extra == 'helm'", specifier = ">=0.5.14" },
{ name = "datamodel-code-generator", extras = ["ruff"], specifier = ">=0.52.2" },
{ name = "duckdb", specifier = ">=1.5.2" },
@@ -892,6 +960,7 @@ requires-dist = [
{ name = "huggingface-hub", specifier = ">=0.36.0,<1.0.0" },
{ name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.160,<0.4.0" },
{ name = "jsonschema", specifier = ">=4.26.0,<5.0.0" },
+ { name = "locust", specifier = "==2.44.5.dev5" },
{ name = "matplotlib", specifier = ">=3.10.8" },
{ name = "numpy", specifier = ">=2.4.1" },
{ name = "pandas", specifier = ">=2.3.3" },
@@ -920,6 +989,49 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" },
]
+[[package]]
+name = "flask"
+version = "3.1.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "blinker" },
+ { name = "click" },
+ { name = "itsdangerous" },
+ { name = "jinja2" },
+ { name = "markupsafe" },
+ { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" },
+]
+
+[[package]]
+name = "flask-cors"
+version = "6.0.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "flask" },
+ { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/47/03/4e464a50860f9adf08b5c1d3479cb8ea1f12af2aa69535c7042c6e628135/flask_cors-6.0.5.tar.gz", hash = "sha256:30c5031552cd59f620ac0c8211dac45b345d3b2df310e7721879e4f46ef9c601", size = 101386, upload-time = "2026-06-08T20:20:17.765Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/49/55/5bb1a2d918e9f02f131e47a59032bae70e48050e986e941511fd737a935c/flask_cors-6.0.5-py3-none-any.whl", hash = "sha256:68fcf75693e961f3af26683b23c4b9a8fb6b64de17d20d0c37b95e8de7ab2ed8", size = 16692, upload-time = "2026-06-08T20:20:16.247Z" },
+]
+
+[[package]]
+name = "flask-login"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "flask" },
+ { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c3/6e/2f4e13e373bb49e68c02c51ceadd22d172715a06716f9299d9df01b6ddb2/Flask-Login-0.6.3.tar.gz", hash = "sha256:5e23d14a607ef12806c699590b89d0f0e0d67baeec599d75947bf9c147330333", size = 48834, upload-time = "2023-10-30T14:53:21.151Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/59/f5/67e9cc5c2036f58115f9fe0f00d203cf6780c3ff8ae0e705e7a9d9e8ff9e/Flask_Login-0.6.3-py3-none-any.whl", hash = "sha256:849b25b82a436bf830a054e74214074af59097171562ab10bfa999e6b78aae5d", size = 17303, upload-time = "2023-10-30T14:53:19.636Z" },
+]
+
[[package]]
name = "fonttools"
version = "4.62.1"
@@ -1082,6 +1194,169 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/5c/e226de133afd8bb267ec27eead9ae3d784b95b39a287ed404caab39a5f50/genson-1.3.0-py3-none-any.whl", hash = "sha256:468feccd00274cc7e4c09e84b08704270ba8d95232aa280f65b986139cec67f7", size = 21470, upload-time = "2024-05-15T22:08:47.056Z" },
]
+[[package]]
+name = "gevent"
+version = "25.9.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" },
+ { name = "greenlet", marker = "platform_python_implementation == 'CPython'" },
+ { name = "zope-event" },
+ { name = "zope-interface" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9e/48/b3ef2673ffb940f980966694e40d6d32560f3ffa284ecaeb5ea3a90a6d3f/gevent-25.9.1.tar.gz", hash = "sha256:adf9cd552de44a4e6754c51ff2e78d9193b7fa6eab123db9578a210e657235dd", size = 5059025, upload-time = "2025-09-17T16:15:34.528Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f7/49/e55930ba5259629eb28ac7ee1abbca971996a9165f902f0249b561602f24/gevent-25.9.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:46b188248c84ffdec18a686fcac5dbb32365d76912e14fda350db5dc0bfd4f86", size = 2955991, upload-time = "2025-09-17T14:52:30.568Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/88/63dc9e903980e1da1e16541ec5c70f2b224ec0a8e34088cb42794f1c7f52/gevent-25.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2b54ea3ca6f0c763281cd3f96010ac7e98c2e267feb1221b5a26e2ca0b9a692", size = 1808503, upload-time = "2025-09-17T15:41:25.59Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/8d/7236c3a8f6ef7e94c22e658397009596fa90f24c7d19da11ad7ab3a9248e/gevent-25.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7a834804ac00ed8a92a69d3826342c677be651b1c3cd66cc35df8bc711057aa2", size = 1890001, upload-time = "2025-09-17T15:49:01.227Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/63/0d7f38c4a2085ecce26b50492fc6161aa67250d381e26d6a7322c309b00f/gevent-25.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:323a27192ec4da6b22a9e51c3d9d896ff20bc53fdc9e45e56eaab76d1c39dd74", size = 1855335, upload-time = "2025-09-17T15:49:20.582Z" },
+ { url = "https://files.pythonhosted.org/packages/95/18/da5211dfc54c7a57e7432fd9a6ffeae1ce36fe5a313fa782b1c96529ea3d/gevent-25.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6ea78b39a2c51d47ff0f130f4c755a9a4bbb2dd9721149420ad4712743911a51", size = 2109046, upload-time = "2025-09-17T15:15:13.817Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/5a/7bb5ec8e43a2c6444853c4a9f955f3e72f479d7c24ea86c95fb264a2de65/gevent-25.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:dc45cd3e1cc07514a419960af932a62eb8515552ed004e56755e4bf20bad30c5", size = 1827099, upload-time = "2025-09-17T15:52:41.384Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/d4/b63a0a60635470d7d986ef19897e893c15326dd69e8fb342c76a4f07fe9e/gevent-25.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34e01e50c71eaf67e92c186ee0196a039d6e4f4b35670396baed4a2d8f1b347f", size = 2172623, upload-time = "2025-09-17T15:24:12.03Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/98/caf06d5d22a7c129c1fb2fc1477306902a2c8ddfd399cd26bbbd4caf2141/gevent-25.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acd6bcd5feabf22c7c5174bd3b9535ee9f088d2bbce789f740ad8d6554b18f3", size = 1682837, upload-time = "2025-09-17T19:48:47.318Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/77/b97f086388f87f8ad3e01364f845004aef0123d4430241c7c9b1f9bde742/gevent-25.9.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:4f84591d13845ee31c13f44bdf6bd6c3dbf385b5af98b2f25ec328213775f2ed", size = 2973739, upload-time = "2025-09-17T14:53:30.279Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/2e/9d5f204ead343e5b27bbb2fedaec7cd0009d50696b2266f590ae845d0331/gevent-25.9.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9cdbb24c276a2d0110ad5c978e49daf620b153719ac8a548ce1250a7eb1b9245", size = 1809165, upload-time = "2025-09-17T15:41:27.193Z" },
+ { url = "https://files.pythonhosted.org/packages/10/3e/791d1bf1eb47748606d5f2c2aa66571f474d63e0176228b1f1fd7b77ab37/gevent-25.9.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:88b6c07169468af631dcf0fdd3658f9246d6822cc51461d43f7c44f28b0abb82", size = 1890638, upload-time = "2025-09-17T15:49:02.45Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/5c/9ad0229b2b4d81249ca41e4f91dd8057deaa0da6d4fbe40bf13cdc5f7a47/gevent-25.9.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b7bb0e29a7b3e6ca9bed2394aa820244069982c36dc30b70eb1004dd67851a48", size = 1857118, upload-time = "2025-09-17T15:49:22.125Z" },
+ { url = "https://files.pythonhosted.org/packages/49/2a/3010ed6c44179a3a5c5c152e6de43a30ff8bc2c8de3115ad8733533a018f/gevent-25.9.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2951bb070c0ee37b632ac9134e4fdaad70d2e660c931bb792983a0837fe5b7d7", size = 2111598, upload-time = "2025-09-17T15:15:15.226Z" },
+ { url = "https://files.pythonhosted.org/packages/08/75/6bbe57c19a7aa4527cc0f9afcdf5a5f2aed2603b08aadbccb5bf7f607ff4/gevent-25.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4e17c2d57e9a42e25f2a73d297b22b60b2470a74be5a515b36c984e1a246d47", size = 1829059, upload-time = "2025-09-17T15:52:42.596Z" },
+ { url = "https://files.pythonhosted.org/packages/06/6e/19a9bee9092be45679cb69e4dd2e0bf5f897b7140b4b39c57cc123d24829/gevent-25.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d94936f8f8b23d9de2251798fcb603b84f083fdf0d7f427183c1828fb64f117", size = 2173529, upload-time = "2025-09-17T15:24:13.897Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/4f/50de9afd879440e25737e63f5ba6ee764b75a3abe17376496ab57f432546/gevent-25.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb51c5f9537b07da673258b4832f6635014fee31690c3f0944d34741b69f92fa", size = 1681518, upload-time = "2025-09-17T19:39:47.488Z" },
+ { url = "https://files.pythonhosted.org/packages/15/1a/948f8167b2cdce573cf01cec07afc64d0456dc134b07900b26ac7018b37e/gevent-25.9.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:1a3fe4ea1c312dbf6b375b416925036fe79a40054e6bf6248ee46526ea628be1", size = 2982934, upload-time = "2025-09-17T14:54:11.302Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/ec/726b146d1d3aad82e03d2e1e1507048ab6072f906e83f97f40667866e582/gevent-25.9.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0adb937f13e5fb90cca2edf66d8d7e99d62a299687400ce2edee3f3504009356", size = 1813982, upload-time = "2025-09-17T15:41:28.506Z" },
+ { url = "https://files.pythonhosted.org/packages/35/5d/5f83f17162301662bd1ce702f8a736a8a8cac7b7a35e1d8b9866938d1f9d/gevent-25.9.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:427f869a2050a4202d93cf7fd6ab5cffb06d3e9113c10c967b6e2a0d45237cb8", size = 1894902, upload-time = "2025-09-17T15:49:03.702Z" },
+ { url = "https://files.pythonhosted.org/packages/83/cd/cf5e74e353f60dab357829069ffc300a7bb414c761f52cf8c0c6e9728b8d/gevent-25.9.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c049880175e8c93124188f9d926af0a62826a3b81aa6d3074928345f8238279e", size = 1861792, upload-time = "2025-09-17T15:49:23.279Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/65/b9a4526d4a4edce26fe4b3b993914ec9dc64baabad625a3101e51adb17f3/gevent-25.9.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b5a67a0974ad9f24721034d1e008856111e0535f1541499f72a733a73d658d1c", size = 2113215, upload-time = "2025-09-17T15:15:16.34Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/be/7d35731dfaf8370795b606e515d964a0967e129db76ea7873f552045dd39/gevent-25.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1d0f5d8d73f97e24ea8d24d8be0f51e0cf7c54b8021c1fddb580bf239474690f", size = 1833449, upload-time = "2025-09-17T15:52:43.75Z" },
+ { url = "https://files.pythonhosted.org/packages/65/58/7bc52544ea5e63af88c4a26c90776feb42551b7555a1c89c20069c168a3f/gevent-25.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddd3ff26e5c4240d3fbf5516c2d9d5f2a998ef87cfb73e1429cfaeaaec860fa6", size = 2176034, upload-time = "2025-09-17T15:24:15.676Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/69/a7c4ba2ffbc7c7dbf6d8b4f5d0f0a421f7815d229f4909854266c445a3d4/gevent-25.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:bb63c0d6cb9950cc94036a4995b9cc4667b8915366613449236970f4394f94d7", size = 1703019, upload-time = "2025-09-17T19:30:55.272Z" },
+]
+
+[[package]]
+name = "geventhttpclient"
+version = "2.3.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "brotli" },
+ { name = "certifi" },
+ { name = "gevent" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d7/ff/cb3db11fca4223b2753ae170d1a09c9d32bfbfa3e8d4a6181324db686830/geventhttpclient-2.3.9.tar.gz", hash = "sha256:16807578dc4a175e8d97e6e39d65a10b04b5237a8c55f7a5ef39044e869baeb8", size = 84353, upload-time = "2026-03-03T08:09:03.336Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1b/8f/a6a787443af8defaae8bab96e056f2e0d48ffcb4eb2724d83727c465335f/geventhttpclient-2.3.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39afb8046fe04358a85956555aa6a1d931710bac386a2fceb5c24bbd4d7c10e7", size = 70141, upload-time = "2026-03-03T08:08:08.415Z" },
+ { url = "https://files.pythonhosted.org/packages/18/c6/043e74e5ce9ce7cfd206f2ba572ded6bfe7278fb73d0d81aef0e913e9b5d/geventhttpclient-2.3.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:04b8feec69fd662eb46b4f81013206f5a23d179b195cbaf590d4a59f641ed0fc", size = 51776, upload-time = "2026-03-03T08:08:09.206Z" },
+ { url = "https://files.pythonhosted.org/packages/16/b2/f65c47ec71278f02c22e5d7120db855fe9c1d8034994c6c4ac8ed9b2328a/geventhttpclient-2.3.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b542025a0c9905c847d1459e598ccbdcd21dc0dd050cc1d3813ce7e01bd350f", size = 51523, upload-time = "2026-03-03T08:08:10.119Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/12/bed280730e754bc8f1691481a58cff71eca16f439dc6629ad6843ffc9988/geventhttpclient-2.3.9-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c582a6697c82a948d3d42094da941544606a0ebee31fc0aa6731e248eeba0e9b", size = 115393, upload-time = "2026-03-03T08:08:11.239Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/eb/def9770ae049a64b698c78186ebe1281900da581401a043c206efe2957ca/geventhttpclient-2.3.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39011c8cdd7ef8b6ab07592525f83018cd1504e8133cce5114bfcee5547b9bb5", size = 116043, upload-time = "2026-03-03T08:08:12.379Z" },
+ { url = "https://files.pythonhosted.org/packages/62/50/877f2ddd8ebebb0e060bfc34ed5d3721689e96c6debb218f5bac9fa04339/geventhttpclient-2.3.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b91fb31523725ddc777c14b444ccedaf2043dcb9af0ede29056a9b8146c79a7", size = 122061, upload-time = "2026-03-03T08:08:13.194Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/00/6ef955f9eb1cfd7412ee20e5f7bee2459f3a9aad3330b4c193729a968ee2/geventhttpclient-2.3.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d91139a4fafd77fa985535966d7a6c2e64753f340ab1395508ee83cd8de70c38", size = 111963, upload-time = "2026-03-03T08:08:15.33Z" },
+ { url = "https://files.pythonhosted.org/packages/28/10/965899a6c557055974b2aca048d478451aa144876171fbe023959fd8f42a/geventhttpclient-2.3.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0ff40ca5b848f96c6390bd8cc3a4c4598c119be08125cf1c30103201adc00940", size = 118841, upload-time = "2026-03-03T08:08:16.173Z" },
+ { url = "https://files.pythonhosted.org/packages/67/e1/e559f1ee8b0d26870cabae401bb32706390ade2f4e540695fbb6ed908bdb/geventhttpclient-2.3.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9091db18eeb53626a81e9280d602ae9e29706ee4c1e7a05edc8b07cc632b3fc", size = 112618, upload-time = "2026-03-03T08:08:17.301Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/25/03b06f42ddb202f2a7da0f8dc3759ac4a09213bad6122e218397268832d0/geventhttpclient-2.3.9-cp312-cp312-win32.whl", hash = "sha256:4110273531fc9ac2ec197a44a90d9c7b4266b51a070747368e38213be281d5c2", size = 48750, upload-time = "2026-03-03T08:08:18.204Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/32/918822056ecc395a1f4e81e292a4549779fd255dd533b036aac80023b691/geventhttpclient-2.3.9-cp312-cp312-win_amd64.whl", hash = "sha256:98f3582a1c9effb56bc2db4f43d382cedd921217a139d5737eeaad3a1e307047", size = 49383, upload-time = "2026-03-03T08:08:19.157Z" },
+ { url = "https://files.pythonhosted.org/packages/12/0c/ec3e7926e5a24780ad0f2d422799966f2f13342c793ed9f37f0c03282f58/geventhttpclient-2.3.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9d0568d38cf74cecd37fd1ef65459f60ecd26dbc0d33bc2a1e0d8df4af24f07d", size = 70144, upload-time = "2026-03-03T08:08:19.932Z" },
+ { url = "https://files.pythonhosted.org/packages/63/d7/d28f76482880f9233de07fb9422db26b983a901cad4670bba8bc1170f988/geventhttpclient-2.3.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:02e06a2f78a225b70e616b493317073f3e2fddd4e51ddfc44569d188f368bd8d", size = 51779, upload-time = "2026-03-03T08:08:20.696Z" },
+ { url = "https://files.pythonhosted.org/packages/35/ff/930be8f0e4f84d1b229b1ec394463ea36701991d888f4856904e292a6b0b/geventhttpclient-2.3.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3eec8e442214d4086e40a3ae7fe1e1e3ecbc422157d8d2118059cf9977336d9f", size = 51516, upload-time = "2026-03-03T08:08:21.802Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/ac/952c51392527f707c1f08401d0b477cdd1840a487dffa6e9fce444d54122/geventhttpclient-2.3.9-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a18b28d2f8bc7fcfc721227733bccb647602399db6b0fd093c00ff9699717b74", size = 115412, upload-time = "2026-03-03T08:08:22.615Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/1f/1b61f8dae1efb670f7728cd727c35ff294b89af727db268f9e2d90102a97/geventhttpclient-2.3.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b16e30dbbc528453a4130210d83638444229357c073eb911421eb44e3367359", size = 116088, upload-time = "2026-03-03T08:08:23.474Z" },
+ { url = "https://files.pythonhosted.org/packages/00/71/941c05d483fe8a95672f8f39e7410292f4b617020d1d595b88da5660b132/geventhttpclient-2.3.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:06df5597edf65d4c691052fce3e37620cbc037879a3b872bc16a7b2a0941d59a", size = 122068, upload-time = "2026-03-03T08:08:24.495Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/75/84400d58934f774cef259c8b49292542313c02224c2f11b1b116d720b464/geventhttpclient-2.3.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:47a303bcac3d69569f025d0c81781c5f0c1a48c9f225e43082d1b56e4c0440f8", size = 112054, upload-time = "2026-03-03T08:08:25.663Z" },
+ { url = "https://files.pythonhosted.org/packages/12/ae/12821cad292235d4db8532f58c8bc93db4211862845bf76a4c06e6ed1416/geventhttpclient-2.3.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e73b25415e83064f5a334e83495d97b138e66f67a98cfcad154068c257733973", size = 118837, upload-time = "2026-03-03T08:08:26.816Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/d3/34e80569f3563eb26f5d7bb971677de0b53b16d720f87373cc7aeee51c04/geventhttpclient-2.3.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:98ff3350d8be75586076140bde565c35ccdd72a6840b88f94037ec6595407383", size = 112643, upload-time = "2026-03-03T08:08:27.666Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/79/94ace94281e40f7258ba4e7166ae846394d2a673dbe47a0a255eb0d53ca8/geventhttpclient-2.3.9-cp313-cp313-win32.whl", hash = "sha256:af7931f55522cddedf84e837769c66d9ceb130b29182ad1e2d0201f501df899f", size = 48741, upload-time = "2026-03-03T08:08:28.507Z" },
+ { url = "https://files.pythonhosted.org/packages/89/67/15b1ba79dfbab515c0d42a01b6545adef7dad00968eaa89ec21cca030c2e/geventhttpclient-2.3.9-cp313-cp313-win_amd64.whl", hash = "sha256:14daf2f0361f19b0221f900d7e9d563c184bb7186676e61fe848495b1f2483d3", size = 49371, upload-time = "2026-03-03T08:08:29.319Z" },
+ { url = "https://files.pythonhosted.org/packages/16/9f/57d5acd0d95417a29661dfa91a8657be8026a9df17cafc6ba4f20bc2a687/geventhttpclient-2.3.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6c06e243de53f54942b098f81622917f4a33c16f44733c9371ea98a2cd5ce12e", size = 70423, upload-time = "2026-03-03T08:08:30.106Z" },
+ { url = "https://files.pythonhosted.org/packages/29/8b/ad6eb43b136fdb2f4954dc21073911d7703ea95fd88a3cc7512714508ce3/geventhttpclient-2.3.9-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:549155d557de403612336ca36cd93a049e67acbf9a29e6b6b971d0f4cb56786d", size = 51902, upload-time = "2026-03-03T08:08:30.892Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/64/2d2cfd9dd9ae0a6d4138b8a88f0b4524657a48a7c81ead6986a3e955deda/geventhttpclient-2.3.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b463324d5fde983657247b2faea77f8f8a40f3f7ac0c2897a2fe3afa27d610", size = 51564, upload-time = "2026-03-03T08:08:31.717Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/f3/4585fabea4f45c4c21cd128d61ebbf43a78c73d520c70471734d41177b1d/geventhttpclient-2.3.9-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e1ac3a39e3c4ae36024ddf1694eb82b0cc22c4516f176477f94f98bcd56ce6cf", size = 115449, upload-time = "2026-03-03T08:08:32.75Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/e8/d7d82f527c632cbdeffa34858557db3da238f68f2fbb9bd80f2ec2c64510/geventhttpclient-2.3.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3d24480c3a2cc88311c41a042bc12ab8e4104dad6029591ecbf5a1e933e8a44", size = 116152, upload-time = "2026-03-03T08:08:33.941Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/63/25ee53c3efa9ded9976e4f5ac8c6f8e8cef941bbbc847290e7f5c0254c40/geventhttpclient-2.3.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b244adcbf5814a29d5cea8b2fc079f9242d92765191faa4dc5eccc0421840ae", size = 122145, upload-time = "2026-03-03T08:08:34.809Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/8c/15c5d71e6011f317f7decb26fd15e1e6caf780b09297af3018599311e6df/geventhttpclient-2.3.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83dc6f037a50b7d2dc45af58a7e7978016a06320a5f823d1bd544c85d69f2058", size = 112134, upload-time = "2026-03-03T08:08:35.716Z" },
+ { url = "https://files.pythonhosted.org/packages/02/5b/fd7b17c37a9f9002a5fd8d690c97ada372393fcfb9358dd62026e089ae96/geventhttpclient-2.3.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:caf8779ca686497e0fab1048b026b4e48fb14fb9e88ddbfd14ca1a1a4c4bfa89", size = 118879, upload-time = "2026-03-03T08:08:36.953Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/b5/c1559f43ef56100d64bf1b227844bf229101fdb58b556e2336b4307bda0d/geventhttpclient-2.3.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cd4efebba798c7f585aa1ceb9aba9524b12ebc51b26ad62de5234b8264d9b94d", size = 112593, upload-time = "2026-03-03T08:08:37.842Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/22/a61a9cb76feede0d4429154d391c275debde78b6602f52c95aeed6dce1ff/geventhttpclient-2.3.9-cp314-cp314-win32.whl", hash = "sha256:7b60c0b650c77d2644374149c38dfee34510e88e569ca85f38fe15f40ecaea1c", size = 49390, upload-time = "2026-03-03T08:08:38.996Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/05/b8d71edd82c9b07b9be0c3e5d6faf94583c6a098e2e9e5ee2b14a6312c5e/geventhttpclient-2.3.9-cp314-cp314-win_amd64.whl", hash = "sha256:c4d5e1b9b1ac9baab42a1789bbfae7e97e40e8e83e09a32b353c6eb985f36071", size = 49881, upload-time = "2026-03-03T08:08:40.228Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/5f/2f7f8f63968d26d7233fb9b9e5b1a5015989b90f95e997e9dc98283b0a86/geventhttpclient-2.3.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ae44cec808193bb70b634fabdfdd89f0850744ace5668dc98063d633cf50c417", size = 70812, upload-time = "2026-03-03T08:08:41.073Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/89/7887f802adee5990c10dd9c44b20a3205e046773061266ce5cffb99e30b9/geventhttpclient-2.3.9-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:53977ca41809eaef73cf38af170484baa53bde5f16bafbca7b77b670c343f48f", size = 52087, upload-time = "2026-03-03T08:08:42.063Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/07/23cc505111abb65cb5a68e5cd123b1ffc1ad7893a1bc46945b9ed3d03245/geventhttpclient-2.3.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0f66a33c95e4d6d343fc6ace458b13c613684bf7cfd6832b61cc9c42eaf394f3", size = 51772, upload-time = "2026-03-03T08:08:42.926Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/97/461fd5c73858b2daaaba2ecefd2ff64aa8f2242c48c939e75caba9ec3cb2/geventhttpclient-2.3.9-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cfe23d419aa676492677374bdd37e364c921895d1090a180173be5d5f87f82b9", size = 118329, upload-time = "2026-03-03T08:08:44.07Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/08/0ede3d90ab92a105f24b758b0bfb2d5e7f34c017d22d76d87524e75e93cc/geventhttpclient-2.3.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e3b279da39ad3eee69a5df9e1b602f87bcd2cec7eb258d3cc801e2170682383", size = 119974, upload-time = "2026-03-03T08:08:45.231Z" },
+ { url = "https://files.pythonhosted.org/packages/98/8f/0ef02946bbbbd91ba4c3da99657d90e250c00409710ed377e4e4540b90c3/geventhttpclient-2.3.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:38535589a564822c64d1b4c2a5d6dcc27159d0d7d76500f2c8c8d21d9dd54880", size = 125764, upload-time = "2026-03-03T08:08:46.446Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/e1/d8f385fd6a3538cf1fd57a3fd47b133fba2e32c6be86e75805117d96ff1f/geventhttpclient-2.3.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a6436cd77885a8ef7cdc6d225cddd732560a17e92969c74e997836cf3135baa0", size = 115599, upload-time = "2026-03-03T08:08:47.393Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/55/ec651647ee2f7fdfee8d7a75ba682064e0e5012696f9aa83c0392d54fdeb/geventhttpclient-2.3.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:5000c9fb0553818c4e4c1de248ee4e9a56de0a245a30ef76b687542a935f4645", size = 122254, upload-time = "2026-03-03T08:08:48.566Z" },
+ { url = "https://files.pythonhosted.org/packages/77/7d/20606d1a4ae085eb3935e4d3625e7208911d0f1a0006c9fd962d88254d92/geventhttpclient-2.3.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:52516d5c153fcef0d3d2447e533244dc6360e8c2a190b958861137db6f227605", size = 115383, upload-time = "2026-03-03T08:08:49.454Z" },
+ { url = "https://files.pythonhosted.org/packages/85/16/d20ac6ac73d63fe326ffe357a8e91c4f43b9e790faeeb9b15774eecf2550/geventhttpclient-2.3.9-cp314-cp314t-win32.whl", hash = "sha256:14eaa836bde26a70952e95ca462018f3a47c1c92642327315aa6502e54141016", size = 49749, upload-time = "2026-03-03T08:08:50.339Z" },
+ { url = "https://files.pythonhosted.org/packages/59/f6/95d1ed1ace7902d8e1ce698db31931cb87d4abe621ec2df24e69daf49ae9/geventhttpclient-2.3.9-cp314-cp314t-win_amd64.whl", hash = "sha256:b9bbcbc7d5d875e5180f2b1f1c6fa8e092ef80d9debfb6ba22a4ec28f0565395", size = 50300, upload-time = "2026-03-03T08:08:51.482Z" },
+]
+
+[[package]]
+name = "greenlet"
+version = "3.5.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5d/6e/4c37d51a2b7f82d2ff11bb6b5f7d766d9a011726624af255e843727627a3/greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2", size = 288685, upload-time = "2026-06-26T18:22:08.977Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" },
+ { url = "https://files.pythonhosted.org/packages/fb/fb/d97dc261209c80744b7c8132693a30d70ec6e7315e632cb0a10b3fec94dd/greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23", size = 622351, upload-time = "2026-06-26T19:24:16.32Z" },
+ { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/ac/e5fee13cbbd0e8de312d9a146584b8a51891c68847330ef9dc8b5109d23f/greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c", size = 425395, upload-time = "2026-06-26T19:25:37.144Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" },
+ { url = "https://files.pythonhosted.org/packages/14/40/c57489acf8e37d74e2913d4eff63aa0dba17acccc4bdeef874dde2dbbec9/greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c", size = 239882, upload-time = "2026-06-26T18:23:27.518Z" },
+ { url = "https://files.pythonhosted.org/packages/71/fd/6fea0e3d6600f785069481ee637e09378dd4118acdfd38ad88ae2db31c98/greenlet-3.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:c4e7b79d83805475f0102008843f6eb45fd3bb0b2e88c774adab5fbaab27117d", size = 238211, upload-time = "2026-06-26T18:22:37.671Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" },
+ { url = "https://files.pythonhosted.org/packages/57/61/2f5b1adf256d039f5dab8005de8d3d7ad2b0070a3219c0e036b3fbfeb440/greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1", size = 625580, upload-time = "2026-06-26T19:24:18.344Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/d9/ab7fc9e543e44d6879b0a6ef9a4b2188940fd180cc65d6f646883ddf7201/greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda", size = 427933, upload-time = "2026-06-26T19:25:38.219Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/7e/220a7f5824a64a60443fc03b39dfac4ea63a7fb6d481efa27eafa928e7f4/greenlet-3.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:dc133a1569ee667b2a6ef56ce551084aeefd87a5acbc4736d336d1e2edc6cfc4", size = 238141, upload-time = "2026-06-26T18:22:48.507Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/93/43e116ee114b28737ba7e12952a0d4e2f55944d0f84e42bc91ba7192a3c9/greenlet-3.5.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:fd2e02fa07485778536a036222d616ab957b1d533f36b3ed98ce725d9c9d3117", size = 288202, upload-time = "2026-06-26T18:23:49.604Z" },
+ { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" },
+ { url = "https://files.pythonhosted.org/packages/86/a9/73fa62893d5b84b4205544e6b673c654cc43aa5b9899bac00f04d64af73d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8d19fe6c39ebff9259f07bcc685d3290f8fa4ea2278e51dd0008e4d6b0f2d814", size = 670657, upload-time = "2026-06-26T19:24:19.967Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" },
+ { url = "https://files.pythonhosted.org/packages/29/7e/2ffce64929fb3cab7b65d5a0b20aaf9764e227681d731b041077fc9a525a/greenlet-3.5.3-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:962c5df2db8cb446da51edf1ca5296c389d93b99c9d8aa2ee4c7d0d8f1218260", size = 473497, upload-time = "2026-06-26T19:25:39.421Z" },
+ { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/94/91aec0030bea75c4b3244251d0de60a1f3432d1ecb53ab6c437fb5c3ba61/greenlet-3.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:7669aa24cf2a1041d6f7899575b494a3ab4cf68bfcc8609b1dc0be7272db835e", size = 240754, upload-time = "2026-06-26T18:22:15.669Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/06/68d0983e79e02138f64b4d303c500c27ddb48e5e77f3debb80888a921eae/greenlet-3.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:5b4807c4082c9d1b6d9eed56fcd041863e37f2228106eef24c30ca096e238605", size = 239549, upload-time = "2026-06-26T18:22:42.996Z" },
+ { url = "https://files.pythonhosted.org/packages/91/95/3e161213d7f1d378d15aa9e792093e9bfe01844680d04b7fd6e0107c9098/greenlet-3.5.3-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:271a8ea7c1024e8a0d7dd2be66dd66dda8a07193f41a17b9e924f7600f5b62be", size = 296389, upload-time = "2026-06-26T18:22:20.657Z" },
+ { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/73/8faec206b851c22b1733545fda900829a1f3f5b1c78ae7e0fb3dba57d9f4/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b897d97759425953f69a9c0fac67f8fe333ec0ce7377ef186fb2b0c3ad5e354d", size = 659582, upload-time = "2026-06-26T19:24:21.357Z" },
+ { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/55/50c19e49f8045834ada71ef12f8ad048eba8517c6aa41161bed676328fae/greenlet-3.5.3-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:3236754d423955ea08e9bb5f6c04a7895f9e22c290b66aa7653fcb922d839eb0", size = 491037, upload-time = "2026-06-26T19:25:40.672Z" },
+ { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/15/7edb977e08f9bff702fe42d6c902702786ff6b9694058b4e6a2a6ac90e57/greenlet-3.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:efc6bd60ea02e085862c74a3ef64b147ffc6f1a5ea7d9f26e7a939943f68c1e3", size = 243626, upload-time = "2026-06-26T18:24:41.485Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/8a/93928dce91e6b3598b5e779e8d1fd6576a504640c58e78627077f6a7a91a/greenlet-3.5.3-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:ea03f2f04367845d6b58eeed276e1e56e51f0b97d8ad5a88a7d20a91dc9056cc", size = 288860, upload-time = "2026-06-26T18:22:48.07Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" },
+ { url = "https://files.pythonhosted.org/packages/25/aa/952cf28c2ff949a8c971134fb43854dd7eaa737218723aaef758f8c9aead/greenlet-3.5.3-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cefa9cef4b371f9844c6053db71f1138bc6807bab1578b0dae5149c1f1141357", size = 674261, upload-time = "2026-06-26T19:24:22.79Z" },
+ { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/f2/b00d6f5e63e531a93562b2ec1a4c320fbee91f580fc42e6417af69d706e5/greenlet-3.5.3-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:6219b6d04dbf6ba6084d77dc609e8473060dc55f759cbf626d512122781fa128", size = 480322, upload-time = "2026-06-26T19:25:41.852Z" },
+ { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" },
+ { url = "https://files.pythonhosted.org/packages/68/5d/00d469daae3c65d2bf620b10eee82eb022127d483c6bc8c69fae6f3fbf17/greenlet-3.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:dd99329bbc15ca78dcc583dba05d0b1b0bae01ab6c2174989f5aaee3e41ac930", size = 241027, upload-time = "2026-06-26T18:22:38.203Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/e8/883785b44c5780ed71e83d3e4437e710470be17a2e181e8b601e2da0dc4a/greenlet-3.5.3-cp315-cp315-win_arm64.whl", hash = "sha256:499fef2acede88c1864a57bb586b4bf533c81e1b82df7ab93451cdb47dfec227", size = 240085, upload-time = "2026-06-26T18:23:54.217Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/da/4f4a8450962fad137c1c8981a3f1b8919d06c829993d4d476f9c525d5173/greenlet-3.5.3-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:176bc16a721fa5fc294d70b87b4dfa5fbdd251b3da5d5372735ecef9bd7d6d0c", size = 297221, upload-time = "2026-06-26T18:23:27.176Z" },
+ { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/39/0e0938a75115b939d42733a2a12e1d349653c9531fe6fe563e8a681f04e6/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d192579ed281051396dddd7f7754dac6259e6b1fb26378c87b66622f8e3f91", size = 663706, upload-time = "2026-06-26T19:24:24.312Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/41/35d1c678cdb3c3b9e6bee691728e563cfb294202b23c7a4c3c2ccc343589/greenlet-3.5.3-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:4399eb8d041f20b68d943918bc55502a93d6fdc0a37c14da7881c04139acee9d", size = 498803, upload-time = "2026-06-26T19:25:43.063Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" },
+ { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/13/1055e1dda7882073eda533e2b96c62e55bbd2db7fda6d5ece992febc7071/greenlet-3.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:8ff8bed3e3baa20a3ea261ce00526f1898ad4801d4886fd2220580ee0ad8fadf", size = 244007, upload-time = "2026-06-26T18:22:04.353Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/0d/ca7d15afbdc397e3401134c9e1800d51d12b829661786187a4ad08fe484f/greenlet-3.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:b7068bd09f761f3f5b4d214c2bed063186b2a86148c740b3873e3f56d79bac31", size = 242586, upload-time = "2026-06-26T18:23:37.93Z" },
+]
+
[[package]]
name = "h11"
version = "0.16.0"
@@ -1317,6 +1592,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" },
]
+[[package]]
+name = "itsdangerous"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" },
+]
+
[[package]]
name = "jinja2"
version = "3.1.6"
@@ -1526,6 +1810,32 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/1e/b832de447dee8b582cac175871d2f6c3d5077cc56d5575cadba1fd1cccfa/linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79", size = 19820, upload-time = "2024-02-04T14:48:02.496Z" },
]
+[[package]]
+name = "locust"
+version = "2.44.5.dev5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "configargparse" },
+ { name = "flask" },
+ { name = "flask-cors" },
+ { name = "flask-login" },
+ { name = "gevent" },
+ { name = "geventhttpclient" },
+ { name = "msgpack" },
+ { name = "psutil" },
+ { name = "pytest" },
+ { name = "python-engineio" },
+ { name = "python-socketio", extra = ["client"] },
+ { name = "pywin32", marker = "sys_platform == 'win32'" },
+ { name = "pyzmq" },
+ { name = "requests" },
+ { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/db/2b4475dcbb571128df6faa9a9cf06f18c3ea5ce7035336be8fe828fbbdbe/locust-2.44.5.dev5.tar.gz", hash = "sha256:72dbdabcfabd3b856b0d186da1e260c802dfe4d04035a90073756eb498dbe3f1", size = 1437351, upload-time = "2026-06-30T17:21:40.189Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f8/5e/b69f264c7ec4544182ed7e799a1dfd9fb29aa286b8d6fdc4393ff2801d84/locust-2.44.5.dev5-py3-none-any.whl", hash = "sha256:279e126f2ae582f6b93c1e9d8e38fd4bb76cb4cdddec45fbca0ae332f59db94c", size = 1458429, upload-time = "2026-06-30T17:21:38.037Z" },
+]
+
[[package]]
name = "mako"
version = "1.3.10"
@@ -1791,6 +2101,58 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" },
]
+[[package]]
+name = "msgpack"
+version = "1.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" },
+ { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" },
+ { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" },
+ { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" },
+ { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/ac/dcddcab6f6c20ecb387ca5e980371cdb3f87ff69aeca388be97eebc4c074/msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064", size = 83151, upload-time = "2026-06-18T16:13:12.173Z" },
+ { url = "https://files.pythonhosted.org/packages/64/71/fbcfa83a1d6a9c6091942d1cfd070962244664b87427a9a49a6897b1b219/msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056", size = 82351, upload-time = "2026-06-18T16:13:13.194Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" },
+ { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" },
+ { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/c5/133f4512a56e983a93445c836c9d94d88f3bc2e0980ff4b9e577bd8416ce/msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707", size = 64471, upload-time = "2026-06-18T16:13:22.293Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/98/577e10b055096a7dd40732358cabaf7180a20c79ed1dcdbb618e4b9deac7/msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9", size = 71274, upload-time = "2026-06-18T16:13:23.455Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/ee/0c0048e7cfbef23c6a94791b8959ab28155232e7956de8a305b5ff588f05/msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a", size = 64795, upload-time = "2026-06-18T16:13:24.687Z" },
+ { url = "https://files.pythonhosted.org/packages/77/58/cce442852c6b9e1639c7c8ac8fd9143121cb32dab0f308df4d1426a8eb9c/msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d", size = 83610, upload-time = "2026-06-18T16:13:25.724Z" },
+ { url = "https://files.pythonhosted.org/packages/60/5c/15b4c7a0182f75ffa90751958ba36a9c01cafee367d49a3edc10ed140b01/msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7", size = 83138, upload-time = "2026-06-18T16:13:26.781Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" },
+ { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" },
+ { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" },
+ { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" },
+ { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" },
+ { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" },
+ { url = "https://files.pythonhosted.org/packages/82/90/8b630fef07d8c5ab457b71ff2c217910c83d333c7a68472c186e87cc504a/msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb", size = 65942, upload-time = "2026-06-18T16:13:36.056Z" },
+ { url = "https://files.pythonhosted.org/packages/16/f1/467b81e98b24dd3885d7b1857728797b4ffc76a7a7483af4fb321a07de3c/msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b", size = 72627, upload-time = "2026-06-18T16:13:37.079Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/1d/5d8c4c89985feb6acefb82a09e501c60392261856d2408d20bfe4f0360b1/msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7", size = 66908, upload-time = "2026-06-18T16:13:38.23Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/02/ad2afb678b4de94496cd432b581759b756a92c1192d8c767edd6b132efdc/msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273", size = 86000, upload-time = "2026-06-18T16:13:39.44Z" },
+ { url = "https://files.pythonhosted.org/packages/54/74/0b797484013128837f3b1cbb6cea019277c4de4e377dc512b4d9a0f92940/msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1", size = 86544, upload-time = "2026-06-18T16:13:40.447Z" },
+ { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" },
+ { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" },
+ { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/e1/aab6c946570496b78e67804721f3d5e2d62a93081b9b37df77764ef56347/msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1", size = 70914, upload-time = "2026-06-18T16:13:49.385Z" },
+ { url = "https://files.pythonhosted.org/packages/13/0a/e608956488a2af014cfe6e3d665e090b8ee42aa14b07f8f95b8880d66b09/msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2", size = 77999, upload-time = "2026-06-18T16:13:50.467Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" },
+]
+
[[package]]
name = "multidict"
version = "6.7.0"
@@ -2739,6 +3101,37 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" },
]
+[[package]]
+name = "python-engineio"
+version = "4.13.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "simple-websocket" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fb/a0/f75491f942184d9960b15e763270f765fe9f239745ca5f9e16289011aed4/python_engineio-4.13.3.tar.gz", hash = "sha256:572b7783e341fed21edbc7cea297ccd378dad79265fdde96aa4664420a7c06c9", size = 79734, upload-time = "2026-06-20T22:53:52.197Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5b/96/82f6328e410515fab21d5602ba35b9377a47b5a141a0c1f9efa00ce21eb4/python_engineio-4.13.3-py3-none-any.whl", hash = "sha256:1f60ecaf1358190f0e26c48c578a60428dc02a8f1295bc3dbf53d1b31116821f", size = 59993, upload-time = "2026-06-20T22:53:50.775Z" },
+]
+
+[[package]]
+name = "python-socketio"
+version = "5.16.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "bidict" },
+ { name = "python-engineio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/32/2d/ffce71017c106b75099fea569df6518c63fee5d6202ce0cfe7b01e6f22c3/python_socketio-5.16.3.tar.gz", hash = "sha256:89b136f677ae65607a84cecda9b4d6c5377b40a97582c504c25df89af16d520e", size = 128095, upload-time = "2026-06-15T22:07:04.003Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0a/38/8c5e72d53ff8eb27497c4f268a7f6d9121e727a50b65248288ad79a93053/python_socketio-5.16.3-py3-none-any.whl", hash = "sha256:e7ad14202a5e6448824c7c2f86161d04e13dec05992257df5c709e6a2798c041", size = 82087, upload-time = "2026-06-15T22:07:02.498Z" },
+]
+
+[package.optional-dependencies]
+client = [
+ { name = "requests" },
+ { name = "websocket-client" },
+]
+
[[package]]
name = "pytokens"
version = "0.3.0"
@@ -2757,6 +3150,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" },
]
+[[package]]
+name = "pywin32"
+version = "312"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" },
+ { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" },
+ { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" },
+ { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" },
+ { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" },
+ { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" },
+]
+
[[package]]
name = "pyyaml"
version = "6.0.3"
@@ -2803,6 +3215,49 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
]
+[[package]]
+name = "pyzmq"
+version = "27.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cffi", marker = "implementation_name == 'pypy'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" },
+ { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" },
+ { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" },
+ { url = "https://files.pythonhosted.org/packages/60/cb/84a13459c51da6cec1b7b1dc1a47e6db6da50b77ad7fd9c145842750a011/pyzmq-27.1.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:93ad4b0855a664229559e45c8d23797ceac03183c7b6f5b4428152a6b06684a5", size = 1122436, upload-time = "2025-09-08T23:08:20.801Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/b6/94414759a69a26c3dd674570a81813c46a078767d931a6c70ad29fc585cb/pyzmq-27.1.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:fbb4f2400bfda24f12f009cba62ad5734148569ff4949b1b6ec3b519444342e6", size = 1156301, upload-time = "2025-09-08T23:08:22.47Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/ad/15906493fd40c316377fd8a8f6b1f93104f97a752667763c9b9c1b71d42d/pyzmq-27.1.0-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:e343d067f7b151cfe4eb3bb796a7752c9d369eed007b91231e817071d2c2fec7", size = 1341197, upload-time = "2025-09-08T23:08:24.286Z" },
+ { url = "https://files.pythonhosted.org/packages/14/1d/d343f3ce13db53a54cb8946594e567410b2125394dafcc0268d8dda027e0/pyzmq-27.1.0-cp313-cp313t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:08363b2011dec81c354d694bdecaef4770e0ae96b9afea70b3f47b973655cc05", size = 897275, upload-time = "2025-09-08T23:08:26.063Z" },
+ { url = "https://files.pythonhosted.org/packages/69/2d/d83dd6d7ca929a2fc67d2c3005415cdf322af7751d773524809f9e585129/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d54530c8c8b5b8ddb3318f481297441af102517602b569146185fa10b63f4fa9", size = 660469, upload-time = "2025-09-08T23:08:27.623Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/cd/9822a7af117f4bc0f1952dbe9ef8358eb50a24928efd5edf54210b850259/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f3afa12c392f0a44a2414056d730eebc33ec0926aae92b5ad5cf26ebb6cc128", size = 847961, upload-time = "2025-09-08T23:08:29.672Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/12/f003e824a19ed73be15542f172fd0ec4ad0b60cf37436652c93b9df7c585/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c65047adafe573ff023b3187bb93faa583151627bc9c51fc4fb2c561ed689d39", size = 1650282, upload-time = "2025-09-08T23:08:31.349Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/4a/e82d788ed58e9a23995cee70dbc20c9aded3d13a92d30d57ec2291f1e8a3/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90e6e9441c946a8b0a667356f7078d96411391a3b8f80980315455574177ec97", size = 2024468, upload-time = "2025-09-08T23:08:33.543Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/94/2da0a60841f757481e402b34bf4c8bf57fa54a5466b965de791b1e6f747d/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:add071b2d25f84e8189aaf0882d39a285b42fa3853016ebab234a5e78c7a43db", size = 1885394, upload-time = "2025-09-08T23:08:35.51Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/6f/55c10e2e49ad52d080dc24e37adb215e5b0d64990b57598abc2e3f01725b/pyzmq-27.1.0-cp313-cp313t-win32.whl", hash = "sha256:7ccc0700cfdf7bd487bea8d850ec38f204478681ea02a582a8da8171b7f90a1c", size = 574964, upload-time = "2025-09-08T23:08:37.178Z" },
+ { url = "https://files.pythonhosted.org/packages/87/4d/2534970ba63dd7c522d8ca80fb92777f362c0f321900667c615e2067cb29/pyzmq-27.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:8085a9fba668216b9b4323be338ee5437a235fe275b9d1610e422ccc279733e2", size = 641029, upload-time = "2025-09-08T23:08:40.595Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/fa/f8aea7a28b0641f31d40dea42d7ef003fded31e184ef47db696bc74cd610/pyzmq-27.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:6bb54ca21bcfe361e445256c15eedf083f153811c37be87e0514934d6913061e", size = 561541, upload-time = "2025-09-08T23:08:42.668Z" },
+ { url = "https://files.pythonhosted.org/packages/87/45/19efbb3000956e82d0331bafca5d9ac19ea2857722fa2caacefb6042f39d/pyzmq-27.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ce980af330231615756acd5154f29813d553ea555485ae712c491cd483df6b7a", size = 1341197, upload-time = "2025-09-08T23:08:44.973Z" },
+ { url = "https://files.pythonhosted.org/packages/48/43/d72ccdbf0d73d1343936296665826350cb1e825f92f2db9db3e61c2162a2/pyzmq-27.1.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1779be8c549e54a1c38f805e56d2a2e5c009d26de10921d7d51cfd1c8d4632ea", size = 897175, upload-time = "2025-09-08T23:08:46.601Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/2e/a483f73a10b65a9ef0161e817321d39a770b2acf8bcf3004a28d90d14a94/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7200bb0f03345515df50d99d3db206a0a6bee1955fbb8c453c76f5bf0e08fb96", size = 660427, upload-time = "2025-09-08T23:08:48.187Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/d2/5f36552c2d3e5685abe60dfa56f91169f7a2d99bbaf67c5271022ab40863/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01c0e07d558b06a60773744ea6251f769cd79a41a97d11b8bf4ab8f034b0424d", size = 847929, upload-time = "2025-09-08T23:08:49.76Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/2a/404b331f2b7bf3198e9945f75c4c521f0c6a3a23b51f7a4a401b94a13833/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:80d834abee71f65253c91540445d37c4c561e293ba6e741b992f20a105d69146", size = 1650193, upload-time = "2025-09-08T23:08:51.7Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/0b/f4107e33f62a5acf60e3ded67ed33d79b4ce18de432625ce2fc5093d6388/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:544b4e3b7198dde4a62b8ff6685e9802a9a1ebf47e77478a5eb88eca2a82f2fd", size = 2024388, upload-time = "2025-09-08T23:08:53.393Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/01/add31fe76512642fd6e40e3a3bd21f4b47e242c8ba33efb6809e37076d9b/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cedc4c68178e59a4046f97eca31b148ddcf51e88677de1ef4e78cf06c5376c9a", size = 1885316, upload-time = "2025-09-08T23:08:55.702Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/59/a5f38970f9bf07cee96128de79590bb354917914a9be11272cfc7ff26af0/pyzmq-27.1.0-cp314-cp314t-win32.whl", hash = "sha256:1f0b2a577fd770aa6f053211a55d1c47901f4d537389a034c690291485e5fe92", size = 587472, upload-time = "2025-09-08T23:08:58.18Z" },
+ { url = "https://files.pythonhosted.org/packages/70/d8/78b1bad170f93fcf5e3536e70e8fadac55030002275c9a29e8f5719185de/pyzmq-27.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:19c9468ae0437f8074af379e986c5d3d7d7bfe033506af442e8c879732bedbe0", size = 661401, upload-time = "2025-09-08T23:08:59.802Z" },
+ { url = "https://files.pythonhosted.org/packages/81/d6/4bfbb40c9a0b42fc53c7cf442f6385db70b40f74a783130c5d0a5aa62228/pyzmq-27.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dc5dbf68a7857b59473f7df42650c621d7e8923fb03fa74a526890f4d33cc4d7", size = 575170, upload-time = "2025-09-08T23:09:01.418Z" },
+]
+
[[package]]
name = "referencing"
version = "0.37.0"
@@ -3253,6 +3708,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c0/44/21d6bf170bf40b41396480d8d49ad640bca3f2b02139cd52aa1e272830a5/shortuuid-1.0.13-py3-none-any.whl", hash = "sha256:a482a497300b49b4953e15108a7913244e1bb0d41f9d332f5e9925dba33a3c5a", size = 10529, upload-time = "2024-03-11T20:11:04.807Z" },
]
+[[package]]
+name = "simple-websocket"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "wsproto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" },
+]
+
[[package]]
name = "six"
version = "1.17.0"
@@ -3863,6 +4330,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a4/74/a148b41572656904a39dfcfed3f84dd1066014eed94e209223ae8e9d088d/weasel-0.4.3-py3-none-any.whl", hash = "sha256:08f65b5d0dbded4879e08a64882de9b9514753d9eaa4c4e2a576e33666ac12cf", size = 50757, upload-time = "2025-11-13T23:52:26.982Z" },
]
+[[package]]
+name = "websocket-client"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" },
+]
+
+[[package]]
+name = "werkzeug"
+version = "3.1.8"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "markupsafe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44", size = 875852, upload-time = "2026-04-02T18:49:14.268Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" },
+]
+
[[package]]
name = "wrapt"
version = "1.17.3"
@@ -3912,6 +4400,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" },
]
+[[package]]
+name = "wsproto"
+version = "1.3.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" },
+]
+
[[package]]
name = "xxhash"
version = "3.6.0"
@@ -4098,6 +4598,50 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
]
+[[package]]
+name = "zope-event"
+version = "6.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" },
+]
+
+[[package]]
+name = "zope-interface"
+version = "8.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/97/cc/b84123a948f3162a34623e188922827cd845244fdd043ed20f8d02228caa/zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9", size = 212165, upload-time = "2026-05-26T06:49:26.566Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/78/cbceec44f1b27208a76c1a688c131302685852406a23df5aab68324109cc/zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c", size = 212341, upload-time = "2026-05-26T06:49:28.182Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" },
+ { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/ea/649908c83aa8fdb7faf2ddca4d3cf6fb8f2157121267dc56e8f72681e26c/zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e", size = 215019, upload-time = "2026-05-26T06:49:35.607Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/97/da13037b4c563e4df32eedbc819f8c00b754af494f68211e3dffd48d52da/zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b", size = 213569, upload-time = "2026-05-26T06:49:37.317Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/8c/4c15755d701f2ec0e80d64a18e1ebaf5be2c584c0ec153fd516f5d13eada/zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52", size = 212512, upload-time = "2026-05-26T06:49:38.996Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/2e/4360c54c465db042cc8fbeeec92abac28b4cedbf6ba63c1f092fd08a190f/zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace", size = 212541, upload-time = "2026-05-26T06:49:41.186Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" },
+ { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" },
+ { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/df/a386940e41469ef615e100a216d8b386521e9e598817147f87932ca203c4/zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3", size = 215021, upload-time = "2026-05-26T06:49:48.478Z" },
+ { url = "https://files.pythonhosted.org/packages/89/75/477eb5669b6b2a7a843decd1a075e9b1971a8720017654143a7183abd3d9/zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e", size = 213610, upload-time = "2026-05-26T06:49:50.01Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/19/5032e954827fdf02db2d2f49737ac4378bb9cfc2cd95a8f2e2a5ae2ec01a/zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784", size = 212597, upload-time = "2026-05-26T06:49:51.63Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/53/3ef644012cf8a6a234a2d6134aab5a5c65ac5467c86296865501d4fbc406/zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f", size = 212626, upload-time = "2026-05-26T06:49:53.236Z" },
+ { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" },
+ { url = "https://files.pythonhosted.org/packages/c1/1d/c420dcd777bb761067ea92879ac766694a5ca78608185f1aecea64cbfc11/zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140", size = 215789, upload-time = "2026-05-26T06:50:00.405Z" },
+ { url = "https://files.pythonhosted.org/packages/62/94/50b5eb8f94e527edceac14f9955e58917424ea79bb572ddc18548561cbc2/zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c", size = 213757, upload-time = "2026-05-26T06:50:01.973Z" },
+ { url = "https://files.pythonhosted.org/packages/17/6f/5d5f32c4dfcdb16ce2ec5363da686840f13c13e1a1214cb70b49e1cd6d9f/zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714", size = 213591, upload-time = "2026-05-26T06:50:03.529Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/55/de0c3459ff717fce3342f9a29464c281fdeb0d36c3171ee88d119d5f0650/zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304", size = 213733, upload-time = "2026-05-26T06:50:05.101Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" },
+ { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/0e/b4c01320859ff1d585438bc231fd60bd258d096359bccf6654fecdf0cffb/zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42", size = 217241, upload-time = "2026-05-26T06:50:12.171Z" },
+]
+
[[package]]
name = "zstandard"
version = "0.18.0"