From 2e862b8c2a015d66ad68b423114801bd96c1c2cf Mon Sep 17 00:00:00 2001 From: Stefanstud Date: Sun, 27 Sep 2026 09:25:59 +0200 Subject: [PATCH 1/3] Fix README quick starts --- README.md | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 68bad7f..a6f191a 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ uv run python **Download the files.** Run this example from the repository root. It downloads all five tables from [Hugging Face](https://huggingface.co/datasets/Andromede-AI/messier/tree/main) into the folders our loaders and analyses read: +> **Note:** Downloading all five tables requires more than 9 GB of disk space. The trajectories account for most of this size. + ```python from huggingface_hub import hf_hub_download @@ -117,23 +119,23 @@ This example streams rows from Hugging Face. Change the filename to stream recor Below we show several use cases and quick starts for using MESSIER. -Combine all five tables into one complete dataset containing the full information. +Collect all stored information for one task. ```python -from collections import defaultdict -import pandas as pd from messier.io import data_path, load_jsonl -tables = {} -for name in ("tasks", "records", "verifiers", "trajectories", "classifications"): - rows_by_task = defaultdict(list) - for row in load_jsonl(data_path(f"{name}.jsonl", source="local")): - rows_by_task[(row["benchmark"], row["task_id"])].append(row) +task = next(load_jsonl(data_path("tasks.jsonl", source="local"))) +key = (task["benchmark"], task["task_id"]) - tables[name] = pd.Series(rows_by_task, dtype=object) +task_data = {} +for name in ("tasks", "records", "verifiers", "trajectories", "classifications"): + task_data[name] = [ + row + for row in load_jsonl(data_path(f"{name}.jsonl", source="local")) + if (row["benchmark"], row["task_id"]) == key + ] -dataset = pd.DataFrame(tables).rename_axis(["benchmark", "task_id"]).reset_index() -print(dataset.head()) +print({name: len(rows) for name, rows in task_data.items()}) ``` Compare agents within each benchmark using trials with a score. @@ -158,12 +160,6 @@ uv run --group dev python analysis/frontier_progress/run.py # Results: analysis/outputs/frontier_progress/ ``` -The script downloads and verifies the frozen raw sources, rebuilds the corpus, and runs validation tests. - -```bash -bash scripts/update.sh -``` - ## License MESSIER is licensed under [MIT](LICENSE). From c06a644545b6adf136cdafa59ab1aab52fcfc36e Mon Sep 17 00:00:00 2001 From: Stefanstud Date: Sun, 27 Sep 2026 09:28:30 +0200 Subject: [PATCH 2/3] docs --- docs/data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data.md b/docs/data.md index be7b19d..7b42191 100644 --- a/docs/data.md +++ b/docs/data.md @@ -11,7 +11,7 @@ | Term | Meaning | | --- | --- | | Model | The underlying language model | -| Scaffold | The system that turns model outputs into actions and parses environment observations | +| Scaffold | The system that turns model outputs into actions and parses environment observations (sometimes called a harness)| | Agent | A model-scaffold pair. If a source reports a reasoning setting, such as `high`, we append it to the agent ID | | Environment | The system with which an agent interacts, defining the states, actions, and observations available | | Task | A given problem consisting of a string instruction (and an initial environment state) | From ebf3b23a6bffb4bac10340cec2afb42bb094389b Mon Sep 17 00:00:00 2001 From: Stefanstud Date: Sun, 27 Sep 2026 09:30:46 +0200 Subject: [PATCH 3/3] Clarify data terminology --- docs/data.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data.md b/docs/data.md index 7b42191..d82840a 100644 --- a/docs/data.md +++ b/docs/data.md @@ -11,7 +11,7 @@ | Term | Meaning | | --- | --- | | Model | The underlying language model | -| Scaffold | The system that turns model outputs into actions and parses environment observations (sometimes called a harness)| +| Scaffold | The system that turns model outputs into actions and parses environment observations (sometimes called a harness) | | Agent | A model-scaffold pair. If a source reports a reasoning setting, such as `high`, we append it to the agent ID | | Environment | The system with which an agent interacts, defining the states, actions, and observations available | | Task | A given problem consisting of a string instruction (and an initial environment state) | @@ -37,7 +37,7 @@ The release files use the same terminology as the paper. Names inside a metadata | **records.jsonl** | One stored trial result, verifier result, or source summary | | **trajectories.jsonl** | One available step-by-step execution trajectory (agent per task) | | **classifications.jsonl** | One task's final SOC and NAICS labels, the labels proposed by each classification model, and adjudication details when the models disagree | -| **task_files/** | Original files available to agents for tasks that provide them | +| **task_files/** | Original files available to agents for tasks that provide them, sometimes omitted because their licenses do not permit redistribution | For example, a trial with three separately reported verifier results produces four records: three verifier-result rows and one trial-result row.