From d0e9800fc853334f931fb937e0e2419cb23bb747 Mon Sep 17 00:00:00 2001 From: Manos Kaparos <> Date: Sat, 29 Aug 2026 16:45:14 +0300 Subject: [PATCH] Add the fairtask-eval skill: run the evaluation tooling from inside an agent session Co-Authored-By: Claude Fable 5 --- .claude-plugin/plugin.json | 2 +- .github/workflows/ci.yml | 2 +- README.md | 9 +++++-- scripts/validate-manifests.mjs | 11 +++++--- skills/fairtask-eval/SKILL.md | 47 ++++++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 skills/fairtask-eval/SKILL.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 4d7a999..5c2539e 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "fairtask", "version": "0.1.0", - "description": "Screen SWE-bench-style coding tasks for fairness before they become benchmark or RL tasks: is the issue specified well enough, and do the graded tests accept every reasonable fix? One skill (/fairtask) that drives an agent pipeline with verified evidence.", + "description": "Screen SWE-bench-style coding tasks for fairness before they become benchmark or RL tasks: is the issue specified well enough, and do the graded tests accept every reasonable fix? Two skills: /fairtask (screen a task) and /fairtask-eval (reproduce and inspect the evaluation) that drives an agent pipeline with verified evidence.", "author": { "name": "Manos Kaparos", "url": "https://github.com/mnkprs" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80d2dd1..6bb333b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,5 +41,5 @@ jobs: - name: plugin manifests are semantically valid and describe one skill run: | npm run validate:manifests - npx --yes @anthropic-ai/claude-code@2.1.250 --plugin-dir . plugin details fairtask | grep -E "Skills \(1\)" + npx --yes @anthropic-ai/claude-code@2.1.250 --plugin-dir . plugin details fairtask | grep -E "Skills \(2\)" npx --yes skills add . --list 2>&1 | grep -q "fairtask" diff --git a/README.md b/README.md index 5fb9f62..ea261db 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,11 @@ The repository also carries a Codex plugin manifest (`.codex-plugin/`). Packagin [ECC](https://github.com/affaan-m/ECC) — plugin manifest, marketplace file, `metadata.origin` in the skill — so the same skill installs the same way on every harness ECC supports. -Then, in a session: `/fairtask django__django-11099`, `/fairtask task.json`, or `/fairtask astropy/astropy 12544` +Then, in a session: `/fairtask django__django-11099`, `/fairtask task.json`, or `/fairtask astropy/astropy 12544`. +A second skill, `/fairtask-eval`, runs the evaluation tooling from inside the session — `/fairtask-eval show the +evaluation set`, `/fairtask-eval score baseline v3-verify`, `/fairtask-eval lay out astropy__astropy-12544` — and +prints the scripts' output verbatim, so the numbers in a conversation come from the code, not from memory. +The screening skill: `/fairtask …` (a pull request — `skills/fairtask/scripts/task-from-pr.sh` turns it into a task: base commit, linked issue text, code diff, test diff, added test functions). The skill runs `npm run screen` from `$FAIRTASK_HOME` (cloning the engine on first use), reports the verdict with every evidence item and its location, and — when the engine cannot @@ -504,7 +508,8 @@ consumer of every verdict; every number in this README is produced by `npm run s ## 9. Layout ``` -src/screen.ts screen ONE task (yours or a SWE-bench id) skills/fairtask/ the agent skill (SKILL.md, method, PR script) +src/screen.ts screen ONE task (yours or a SWE-bench id) skills/fairtask/ the screening skill (SKILL.md, method, PR script) +skills/fairtask-eval/ the evaluation skill (runs the scripts in-chat) src/show.ts lay out one eval instance as readable files examples/ committed screenings + the challenging case src/run.ts run one system over the eval set src/score.ts metrics vs. human labels src/code-check.ts zero-LLM pre-check for gold-only identifiers diff --git a/scripts/validate-manifests.mjs b/scripts/validate-manifests.mjs index e3e3a44..6fbe2b7 100644 --- a/scripts/validate-manifests.mjs +++ b/scripts/validate-manifests.mjs @@ -19,11 +19,14 @@ if (codex.version !== claude.version) fail("codex plugin.json: version differs f if (typeof codex.skills !== "string" || !existsSync(codex.skills)) fail("codex plugin.json: skills must point at an existing directory"); const caps = codex.interface?.capabilities ?? []; for (const c of ["Read", "Write", "Network", "Execute"]) if (!caps.includes(c)) fail(`codex plugin.json: capabilities must disclose ${c} (the skill clones, installs and runs the engine)`); +for (const name of ["fairtask", "fairtask-eval"]) { + const text = readFileSync(`skills/${name}/SKILL.md`, "utf8"); + const front = text.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? ""; + if (!new RegExp(`^name: ${name}$`, "m").test(front)) fail(`skills/${name}/SKILL.md: frontmatter name must be ${name}`); + if (!/^description: .{40,}/m.test(front)) fail(`skills/${name}/SKILL.md: description missing or too short`); + if (/\$\d/.test(text)) fail(`skills/${name}/SKILL.md: contains $ which the skill loader substitutes with invocation words`); +} const skill = readFileSync("skills/fairtask/SKILL.md", "utf8"); -const fm = skill.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? ""; -if (!/^name: fairtask$/m.test(fm)) fail("SKILL.md: frontmatter name must be fairtask"); -if (!/^description: .{40,}/m.test(fm)) fail("SKILL.md: description missing or too short"); -if (/\$\d/.test(skill)) fail("SKILL.md: contains $ which the skill loader substitutes with invocation words"); if (!/--branch v\d+\.\d+\.\d+/.test(skill)) fail("SKILL.md: engine install must pin a release tag"); const pkg = read("package.json"); if (pkg.version !== claude.version) fail(`package.json version ${pkg.version} differs from plugin version ${claude.version}`); diff --git a/skills/fairtask-eval/SKILL.md b/skills/fairtask-eval/SKILL.md new file mode 100644 index 0000000..35d90bc --- /dev/null +++ b/skills/fairtask-eval/SKILL.md @@ -0,0 +1,47 @@ +--- +name: fairtask-eval +description: Reproduce, inspect or explain the fairtask evaluation from inside an agent session — build the 30-case evaluation set, score runs against the human labels, regenerate the comparison report, audit cited evidence, run the zero-LLM pre-check, verify the data provenance, or lay out one evaluation instance as readable files. Use when asked to show, rerun, check or explain the evaluation numbers, the eval set, a run's results, or a specific evaluation instance of the fairtask project. Read-only and offline except for the data fetch; makes no model calls. +license: MIT +metadata: + origin: fairtask + repository: https://github.com/mnkprs/fairtask +--- + +# fairtask-eval + +Run the evaluation tooling of the `fairtask` repository and bring its output into the conversation, so the numbers a +reader sees come from the scripts, never from memory. Every operation is offline (the scorer reads committed results) +except `data` and `annotations`, which download pinned public files. None calls a model. + +## Steps + +1. **Locate the repository.** The current directory if its `package.json` is named `fairtask`; else `FAIRTASK_HOME`; else + `~/.fairtask`. If none exists, say so and stop — this skill does not clone. +2. **Pick the operation** from the request (one per invocation; ask if two are equally plausible): + + | Request mentions | Run from the repository root | + |---|---| + | the evaluation set, the thirty cases, strata, which tasks | `npm run data:eval-set` (needs `data/raw/swebench_test.parquet`; if missing, run `curl -L -o data/raw/swebench_test.parquet https://huggingface.co/datasets/princeton-nlp/SWE-bench/resolve/main/data/test-00000-of-00001.parquet` first and say you did) | + | score, accuracy, kappa, recall, a run id, compare runs | `npm run score -- ` (default: `baseline v3-verify`; add `--detail` for per-instance rows) | + | the report, the headline table, baseline versus final | `node src/report.ts --baseline baseline --final v3-verify --final-repeat v5-rerun --runs ` — the run ids are the directory names under `results/` | + | evidence audit, bad evidence, fabricated quotes | `npm run audit -- ` (needs cloned workspaces: `npm run data:workspaces` first, ~20 s) | + | code check, novel identifiers, pre-check | `npm run code-check` (needs workspaces) | + | provenance, annotations, checksum, data source | `npm run data:annotations -- --check` | + | show an instance, lay out, issue text, test patch of `` | `npm run show -- ` then read back `examples//issue.md`, `test.patch`, `human-labels.md` | + | trajectory, what the agent did on `` in run `` | `npm run trajectory -- trajectories//.jsonl` and open the rendered `.md` | + +3. **Show the output verbatim** in a code block. Do not round, reorder or summarise numbers before the block. +4. **Add one or two sentences** on how to read it — which row is the primary metric (decision accuracy), that + TPR/TNR are over scored cases only, that "bad evidence" is the share of cited quotes not found where cited. For the + evaluation set, name the four strata and the count per stratum. Stop there; the reader asked for the artifact, not + an essay. + +## Gotchas + +- Committed run ids are complete: `npm run run -- --run-id baseline` does nothing. Reproduction runs use fresh ids + (`baseline-repro`), cost money and need `claude login` or `ANTHROPIC_API_KEY`; this skill does not start them. + Point the user at `REPRODUCE.md` instead. +- `audit`, `code-check` and `finalize-report.py` refuse to run without workspaces at the right commits; that is + intended. Run `npm run data:workspaces`, do not work around it. +- `data:eval-set` prints the table; it also rewrites `data/eval/instances.json` deterministically. In a clean checkout + `git status` stays clean afterwards — if it does not, say so, because that is a reproducibility failure.