MIT-618: three verified GitHub tasks (first batch, partial)

docs/eval/README.md

@@ -77,7 +77,7 @@ with a `repo`, so `run`, grading, and the ledger work as for every other scenari
 | `repo` | https URL of the repository |
 | `base_commit` | full sha of the fix's parent: the tree the agent starts from |
 | `fix_commit` | full sha of the fix; used only by `check`, never shown to the agent |
-| `issue_url`, `fix_url` | the issue, and the pull request or commit that fixed it |
+| `issue_url`, `fix_url` | the issue, and the pull request or commit that fixed it; when the repository has no issue for it, `issue_url` is the fix's pull request, which reports the bug, and the task's comment says so |
 | `test_paths` | the test files the fix added or changed, laid over the base so the tests fail |
 | `python` | the interpreter version, e.g. `"3.12"` |
 | `pins` | exact dependency versions (`name==version`), installed after `setup` |

tools/eval/github_tasks.py

@@ -7,7 +7,151 @@
 above it: the issue, the fix, and the upstream release notes.
 """
 
-from tools.eval.model import Expect, GitHubTask, Scenario
+from tools.eval.model import Discovery, Expect, GitHubTask, Scenario, Upstream
+
+# The measured models' training cutoff, from Anthropic's models overview
+# (https://platform.claude.com/docs/en/about-claude/models/overview, read 2026-11-08): Claude
+# Sonnet 5.5 and Claude Opus 5.5 both list a training data cutoff, and a reliable knowledge
+# cutoff, of June 2026. Every upstream release below came out on or after 2026-07-01, and every
+# issue and fix after that release.
+#
+# Each setup resolves the project's own dependencies as of 2026-11-08, so a task installs the
+# same versions whenever it runs; the pins then install the upstream release that broke it.
+EXCLUDE_NEWER = "--exclude-newer=2026-11-08T00:00:00Z"
+SETUP = f"uv pip install --quiet {EXCLUDE_NEWER} -e ."
+
+
+def tests(*ids: str) -> str:
+    """Return a test command for ids, run from the project's .venv with no network or sync."""
+    return "uv run --no-sync pytest -q -p no:cacheprovider " + " ".join(ids)
+
+
+# pytest 9.2.0 (2026-10-27) replaced the private `FixtureManager._arg2fixturedefs` (name -> defs)
+# with `_arg2node2fixturedefs` (name -> node -> defs), pytest-dev/pytest#14984, merged 2026-09-14.
+# Release notes: https://github.com/pytest-dev/pytest/releases/tag/9.2.0
+PYTEST_FIXTUREDEFS = Upstream(
+    library="pytest",
+    version="9.2.0",
+    released="2026-10-27",
+    changelog="https://github.com/pytest-dev/pytest/releases/tag/9.2.0",
+    upstream_change="pytest-9.2-arg2node2fixturedefs",
+)
+
+# Issue: https://github.com/pytest-dev/pytest-bdd/issues/850 (2026-10-28): every scenario fails in
+# step lookup with AttributeError on `_arg2fixturedefs`.
+# Fix: https://github.com/pytest-dev/pytest-bdd/pull/851 (merged 2026-10-29), a compat layer in
+# src/pytest_bdd/compat.py used by scenario.py; tests/feature/test_report.py changed with it.
+# The error's "Did you mean" names the new attribute, so it points at the rename: not a
+# hard-set task under decision 0023. Cause words name the new keying, not the attribute.
+PYTEST_BDD = GitHubTask(
+    name="gh-pytest-bdd-fixturedefs",
+    repo="https://github.com/pytest-dev/pytest-bdd",
+    base_commit="23aea7e8fe40c2d44c6e77de7f78b8752eb681ee",
+    fix_commit="572f7b55d5e4745c42d13d832e44147da5989319",
+    issue_url="https://github.com/pytest-dev/pytest-bdd/issues/850",
+    fix_url="https://github.com/pytest-dev/pytest-bdd/pull/851",
+    test_paths=("tests/feature/test_report.py",),
+    python="3.12",
+    pins=("pytest==9.2.0",),
+    setup=SETUP,
+    test_command=tests("tests/feature/test_report.py"),
+    upstream=PYTEST_FIXTUREDEFS,
+    discovery=Discovery(
+        symptom_words=("_arg2fixturedefs", "FixtureManager"),
+        cause_words=("9.2", "per node", "per-node", "by node"),
+        library="pytest",
+    ),
+    trigger="version_specific",
+    license="MIT",
+)
+
+
+# Pydantic 2.14.0 (2026-10-08) calls an enum's constructor, and so its `_missing_`, for a value
+# it can't match, in both validation modes: pydantic/pydantic#13792, merged 2026-09-11, after
+# the last 2.14 beta. Release notes: https://github.com/pydantic/pydantic/releases/tag/v2.14.0
+PYDANTIC_ENUM_MISSING = Upstream(
+    library="pydantic",
+    version="2.14.0",
+    released="2026-10-08",
+    changelog="https://github.com/pydantic/pydantic/releases/tag/v2.14.0",
+    upstream_change="pydantic-2.14-enum-missing-called",
+)
+
+# Issue: none in the repository; the fix's pull request (opened 2026-11-01) reports it: a
+# playlist holding a local file fails to parse with AttributeError ('NoneType' object has no
+# attribute 'lower'), first seen downstream in https://github.com/Ch00k/mottle/pull/110.
+# Fix: https://github.com/felix-hilden/tekore/pull/351 (merged 2026-11-06): StrEnum._missing_
+# returns None for a value it can't match. Rebase-merged: the base is the parent of the fix's
+# first commit (3ae6b06). The tests below pass at the base on Pydantic 2.13.5 (checked) and fail
+# on 2.14.0; the PR's bare `E(None)` tests fail at the base on any Pydantic, so they're left out.
+TEKORE = GitHubTask(
+    name="gh-tekore-enum-missing",
+    repo="https://github.com/felix-hilden/tekore",
+    base_commit="2438a90a5fb2157b805814d5cb7aca920883ed86",
+    fix_commit="ef00b137566847333501d037f3f87beaab90a989",
+    issue_url="https://github.com/felix-hilden/tekore/pull/351",
+    fix_url="https://github.com/felix-hilden/tekore/pull/351",
+    test_paths=("tests/model.py",),
+    python="3.12",
+    pins=("pydantic==2.14.0",),
+    setup=f"uv pip install --quiet {EXCLUDE_NEWER} -e . -r requirements/tests",
+    test_command=tests(
+        "tests/model.py::TestEnumUnknownValue::test_unknown_value_in_model_raises_validation_error",
+        "tests/model.py::TestEnumUnknownValue::test_unknown_value_falls_back_to_next_union_member",
+    ),
+    upstream=PYDANTIC_ENUM_MISSING,
+    discovery=Discovery(
+        symptom_words=("lower", "AttributeError", "NoneType"),
+        # Not `_missing_`: the traceback ends in it and every fix edits it.
+        cause_words=("enum constructor", "calls the enum", "2.14"),
+        library="pydantic",
+    ),
+    trigger="version_specific",
+    license="MIT",
+)
+
+
+# Polars 2.0.0 (2026-10-06; first release candidate 2026-09-02): a lazy query's collect now
+# runs on the streaming engine, which doesn't keep a join's row order unless the join passes
+# `maintain_order`. Upgrade guide, "The Lazy API defaults to the streaming engine":
+# https://docs.pola.rs/releases/upgrade/2/ (release: https://github.com/pola-rs/polars/releases/tag/py-2.0.0)
+POLARS_JOIN_ORDER = Upstream(
+    library="polars",
+    version="2.0.0",
+    released="2026-10-06",
+    changelog="https://docs.pola.rs/releases/upgrade/2/",
+    upstream_change="polars-2.0-streaming-join-order",
+)
+
+# Issue: https://github.com/mloda-ai/mloda/issues/2252 (2026-10-24): the polars lazy merge_left
+# test fails on polars 2.0, frames equal but for row order.
+# Fix: https://github.com/mloda-ai/mloda/pull/2262 (merged 2026-10-24), maintain_order per join
+# type in the polars merge engine, with a new row-order test on large shuffled frames.
+# Only the new test is laid over: the fix's edit of test_polars_lazy_dataframe.py adds
+# `maintain_order` to a fixture, which would show the agent the fix.
+MLODA_TESTS = "tests/test_plugins/compute_framework/base_implementations/polars"
+MLODA = GitHubTask(
+    name="gh-mloda-join-order",
+    repo="https://github.com/mloda-ai/mloda",
+    base_commit="fa0a1d1ccbcf8a147358807fcd1437edb6438a13",
+    fix_commit="28998b818c0e2e111bbea9bccca1897646a7fdbf",
+    issue_url="https://github.com/mloda-ai/mloda/issues/2252",
+    fix_url="https://github.com/mloda-ai/mloda/pull/2262",
+    test_paths=(f"{MLODA_TESTS}/test_polars_merge_row_order.py",),
+    python="3.12",
+    pins=("polars==2.0.0", "pytest==9.1.1"),
+    setup=f"uv pip install --quiet {EXCLUDE_NEWER} -e .[polars]",
+    test_command=tests(f"{MLODA_TESTS}/test_polars_merge_row_order.py"),
+    upstream=POLARS_JOIN_ORDER,
+    discovery=Discovery(
+        symptom_words=("order", "merge", "join"),
+        # The cause, not the symptom: the streaming engine and the join's maintain_order.
+        cause_words=("maintain_order", "streaming"),
+        library="polars",
+    ),
+    trigger="version_specific",
+    license="Apache-2.0",
+)
 
 
 def scenario(task: GitHubTask) -> Scenario:
@@ -23,6 +167,6 @@ def scenario(task: GitHubTask) -> Scenario:
     )
 
 
-TASKS: list[GitHubTask] = []
+TASKS: list[GitHubTask] = [PYTEST_BDD, TEKORE, MLODA]
 
 GITHUB = {task.name: scenario(task) for task in TASKS}