From 5ec890cede60c8ccbb52323b3b3f01d2ee3a83d9 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Tue, 10 Nov 2026 18:54:12 -0500 Subject: [PATCH 1/2] feat(eval): MIT-618 add the UltraPlot matplotlib 3.11.2 GitHub task Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- server/tests/test_eval_grader.py | 1 + tools/eval/github_tasks.py | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/server/tests/test_eval_grader.py b/server/tests/test_eval_grader.py index e6a9953..9775212 100644 --- a/server/tests/test_eval_grader.py +++ b/server/tests/test_eval_grader.py @@ -270,6 +270,7 @@ def test_the_cause_words_grader_v2_skips_are_exactly_the_versions() -> None: "8.5", "3.4", "0.147", + "3.11.2", } diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index 01e8d62..a5fb7dc 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -508,6 +508,71 @@ def tests(*ids: str) -> str: ) +# Matplotlib 3.11.2 (2026-09-11) gave `ColormapRegistry` its own `__contains__`, which looks the +# name up in the registry's dict: before, `in` was `Mapping.__contains__`, which calls +# `__getitem__`, so a subclass's `__getitem__` decided membership. matplotlib/matplotlib#32172 +# ("PERF: Avoid `difflib` lookup in `ColormapRegistry.register`"), merged 2026-08-05, backported +# to 3.11.x as eb3e50c (authored and committed 2026-08-05). +# Release notes: https://github.com/matplotlib/matplotlib/releases/tag/v3.11.2 +MATPLOTLIB_REGISTRY_CONTAINS = Upstream( + library="matplotlib", + version="3.11.2", + released="2026-09-11", + changelog="https://github.com/matplotlib/matplotlib/releases/tag/v3.11.2", + upstream_change="matplotlib-3.11.2-colormap-registry-contains", +) + +# Issue: https://github.com/Ultraplot/UltraPlot/issues/831 (2026-09-29): on matplotlib 3.11.2 a +# plain `ax.scatter` fails with "None is not a valid value for cmap"; 3.11.1 works. +# Fix: https://github.com/Ultraplot/UltraPlot/pull/832 (squash-merged 2026-09-30): +# `ColormapDatabase`, a `ColormapRegistry` subclass, gets a `__contains__` that translates names +# as its `__getitem__` does. The base is the squash commit's parent. The new test fails at the +# base on matplotlib 3.11.2 (`'Fire' in uplt.colormaps`) and passes there on 3.11.1 (checked). +# The laid-over file's only comment is about an older reversal bug; nothing names matplotlib's +# change. The fix's source comment names 3.11.2, but the agent never sees the fix tree. The +# error doesn't name its fix; not claimed hard-set. +# No bare `__contains__` or `__getitem__`: the fix adds the one, and the base docstring cites +# the other. Bare "ColormapRegistry" is in the base's class line, and "colormap" fills the test. +ULTRAPLOT = GitHubTask( + name="gh-ultraplot-colormap-contains", + repo="https://github.com/Ultraplot/UltraPlot", + base_commit="74e28e2aeb684b013f6f732a63b4d19b68cb93dc", + fix_commit="d48479c2196368945b9e920300cc43d651ed5325", + issue_url="https://github.com/Ultraplot/UltraPlot/issues/831", + fix_url="https://github.com/Ultraplot/UltraPlot/pull/832", + test_paths=("ultraplot/tests/test_colormap.py",), + python="3.12", + pins=("matplotlib==3.11.2",), + # The version comes from setuptools-scm, which needs a git checkout. + setup=( + "env SETUPTOOLS_SCM_PRETEND_VERSION=2.7.0" + f" uv pip install --quiet {EXCLUDE_NEWER} -e . pytest==9.1.1 pytest-mpl" + ), + test_command=tests("ultraplot/tests/test_colormap.py"), + upstream=MATPLOTLIB_REGISTRY_CONTAINS, + discovery=Discovery( + symptom_words=("ColormapDatabase", "colormaps", "Fire"), + # Every phrase names matplotlib's object, `ColormapRegistry` or its `__contains__`, with + # what changed: membership stopped going through `__getitem__`. + cause_words=( + "colormapregistry.__contains__", + "colormapregistry's __contains__", + "colormapregistry defines __contains__", + "colormapregistry now defines", + "__contains__ no longer calls __getitem__", + "__contains__ no longer goes through __getitem__", + "__contains__ bypasses __getitem__", + "__contains__ skips __getitem__", + "__contains__ checks _cmaps", + "3.11.2", + ), + library="matplotlib", + ), + trigger="version_specific", + license="MIT", +) + + def scenario(task: GitHubTask) -> Scenario: """Return the scenario that replays a task: its project comes from the repository.""" return Scenario( @@ -531,6 +596,7 @@ def scenario(task: GitHubTask) -> Scenario: MARKLASSIAN, MLODA_DUCKDB, TABLE_EXTRACTION, + ULTRAPLOT, ] GITHUB = {task.name: scenario(task) for task in TASKS} From 0babf6f457de417ee4fdec8a46767fe9522d0f59 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Tue, 10 Nov 2026 19:02:12 -0500 Subject: [PATCH 2/2] fix(eval): MIT-618 tighten the UltraPlot task's cause words Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- tools/eval/github_tasks.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index a5fb7dc..1953b41 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -512,7 +512,9 @@ def tests(*ids: str) -> str: # name up in the registry's dict: before, `in` was `Mapping.__contains__`, which calls # `__getitem__`, so a subclass's `__getitem__` decided membership. matplotlib/matplotlib#32172 # ("PERF: Avoid `difflib` lookup in `ColormapRegistry.register`"), merged 2026-08-05, backported -# to 3.11.x as eb3e50c (authored and committed 2026-08-05). +# to 3.11.x as eb3e50c (authored and committed 2026-08-05). The PERF title is this change: the +# new `__contains__` in lib/matplotlib/cm.py keeps `register`'s `name in self` from going +# through `__getitem__` and its difflib suggestions. # Release notes: https://github.com/matplotlib/matplotlib/releases/tag/v3.11.2 MATPLOTLIB_REGISTRY_CONTAINS = Upstream( library="matplotlib", @@ -552,18 +554,21 @@ def tests(*ids: str) -> str: upstream=MATPLOTLIB_REGISTRY_CONTAINS, discovery=Discovery( symptom_words=("ColormapDatabase", "colormaps", "Fire"), - # Every phrase names matplotlib's object, `ColormapRegistry` or its `__contains__`, with - # what changed: membership stopped going through `__getitem__`. + # Every phrase states what changed in matplotlib: `ColormapRegistry` gained its own + # `__contains__`, or membership stopped going through `__getitem__`. No bare object + # nouns ("ColormapRegistry.__contains__"): a symptom-only fix says "overrides + # ColormapRegistry.__contains__". Nothing that describes what a `__contains__` checks + # ("checks _cmaps"): the fix's own `__contains__` checks the translated key. cause_words=( - "colormapregistry.__contains__", - "colormapregistry's __contains__", "colormapregistry defines __contains__", "colormapregistry now defines", + "colormapregistry now has", + "colormapregistry gained", "__contains__ no longer calls __getitem__", "__contains__ no longer goes through __getitem__", + "no longer falls back to __getitem__", "__contains__ bypasses __getitem__", "__contains__ skips __getitem__", - "__contains__ checks _cmaps", "3.11.2", ), library="matplotlib",