From e3ac2d8d7216ed752d432c51f6a9b57f73038374 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Sun, 8 Nov 2026 15:40:01 -0500 Subject: [PATCH 1/3] feat(eval): GitHub task pip-tools on Click 8.5 flag defaults (WIP) MIT-618: verified in containers, fails at the base on click 8.5.0, passes with the fix, and the base passes on click 8.4.2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- tools/eval/github_tasks.py | 49 +++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index 8f31851..4a024ca 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -206,6 +206,53 @@ def tests(*ids: str) -> str: ) +# Click 8.5.0 (2026-08-26) no longer sets a boolean flag's `default` to False when the option is +# built: `Option.default` stays the UNSET sentinel unless read through `get_default()`, +# pallets/click#3641, merged 2026-07-16. Release notes: +# https://github.com/pallets/click/releases/tag/8.5.0 +CLICK_FLAG_DEFAULT = Upstream( + library="click", + version="8.5.0", + released="2026-08-26", + changelog="https://github.com/pallets/click/releases/tag/8.5.0", + upstream_change="click-8.5-flag-default-unset", +) + +# Issue: https://github.com/pypa/pip-tools/issues/2472 (2026-09-02): the command pip-compile +# records in its output gains `--no-index` though it was never passed. +# Fix: https://github.com/pypa/pip-tools/pull/2475 (merged 2026-09-18), `get_compile_command` +# compares a value with the option's resolved default. A merge commit: the base is its first +# parent. The new test passes at the base on Click 8.4.2 (checked) and fails on 8.5.0. Its +# docstring states only the symptom, with "See GH-2472". The version comes from setuptools-scm, +# which needs a git checkout, hence the pretend version. +PIP_TOOLS = GitHubTask( + name="gh-pip-tools-flag-default", + repo="https://github.com/pypa/pip-tools", + base_commit="c58d5d945c0f20e9713e10f5fea2491a5c984f39", + fix_commit="e4187bda8ba559e4bc1991db0635b76a86ab079c", + issue_url="https://github.com/pypa/pip-tools/issues/2472", + fix_url="https://github.com/pypa/pip-tools/pull/2475", + test_paths=("tests/test_utils.py",), + python="3.12", + pins=("click==8.5.0",), + setup=( + "env SETUPTOOLS_SCM_PRETEND_VERSION=7.6.1" + f" uv pip install --quiet {EXCLUDE_NEWER} -e .[testing]" + ), + test_command=tests( + "-n0", "tests/test_utils.py::test_get_compile_command_no_index_not_added_by_default" + ), + upstream=CLICK_FLAG_DEFAULT, + discovery=Discovery( + symptom_words=("--no-index", "recorded command"), + cause_words=("unset", "sentinel", "8.5"), + library="click", + ), + trigger="version_specific", + license="BSD-3-Clause", +) + + def scenario(task: GitHubTask) -> Scenario: """Return the scenario that replays a task: its project comes from the repository.""" return Scenario( @@ -219,6 +266,6 @@ def scenario(task: GitHubTask) -> Scenario: ) -TASKS: list[GitHubTask] = [PYTEST_BDD, TEKORE, PYDANTIC_SETTINGS, MLODA] +TASKS: list[GitHubTask] = [PYTEST_BDD, TEKORE, PYDANTIC_SETTINGS, MLODA, PIP_TOOLS] GITHUB = {task.name: scenario(task) for task in TASKS} From 537630afbd6ebfdf3fb89d4d4b401da90b164db6 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Sun, 8 Nov 2026 15:55:41 -0500 Subject: [PATCH 2/3] test(eval): pin "8.5" among the version words grading v2 skips MIT-618: the pip-tools task's cause words include the Click version "8.5", which grading v2 never counts toward cause_right; its other cause words, "unset" and "sentinel", are not versions. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- server/tests/test_eval_grader.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/tests/test_eval_grader.py b/server/tests/test_eval_grader.py index d8a0dc0..a80ab5e 100644 --- a/server/tests/test_eval_grader.py +++ b/server/tests/test_eval_grader.py @@ -257,7 +257,18 @@ def test_the_cause_words_grader_v2_skips_are_exactly_the_versions() -> None: for word in discovery.cause_words if is_version_word(word, discovery.library) } - assert skipped == {"9.2", "2.14", "0.0.45", "0.11", "polars 2", "3.14", "3.0", "2.1", "2.0"} + assert skipped == { + "9.2", + "2.14", + "0.0.45", + "0.11", + "polars 2", + "3.14", + "3.0", + "2.1", + "2.0", + "8.5", + } # A recorded session (2026-10-25, hard-sqlalchemy-autoflush), trimmed to what grading reads. From 85be179283d7da9465511c34ae22b9acd9dd1c8f Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Sun, 8 Nov 2026 16:03:01 -0500 Subject: [PATCH 3/3] fix(eval): pip-tools cause words name the default, not bare "unset" MIT-618: bare "unset" is also what a symptom-only fix says ("skip options left unset"). The cause words are now "sentinel", phrases about the option's default staying unset, "`unset" for UNSET written as code, and "8.5". The record's comment says why, and that the issue the test cites states the cause while eval sessions have no web tools. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- tools/eval/github_tasks.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index 4a024ca..e58eef4 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -223,8 +223,11 @@ def tests(*ids: str) -> str: # Fix: https://github.com/pypa/pip-tools/pull/2475 (merged 2026-09-18), `get_compile_command` # compares a value with the option's resolved default. A merge commit: the base is its first # parent. The new test passes at the base on Click 8.4.2 (checked) and fails on 8.5.0. Its -# docstring states only the symptom, with "See GH-2472". The version comes from setuptools-scm, -# which needs a git checkout, hence the pretend version. +# docstring states only the symptom, with "See GH-2472"; that issue states the cause, but eval +# sessions have no web tools. The version comes from setuptools-scm, which needs a git +# checkout, hence the pretend version. +# Not bare "unset": a fix that only treats the symptom says it too ("skip options left unset"). +# The phrases name the option's default, and "`unset" matches `UNSET` written as code. PIP_TOOLS = GitHubTask( name="gh-pip-tools-flag-default", repo="https://github.com/pypa/pip-tools", @@ -245,7 +248,14 @@ def tests(*ids: str) -> str: upstream=CLICK_FLAG_DEFAULT, discovery=Discovery( symptom_words=("--no-index", "recorded command"), - cause_words=("unset", "sentinel", "8.5"), + cause_words=( + "sentinel", + "default is unset", + "default is now unset", + "default stays unset", + "`unset", + "8.5", + ), library="click", ), trigger="version_specific",