From 3a31a1a16af36ddf26685e46e30845e98233c789 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Tue, 10 Nov 2026 18:11:36 -0500 Subject: [PATCH 1/3] feat(eval): add the table-extraction-pro FastAPI 0.147 GitHub task (MIT-618 WIP) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- server/tests/test_eval_grader.py | 1 + tools/eval/github_tasks.py | 59 +++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/server/tests/test_eval_grader.py b/server/tests/test_eval_grader.py index 38ea72a..e6a9953 100644 --- a/server/tests/test_eval_grader.py +++ b/server/tests/test_eval_grader.py @@ -269,6 +269,7 @@ def test_the_cause_words_grader_v2_skips_are_exactly_the_versions() -> None: "2.0", "8.5", "3.4", + "0.147", } diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index f93fd66..0f2d6ab 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -315,6 +315,55 @@ def tests(*ids: str) -> str: ) +# FastAPI 0.147.0 (2026-10-19) reads an omitted `bool` form field as False, like an unchecked +# HTML checkbox, whatever default the parameter declares: fastapi/fastapi#16523, merged +# 2026-10-19 (fastapi/fastapi@19c0b7a, authored and committed 2026-10-19). 0.162.0 (2026-10-23) +# went back to the declared default. Release notes: +# https://github.com/fastapi/fastapi/releases/tag/0.147.0 +FASTAPI_OMITTED_BOOL = Upstream( + library="fastapi", + version="0.147.0", + released="2026-10-19", + changelog="https://github.com/fastapi/fastapi/releases/tag/0.147.0", + upstream_change="fastapi-0.147-omitted-bool-form-false", +) + +# Issue: https://github.com/huss-huss/table-extraction-pro/issues/6 (2026-10-19): `/extract` and +# `/jobs` read an omitted `include_metadata: bool = Form(True)` as False on FastAPI 0.147. +# Fix: https://github.com/huss-huss/table-extraction-pro/pull/7 (merged 2026-10-19): the routes +# take the boolean fields as optional strings and parse them, applying the default themselves. +# A merge commit: the base is its first parent. Only tests/test_jobs.py is laid over: the fix's +# tests/test_api.py has a comment stating the cause ("FastAPI 0.147 reads a missing ..."). +# The laid-over file's new omitted-field test and `test_result_formats` fail at the base on +# 0.147.0; the file passes at the base on 0.146.0 (checked). Its new test's comment names only +# the symptom, with "Regression for #6"; that issue states the cause, but eval sessions have no +# web tools. The error doesn't name its fix; not claimed hard-set. +TABLE_EXTRACTION = GitHubTask( + name="gh-table-extraction-bool-form", + repo="https://github.com/huss-huss/table-extraction-pro", + base_commit="16908b90dc506ea42666c95e1a16415fd841dfd6", + fix_commit="3c30a22452397abb0adbc16f468f2bb17868675e", + issue_url="https://github.com/huss-huss/table-extraction-pro/issues/6", + fix_url="https://github.com/huss-huss/table-extraction-pro/pull/7", + test_paths=("tests/test_jobs.py",), + python="3.12", + pins=("fastapi==0.147.0",), + # The project's CI constraints: without them PyMuPDF 2.1 pulls in onnxruntime (over 300 MB). + setup=f"uv pip install --quiet {EXCLUDE_NEWER} -c requirements/ci-constraints.txt -e .[dev]", + test_command=tests("tests/test_jobs.py"), + upstream=FASTAPI_OMITTED_BOOL, + discovery=Discovery( + symptom_words=("include_metadata", "metadata"), + # FastAPI's own words for the change ("unchecked HTML checkboxes"). Not "omitted bool" + # or "default": the failing test's name and a symptom-only fix say them. + cause_words=("checkbox", "unchecked", "bool form fields as false", "0.147"), + library="fastapi", + ), + trigger="version_specific", + license="MIT", +) + + def scenario(task: GitHubTask) -> Scenario: """Return the scenario that replays a task: its project comes from the repository.""" return Scenario( @@ -328,6 +377,14 @@ def scenario(task: GitHubTask) -> Scenario: ) -TASKS: list[GitHubTask] = [PYTEST_BDD, TEKORE, PYDANTIC_SETTINGS, MLODA, PIP_TOOLS, MARKLASSIAN] +TASKS: list[GitHubTask] = [ + PYTEST_BDD, + TEKORE, + PYDANTIC_SETTINGS, + MLODA, + PIP_TOOLS, + MARKLASSIAN, + TABLE_EXTRACTION, +] GITHUB = {task.name: scenario(task) for task in TASKS} From aeb16bfdd7b830974891ace482224ff7cc3b1a66 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Tue, 10 Nov 2026 18:23:14 -0500 Subject: [PATCH 2/3] fix(eval): name FastAPI's form field in the table-extraction cause words (MIT-618) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- tools/eval/github_tasks.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index 0f2d6ab..2461843 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -348,15 +348,18 @@ def tests(*ids: str) -> str: test_paths=("tests/test_jobs.py",), python="3.12", pins=("fastapi==0.147.0",), - # The project's CI constraints: without them PyMuPDF 2.1 pulls in onnxruntime (over 300 MB). + # The project's CI constraints: without them PyMuPDF 2.1 pulls in onnxruntime (over 300 MB); + # with them the project is about 226 MB. They pin fastapi 0.146.0; the pin installs over it. setup=f"uv pip install --quiet {EXCLUDE_NEWER} -c requirements/ci-constraints.txt -e .[dev]", test_command=tests("tests/test_jobs.py"), upstream=FASTAPI_OMITTED_BOOL, discovery=Discovery( symptom_words=("include_metadata", "metadata"), - # FastAPI's own words for the change ("unchecked HTML checkboxes"). Not "omitted bool" - # or "default": the failing test's name and a symptom-only fix say them. - cause_words=("checkbox", "unchecked", "bool form fields as false", "0.147"), + # Each names FastAPI's object that changed: its `bool` form field, read by the new rule + # its release notes call "unchecked HTML checkboxes". Not "boolean form field", + # "omitted bool" or "default": the laid-over test names say them, and so would a fix + # that only treats the symptom. + cause_words=("bool form field", "html checkbox", "unchecked checkbox", "0.147"), library="fastapi", ), trigger="version_specific", From 9666add05eba8dba61c5fdef32212f4560b96940 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Tue, 10 Nov 2026 18:34:27 -0500 Subject: [PATCH 3/3] fix(eval): pair the form field with FastAPI's rule in cause words (MIT-618) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- tools/eval/github_tasks.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/eval/github_tasks.py b/tools/eval/github_tasks.py index 17efab9..01e8d62 100644 --- a/tools/eval/github_tasks.py +++ b/tools/eval/github_tasks.py @@ -426,11 +426,20 @@ def tests(*ids: str) -> str: upstream=FASTAPI_OMITTED_BOOL, discovery=Discovery( symptom_words=("include_metadata", "metadata"), - # Each names FastAPI's object that changed: its `bool` form field, read by the new rule - # its release notes call "unchecked HTML checkboxes". Not "boolean form field", - # "omitted bool" or "default": the laid-over test names say them, and so would a fix - # that only treats the symptom. - cause_words=("bool form field", "html checkbox", "unchecked checkbox", "0.147"), + # Every phrase pairs FastAPI's object, its `bool` form field, with its new rule: an + # omitted one is read as False, the "unchecked HTML checkboxes" of its release notes. + # Not a bare "bool form field", "boolean form field" or "default": the laid-over test + # names say them, and so would a fix that only treats the symptom. + cause_words=( + "omitted bool form fields as false", + "omitted bool form field as false", + "omitted boolean form fields as false", + "omitted boolean form field as false", + "html checkbox", + "unchecked checkbox", + "checkbox semantics", + "0.147", + ), library="fastapi", ), trigger="version_specific",