Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/tests/test_eval_grader.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ def test_the_cause_words_grader_v2_skips_are_exactly_the_versions() -> None:
"2.0",
"8.5",
"3.4",
"0.147",
}


Expand Down
62 changes: 62 additions & 0 deletions tools/eval/github_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,67 @@ def tests(*ids: str) -> str:
)


# FastAPI 0.147.0 (2026-10-19) reads an omitted `bool` form field as False, like an unchecked
# HTML checkbox, whatever default the parameter declares: fastapi/fastapi#16523, merged
# 2026-10-19 (fastapi/fastapi@19c0b7a, authored and committed 2026-10-19). 0.162.0 (2026-10-23)
# went back to the declared default. Release notes:
# https://github.com/fastapi/fastapi/releases/tag/0.147.0
FASTAPI_OMITTED_BOOL = Upstream(
library="fastapi",
version="0.147.0",
released="2026-10-19",
changelog="https://github.com/fastapi/fastapi/releases/tag/0.147.0",
upstream_change="fastapi-0.147-omitted-bool-form-false",
)

# Issue: https://github.com/huss-huss/table-extraction-pro/issues/6 (2026-10-19): `/extract` and
# `/jobs` read an omitted `include_metadata: bool = Form(True)` as False on FastAPI 0.147.
# Fix: https://github.com/huss-huss/table-extraction-pro/pull/7 (merged 2026-10-19): the routes
# take the boolean fields as optional strings and parse them, applying the default themselves.
# A merge commit: the base is its first parent. Only tests/test_jobs.py is laid over: the fix's
# tests/test_api.py has a comment stating the cause ("FastAPI 0.147 reads a missing ...").
# The laid-over file's new omitted-field test and `test_result_formats` fail at the base on
# 0.147.0; the file passes at the base on 0.146.0 (checked). Its new test's comment names only
# the symptom, with "Regression for #6"; that issue states the cause, but eval sessions have no
# web tools. The error doesn't name its fix; not claimed hard-set.
TABLE_EXTRACTION = GitHubTask(
name="gh-table-extraction-bool-form",
repo="https://github.com/huss-huss/table-extraction-pro",
base_commit="16908b90dc506ea42666c95e1a16415fd841dfd6",
fix_commit="3c30a22452397abb0adbc16f468f2bb17868675e",
issue_url="https://github.com/huss-huss/table-extraction-pro/issues/6",
fix_url="https://github.com/huss-huss/table-extraction-pro/pull/7",
test_paths=("tests/test_jobs.py",),
python="3.12",
pins=("fastapi==0.147.0",),
# The project's CI constraints: without them PyMuPDF 2.1 pulls in onnxruntime (over 300 MB);
# with them the project is about 226 MB. They pin fastapi 0.146.0; the pin installs over it.
setup=f"uv pip install --quiet {EXCLUDE_NEWER} -c requirements/ci-constraints.txt -e .[dev]",
test_command=tests("tests/test_jobs.py"),
upstream=FASTAPI_OMITTED_BOOL,
discovery=Discovery(
symptom_words=("include_metadata", "metadata"),
# Every phrase pairs FastAPI's object, its `bool` form field, with its new rule: an
# omitted one is read as False, the "unchecked HTML checkboxes" of its release notes.
# Not a bare "bool form field", "boolean form field" or "default": the laid-over test
# names say them, and so would a fix that only treats the symptom.
cause_words=(
"omitted bool form fields as false",
"omitted bool form field as false",
"omitted boolean form fields as false",
"omitted boolean form field as false",
"html checkbox",
"unchecked checkbox",
"checkbox semantics",
"0.147",
),
library="fastapi",
),
trigger="version_specific",
license="MIT",
)


# DuckDB 2.0.0 (2026-10-21) rebuilt the Python relational API on the engine's BindStatement:
# after a join, an input's alias stops resolving once a later step (such as `order()`) starts a
# new SELECT. duckdb/duckdb-python#697, "Move Relational API to BindStatement", opened 2026-10-19,
Expand Down Expand Up @@ -469,6 +530,7 @@ def scenario(task: GitHubTask) -> Scenario:
PIP_TOOLS,
MARKLASSIAN,
MLODA_DUCKDB,
TABLE_EXTRACTION,
]

GITHUB = {task.name: scenario(task) for task in TASKS}
Loading