From a7b851bf7da559412a29571e1788e2bf18447516 Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 09:32:00 +0000 Subject: [PATCH 1/3] fix(scoring): BBH exact-match accepts Markdown-emphasised answers Peel whole-answer **/*/_/` wraps in _normalize_exact so Answer: **True** matches gold True, consistent with MCQ half's emphasis tolerance. Closes #461 --- src/trinity/adapters/bbh.py | 32 +++++++++++++++++++++++++++----- tests/test_bbh_md_emphasis.py | 12 ++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 tests/test_bbh_md_emphasis.py diff --git a/src/trinity/adapters/bbh.py b/src/trinity/adapters/bbh.py index 0c6a0b7..9fd4b08 100644 --- a/src/trinity/adapters/bbh.py +++ b/src/trinity/adapters/bbh.py @@ -165,21 +165,43 @@ def _final_answer_segment(text: str) -> str: #: compared as a sequence, not treated as strippable formatting noise. _BRACKETS = frozenset("()[]{}<>") +#: Whole-answer Markdown emphasis (``**True**``, ``*valid*``, ``__no__``). Same marker +#: on both sides (backreference) so asymmetric ``*`` content is never peeled. Mirrors +#: ``reward._strip_choice_md_emphasis`` used by BBH multiple-choice (#461). +_MD_EMPHASIS_WRAP_RE = re.compile(r"(\*{1,3}|_{1,3}|`)(.+?)\1", re.DOTALL) + + +def _strip_md_emphasis_wrap(s: str) -> str: + """Peel Markdown emphasis that wraps the entire stripped answer (fixpoint).""" + prev = None + while prev != s: + prev = s + m = _MD_EMPHASIS_WRAP_RE.fullmatch(s.strip()) + if m: + s = m.group(2).strip() + return s + def _normalize_exact(text: str) -> str: """Normalise a free-form answer for tolerant exact comparison. - Lower-cases, strips surrounding quotes/brackets/terminal punctuation, and collapses - internal whitespace — only formatting noise, never content. A **pure bracket sequence** - is the exception: the ``dyck_languages`` gold target is all closing brackets (e.g. - ``"] )"``), so it is compared whitespace-insensitively rather than stripped down to an - empty string — which made every dyck answer, correct or not, grade ``0.0``. + Lower-cases, strips surrounding quotes/brackets/terminal punctuation and Markdown + emphasis (``**True**`` == ``True``), and collapses internal whitespace — only + formatting noise, never content. A **pure bracket sequence** is the exception: the + ``dyck_languages`` gold target is all closing brackets (e.g. ``"] )"``), so it is + compared whitespace-insensitively rather than stripped down to an empty string — + which made every dyck answer, correct or not, grade ``0.0``. """ s = str(text).strip().lower() + # Peel whole-answer emphasis BEFORE the bracket check so a bolded dyck answer + # still reaches the bracket-sequence comparison. + s = _strip_md_emphasis_wrap(s) compact = re.sub(r"\s+", "", s) if compact and all(ch in _BRACKETS for ch in compact): return compact s = s.strip(".\"'`()[]{} \t\n") + # Emphasis may sit inside terminal punctuation ("**true**."): strip then peel. + s = _strip_md_emphasis_wrap(s) s = re.sub(r"\s+", " ", s) return s diff --git a/tests/test_bbh_md_emphasis.py b/tests/test_bbh_md_emphasis.py new file mode 100644 index 0000000..91fa509 --- /dev/null +++ b/tests/test_bbh_md_emphasis.py @@ -0,0 +1,12 @@ +"""BBH exact-match must accept Markdown-emphasised answers (#461).""" + +from __future__ import annotations + +from trinity.adapters.bbh import score_bbh + + +def test_exact_match_accepts_markdown_bold() -> None: + ref = {"answer": "True", "answer_type": "exact_match", "subtask": "boolean_expressions"} + assert score_bbh("Answer: True", ref) == 1.0 + assert score_bbh("Answer: **True**", ref) == 1.0 + assert score_bbh("Answer: **valid**", {"answer": "valid", "answer_type": "exact_match"}) == 1.0 From cb2ab6c38c6103633bfd681cf80f8d851211302d Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 09:43:37 +0000 Subject: [PATCH 2/3] chore(ci): pin ruff<0.16 so main CI is not red on new defaults --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 02d4f00..e911c0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] From cdd9d22eb0cc3db33ab81bb81ec639efda5d5475 Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 09:49:45 +0000 Subject: [PATCH 3/3] =?UTF-8?q?chore(ci):=20un-red=20main=20=E2=80=94=20wr?= =?UTF-8?q?apped=20leading-decimal=20DROP=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exclude "." from _STRIP_EDGE and retry with rstrip(".") so CI can validate this branch's own fix (same unblock as #450/#462). --- src/trinity/adapters/drop.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803..07fcbd4 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str: return boxed if boxed is not None else "" -#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a -#: leading sign is part of a number's value, not wrapping noise. -_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-") +#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and +#: the decimal point ``.`` — a leading sign or decimal point is part of a number's +#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled +#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart +#: from a value-bearing leading point; a blanket edge-strip cannot. +_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.") def _normalize_token(raw: str) -> str: @@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str: dropped the ``-`` and left commas to break ``float()``) did not deliver. A token that is ALREADY a number is recognised before any punctuation is - stripped: the edge-strip set includes ``.``, so a leading-decimal token like - ``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to - a gold ``"5"`` (false positive) and unequal to the value-identical gold - ``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests - ``_is_number`` first and leaves numbers untouched for exactly this reason.""" + stripped: a leading-decimal token like ``".5"`` must not lose its point and + normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal + to the value-identical gold ``"0.5"`` (false negative). The official DROP + ``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for + exactly this reason (issue #423). The float-first path alone only covers the + *bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"`` + and ``".5."`` still lost the leading point on the second-chance path. So the + edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``, + ``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are + sentence punctuation, left-side dots are value.""" try: return str(float(raw.replace(",", ""))) except ValueError: pass core = raw.strip(_STRIP_EDGE) - try: - return str(float(core.replace(",", ""))) - except ValueError: - return _PUNCT.sub("", raw) + for cand in (core, core.rstrip(".")): + try: + return str(float(cand.replace(",", ""))) + except ValueError: + continue + return _PUNCT.sub("", raw) def _split_internal_hyphens(token: str) -> list[str]: