diff --git a/examples/camerontjs-dot_compare_fn/.gitignore b/examples/camerontjs-dot_compare_fn/.gitignore new file mode 100644 index 0000000..3b280be --- /dev/null +++ b/examples/camerontjs-dot_compare_fn/.gitignore @@ -0,0 +1,12 @@ +.env +.env.* +__pycache__/ +*.py[cod] +.venv/ +venv/ +env/ +*.log +*.stderr +*.jsonl +.cache/ +.cache_huggingface/ diff --git a/examples/camerontjs-dot_compare_fn/RESULTS.md b/examples/camerontjs-dot_compare_fn/RESULTS.md new file mode 100644 index 0000000..197afd3 --- /dev/null +++ b/examples/camerontjs-dot_compare_fn/RESULTS.md @@ -0,0 +1,133 @@ +# BOOTH comparator example: lexical overlap vs pinned DeBERTa NLI + +## Question and scope + +Does choosing a semantic natural-language-inference (NLI) `compare_fn` instead of a weak +word-overlap `compare_fn` change the acceptance decisions returned by BOOTH for the +**same fixed claims and evidence**? + +This is an example for [issue #18](https://github.com/Vedantgitbot/booth/issues/18), +related to [provider example issue #2](https://github.com/Vedantgitbot/booth/issues/2). +No BOOTH source code, dependency declarations, or public behavior changes are proposed. + +## Real run details + +- **Date:** 2026-10-19. +- **Upstream BOOTH source base:** `dfe29eb95406bce9822d40cf36babe9e6b753c26`. +- **BOOTH package:** `boothpy==0.5.4` from PyPI, verified with `pip show boothpy`. +- **Python / OS:** CPython 3.9.6, macOS arm64; local CPU inference. +- **Dependencies for this example only:** torch 2.8.0, transformers 4.57.6, sentencepiece 0.2.2. +- **NLI model:** `MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli`, revision `6f5cf0a2b59cabb106aca4c287eed12e357e90eb`. +- **Functions:** `booth.check_with_evidence()`; neither BOOTH `check()` nor an LLM generator is involved. +- **Runs:** 2 initial executions plus a review-revision rerun on the same 10 authored cases in the recorded local environment; no retries or model-generated answers. +- **Script SHA-256 (review revision):** `f8e668163cfd5718c953c6cc27fd1143951169e62e77550803663884d8ccb337`. The original submitted script was `f5079fdbdd2870fa4e563eb21cc2bfa2a49be7a1f6d0b8097ef1967703e0fb93`; only its docstring changed. +- **Standard output SHA-256, both initial local runs and review rerun:** `271bec75f9d7bd435e5b3eaf5b5801f5b0b7fa1c2559a0d9bbaf13f0712820e2`. +- **Pinned model cache artifact size:** 380,001,591 bytes (approximately **380 MB** in decimal units): 368,877,646-byte `model.safetensors` plus approximately 11 MB of tokenizer/configuration files, inspected at the pinned revision. This is the approximate first-run model/tokenizer download payload, excluding environment installation, caching, and network overhead. + +Byte-identical stdout applies to the **two recorded runs on this macOS arm64 environment** only. Full-precision floating-point values, including NLI scores printed in JSON, may differ across CPU/hardware/software platforms. Check the statuses and substantive controls rather than expecting the documented stdout hash to match across machines. + +The answer and evidence sequence for each case are held constant between comparator arms. +The lexical checker counts unique matching words and accepts at a **0.5 overlap threshold**. +DeBERTa scores each passage as a premise against the complete claim as hypothesis, +then passes its **maximum entailment softmax score** to BOOTH at a **0.7 threshold**. +Both thresholds were selected for this demonstration, **not independently calibrated**. +Scores from the two methods do not represent equivalent probability scales. + +All cases are hand-authored demonstrations, not a held-out or source-disjoint evaluation. +Expected `BLOCKED` means *not acceptable under this fixture's intended checkpoint*; +it does **not** mean the evidence logically refutes the claim. An unrelated source, for +example, justifies no acceptance, not a contradiction verdict. + +## Summary + +| Outcome against the authored expected checkpoint status | Weak lexical | Pinned DeBERTa | +|---|---:|---:| +| Cases matching expected status | 4 / 10 | 9 / 10 | +| False accepts (expected blocked, accepted) | 5 | 1 | +| False blocks (expected accepted, blocked) | 1 | 0 | +| Uncertain | 0 | 0 | + +The lexical checker accepts a wrong **90-day** refund window because nearly every word +overlaps a 30-day rule; accepts flipped refund negation and an Enterprise/Standard +scope swap; and rejects a supported paraphrase with different vocabulary. +DeBERTa's checkpoint decisions are better on these examples. + +The important DeBERTa-side failure is **conflicting evidence**: +one passage has `p(entail)=0.993838`, while another has +`p(contradict)=0.996633`. DeBERTa *identified both relations*, but the +simple `max(p_entail)` comparator discarded contradiction information, +so BOOTH received a high score and returned `ACCEPTED`. +This is an adapter/aggregation limitation, **not a BOOTH core defect**. + +Another limitation appears in the **scope swap** case. The evidence is about +Enterprise plans but the claim is about Standard plans. DeBERTa assigns the +passage its top label `contradiction` (about 0.752), although the evidence +does not logically refute an unrelated Standard-plan rule. BOOTH correctly +does not accept the claim in this fixture, but that does **not** validate +DeBERTa's finer contradiction labeling. + +The current comparison provides an illustrative **capability contrast**. +It does not establish general comparator accuracy, model calibration, actual +source legitimacy, corpus sufficiency, or safe deployment in an evidence pipeline. + +## Actual stdout from run 1 + +The following is the complete output printed by `run_example.py` (stdout, +not manually recomposed predictions). Run 2 and the review-revision rerun produced the +same stdout bytes **locally**; this is not a cross-platform byte-identity guarantee. + +```text +CONFIG {"booth": "0.5.4", "lexical_threshold": 0.5, "machine": "arm64", "model": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli", "model_revision": "6f5cf0a2b59cabb106aca4c287eed12e357e90eb", "nli_threshold": 0.7, "python": "3.9.6", "system": "Darwin", "torch": "2.8.0", "transformers": "4.57.6"} +CASE {"control": "positive", "expected": "ACCEPTED", "id": "exact_support", "lexical": {"detail": null, "evidence_agreement": 1.0, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": null, "evidence_agreement": 0.993838369846344, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli_passages": [{"label": "entailment", "p_contradict": 0.0009218579507432878, "p_entail": 0.993838369846344, "p_neutral": 0.005239792633801699}]} +CASE {"control": "paraphrase-invariance", "expected": "ACCEPTED", "id": "paraphrase_support", "lexical": {"detail": "score 0.3076923076923077 below evidence_threshold 0.5", "evidence_agreement": 0.3076923076923077, "matches_expected": false, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli": {"detail": null, "evidence_agreement": 0.9941997528076172, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli_passages": [{"label": "entailment", "p_contradict": 0.00045581915765069425, "p_entail": 0.9941997528076172, "p_neutral": 0.005344494711607695}]} +CASE {"control": "quantity-mutation", "expected": "BLOCKED", "id": "wrong_number", "lexical": {"detail": null, "evidence_agreement": 0.9166666666666666, "matches_expected": false, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": "score 0.001540952012874186 below evidence_threshold 0.7", "evidence_agreement": 0.001540952012874186, "matches_expected": true, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli_passages": [{"label": "contradiction", "p_contradict": 0.995970606803894, "p_entail": 0.001540952012874186, "p_neutral": 0.0024884925223886967}]} +CASE {"control": "negation-mutation", "expected": "BLOCKED", "id": "negation_flip", "lexical": {"detail": null, "evidence_agreement": 1.0, "matches_expected": false, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": "score 0.0007738820859231055 below evidence_threshold 0.7", "evidence_agreement": 0.0007738820859231055, "matches_expected": true, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli_passages": [{"label": "contradiction", "p_contradict": 0.9982706308364868, "p_entail": 0.0007738820859231055, "p_neutral": 0.0009555029100738466}]} +CASE {"control": "entity-scope-mutation", "expected": "BLOCKED", "id": "scope_swap", "lexical": {"detail": null, "evidence_agreement": 0.8571428571428571, "matches_expected": false, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": "score 0.07546699792146683 below evidence_threshold 0.7", "evidence_agreement": 0.07546699792146683, "matches_expected": true, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli_passages": [{"label": "contradiction", "p_contradict": 0.751786470413208, "p_entail": 0.07546699792146683, "p_neutral": 0.17274649441242218}]} +CASE {"control": "policy-negation-mutation", "expected": "BLOCKED", "id": "cancellation_refund_inversion", "lexical": {"detail": null, "evidence_agreement": 0.6363636363636364, "matches_expected": false, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": "score 0.0005258059245534241 below evidence_threshold 0.7", "evidence_agreement": 0.0005258059245534241, "matches_expected": true, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli_passages": [{"label": "contradiction", "p_contradict": 0.9978524446487427, "p_entail": 0.0005258059245534241, "p_neutral": 0.001621777075342834}]} +CASE {"control": "irrelevant-evidence-negative", "expected": "BLOCKED", "id": "unrelated_evidence", "lexical": {"detail": "score 0.0 below evidence_threshold 0.5", "evidence_agreement": 0.0, "matches_expected": true, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli": {"detail": "score 0.0009648687555454671 below evidence_threshold 0.7", "evidence_agreement": 0.0009648687555454671, "matches_expected": true, "reason": "EVIDENCE_DISAGREES", "status": "BLOCKED"}, "nli_passages": [{"label": "neutral", "p_contradict": 0.0003691909369081259, "p_entail": 0.0009648687555454671, "p_neutral": 0.9986659288406372}]} +CASE {"control": "conflict-negative", "expected": "BLOCKED", "id": "conflicting_evidence", "lexical": {"detail": null, "evidence_agreement": 1.0, "matches_expected": false, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": null, "evidence_agreement": 0.993838369846344, "matches_expected": false, "reason": null, "status": "ACCEPTED"}, "nli_passages": [{"label": "entailment", "p_contradict": 0.0009218579507432878, "p_entail": 0.993838369846344, "p_neutral": 0.005239792633801699}, {"label": "contradiction", "p_contradict": 0.9966332316398621, "p_entail": 0.0006440724246203899, "p_neutral": 0.0027227369137108326}]} +CASE {"control": "irrelevant-evidence-invariance", "expected": "ACCEPTED", "id": "support_plus_irrelevant", "lexical": {"detail": null, "evidence_agreement": 1.0, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": null, "evidence_agreement": 0.993838369846344, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli_passages": [{"label": "entailment", "p_contradict": 0.0009218579507432878, "p_entail": 0.993838369846344, "p_neutral": 0.005239792633801699}, {"label": "neutral", "p_contradict": 0.0006168576655909419, "p_entail": 0.0011228114599362016, "p_neutral": 0.9982603192329407}]} +CASE {"control": "duplicate-invariance", "expected": "ACCEPTED", "id": "duplicate_support", "lexical": {"detail": null, "evidence_agreement": 1.0, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli": {"detail": null, "evidence_agreement": 0.993838369846344, "matches_expected": true, "reason": null, "status": "ACCEPTED"}, "nli_passages": [{"label": "entailment", "p_contradict": 0.0009218579507432878, "p_entail": 0.993838369846344, "p_neutral": 0.005239792633801699}, {"label": "entailment", "p_contradict": 0.0009218579507432878, "p_entail": 0.993838369846344, "p_neutral": 0.005239792633801699}]} +SUMMARY {"lexical": {"correct": 4, "false_accept": 5, "false_block": 1, "total": 10, "uncertain": 0}, "nli": {"correct": 9, "false_accept": 1, "false_block": 0, "total": 10, "uncertain": 0}, "note": "Small author-designed controls; no source-disjoint evaluation, model calibration, or factual-truth warrant."} +``` + +Both runs exited 0. Their stderr contained a local Python LibreSSL/urllib3 +compatibility warning and a transformers warning that the tokenizer has no +predefined maximum sequence length. Neither warning changed this short-fixture +run. Long-input truncation and source-context loss are **not evaluated** here. + +## Reproduction + +From the BOOTH checkout root containing this example (base at creation: +`dfe29eb95406bce9822d40cf36babe9e6b753c26`), use a **clean Python 3.9+ environment**. +For macOS/Linux: + +```bash +python3 -m venv .venv +. .venv/bin/activate +python -m pip install boothpy==0.5.4 torch==2.8.0 transformers==4.57.6 sentencepiece==0.2.2 +python examples/camerontjs-dot_compare_fn/run_example.py +``` + +The example's `import booth` must resolve to **`boothpy==0.5.4` installed from PyPI**, **not to BOOTH imported from the checkout**. The checkout supplies the example script and the separately tested upstream suite; do not `pip install -e .` or put the checkout's `src/` on `PYTHONPATH` in the example environment. The recorded local import resolved to the virtual environment's `site-packages/booth`. + +The first run downloads approximately **380 MB** of model and tokenizer files at the pinned revision unless already cached (approximately **369 MB model weights + 11 MB tokenizer/configuration**); allow additional disk space for installed dependencies and cache overhead. An API key is not needed. The example adds no dependency to BOOTH itself. +The script reports missing packages and model-load failures explicitly. + +The **existing upstream pytest suite** was run from this BOOTH checkout +under the same Python environment, with `PYTHONPATH="$PWD/src" python -m pytest -v` +so suite imports resolve to the checkout source: **348 passed** on the review +revision. The final script rerun exited 0, and its stdout matched the recorded +block byte-for-byte; the negative missing-dependency check exited 1 with a +clear installation message. Only example-folder files are changed. + +## What this example teaches + +BOOTH is a checkpoint wrapper: its `ACCEPTED` and `BLOCKED` reflect +the comparator the application supplies. A stronger semantic comparator +can improve useful acceptance and rejection on fixed claims, while still +misreading scope or hiding a material conflicting passage if the adapter +collapses richer outputs to a scalar. A valid comparison result is not proof +of objective truth. The minimum next check before treating an NLI-based +comparator as dependable would be fresh, independently adjudicated claims +with measured false acceptances, false blocks, neutral cases, and conflicts. diff --git a/examples/camerontjs-dot_compare_fn/run_example.py b/examples/camerontjs-dot_compare_fn/run_example.py new file mode 100644 index 0000000..9613eaf --- /dev/null +++ b/examples/camerontjs-dot_compare_fn/run_example.py @@ -0,0 +1,286 @@ +""" +Compare a weak lexical BOOTH evidence checker against a pinned DeBERTa NLI checker. + +This is an illustration of compare_fn behavior, not a factual-verification benchmark. +No LLM generator, retrieval system, API key, or BOOTH code changes are required. +Both comparators receive exactly the same authored answer and evidence per case. + +From the BOOTH checkout root on macOS/Linux, in a clean Python 3.9+ +environment (tested on Python 3.9.6, macOS arm64): + + python3 -m venv .venv + . .venv/bin/activate + python -m pip install boothpy==0.5.4 torch==2.8.0 transformers==4.57.6 sentencepiece==0.2.2 + python examples/camerontjs-dot_compare_fn/run_example.py + +This example imports boothpy==0.5.4 installed from PyPI, NOT BOOTH from the +checkout. Do not install the checkout in editable mode or put its src/ on +PYTHONPATH in the example environment. + +The first execution downloads about 380 MB of pinned model/tokenizer files if +not cached (observed: 369 MB weights plus 11 MB tokenizer/configuration). +Allow extra disk space for environment and cache overhead. No API key needed. +The example dependencies do not become BOOTH runtime dependencies. +""" +from __future__ import annotations + +import json +import platform +import re + +try: + import booth + import torch + import transformers + from transformers import AutoModelForSequenceClassification, AutoTokenizer +except ImportError as exc: + raise SystemExit( + f"Missing example dependency ({exc.name}). Install boothpy, torch, " + "transformers, and sentencepiece as shown in this script's docstring." + ) from exc + +MODEL_ID = "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli" +REVISION = "6f5cf0a2b59cabb106aca4c287eed12e357e90eb" + +# Intentionally different, uncalibrated defaults for differently scaled scores. +# Do not compare the numeric scores as if they were calibrated probabilities. +WEAK_THRESHOLD = 0.5 +NLI_THRESHOLD = 0.7 + +# Illustrative development cases, not independent or held-out accuracy evidence. +CASES = [ + { + "id": "exact_support", + "answer": "Standard plans can be refunded within 30 days of purchase.", + "evidence": ["Standard plans can be refunded within 30 days of the initial purchase."], + "expected": "ACCEPTED", + "control": "positive", + }, + { + "id": "paraphrase_support", + "answer": "Customers with a Standard plan may obtain a refund during the first 30 days.", + "evidence": ["Standard plans can be refunded within 30 days of the initial purchase."], + "expected": "ACCEPTED", + "control": "paraphrase-invariance", + }, + { + "id": "wrong_number", + "answer": "Standard plans can be refunded within 90 days of the initial purchase.", + "evidence": ["Standard plans can be refunded within 30 days of the initial purchase."], + "expected": "BLOCKED", + "control": "quantity-mutation", + }, + { + "id": "negation_flip", + "answer": "Enterprise plans are refundable after activation.", + "evidence": ["Enterprise plans are non-refundable after activation."], + "expected": "BLOCKED", + "control": "negation-mutation", + }, + { + "id": "scope_swap", + "answer": "Standard plans are non-refundable after activation.", + "evidence": ["Enterprise plans are non-refundable after activation."], + "expected": "BLOCKED", + "control": "entity-scope-mutation", + }, + { + "id": "cancellation_refund_inversion", + "answer": "Cancelling an Annual plan at any time automatically creates a refund.", + "evidence": ["Annual plans may be cancelled at any time, but cancellation does not automatically create a refund."], + "expected": "BLOCKED", + "control": "policy-negation-mutation", + }, + { + "id": "unrelated_evidence", + "answer": "Standard plans can be refunded within 30 days.", + "evidence": ["Technical support is available 24 hours a day."], + "expected": "BLOCKED", + "control": "irrelevant-evidence-negative", + }, + { + "id": "conflicting_evidence", + "answer": "Standard plans can be refunded within 30 days of purchase.", + "evidence": [ + "Standard plans can be refunded within 30 days of the initial purchase.", + "Standard plans are non-refundable after purchase, including during the first 30 days.", + ], + "expected": "BLOCKED", + "control": "conflict-negative", + }, + { + "id": "support_plus_irrelevant", + "answer": "Standard plans can be refunded within 30 days of purchase.", + "evidence": [ + "Standard plans can be refunded within 30 days of the initial purchase.", + "Technical support is available 24 hours a day.", + ], + "expected": "ACCEPTED", + "control": "irrelevant-evidence-invariance", + }, + { + "id": "duplicate_support", + "answer": "Standard plans can be refunded within 30 days of purchase.", + "evidence": [ + "Standard plans can be refunded within 30 days of the initial purchase.", + "Standard plans can be refunded within 30 days of the initial purchase.", + ], + "expected": "ACCEPTED", + "control": "duplicate-invariance", + }, +] + + + +WORD_RE = re.compile(r"[A-Za-z0-9]+") + + +def lexical_overlap(answer: str, evidence: list[str]) -> float: + """Fraction of unique answer tokens present in the evidence, ignoring meaning.""" + answer_words = set(WORD_RE.findall(answer.lower())) + evidence_words = set(WORD_RE.findall(" ".join(evidence).lower())) + return len(answer_words & evidence_words) / len(answer_words) if answer_words else 0.0 + + +class PinnedDeBERTaComparator: + """Passage-level three-way NLI, adapted to BOOTH's scalar compare_fn.""" + + def __init__(self) -> None: + torch.set_num_threads(1) + torch.manual_seed(0) + try: + tokenizer = AutoTokenizer.from_pretrained( + MODEL_ID, revision=REVISION, trust_remote_code=False + ) + model = AutoModelForSequenceClassification.from_pretrained( + MODEL_ID, revision=REVISION, trust_remote_code=False + ) + except (OSError, ValueError) as exc: + raise SystemExit( + f"Could not load pinned NLI model {MODEL_ID}@{REVISION}. " + "Check network access, available disk space, and installed dependencies. " + f"Original error: {exc}" + ) from exc + + self.tokenizer = tokenizer + self.model = model.to("cpu").eval() + labels = [model.config.id2label[i].lower() for i in range(len(model.config.id2label))] + expected_labels = {"entailment", "neutral", "contradiction"} + if set(labels) != expected_labels: + raise SystemExit(f"Unexpected NLI labels: {labels}; expected {expected_labels}") + self.labels = labels + self.indices = {label: index for index, label in enumerate(labels)} + self.last_passages: list[dict] = [] + + def __call__(self, answer: str, evidence: list[str]) -> float: + self.last_passages = [] + for passage in evidence: + # NLI order is premise (evidence), hypothesis (answer). + tokens = self.tokenizer(passage, answer, return_tensors="pt", truncation=True) + with torch.no_grad(): + logits = self.model(**tokens).logits[0] + probs = torch.softmax(logits, dim=-1) + label = self.labels[int(torch.argmax(logits))] + record = { + "label": label, + "p_entail": float(probs[self.indices["entailment"]]), + "p_neutral": float(probs[self.indices["neutral"]]), + "p_contradict": float(probs[self.indices["contradiction"]]), + } + self.last_passages.append(record) + + # Common simple adapter: a claim passes if any passage strongly entails it. + # It intentionally does NOT resolve conflicting evidence. The conflict + # control shows how this scalar projection can hide contradiction. + return max((record["p_entail"] for record in self.last_passages), default=0.0) + + +def assess(result: booth.BoothResult, expected: str) -> dict: + return { + "status": result.status, + "reason": result.reason, + "detail": result.detail, + "evidence_agreement": result.evidence_agreement, + "matches_expected": result.status == expected, + } + + +def summarize(rows: list[dict], arm: str) -> dict: + return { + "correct": sum(row[arm]["matches_expected"] for row in rows), + "total": len(rows), + "false_accept": sum( + row["expected"] == booth.BLOCKED and row[arm]["status"] == booth.ACCEPTED + for row in rows + ), + "false_block": sum( + row["expected"] == booth.ACCEPTED and row[arm]["status"] == booth.BLOCKED + for row in rows + ), + "uncertain": sum(row[arm]["status"] == booth.UNCERTAIN for row in rows), + } + + +def main() -> None: + print( + "CONFIG " + + json.dumps( + { + "python": platform.python_version(), + "system": platform.system(), + "machine": platform.machine(), + "booth": booth.__version__, + "torch": torch.__version__, + "transformers": transformers.__version__, + "model": MODEL_ID, + "model_revision": REVISION, + "lexical_threshold": WEAK_THRESHOLD, + "nli_threshold": NLI_THRESHOLD, + }, + sort_keys=True, + ) + ) + nli_compare = PinnedDeBERTaComparator() + rows = [] + for case in CASES: + weak_result = booth.check_with_evidence( + answer=case["answer"], + evidence=case["evidence"], + compare_fn=lexical_overlap, + evidence_threshold=WEAK_THRESHOLD, + ) + nli_result = booth.check_with_evidence( + answer=case["answer"], + evidence=case["evidence"], + compare_fn=nli_compare, + evidence_threshold=NLI_THRESHOLD, + ) + row = { + "id": case["id"], + "control": case["control"], + "expected": case["expected"], + "lexical": assess(weak_result, case["expected"]), + "nli": assess(nli_result, case["expected"]), + "nli_passages": list(nli_compare.last_passages), + } + rows.append(row) + print("CASE " + json.dumps(row, sort_keys=True)) + + print( + "SUMMARY " + + json.dumps( + { + "lexical": summarize(rows, "lexical"), + "nli": summarize(rows, "nli"), + "note": ( + "Small author-designed controls; no source-disjoint evaluation, " + "model calibration, or factual-truth warrant." + ), + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main()