From 65ed673e126e273d6d4eb6ee3940438e0b8978a2 Mon Sep 17 00:00:00 2001 From: jordanyushen <245894745+jordanyushen@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:07:03 +0800 Subject: [PATCH] feat: establish verification framework --- .github/workflows/tests.yml | 5 + beamcheck/verification/__init__.py | 11 + beamcheck/verification/__main__.py | 7 + beamcheck/verification/compare.py | 63 +++++ beamcheck/verification/loader.py | 249 ++++++++++++++++++ beamcheck/verification/models.py | 97 +++++++ beamcheck/verification/runner.py | 99 +++++++ requirements.txt | 1 + tests/test_verification_corpus.py | 197 ++++++++++++++ verification/README.md | 37 +++ verification/cases/BC-0001/case.json | 33 +++ verification/cases/BC-0001/expected.json | 5 + verification/cases/BC-0001/reference.md | 28 ++ verification/cases/BC-0002/case.json | 32 +++ verification/cases/BC-0002/expected.json | 5 + verification/cases/BC-0002/reference.md | 28 ++ verification/cases/BC-0003/case.json | 33 +++ verification/cases/BC-0003/expected.json | 5 + verification/cases/BC-0003/reference.md | 28 ++ verification/cases/BC-0004/case.json | 32 +++ verification/cases/BC-0004/expected.json | 5 + verification/cases/BC-0004/reference.md | 28 ++ verification/generated/README.md | 5 + .../schema/verification_case.schema.json | 139 ++++++++++ .../schema/verification_result.schema.json | 50 ++++ 25 files changed, 1222 insertions(+) create mode 100644 beamcheck/verification/__init__.py create mode 100644 beamcheck/verification/__main__.py create mode 100644 beamcheck/verification/compare.py create mode 100644 beamcheck/verification/loader.py create mode 100644 beamcheck/verification/models.py create mode 100644 beamcheck/verification/runner.py create mode 100644 tests/test_verification_corpus.py create mode 100644 verification/README.md create mode 100644 verification/cases/BC-0001/case.json create mode 100644 verification/cases/BC-0001/expected.json create mode 100644 verification/cases/BC-0001/reference.md create mode 100644 verification/cases/BC-0002/case.json create mode 100644 verification/cases/BC-0002/expected.json create mode 100644 verification/cases/BC-0002/reference.md create mode 100644 verification/cases/BC-0003/case.json create mode 100644 verification/cases/BC-0003/expected.json create mode 100644 verification/cases/BC-0003/reference.md create mode 100644 verification/cases/BC-0004/case.json create mode 100644 verification/cases/BC-0004/expected.json create mode 100644 verification/cases/BC-0004/reference.md create mode 100644 verification/generated/README.md create mode 100644 verification/schema/verification_case.schema.json create mode 100644 verification/schema/verification_result.schema.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e90f4db..a88b479 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,3 +28,8 @@ jobs: env: MPLBACKEND: Agg run: python -m pytest -q + + - name: Run verification corpus + env: + MPLBACKEND: Agg + run: python -m beamcheck.verification diff --git a/beamcheck/verification/__init__.py b/beamcheck/verification/__init__.py new file mode 100644 index 0000000..2ca572e --- /dev/null +++ b/beamcheck/verification/__init__.py @@ -0,0 +1,11 @@ +"""Versioned verification corpus loading and execution.""" + +from .loader import DEFAULT_CORPUS_ROOT, VerificationDataError, load_corpus +from .runner import run_verification + +__all__ = [ + "DEFAULT_CORPUS_ROOT", + "VerificationDataError", + "load_corpus", + "run_verification", +] diff --git a/beamcheck/verification/__main__.py b/beamcheck/verification/__main__.py new file mode 100644 index 0000000..7255553 --- /dev/null +++ b/beamcheck/verification/__main__.py @@ -0,0 +1,7 @@ +"""Run the BeamCheck verification corpus with ``python -m``.""" + +from .runner import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/beamcheck/verification/compare.py b/beamcheck/verification/compare.py new file mode 100644 index 0000000..e9b8b87 --- /dev/null +++ b/beamcheck/verification/compare.py @@ -0,0 +1,63 @@ +"""Compare solver results with independently supplied expected quantities.""" + +from __future__ import annotations + +from collections.abc import Callable +from math import inf, isfinite + +from beamcheck.core.models import CalculationResult + +from .models import QuantityComparison, ToleranceType, VerificationCase + + +QuantityAccessor = Callable[[CalculationResult], float] + + +QUANTITY_ACCESSORS: dict[str, QuantityAccessor] = { + "left_reaction_n": lambda result: result.reactions.left_vertical, + "right_reaction_n": lambda result: result.reactions.right_vertical, + "fixed_vertical_n": lambda result: result.reactions.fixed_vertical, + "fixed_moment_nm": lambda result: result.reactions.fixed_moment, + "max_shear_n": lambda result: result.max_abs_shear, + "max_moment_nm": lambda result: result.max_abs_moment, + "max_bending_stress_pa": lambda result: result.max_bending_stress, + "max_deflection_m": lambda result: result.max_deflection, + "deflection_location_m": lambda result: result.deflection_location, + "factor_of_safety": lambda result: result.factor_of_safety, + "allowable_deflection_m": lambda result: result.deflection_check.allowable, +} + + +def compare_case( + case: VerificationCase, result: CalculationResult +) -> tuple[QuantityComparison, ...]: + comparisons: list[QuantityComparison] = [] + for quantity, expected in case.expected.items(): + try: + actual = float(QUANTITY_ACCESSORS[quantity](result)) + except KeyError as exc: + raise ValueError(f"Unsupported expected quantity: {quantity}") from exc + if not isfinite(actual): + raise ValueError(f"Solver produced a non-finite value for {quantity}.") + + absolute_difference = abs(actual - expected.value) + relative_difference = ( + absolute_difference / abs(expected.value) if expected.value != 0 else inf + ) + if expected.tolerance.type is ToleranceType.ABSOLUTE: + passed = absolute_difference <= expected.tolerance.value + else: + passed = relative_difference <= expected.tolerance.value + comparisons.append( + QuantityComparison( + case_id=case.case_id, + quantity=quantity, + expected=expected.value, + actual=actual, + tolerance=expected.tolerance, + absolute_difference=absolute_difference, + relative_difference=relative_difference, + passed=passed, + ) + ) + return tuple(comparisons) diff --git a/beamcheck/verification/loader.py b/beamcheck/verification/loader.py new file mode 100644 index 0000000..b9a4316 --- /dev/null +++ b/beamcheck/verification/loader.py @@ -0,0 +1,249 @@ +"""Strict loading of versioned verification data assets.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator + +from beamcheck.core.loads import FullSpanUDL, PointLoad +from beamcheck.core.materials import Material +from beamcheck.core.models import ( + Beam, + BeamType, + CalculationInput, + DeflectionCriterion, + ProjectInfo, +) +from beamcheck.core.sections import ( + CircularSection, + HollowRectangularSection, + RectangularSection, + SquareHollowSection, +) + +from .models import ( + ExpectedQuantity, + Tolerance, + ToleranceType, + VerificationCase, + VerificationStatus, +) + + +DATASET_SCHEMA_VERSION = "1.0" +DEFAULT_CORPUS_ROOT = Path(__file__).resolve().parents[2] / "verification" + + +class VerificationDataError(ValueError): + """Raised when corpus data is malformed or semantically inconsistent.""" + + +def _reject_json_constant(token: str) -> None: + raise VerificationDataError(f"Non-finite JSON number is not allowed: {token}") + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads( + path.read_text(encoding="utf-8"), parse_constant=_reject_json_constant + ) + except VerificationDataError: + raise + except (OSError, json.JSONDecodeError) as exc: + raise VerificationDataError(f"Cannot read valid JSON from {path}: {exc}") from exc + if not isinstance(value, dict): + raise VerificationDataError(f"Top-level JSON value must be an object: {path}") + return value + + +def _validate_schema(data: dict[str, Any], schema_path: Path, data_path: Path) -> None: + schema = _read_json(schema_path) + errors = sorted( + Draft202012Validator(schema).iter_errors(data), + key=lambda error: tuple(str(part) for part in error.absolute_path), + ) + if errors: + error = errors[0] + location = ".".join(str(part) for part in error.absolute_path) or "" + raise VerificationDataError( + f"Schema validation failed for {data_path} at {location}: {error.message}" + ) + + +def _build_input(data: dict[str, Any], case_id: str, title: str) -> CalculationInput: + beam_data = data["beam"] + beam_type = { + "simply_supported": BeamType.SIMPLY_SUPPORTED, + "cantilever": BeamType.CANTILEVER, + }[beam_data["type"]] + beam = Beam(beam_type=beam_type, length=float(beam_data["length_m"])) + + material_data = data["material"] + material = Material( + name=material_data["name"], + youngs_modulus=float(material_data["youngs_modulus_pa"]), + yield_strength=float(material_data["yield_strength_pa"]), + density=float(material_data["density_kg_m3"]), + ) + + section_data = data["section"] + section_type = section_data["type"] + if section_type == "rectangular": + section = RectangularSection( + width=float(section_data["width_m"]), + height=float(section_data["height_m"]), + ) + elif section_type == "circular": + section = CircularSection(diameter=float(section_data["diameter_m"])) + elif section_type == "rectangular_hollow": + section = HollowRectangularSection( + outer_width=float(section_data["outer_width_m"]), + outer_height=float(section_data["outer_height_m"]), + wall_thickness=float(section_data["wall_thickness_m"]), + ) + else: + outer_size = float(section_data["outer_size_m"]) + section = SquareHollowSection( + outer_width=outer_size, + outer_height=outer_size, + wall_thickness=float(section_data["wall_thickness_m"]), + ) + + loads = [] + for load_data in data["loads"]: + if load_data["type"] == "point": + loads.append( + PointLoad( + magnitude=float(load_data["magnitude_n"]), + position=float(load_data["position_m"]), + ) + ) + else: + loads.append(FullSpanUDL(magnitude=float(load_data["magnitude_n_per_m"]))) + + criterion_data = data["criterion"] + if criterion_data["type"] == "span_ratio": + criterion = DeflectionCriterion(ratio=float(criterion_data["ratio"])) + else: + criterion = DeflectionCriterion( + ratio=None, + custom_allowable=float(criterion_data["allowable_deflection_m"]), + ) + + calculation_input = CalculationInput( + beam=beam, + material=material, + section=section, + loads=tuple(loads), + criterion=criterion, + project=ProjectInfo(project_name=case_id, calculation_title=title), + ) + calculation_input.validate() + return calculation_input + + +def load_case(case_directory: Path, corpus_root: Path = DEFAULT_CORPUS_ROOT) -> VerificationCase: + case_path = case_directory / "case.json" + expected_path = case_directory / "expected.json" + reference_path = case_directory / "reference.md" + schema_root = corpus_root / "schema" + + case_data = _read_json(case_path) + expected_data = _read_json(expected_path) + _validate_schema(case_data, schema_root / "verification_case.schema.json", case_path) + _validate_schema( + expected_data, schema_root / "verification_result.schema.json", expected_path + ) + + try: + reference_text = reference_path.read_text(encoding="utf-8") + except OSError as exc: + raise VerificationDataError(f"Cannot read reference file {reference_path}: {exc}") from exc + + if case_data["schema_version"] != DATASET_SCHEMA_VERSION: + raise VerificationDataError( + f"Unsupported case schema version in {case_path}: {case_data['schema_version']}" + ) + if expected_data["schema_version"] != DATASET_SCHEMA_VERSION: + raise VerificationDataError( + f"Unsupported expected schema version in {expected_path}: " + f"{expected_data['schema_version']}" + ) + if case_data["case_id"] != expected_data["case_id"]: + raise VerificationDataError( + f"case_id mismatch between {case_path} and {expected_path}." + ) + if case_data["case_id"] != case_directory.name: + raise VerificationDataError( + f"case_id {case_data['case_id']} must match directory {case_directory.name}." + ) + if case_data["expected_reference_id"] not in reference_text: + raise VerificationDataError( + f"Reference file for {case_data['case_id']} must contain expected_reference_id " + f"{case_data['expected_reference_id']}." + ) + + try: + status = VerificationStatus(case_data["status"]) + expected = { + name: ExpectedQuantity( + value=float(quantity["value"]), + tolerance=Tolerance( + type=ToleranceType(quantity["tolerance"]["type"]), + value=float(quantity["tolerance"]["value"]), + ), + ) + for name, quantity in expected_data["expected"].items() + } + calculation_input = _build_input( + case_data, case_data["case_id"], case_data["title"] + ) + except (KeyError, TypeError, ValueError) as exc: + raise VerificationDataError( + f"Invalid verification semantics in {case_directory}: {exc}" + ) from exc + + if status is VerificationStatus.VERIFIED: + if not expected: + raise VerificationDataError( + f"Verified case {case_data['case_id']} must declare expected quantities." + ) + if "STATUS: OWNER_REFERENCE_REQUIRED" in reference_text: + raise VerificationDataError( + f"Verified case {case_data['case_id']} cannot use an owner-reference placeholder." + ) + if "OWNER-REQUIRED" in case_data["expected_reference_id"]: + raise VerificationDataError( + f"Verified case {case_data['case_id']} must use a reviewed reference ID." + ) + + return VerificationCase( + schema_version=case_data["schema_version"], + case_id=case_data["case_id"], + title=case_data["title"], + status=status, + expected_reference_id=case_data["expected_reference_id"], + calculation_input=calculation_input, + expected=expected, + case_directory=case_directory, + reference_text=reference_text, + ) + + +def load_corpus(corpus_root: Path = DEFAULT_CORPUS_ROOT) -> tuple[VerificationCase, ...]: + cases_root = corpus_root / "cases" + if not cases_root.is_dir(): + raise VerificationDataError(f"Verification cases directory does not exist: {cases_root}") + case_directories = sorted( + path for path in cases_root.iterdir() if path.is_dir() and path.name.startswith("BC-") + ) + if not case_directories: + raise VerificationDataError(f"No verification cases found in {cases_root}") + cases = tuple(load_case(path, corpus_root) for path in case_directories) + case_ids = [case.case_id for case in cases] + if len(case_ids) != len(set(case_ids)): + raise VerificationDataError("Verification corpus contains duplicate case IDs.") + return cases diff --git a/beamcheck/verification/models.py b/beamcheck/verification/models.py new file mode 100644 index 0000000..ba4378f --- /dev/null +++ b/beamcheck/verification/models.py @@ -0,0 +1,97 @@ +"""Typed models for verification cases and comparison results.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from math import isfinite +from pathlib import Path + +from beamcheck.core.models import CalculationInput + + +class VerificationStatus(str, Enum): + DRAFT = "draft" + VERIFIED = "verified" + + +class ToleranceType(str, Enum): + ABSOLUTE = "absolute" + RELATIVE = "relative" + + +@dataclass(frozen=True) +class Tolerance: + type: ToleranceType + value: float + + def __post_init__(self) -> None: + if not isfinite(self.value) or self.value < 0: + raise ValueError("Tolerance must be a finite, non-negative number.") + + +@dataclass(frozen=True) +class ExpectedQuantity: + value: float + tolerance: Tolerance + + def __post_init__(self) -> None: + if not isfinite(self.value): + raise ValueError("Expected values must be finite numbers.") + if self.value == 0 and self.tolerance.type is ToleranceType.RELATIVE: + raise ValueError("An expected value of zero requires an absolute tolerance.") + + +@dataclass(frozen=True) +class VerificationCase: + schema_version: str + case_id: str + title: str + status: VerificationStatus + expected_reference_id: str + calculation_input: CalculationInput + expected: dict[str, ExpectedQuantity] + case_directory: Path + reference_text: str + + +@dataclass(frozen=True) +class QuantityComparison: + case_id: str + quantity: str + expected: float + actual: float + tolerance: Tolerance + absolute_difference: float + relative_difference: float + passed: bool + + def failure_message(self) -> str: + return ( + f"{self.case_id} {self.quantity}: expected={self.expected:.17g}, " + f"actual={self.actual:.17g}, tolerance={self.tolerance.type.value} " + f"{self.tolerance.value:.17g}, absolute_difference={self.absolute_difference:.17g}, " + f"relative_difference={self.relative_difference:.17g}" + ) + + +@dataclass(frozen=True) +class CaseRunResult: + case_id: str + status: str + comparisons: tuple[QuantityComparison, ...] = () + reason: str = "" + + +@dataclass(frozen=True) +class VerificationSummary: + discovered: int + verified: int + draft: int + passed: int + failed: int + results: tuple[CaseRunResult, ...] + + @property + def exit_code(self) -> int: + return 1 if self.failed else 0 diff --git a/beamcheck/verification/runner.py b/beamcheck/verification/runner.py new file mode 100644 index 0000000..2084c38 --- /dev/null +++ b/beamcheck/verification/runner.py @@ -0,0 +1,99 @@ +"""Command-line runner for the versioned verification corpus.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import TextIO + +from beamcheck.core.solver import solve + +from .compare import compare_case +from .loader import DEFAULT_CORPUS_ROOT, VerificationDataError, load_corpus +from .models import CaseRunResult, VerificationStatus, VerificationSummary + + +def run_verification( + corpus_root: Path = DEFAULT_CORPUS_ROOT, stream: TextIO = sys.stdout +) -> VerificationSummary: + cases = load_corpus(corpus_root) + verified = sum(case.status is VerificationStatus.VERIFIED for case in cases) + draft = len(cases) - verified + results: list[CaseRunResult] = [] + + print("BeamCheck Verification", file=stream) + print(file=stream) + print(f"Cases discovered: {len(cases)}", file=stream) + print(f"Verified cases: {verified}", file=stream) + print(f"Draft/unreferenced: {draft}", file=stream) + print(file=stream) + + passed = 0 + failed = 0 + for case in cases: + if case.status is VerificationStatus.DRAFT: + reason = "owner reference required" + print(f"SKIP {case.case_id} - {reason}", file=stream) + results.append(CaseRunResult(case.case_id, "SKIP", reason=reason)) + continue + try: + comparisons = compare_case(case, solve(case.calculation_input)) + failures = tuple(comparison for comparison in comparisons if not comparison.passed) + if failures: + failed += 1 + print(f"FAIL {case.case_id}", file=stream) + for failure in failures: + print(f" {failure.failure_message()}", file=stream) + results.append( + CaseRunResult( + case.case_id, + "FAIL", + comparisons=comparisons, + reason="one or more quantities exceeded tolerance", + ) + ) + else: + passed += 1 + print(f"PASS {case.case_id}", file=stream) + results.append( + CaseRunResult(case.case_id, "PASS", comparisons=comparisons) + ) + except Exception as exc: + failed += 1 + reason = f"{type(exc).__name__}: {exc}" + print(f"FAIL {case.case_id} - {reason}", file=stream) + results.append(CaseRunResult(case.case_id, "FAIL", reason=reason)) + + print(file=stream) + print(f"Verified passed: {passed}/{verified}", file=stream) + if failed: + print(f"Verified failed: {failed}", file=stream) + return VerificationSummary( + discovered=len(cases), + verified=verified, + draft=draft, + passed=passed, + failed=failed, + results=tuple(results), + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run the BeamCheck verification corpus.") + parser.add_argument( + "--corpus", + type=Path, + default=DEFAULT_CORPUS_ROOT, + help="Path to the verification corpus root.", + ) + args = parser.parse_args(argv) + try: + return run_verification(args.corpus).exit_code + except VerificationDataError as exc: + print(f"Verification data error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/requirements.txt b/requirements.txt index b796550..15a0748 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ PySide6>=6.7,<7 matplotlib>=3.8,<4 reportlab>=4.2,<6 +jsonschema>=4.23,<5 pytest>=8,<10 pyinstaller>=6.10,<7 diff --git a/tests/test_verification_corpus.py b/tests/test_verification_corpus.py new file mode 100644 index 0000000..0332470 --- /dev/null +++ b/tests/test_verification_corpus.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import io +import json +import shutil +from dataclasses import replace + +import pytest +from jsonschema import Draft202012Validator + +from beamcheck.core.solver import solve +from beamcheck.verification.compare import compare_case +from beamcheck.verification.loader import ( + DEFAULT_CORPUS_ROOT, + VerificationDataError, + load_corpus, +) +from beamcheck.verification.models import ( + ExpectedQuantity, + Tolerance, + ToleranceType, + VerificationStatus, +) +from beamcheck.verification.runner import run_verification + + +def _synthetic_verified_corpus(tmp_path, expected_moment: float): + corpus = tmp_path / "verification" + shutil.copytree(DEFAULT_CORPUS_ROOT, corpus) + case_directory = corpus / "cases" / "BC-0001" + case_path = case_directory / "case.json" + case_data = json.loads(case_path.read_text(encoding="utf-8")) + case_data["status"] = "verified" + case_data["expected_reference_id"] = "REF-SYNTHETIC-UNIT-TEST" + case_path.write_text(json.dumps(case_data), encoding="utf-8") + expected_path = case_directory / "expected.json" + expected_data = json.loads(expected_path.read_text(encoding="utf-8")) + expected_data["expected"] = { + "max_moment_nm": { + "value": expected_moment, + "tolerance": {"type": "absolute", "value": 0.0}, + } + } + expected_path.write_text(json.dumps(expected_data), encoding="utf-8") + reference_path = case_directory / "reference.md" + reference_path.write_text( + "# Synthetic unit-test reference\n\n" + "Reference ID: REF-SYNTHETIC-UNIT-TEST\n\n" + "This temporary file tests runner behavior and is not corpus evidence.\n", + encoding="utf-8", + ) + return corpus + + +def test_versioned_corpus_is_loadable_and_drafts_are_explicit() -> None: + cases = load_corpus() + assert [case.case_id for case in cases] == [ + "BC-0001", + "BC-0002", + "BC-0003", + "BC-0004", + ] + assert all(case.status is VerificationStatus.DRAFT for case in cases) + assert all(not case.expected for case in cases) + assert all("STATUS: OWNER_REFERENCE_REQUIRED" in case.reference_text for case in cases) + + +def test_corpus_schemas_are_valid_draft_2020_12_documents() -> None: + for schema_path in sorted((DEFAULT_CORPUS_ROOT / "schema").glob("*.json")): + schema = json.loads(schema_path.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(schema) + + +def test_all_verified_corpus_cases_use_the_production_solver() -> None: + verified_cases = [ + case for case in load_corpus() if case.status is VerificationStatus.VERIFIED + ] + failures: list[str] = [] + for case in verified_cases: + comparisons = compare_case(case, solve(case.calculation_input)) + failures.extend( + comparison.failure_message() + for comparison in comparisons + if not comparison.passed + ) + assert not failures, "\n".join(failures) + + +def test_runner_reports_drafts_without_counting_them_as_verified() -> None: + output = io.StringIO() + summary = run_verification(stream=output) + assert summary.discovered == 4 + assert summary.verified == 0 + assert summary.draft == 4 + assert summary.passed == 0 + assert summary.failed == 0 + assert summary.exit_code == 0 + assert "Verified passed: 0/0" in output.getvalue() + assert "SKIP BC-0001 - owner reference required" in output.getvalue() + + +def test_verified_status_rejects_empty_expected_data(tmp_path) -> None: + corpus = tmp_path / "verification" + shutil.copytree(DEFAULT_CORPUS_ROOT, corpus) + case_path = corpus / "cases" / "BC-0001" / "case.json" + data = json.loads(case_path.read_text(encoding="utf-8")) + data["status"] = "verified" + case_path.write_text(json.dumps(data), encoding="utf-8") + + with pytest.raises(VerificationDataError, match="must declare expected quantities"): + load_corpus(corpus) + + +def test_verified_status_rejects_owner_reference_placeholder(tmp_path) -> None: + corpus = tmp_path / "verification" + shutil.copytree(DEFAULT_CORPUS_ROOT, corpus) + case_path = corpus / "cases" / "BC-0001" / "case.json" + case_data = json.loads(case_path.read_text(encoding="utf-8")) + case_data["status"] = "verified" + case_path.write_text(json.dumps(case_data), encoding="utf-8") + expected_path = corpus / "cases" / "BC-0001" / "expected.json" + expected_data = json.loads(expected_path.read_text(encoding="utf-8")) + expected_data["expected"] = { + "max_moment_nm": { + "value": 10000.0, + "tolerance": {"type": "relative", "value": 1e-10}, + } + } + expected_path.write_text(json.dumps(expected_data), encoding="utf-8") + + with pytest.raises(VerificationDataError, match="owner-reference placeholder"): + load_corpus(corpus) + + +def test_loader_rejects_non_finite_json_numbers(tmp_path) -> None: + corpus = tmp_path / "verification" + shutil.copytree(DEFAULT_CORPUS_ROOT, corpus) + case_path = corpus / "cases" / "BC-0001" / "case.json" + text = case_path.read_text(encoding="utf-8").replace('"length_m": 4.0', '"length_m": NaN') + case_path.write_text(text, encoding="utf-8") + + with pytest.raises(VerificationDataError, match="Non-finite JSON number"): + load_corpus(corpus) + + +def test_comparison_failure_contains_auditable_details() -> None: + draft = load_corpus()[0] + result = solve(draft.calculation_input) + synthetic = replace( + draft, + status=VerificationStatus.VERIFIED, + expected={ + "max_moment_nm": ExpectedQuantity( + value=result.max_abs_moment + 1.0, + tolerance=Tolerance(ToleranceType.ABSOLUTE, 0.0), + ) + }, + ) + comparison = compare_case(synthetic, result)[0] + + assert not comparison.passed + message = comparison.failure_message() + assert "BC-0001" in message + assert "max_moment_nm" in message + assert "expected=" in message + assert "actual=" in message + assert "tolerance=absolute" in message + assert "absolute_difference=" in message + assert "relative_difference=" in message + + +def test_runner_returns_nonzero_when_verified_quantity_exceeds_tolerance(tmp_path) -> None: + corpus = _synthetic_verified_corpus(tmp_path, expected_moment=9999.0) + + output = io.StringIO() + summary = run_verification(corpus, output) + + assert summary.verified == 1 + assert summary.failed == 1 + assert summary.exit_code == 1 + assert "FAIL BC-0001" in output.getvalue() + assert "expected=" in output.getvalue() + assert "actual=" in output.getvalue() + + +def test_runner_passes_a_synthetic_verified_case(tmp_path) -> None: + corpus = _synthetic_verified_corpus(tmp_path, expected_moment=10000.0) + + output = io.StringIO() + summary = run_verification(corpus, output) + + assert summary.verified == 1 + assert summary.passed == 1 + assert summary.failed == 0 + assert summary.exit_code == 0 + assert "PASS BC-0001" in output.getvalue() + assert "Verified passed: 1/1" in output.getvalue() diff --git a/verification/README.md b/verification/README.md new file mode 100644 index 0000000..72f4b02 --- /dev/null +++ b/verification/README.md @@ -0,0 +1,37 @@ +# BeamCheck Verification Corpus + +This directory is a versioned, machine-readable collection of BeamCheck calculation cases. It is separate from ordinary unit tests so inputs, expected quantities, tolerances, and independent references can be reviewed as data assets. + +## Status model + +- `draft` — the input is valid, but an owner-reviewed independent reference and expected quantities are not yet complete. Draft cases are discovered and reported, then skipped. +- `verified` — the case has an independent reference, declared expected quantities, and fixed tolerances. Verified cases must pass in pytest and `python -m beamcheck.verification`. + +A case cannot be loaded as `verified` while its `reference.md` contains `STATUS: OWNER_REFERENCE_REQUIRED`, or while `expected.json` contains no quantities. + +## Layout + +```text +verification/ +├── schema/ +│ ├── verification_case.schema.json +│ └── verification_result.schema.json +└── cases/ + └── BC-NNNN/ + ├── case.json + ├── expected.json + └── reference.md +``` + +All numeric solver inputs in `case.json` use canonical SI units. JSON `NaN` and infinity values are rejected. Expected values are never rewritten by the runner. + +## Run + +```powershell +.\.venv\Scripts\python.exe -m pytest -q +.\.venv\Scripts\python.exe -m beamcheck.verification +``` + +## Initial migration state + +`BC-0001` through `BC-0004` preserve the input definitions of the existing textbook-form regression tests. They are intentionally `draft` because repository-owner-approved independent sources have not yet been supplied. They do not count as independently verified cases. diff --git a/verification/cases/BC-0001/case.json b/verification/cases/BC-0001/case.json new file mode 100644 index 0000000..a80976a --- /dev/null +++ b/verification/cases/BC-0001/case.json @@ -0,0 +1,33 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0001", + "title": "Simply supported beam — centre point load", + "status": "draft", + "beam": { + "type": "simply_supported", + "length_m": 4.0 + }, + "section": { + "type": "rectangular", + "width_m": 0.1, + "height_m": 0.2 + }, + "material": { + "name": "Test steel", + "youngs_modulus_pa": 200000000000.0, + "yield_strength_pa": 250000000.0, + "density_kg_m3": 7850.0 + }, + "loads": [ + { + "type": "point", + "magnitude_n": 10000.0, + "position_m": 2.0 + } + ], + "criterion": { + "type": "span_ratio", + "ratio": 250.0 + }, + "expected_reference_id": "REF-OWNER-REQUIRED-BC-0001" +} diff --git a/verification/cases/BC-0001/expected.json b/verification/cases/BC-0001/expected.json new file mode 100644 index 0000000..e8ce89d --- /dev/null +++ b/verification/cases/BC-0001/expected.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0001", + "expected": {} +} diff --git a/verification/cases/BC-0001/reference.md b/verification/cases/BC-0001/reference.md new file mode 100644 index 0000000..e2b16d8 --- /dev/null +++ b/verification/cases/BC-0001/reference.md @@ -0,0 +1,28 @@ +# BC-0001 Reference + +STATUS: OWNER_REFERENCE_REQUIRED + +Reference ID: REF-OWNER-REQUIRED-BC-0001 + +## Source + +- Source type: Owner review required +- Title: Not supplied +- Author / organization: Not supplied +- Edition / revision: Not supplied +- Page / equation / section: Not supplied +- URL or identifier: Not supplied +- Accessed: Not applicable +- License / quotation note: No source content has been copied + +## Assumptions + +The input model is recorded in `case.json`. Engineering assumptions must be reconciled with an independent source before this case can be promoted to `verified`. + +## Independent derivation + +Not yet supplied or owner-reviewed. + +## Quantities used for verification + +None. `expected.json` is intentionally empty while this case is `draft`. diff --git a/verification/cases/BC-0002/case.json b/verification/cases/BC-0002/case.json new file mode 100644 index 0000000..d0ab337 --- /dev/null +++ b/verification/cases/BC-0002/case.json @@ -0,0 +1,32 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0002", + "title": "Simply supported beam — full-span uniformly distributed load", + "status": "draft", + "beam": { + "type": "simply_supported", + "length_m": 4.0 + }, + "section": { + "type": "rectangular", + "width_m": 0.1, + "height_m": 0.2 + }, + "material": { + "name": "Test steel", + "youngs_modulus_pa": 200000000000.0, + "yield_strength_pa": 250000000.0, + "density_kg_m3": 7850.0 + }, + "loads": [ + { + "type": "full_span_udl", + "magnitude_n_per_m": 2000.0 + } + ], + "criterion": { + "type": "span_ratio", + "ratio": 250.0 + }, + "expected_reference_id": "REF-OWNER-REQUIRED-BC-0002" +} diff --git a/verification/cases/BC-0002/expected.json b/verification/cases/BC-0002/expected.json new file mode 100644 index 0000000..d131f25 --- /dev/null +++ b/verification/cases/BC-0002/expected.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0002", + "expected": {} +} diff --git a/verification/cases/BC-0002/reference.md b/verification/cases/BC-0002/reference.md new file mode 100644 index 0000000..0880602 --- /dev/null +++ b/verification/cases/BC-0002/reference.md @@ -0,0 +1,28 @@ +# BC-0002 Reference + +STATUS: OWNER_REFERENCE_REQUIRED + +Reference ID: REF-OWNER-REQUIRED-BC-0002 + +## Source + +- Source type: Owner review required +- Title: Not supplied +- Author / organization: Not supplied +- Edition / revision: Not supplied +- Page / equation / section: Not supplied +- URL or identifier: Not supplied +- Accessed: Not applicable +- License / quotation note: No source content has been copied + +## Assumptions + +The input model is recorded in `case.json`. Engineering assumptions must be reconciled with an independent source before this case can be promoted to `verified`. + +## Independent derivation + +Not yet supplied or owner-reviewed. + +## Quantities used for verification + +None. `expected.json` is intentionally empty while this case is `draft`. diff --git a/verification/cases/BC-0003/case.json b/verification/cases/BC-0003/case.json new file mode 100644 index 0000000..97a2c25 --- /dev/null +++ b/verification/cases/BC-0003/case.json @@ -0,0 +1,33 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0003", + "title": "Cantilever beam — free-end point load", + "status": "draft", + "beam": { + "type": "cantilever", + "length_m": 4.0 + }, + "section": { + "type": "rectangular", + "width_m": 0.1, + "height_m": 0.2 + }, + "material": { + "name": "Test steel", + "youngs_modulus_pa": 200000000000.0, + "yield_strength_pa": 250000000.0, + "density_kg_m3": 7850.0 + }, + "loads": [ + { + "type": "point", + "magnitude_n": 10000.0, + "position_m": 4.0 + } + ], + "criterion": { + "type": "span_ratio", + "ratio": 250.0 + }, + "expected_reference_id": "REF-OWNER-REQUIRED-BC-0003" +} diff --git a/verification/cases/BC-0003/expected.json b/verification/cases/BC-0003/expected.json new file mode 100644 index 0000000..7305c14 --- /dev/null +++ b/verification/cases/BC-0003/expected.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0003", + "expected": {} +} diff --git a/verification/cases/BC-0003/reference.md b/verification/cases/BC-0003/reference.md new file mode 100644 index 0000000..0b9c382 --- /dev/null +++ b/verification/cases/BC-0003/reference.md @@ -0,0 +1,28 @@ +# BC-0003 Reference + +STATUS: OWNER_REFERENCE_REQUIRED + +Reference ID: REF-OWNER-REQUIRED-BC-0003 + +## Source + +- Source type: Owner review required +- Title: Not supplied +- Author / organization: Not supplied +- Edition / revision: Not supplied +- Page / equation / section: Not supplied +- URL or identifier: Not supplied +- Accessed: Not applicable +- License / quotation note: No source content has been copied + +## Assumptions + +The input model is recorded in `case.json`. Engineering assumptions must be reconciled with an independent source before this case can be promoted to `verified`. + +## Independent derivation + +Not yet supplied or owner-reviewed. + +## Quantities used for verification + +None. `expected.json` is intentionally empty while this case is `draft`. diff --git a/verification/cases/BC-0004/case.json b/verification/cases/BC-0004/case.json new file mode 100644 index 0000000..4b5915c --- /dev/null +++ b/verification/cases/BC-0004/case.json @@ -0,0 +1,32 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0004", + "title": "Cantilever beam — full-span uniformly distributed load", + "status": "draft", + "beam": { + "type": "cantilever", + "length_m": 4.0 + }, + "section": { + "type": "rectangular", + "width_m": 0.1, + "height_m": 0.2 + }, + "material": { + "name": "Test steel", + "youngs_modulus_pa": 200000000000.0, + "yield_strength_pa": 250000000.0, + "density_kg_m3": 7850.0 + }, + "loads": [ + { + "type": "full_span_udl", + "magnitude_n_per_m": 2000.0 + } + ], + "criterion": { + "type": "span_ratio", + "ratio": 250.0 + }, + "expected_reference_id": "REF-OWNER-REQUIRED-BC-0004" +} diff --git a/verification/cases/BC-0004/expected.json b/verification/cases/BC-0004/expected.json new file mode 100644 index 0000000..f2c2e4e --- /dev/null +++ b/verification/cases/BC-0004/expected.json @@ -0,0 +1,5 @@ +{ + "schema_version": "1.0", + "case_id": "BC-0004", + "expected": {} +} diff --git a/verification/cases/BC-0004/reference.md b/verification/cases/BC-0004/reference.md new file mode 100644 index 0000000..857365e --- /dev/null +++ b/verification/cases/BC-0004/reference.md @@ -0,0 +1,28 @@ +# BC-0004 Reference + +STATUS: OWNER_REFERENCE_REQUIRED + +Reference ID: REF-OWNER-REQUIRED-BC-0004 + +## Source + +- Source type: Owner review required +- Title: Not supplied +- Author / organization: Not supplied +- Edition / revision: Not supplied +- Page / equation / section: Not supplied +- URL or identifier: Not supplied +- Accessed: Not applicable +- License / quotation note: No source content has been copied + +## Assumptions + +The input model is recorded in `case.json`. Engineering assumptions must be reconciled with an independent source before this case can be promoted to `verified`. + +## Independent derivation + +Not yet supplied or owner-reviewed. + +## Quantities used for verification + +None. `expected.json` is intentionally empty while this case is `draft`. diff --git a/verification/generated/README.md b/verification/generated/README.md new file mode 100644 index 0000000..bbc0706 --- /dev/null +++ b/verification/generated/README.md @@ -0,0 +1,5 @@ +# Generated verification summaries + +This directory is reserved for machine-generated verification summaries. Generated files must report their dataset version, generation time, Git commit when available, verified/draft/pass/fail counts, case IDs, and maximum observed errors. + +No summary is committed in the foundation change because the initial corpus contains zero independently referenced cases. A later change will add deterministic summary generation before README statistics are published. diff --git a/verification/schema/verification_case.schema.json b/verification/schema/verification_case.schema.json new file mode 100644 index 0000000..677f3c2 --- /dev/null +++ b/verification/schema/verification_case.schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/jordanyushen/BeamCheck/verification/schema/verification_case.schema.json", + "title": "BeamCheck verification case", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "case_id", + "title", + "status", + "beam", + "section", + "material", + "loads", + "criterion", + "expected_reference_id" + ], + "properties": { + "schema_version": { "const": "1.0" }, + "case_id": { "type": "string", "pattern": "^BC-[0-9]{4}$" }, + "title": { "type": "string", "minLength": 1 }, + "status": { "enum": ["draft", "verified"] }, + "expected_reference_id": { "type": "string", "minLength": 1 }, + "beam": { + "type": "object", + "additionalProperties": false, + "required": ["type", "length_m"], + "properties": { + "type": { "enum": ["simply_supported", "cantilever"] }, + "length_m": { "type": "number", "exclusiveMinimum": 0 } + } + }, + "material": { + "type": "object", + "additionalProperties": false, + "required": ["name", "youngs_modulus_pa", "yield_strength_pa", "density_kg_m3"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "youngs_modulus_pa": { "type": "number", "exclusiveMinimum": 0 }, + "yield_strength_pa": { "type": "number", "exclusiveMinimum": 0 }, + "density_kg_m3": { "type": "number", "exclusiveMinimum": 0 } + } + }, + "section": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["type", "width_m", "height_m"], + "properties": { + "type": { "const": "rectangular" }, + "width_m": { "type": "number", "exclusiveMinimum": 0 }, + "height_m": { "type": "number", "exclusiveMinimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "diameter_m"], + "properties": { + "type": { "const": "circular" }, + "diameter_m": { "type": "number", "exclusiveMinimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "outer_width_m", "outer_height_m", "wall_thickness_m"], + "properties": { + "type": { "const": "rectangular_hollow" }, + "outer_width_m": { "type": "number", "exclusiveMinimum": 0 }, + "outer_height_m": { "type": "number", "exclusiveMinimum": 0 }, + "wall_thickness_m": { "type": "number", "exclusiveMinimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "outer_size_m", "wall_thickness_m"], + "properties": { + "type": { "const": "square_hollow" }, + "outer_size_m": { "type": "number", "exclusiveMinimum": 0 }, + "wall_thickness_m": { "type": "number", "exclusiveMinimum": 0 } + } + } + ] + }, + "loads": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["type", "magnitude_n", "position_m"], + "properties": { + "type": { "const": "point" }, + "magnitude_n": { "type": "number", "minimum": 0 }, + "position_m": { "type": "number", "minimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "magnitude_n_per_m"], + "properties": { + "type": { "const": "full_span_udl" }, + "magnitude_n_per_m": { "type": "number", "minimum": 0 } + } + } + ] + } + }, + "criterion": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["type", "ratio"], + "properties": { + "type": { "const": "span_ratio" }, + "ratio": { "type": "number", "exclusiveMinimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["type", "allowable_deflection_m"], + "properties": { + "type": { "const": "custom_allowable" }, + "allowable_deflection_m": { "type": "number", "exclusiveMinimum": 0 } + } + } + ] + } + } +} diff --git a/verification/schema/verification_result.schema.json b/verification/schema/verification_result.schema.json new file mode 100644 index 0000000..a89c1c1 --- /dev/null +++ b/verification/schema/verification_result.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/jordanyushen/BeamCheck/verification/schema/verification_result.schema.json", + "title": "BeamCheck verification expected result", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "case_id", "expected"], + "properties": { + "schema_version": { "const": "1.0" }, + "case_id": { "type": "string", "pattern": "^BC-[0-9]{4}$" }, + "expected": { + "type": "object", + "propertyNames": { + "enum": [ + "left_reaction_n", + "right_reaction_n", + "fixed_vertical_n", + "fixed_moment_nm", + "max_shear_n", + "max_moment_nm", + "max_bending_stress_pa", + "max_deflection_m", + "deflection_location_m", + "factor_of_safety", + "allowable_deflection_m" + ] + }, + "additionalProperties": { "$ref": "#/$defs/expected_quantity" } + } + }, + "$defs": { + "expected_quantity": { + "type": "object", + "additionalProperties": false, + "required": ["value", "tolerance"], + "properties": { + "value": { "type": "number" }, + "tolerance": { + "type": "object", + "additionalProperties": false, + "required": ["type", "value"], + "properties": { + "type": { "enum": ["absolute", "relative"] }, + "value": { "type": "number", "minimum": 0 } + } + } + } + } + } +}