diff --git a/benchmark/report.py b/benchmark/report.py index dcafc7c..22132a2 100644 --- a/benchmark/report.py +++ b/benchmark/report.py @@ -8,6 +8,7 @@ from .contract_test import load_cases from .definition import PROFILE from .healthcheck import CONTAINER_HEALTHCHECK, validate_policy +from .java_versions import JAVA_VERSION from .results import ( number, object_fields, @@ -183,9 +184,15 @@ def validate_report(report: dict, *, expected_context: dict | None = None) -> No keys = registry.implementation(implementation)["version_fields"] values = versions[implementation] require(type(values) is dict and set(keys) <= set(values), "missing stack versions") - for value in values.values(): + for key, value in values.items(): + # Only the Java runtime uses an exact build; dependencies stay triplets. + pattern = ( + JAVA_VERSION + if implementation == "java-spring-boot" and key == "java" + else r"[0-9]+\.[0-9]+\.[0-9]+" + ) require( - type(value) is str and re.fullmatch(r"\d+\.\d+\.\d+", value) is not None, + type(value) is str and re.fullmatch(pattern, value) is not None, "invalid stack version", ) backends = report["implementations"] diff --git a/docs/IMPLEMENTATIONS.md b/docs/IMPLEMENTATIONS.md index 6150835..379b21f 100644 --- a/docs/IMPLEMENTATIONS.md +++ b/docs/IMPLEMENTATIONS.md @@ -228,6 +228,20 @@ implementations, while official measurement selects the active cohort. Local or partial data cannot become a complete official result by setting a flag or supplying a smaller list. No partial benchmark publication mode is introduced. +Stack version values are preserved as strings, never normalized or truncated. +The `java-spring-boot` runtime field `java` follows the extractor's exact-build +contract: three or four dot-separated ASCII digit components, then `+` and an +ASCII digit build number (for example, `25.0.4.1+1` or `25.0.4+1`). The build is +mandatory. This intentionally narrow release format excludes early-access/vendor +suffixes, whitespace and control characters; it is not a general Java version +parser. Existing numeric spelling, including leading zeroes, is preserved. +All other required and additional version fields keep the three-component ASCII +numeric contract, including Java dependencies. A `java` key on a different stack +does not enable the Java exception. Both validators exercise the same valid and +invalid cases in `tests/fixtures/stack-versions.json`. Java-inclusive validation +and rendering tests use only a synthetic trusted cohort, which production rejects; +this support neither admits Java to an official cohort nor publishes a result. + JavaScript performs presentation validation, not publication authorization. It renders the report's known cohort rather than the current active list. Missing or invalid results remain empty/unavailable, never zero. The result JSON link stays diff --git a/site/app.mjs b/site/app.mjs index 7521ca2..a2b906d 100644 --- a/site/app.mjs +++ b/site/app.mjs @@ -119,6 +119,15 @@ function validateMeasuredRun(run, expectedRun) { return { run: run.run, rps, mean, memory: memoryBytes / 1048576 }; } +function validStackVersion(id, key, value) { + // Match java_versions.py's exact runtime build, never arbitrary dependency suffixes. + const pattern = id === "java-spring-boot" && key === "java" + ? /^[0-9]+\.[0-9]+\.[0-9]+(?:\.[0-9]+)?\+[0-9]+$/ + : /^[0-9]+\.[0-9]+\.[0-9]+$/; + // JavaScript's $ also matches before a final newline; require the whole string. + return typeof value === "string" && pattern.exec(value)?.[0] === value; +} + function validateReport(report) { object(report, "report"); const cohort = reportCohort(report); @@ -148,9 +157,9 @@ function validateReport(report) { assert(id === members[backendIndex], "implementation identity/order mismatch"); const versionSet = object(versions[id], `versions for ${id}`); for (const key of STACKS[id].version_fields) { - assert(typeof versionSet[key] === "string" && /^\d+\.\d+\.\d+$/.test(versionSet[key]), `invalid ${id} ${key} version`); + assert(Object.hasOwn(versionSet, key) && validStackVersion(id, key, versionSet[key]), `invalid ${id} ${key} version`); } - assert(Object.values(versionSet).every((value) => typeof value === "string" && /^\d+\.\d+\.\d+$/.test(value)), "invalid stack version"); + assert(Object.entries(versionSet).every(([key, value]) => validStackVersion(id, key, value)), "invalid stack version"); assert(Array.isArray(backend.endpoints) && backend.endpoints.length === ENDPOINTS.length, "three endpoints required"); backend.endpoints.forEach((entry, endpointIndex) => { object(entry, "endpoint result"); diff --git a/tests/fixtures/stack-versions.json b/tests/fixtures/stack-versions.json new file mode 100644 index 0000000..4f4e4c3 --- /dev/null +++ b/tests/fixtures/stack-versions.json @@ -0,0 +1,158 @@ +[ + {"name": "exact Java 25.0.4.1+1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4.1+1", "valid": true}, + {"name": "exact Java 25.0.4+1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1", "valid": true}, + {"name": "exact Java 21.0.8+9", "implementation": "java-spring-boot", "field": "java", "value": "21.0.8+9", "valid": true}, + {"name": "exact Java 025.00.04.01+01", "implementation": "java-spring-boot", "field": "java", "value": "025.00.04.01+01", "valid": true}, + {"name": "malformed Java 25", "implementation": "java-spring-boot", "field": "java", "value": "25", "valid": false}, + {"name": "malformed Java 25.0", "implementation": "java-spring-boot", "field": "java", "value": "25.0", "valid": false}, + {"name": "malformed Java 25.0.4", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4", "valid": false}, + {"name": "malformed Java 25.0.4.1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4.1", "valid": false}, + {"name": "malformed Java 25.0.4.1.2+1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4.1.2+1", "valid": false}, + {"name": "malformed Java 25.0.4+", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+", "valid": false}, + {"name": "malformed Java 25.0.4+-1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+-1", "valid": false}, + {"name": "malformed Java 25.0.4+1.1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1.1", "valid": false}, + {"name": "malformed Java 25.0.4+1+2", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1+2", "valid": false}, + {"name": "malformed Java 25.0.4-ea+1", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4-ea+1", "valid": false}, + {"name": "malformed Java 25.0.4+1-LTS", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1-LTS", "valid": false}, + {"name": "malformed Java v25.0.4+1", "implementation": "java-spring-boot", "field": "java", "value": "v25.0.4+1", "valid": false}, + {"name": "malformed Java 25..4+1", "implementation": "java-spring-boot", "field": "java", "value": "25..4+1", "valid": false}, + {"name": "malformed Java \uff12\uff15.0.4+1", "implementation": "java-spring-boot", "field": "java", "value": "\uff12\uff15.0.4+1", "valid": false}, + {"name": "java-spring-boot/java rejects ''", "implementation": "java-spring-boot", "field": "java", "value": "", "valid": false}, + {"name": "java-spring-boot/java rejects None", "implementation": "java-spring-boot", "field": "java", "value": null, "valid": false}, + {"name": "java-spring-boot/java rejects True", "implementation": "java-spring-boot", "field": "java", "value": true, "valid": false}, + {"name": "java-spring-boot/java rejects 25", "implementation": "java-spring-boot", "field": "java", "value": 25, "valid": false}, + {"name": "java-spring-boot/java rejects 25.4", "implementation": "java-spring-boot", "field": "java", "value": 25.4, "valid": false}, + {"name": "java-spring-boot/java rejects []", "implementation": "java-spring-boot", "field": "java", "value": [], "valid": false}, + {"name": "java-spring-boot/java rejects {}", "implementation": "java-spring-boot", "field": "java", "value": {}, "valid": false}, + {"name": "java-spring-boot/java rejects ' 25.0.4+1'", "implementation": "java-spring-boot", "field": "java", "value": " 25.0.4+1", "valid": false}, + {"name": "java-spring-boot/java rejects '25.0.4+1 '", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1 ", "valid": false}, + {"name": "java-spring-boot/java rejects '25.0.4+1\\n'", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1\n", "valid": false}, + {"name": "java-spring-boot/java rejects '25.0.4+1\\r'", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1\r", "valid": false}, + {"name": "java-spring-boot/java rejects '25.0.4+1\\t'", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1\t", "valid": false}, + {"name": "java-spring-boot/java rejects '25.0.4+1\\x00'", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1\u0000", "valid": false}, + {"name": "java-spring-boot/java rejects '25.0.4+1\\u2028'", "implementation": "java-spring-boot", "field": "java", "value": "25.0.4+1\u2028", "valid": false}, + {"name": "java-spring-boot/java rejects '1.2.3\\n'", "implementation": "java-spring-boot", "field": "java", "value": "1.2.3\n", "valid": false}, + {"name": "java-spring-boot/java rejects '\uff11.2.3'", "implementation": "java-spring-boot", "field": "java", "value": "\uff11.2.3", "valid": false}, + {"name": "java-spring-boot/java rejects '1.2.3-rc1'", "implementation": "java-spring-boot", "field": "java", "value": "1.2.3-rc1", "valid": false}, + {"name": "java-spring-boot/java rejects ''", "implementation": "java-spring-boot", "field": "java", "value": "", "valid": false}, + {"name": "java-spring-boot/tomcat rejects ''", "implementation": "java-spring-boot", "field": "tomcat", "value": "", "valid": false}, + {"name": "java-spring-boot/tomcat rejects None", "implementation": "java-spring-boot", "field": "tomcat", "value": null, "valid": false}, + {"name": "java-spring-boot/tomcat rejects True", "implementation": "java-spring-boot", "field": "tomcat", "value": true, "valid": false}, + {"name": "java-spring-boot/tomcat rejects 25", "implementation": "java-spring-boot", "field": "tomcat", "value": 25, "valid": false}, + {"name": "java-spring-boot/tomcat rejects 25.4", "implementation": "java-spring-boot", "field": "tomcat", "value": 25.4, "valid": false}, + {"name": "java-spring-boot/tomcat rejects []", "implementation": "java-spring-boot", "field": "tomcat", "value": [], "valid": false}, + {"name": "java-spring-boot/tomcat rejects {}", "implementation": "java-spring-boot", "field": "tomcat", "value": {}, "valid": false}, + {"name": "java-spring-boot/tomcat rejects ' 25.0.4+1'", "implementation": "java-spring-boot", "field": "tomcat", "value": " 25.0.4+1", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '25.0.4+1 '", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1 ", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '25.0.4+1\\n'", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1\n", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '25.0.4+1\\r'", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1\r", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '25.0.4+1\\t'", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1\t", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '25.0.4+1\\x00'", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1\u0000", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '25.0.4+1\\u2028'", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1\u2028", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '1.2.3\\n'", "implementation": "java-spring-boot", "field": "tomcat", "value": "1.2.3\n", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '\uff11.2.3'", "implementation": "java-spring-boot", "field": "tomcat", "value": "\uff11.2.3", "valid": false}, + {"name": "java-spring-boot/tomcat rejects '1.2.3-rc1'", "implementation": "java-spring-boot", "field": "tomcat", "value": "1.2.3-rc1", "valid": false}, + {"name": "java-spring-boot/tomcat rejects ''", "implementation": "java-spring-boot", "field": "tomcat", "value": "", "valid": false}, + {"name": "java-spring-boot/tomcat triplet 1.2.3", "implementation": "java-spring-boot", "field": "tomcat", "value": "1.2.3", "valid": true}, + {"name": "java-spring-boot/tomcat triplet 0.0.0", "implementation": "java-spring-boot", "field": "tomcat", "value": "0.0.0", "valid": true}, + {"name": "java-spring-boot/tomcat triplet 01.02.03", "implementation": "java-spring-boot", "field": "tomcat", "value": "01.02.03", "valid": true}, + {"name": "java-spring-boot/tomcat no Java exception 25.0.4.1+1", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4.1+1", "valid": false}, + {"name": "java-spring-boot/tomcat no Java exception 25.0.4+1", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4+1", "valid": false}, + {"name": "java-spring-boot/tomcat no Java exception 25.0.4.1", "implementation": "java-spring-boot", "field": "tomcat", "value": "25.0.4.1", "valid": false}, + {"name": "java-spring-boot/additional rejects ''", "implementation": "java-spring-boot", "field": "additional", "value": "", "valid": false}, + {"name": "java-spring-boot/additional rejects None", "implementation": "java-spring-boot", "field": "additional", "value": null, "valid": false}, + {"name": "java-spring-boot/additional rejects True", "implementation": "java-spring-boot", "field": "additional", "value": true, "valid": false}, + {"name": "java-spring-boot/additional rejects 25", "implementation": "java-spring-boot", "field": "additional", "value": 25, "valid": false}, + {"name": "java-spring-boot/additional rejects 25.4", "implementation": "java-spring-boot", "field": "additional", "value": 25.4, "valid": false}, + {"name": "java-spring-boot/additional rejects []", "implementation": "java-spring-boot", "field": "additional", "value": [], "valid": false}, + {"name": "java-spring-boot/additional rejects {}", "implementation": "java-spring-boot", "field": "additional", "value": {}, "valid": false}, + {"name": "java-spring-boot/additional rejects ' 25.0.4+1'", "implementation": "java-spring-boot", "field": "additional", "value": " 25.0.4+1", "valid": false}, + {"name": "java-spring-boot/additional rejects '25.0.4+1 '", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1 ", "valid": false}, + {"name": "java-spring-boot/additional rejects '25.0.4+1\\n'", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1\n", "valid": false}, + {"name": "java-spring-boot/additional rejects '25.0.4+1\\r'", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1\r", "valid": false}, + {"name": "java-spring-boot/additional rejects '25.0.4+1\\t'", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1\t", "valid": false}, + {"name": "java-spring-boot/additional rejects '25.0.4+1\\x00'", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1\u0000", "valid": false}, + {"name": "java-spring-boot/additional rejects '25.0.4+1\\u2028'", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1\u2028", "valid": false}, + {"name": "java-spring-boot/additional rejects '1.2.3\\n'", "implementation": "java-spring-boot", "field": "additional", "value": "1.2.3\n", "valid": false}, + {"name": "java-spring-boot/additional rejects '\uff11.2.3'", "implementation": "java-spring-boot", "field": "additional", "value": "\uff11.2.3", "valid": false}, + {"name": "java-spring-boot/additional rejects '1.2.3-rc1'", "implementation": "java-spring-boot", "field": "additional", "value": "1.2.3-rc1", "valid": false}, + {"name": "java-spring-boot/additional rejects ''", "implementation": "java-spring-boot", "field": "additional", "value": "", "valid": false}, + {"name": "java-spring-boot/additional triplet 1.2.3", "implementation": "java-spring-boot", "field": "additional", "value": "1.2.3", "valid": true}, + {"name": "java-spring-boot/additional triplet 0.0.0", "implementation": "java-spring-boot", "field": "additional", "value": "0.0.0", "valid": true}, + {"name": "java-spring-boot/additional triplet 01.02.03", "implementation": "java-spring-boot", "field": "additional", "value": "01.02.03", "valid": true}, + {"name": "java-spring-boot/additional no Java exception 25.0.4.1+1", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4.1+1", "valid": false}, + {"name": "java-spring-boot/additional no Java exception 25.0.4+1", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4+1", "valid": false}, + {"name": "java-spring-boot/additional no Java exception 25.0.4.1", "implementation": "java-spring-boot", "field": "additional", "value": "25.0.4.1", "valid": false}, + {"name": "go-gin/go rejects ''", "implementation": "go-gin", "field": "go", "value": "", "valid": false}, + {"name": "go-gin/go rejects None", "implementation": "go-gin", "field": "go", "value": null, "valid": false}, + {"name": "go-gin/go rejects True", "implementation": "go-gin", "field": "go", "value": true, "valid": false}, + {"name": "go-gin/go rejects 25", "implementation": "go-gin", "field": "go", "value": 25, "valid": false}, + {"name": "go-gin/go rejects 25.4", "implementation": "go-gin", "field": "go", "value": 25.4, "valid": false}, + {"name": "go-gin/go rejects []", "implementation": "go-gin", "field": "go", "value": [], "valid": false}, + {"name": "go-gin/go rejects {}", "implementation": "go-gin", "field": "go", "value": {}, "valid": false}, + {"name": "go-gin/go rejects ' 25.0.4+1'", "implementation": "go-gin", "field": "go", "value": " 25.0.4+1", "valid": false}, + {"name": "go-gin/go rejects '25.0.4+1 '", "implementation": "go-gin", "field": "go", "value": "25.0.4+1 ", "valid": false}, + {"name": "go-gin/go rejects '25.0.4+1\\n'", "implementation": "go-gin", "field": "go", "value": "25.0.4+1\n", "valid": false}, + {"name": "go-gin/go rejects '25.0.4+1\\r'", "implementation": "go-gin", "field": "go", "value": "25.0.4+1\r", "valid": false}, + {"name": "go-gin/go rejects '25.0.4+1\\t'", "implementation": "go-gin", "field": "go", "value": "25.0.4+1\t", "valid": false}, + {"name": "go-gin/go rejects '25.0.4+1\\x00'", "implementation": "go-gin", "field": "go", "value": "25.0.4+1\u0000", "valid": false}, + {"name": "go-gin/go rejects '25.0.4+1\\u2028'", "implementation": "go-gin", "field": "go", "value": "25.0.4+1\u2028", "valid": false}, + {"name": "go-gin/go rejects '1.2.3\\n'", "implementation": "go-gin", "field": "go", "value": "1.2.3\n", "valid": false}, + {"name": "go-gin/go rejects '\uff11.2.3'", "implementation": "go-gin", "field": "go", "value": "\uff11.2.3", "valid": false}, + {"name": "go-gin/go rejects '1.2.3-rc1'", "implementation": "go-gin", "field": "go", "value": "1.2.3-rc1", "valid": false}, + {"name": "go-gin/go rejects ''", "implementation": "go-gin", "field": "go", "value": "", "valid": false}, + {"name": "go-gin/go triplet 1.2.3", "implementation": "go-gin", "field": "go", "value": "1.2.3", "valid": true}, + {"name": "go-gin/go triplet 0.0.0", "implementation": "go-gin", "field": "go", "value": "0.0.0", "valid": true}, + {"name": "go-gin/go triplet 01.02.03", "implementation": "go-gin", "field": "go", "value": "01.02.03", "valid": true}, + {"name": "go-gin/go no Java exception 25.0.4.1+1", "implementation": "go-gin", "field": "go", "value": "25.0.4.1+1", "valid": false}, + {"name": "go-gin/go no Java exception 25.0.4+1", "implementation": "go-gin", "field": "go", "value": "25.0.4+1", "valid": false}, + {"name": "go-gin/go no Java exception 25.0.4.1", "implementation": "go-gin", "field": "go", "value": "25.0.4.1", "valid": false}, + {"name": "go-gin/java rejects ''", "implementation": "go-gin", "field": "java", "value": "", "valid": false}, + {"name": "go-gin/java rejects None", "implementation": "go-gin", "field": "java", "value": null, "valid": false}, + {"name": "go-gin/java rejects True", "implementation": "go-gin", "field": "java", "value": true, "valid": false}, + {"name": "go-gin/java rejects 25", "implementation": "go-gin", "field": "java", "value": 25, "valid": false}, + {"name": "go-gin/java rejects 25.4", "implementation": "go-gin", "field": "java", "value": 25.4, "valid": false}, + {"name": "go-gin/java rejects []", "implementation": "go-gin", "field": "java", "value": [], "valid": false}, + {"name": "go-gin/java rejects {}", "implementation": "go-gin", "field": "java", "value": {}, "valid": false}, + {"name": "go-gin/java rejects ' 25.0.4+1'", "implementation": "go-gin", "field": "java", "value": " 25.0.4+1", "valid": false}, + {"name": "go-gin/java rejects '25.0.4+1 '", "implementation": "go-gin", "field": "java", "value": "25.0.4+1 ", "valid": false}, + {"name": "go-gin/java rejects '25.0.4+1\\n'", "implementation": "go-gin", "field": "java", "value": "25.0.4+1\n", "valid": false}, + {"name": "go-gin/java rejects '25.0.4+1\\r'", "implementation": "go-gin", "field": "java", "value": "25.0.4+1\r", "valid": false}, + {"name": "go-gin/java rejects '25.0.4+1\\t'", "implementation": "go-gin", "field": "java", "value": "25.0.4+1\t", "valid": false}, + {"name": "go-gin/java rejects '25.0.4+1\\x00'", "implementation": "go-gin", "field": "java", "value": "25.0.4+1\u0000", "valid": false}, + {"name": "go-gin/java rejects '25.0.4+1\\u2028'", "implementation": "go-gin", "field": "java", "value": "25.0.4+1\u2028", "valid": false}, + {"name": "go-gin/java rejects '1.2.3\\n'", "implementation": "go-gin", "field": "java", "value": "1.2.3\n", "valid": false}, + {"name": "go-gin/java rejects '\uff11.2.3'", "implementation": "go-gin", "field": "java", "value": "\uff11.2.3", "valid": false}, + {"name": "go-gin/java rejects '1.2.3-rc1'", "implementation": "go-gin", "field": "java", "value": "1.2.3-rc1", "valid": false}, + {"name": "go-gin/java rejects ''", "implementation": "go-gin", "field": "java", "value": "", "valid": false}, + {"name": "go-gin/java triplet 1.2.3", "implementation": "go-gin", "field": "java", "value": "1.2.3", "valid": true}, + {"name": "go-gin/java triplet 0.0.0", "implementation": "go-gin", "field": "java", "value": "0.0.0", "valid": true}, + {"name": "go-gin/java triplet 01.02.03", "implementation": "go-gin", "field": "java", "value": "01.02.03", "valid": true}, + {"name": "go-gin/java no Java exception 25.0.4.1+1", "implementation": "go-gin", "field": "java", "value": "25.0.4.1+1", "valid": false}, + {"name": "go-gin/java no Java exception 25.0.4+1", "implementation": "go-gin", "field": "java", "value": "25.0.4+1", "valid": false}, + {"name": "go-gin/java no Java exception 25.0.4.1", "implementation": "go-gin", "field": "java", "value": "25.0.4.1", "valid": false}, + {"name": "go-gin/additional rejects ''", "implementation": "go-gin", "field": "additional", "value": "", "valid": false}, + {"name": "go-gin/additional rejects None", "implementation": "go-gin", "field": "additional", "value": null, "valid": false}, + {"name": "go-gin/additional rejects True", "implementation": "go-gin", "field": "additional", "value": true, "valid": false}, + {"name": "go-gin/additional rejects 25", "implementation": "go-gin", "field": "additional", "value": 25, "valid": false}, + {"name": "go-gin/additional rejects 25.4", "implementation": "go-gin", "field": "additional", "value": 25.4, "valid": false}, + {"name": "go-gin/additional rejects []", "implementation": "go-gin", "field": "additional", "value": [], "valid": false}, + {"name": "go-gin/additional rejects {}", "implementation": "go-gin", "field": "additional", "value": {}, "valid": false}, + {"name": "go-gin/additional rejects ' 25.0.4+1'", "implementation": "go-gin", "field": "additional", "value": " 25.0.4+1", "valid": false}, + {"name": "go-gin/additional rejects '25.0.4+1 '", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1 ", "valid": false}, + {"name": "go-gin/additional rejects '25.0.4+1\\n'", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1\n", "valid": false}, + {"name": "go-gin/additional rejects '25.0.4+1\\r'", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1\r", "valid": false}, + {"name": "go-gin/additional rejects '25.0.4+1\\t'", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1\t", "valid": false}, + {"name": "go-gin/additional rejects '25.0.4+1\\x00'", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1\u0000", "valid": false}, + {"name": "go-gin/additional rejects '25.0.4+1\\u2028'", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1\u2028", "valid": false}, + {"name": "go-gin/additional rejects '1.2.3\\n'", "implementation": "go-gin", "field": "additional", "value": "1.2.3\n", "valid": false}, + {"name": "go-gin/additional rejects '\uff11.2.3'", "implementation": "go-gin", "field": "additional", "value": "\uff11.2.3", "valid": false}, + {"name": "go-gin/additional rejects '1.2.3-rc1'", "implementation": "go-gin", "field": "additional", "value": "1.2.3-rc1", "valid": false}, + {"name": "go-gin/additional rejects ''", "implementation": "go-gin", "field": "additional", "value": "", "valid": false}, + {"name": "go-gin/additional triplet 1.2.3", "implementation": "go-gin", "field": "additional", "value": "1.2.3", "valid": true}, + {"name": "go-gin/additional triplet 0.0.0", "implementation": "go-gin", "field": "additional", "value": "0.0.0", "valid": true}, + {"name": "go-gin/additional triplet 01.02.03", "implementation": "go-gin", "field": "additional", "value": "01.02.03", "valid": true}, + {"name": "go-gin/additional no Java exception 25.0.4.1+1", "implementation": "go-gin", "field": "additional", "value": "25.0.4.1+1", "valid": false}, + {"name": "go-gin/additional no Java exception 25.0.4+1", "implementation": "go-gin", "field": "additional", "value": "25.0.4+1", "valid": false}, + {"name": "go-gin/additional no Java exception 25.0.4.1", "implementation": "go-gin", "field": "additional", "value": "25.0.4.1", "valid": false} +] diff --git a/tests/registry_fixtures.py b/tests/registry_fixtures.py index 7783d07..0052360 100644 --- a/tests/registry_fixtures.py +++ b/tests/registry_fixtures.py @@ -87,3 +87,29 @@ def real_eight_report(report, root=None): target = directory / path.name.replace(original["implementation"], identifier, 1) target.write_bytes(path.read_bytes()) return result + + +def java_registry(): + """Trust a Java-only cohort in isolated tests, never in production projections.""" + from benchmark.registry import load_registry + + result = copy.deepcopy(load_registry()) + result["cohorts"]["synthetic-java-v1"] = { + "definition": "simple-api-v1", + "members": ["java-spring-boot"], + } + return result + + +def java_report(report): + """Use synthetic measurements and actual extracted pins, without publication inputs.""" + from benchmark.java_versions import pinned_versions + + result = explicit_report(report, "synthetic-java-v1") + backend = copy.deepcopy(result["implementations"][0]) + backend["implementation"] = "java-spring-boot" + result["implementations"] = [backend] + result["metadata"]["versions"] = { + "java-spring-boot": pinned_versions(ROOT / "apps/java-spring-boot") + } + return result diff --git a/tests/test_benchmark_version_conformance.py b/tests/test_benchmark_version_conformance.py new file mode 100644 index 0000000..feb67fb --- /dev/null +++ b/tests/test_benchmark_version_conformance.py @@ -0,0 +1,74 @@ +"""Shared version contract exercised through synthetic, never published reports.""" + +import copy +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from registry_fixtures import java_registry, java_report +from test_benchmark_publication import synthetic_report + +from benchmark import registry +from benchmark.report import validate_report +from benchmark.results import BenchmarkFailure + +ROOT = Path(__file__).resolve().parents[1] + + +class VersionConformanceTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.legacy = synthetic_report(Path(temporary.name)) + self.java = java_report(self.legacy) + self.cases = json.loads((ROOT / "tests/fixtures/stack-versions.json").read_text()) + + def test_shared_version_contract(self): + for case in self.cases: + with self.subTest(case=case["name"]): + source = self.java if case["implementation"] == "java-spring-boot" else self.legacy + report = copy.deepcopy(source) + report["metadata"]["versions"][case["implementation"]][case["field"]] = case[ + "value" + ] + before = copy.deepcopy(report) + with patch.object(registry, "REGISTRY", java_registry()): + if case["valid"]: + validate_report(report) + else: + with self.assertRaises(BenchmarkFailure): + validate_report(report) + self.assertEqual(report, before) + + def test_extracted_java_build_survives_full_report_validation(self): + self.assertEqual( + self.java["metadata"]["versions"]["java-spring-boot"]["java"], "25.0.4.1+1" + ) + before = copy.deepcopy(self.java) + with patch.object(registry, "REGISTRY", java_registry()): + validate_report(self.java) + self.assertEqual(self.java, before) + with self.assertRaises(BenchmarkFailure): + validate_report(self.java) + + def test_java_fixture_does_not_bypass_integrity_checks(self): + changes = [ + lambda r: r.update(mode="smoke"), + lambda r: r.update(official=False), + lambda r: r.update(status="failed"), + lambda r: r["benchmark"].update(cohort="unknown-v1"), + lambda r: r["benchmark"].update(definition="unknown-v1"), + lambda r: r["implementations"].clear(), + lambda r: r["implementations"][0]["endpoints"].pop(), + lambda r: r["implementations"][0]["endpoints"][0]["runs"].pop(), + lambda r: r["metadata"]["versions"]["java-spring-boot"].pop("java"), + lambda r: r["metadata"]["versions"]["java-spring-boot"].pop("spring-boot"), + ] + with patch.object(registry, "REGISTRY", java_registry()): + for change in changes: + report = copy.deepcopy(self.java) + change(report) + with self.subTest(change=change), self.assertRaises(BenchmarkFailure): + validate_report(report) diff --git a/tests/test_site.mjs b/tests/test_site.mjs index 656e6f1..452f8a7 100644 --- a/tests/test_site.mjs +++ b/tests/test_site.mjs @@ -366,3 +366,68 @@ print(json.dumps({"legacy": legacy, "expanded": expanded})) // Exercise real cohort IDs without substituting the production viewer registry. registerEightStackTests(); + +test('shared version conformance and exact Java rendering use only a synthetic trusted cohort', async () => { + const temporary = await mkdtemp(join(tmpdir(), 'sab-java-version-fixture-')); + try { + await cp(new URL('../site/app.mjs', import.meta.url), join(temporary, 'app.mjs')); + const source = ` +import json, sys +from pathlib import Path +from unittest.mock import patch +sys.path.insert(0, 'tests') +from registry_fixtures import java_registry, java_report +from test_benchmark_publication import synthetic_report +from benchmark import registry +root = Path(sys.argv[1]) +legacy = synthetic_report(root) +data = java_registry() +with patch.object(registry, 'REGISTRY', data): + (root / 'registry.mjs').write_text(registry.generated_files()['site/registry.mjs']) +print(json.dumps({'legacy': legacy, 'java': java_report(legacy)})) +`; + const fixtures = JSON.parse(execFileSync(process.env.PYTHON || 'python3', ['-c', source, temporary], { + cwd: fileURLToPath(new URL('..', import.meta.url)), encoding: 'utf8', + env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' }, + })); + const { viewModel, renderReport } = await import(pathToFileURL(join(temporary, 'app.mjs')).href); + const cases = JSON.parse(await readFile(new URL('./fixtures/stack-versions.json', import.meta.url), 'utf8')); + for (const entry of cases) { + const report = structuredClone(entry.implementation === 'java-spring-boot' ? fixtures.java : fixtures.legacy); + report.metadata.versions[entry.implementation][entry.field] = entry.value; + const before = structuredClone(report); + if (entry.valid) { + assert.doesNotThrow(() => viewModel(report), entry.name); + assert.ok(renderReport(report).includes(entry.value), entry.name); + } else { + assert.throws(() => viewModel(report), undefined, entry.name); + } + assert.deepEqual(report, before, entry.name); + } + const model = viewModel(fixtures.java); + assert.equal(model.versions['java-spring-boot'].java, '25.0.4.1+1'); + assert.equal(model.rows.length, 3); + const escaped = structuredClone(fixtures.java); + escaped.metadata.versions['java-spring-boot'][''] = '1.2.3'; + const html = renderReport(escaped); + assert.ok(html.includes('25.0.4.1+1')); + assert.ok(html.includes('<script>extra</script>')); + assert.ok(!html.includes('