diff --git a/README.md b/README.md index 7180647..6a779a0 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,27 @@ var result = schema.validate( // result.valid() => true ``` -Compatibility: runs the official 2020‑12 JSON Schema Test Suite on `verify`; in strict mode it currently passes about 71% of applicable cases. +Compatibility: runs the official 2020‑12 JSON Schema Test Suite on `verify`; **measured compatibility is 63.3%** (1,153 of 1,822 tests pass) with comprehensive metrics reporting. + +### JSON Schema Test Suite Metrics + +The validator now provides defensible compatibility statistics: + +```bash +# Run with console metrics (default) +mvn verify -pl json-java21-schema + +# Export detailed JSON metrics +mvn verify -pl json-java21-schema -Djson.schema.metrics=json + +# Export CSV metrics for analysis +mvn verify -pl json-java21-schema -Djson.schema.metrics=csv +``` + +**Current measured compatibility**: +- **Overall**: 63.3% (1,153 of 1,822 tests pass) +- **Test coverage**: 420 test groups, 1,657 validation attempts +- **Skip breakdown**: 70 unsupported schema groups, 2 test exceptions, 504 lenient mismatches ## Building diff --git a/json-java21-schema/AGENTS.md b/json-java21-schema/AGENTS.md index f08d03e..53766fd 100644 --- a/json-java21-schema/AGENTS.md +++ b/json-java21-schema/AGENTS.md @@ -47,6 +47,32 @@ The project uses `java.util.logging` with levels: - **JSON Schema Test Suite**: Official tests from json-schema-org - **Real-world schemas**: Complex nested validation scenarios - **Performance tests**: Large schema compilation +- **Metrics reporting**: Comprehensive compatibility statistics with detailed skip categorization + +### JSON Schema Test Suite Metrics + +The integration test now provides defensible compatibility metrics: + +```bash +# Run with console metrics (default) +mvnd verify -pl json-java21-schema + +# Export detailed JSON metrics +mvnd verify -pl json-java21-schema -Djson.schema.metrics=json + +# Export CSV metrics for analysis +mvnd verify -pl json-java21-schema -Djson.schema.metrics=csv +``` + +**Current measured compatibility** (as of implementation): +- **Overall**: 63.3% (1,153 of 1,822 tests pass) +- **Test coverage**: 420 test groups, 1,657 validation attempts +- **Skip breakdown**: 70 unsupported schema groups, 2 test exceptions, 504 lenient mismatches + +The metrics distinguish between: +- **unsupportedSchemaGroup**: Whole groups skipped due to unsupported features (e.g., $ref, anchors) +- **testException**: Individual tests that threw exceptions during validation +- **lenientMismatch**: Expected≠actual results in lenient mode (counted as failures in strict mode) #### Annotation Tests (`JsonSchemaAnnotationsTest.java`) - **Annotation processing**: Compile-time schema generation diff --git a/json-java21-schema/README.md b/json-java21-schema/README.md index 9a249ff..813c438 100644 --- a/json-java21-schema/README.md +++ b/json-java21-schema/README.md @@ -22,7 +22,10 @@ Compatibility and verify - The module runs the official JSON Schema Test Suite during Maven verify. - Default mode is lenient: unsupported groups/tests are skipped to avoid build breaks while still logging. -- Strict mode: enable with -Djson.schema.strict=true to enforce full assertions. In strict mode it currently passes about 71% of applicable cases. +- Strict mode: enable with -Djson.schema.strict=true to enforce full assertions. +- **Measured compatibility**: 63.3% (1,153 of 1,822 tests pass in lenient mode) +- **Test coverage**: 420 test groups, 1,657 validation attempts, 70 unsupported schema groups, 2 test exceptions +- Detailed metrics available via `-Djson.schema.metrics=json|csv` How to run diff --git a/json-java21-schema/src/test/java/io/github/simbo1905/json/schema/JsonSchemaCheckIT.java b/json-java21-schema/src/test/java/io/github/simbo1905/json/schema/JsonSchemaCheckIT.java index 3c75bf3..6a16836 100644 --- a/json-java21-schema/src/test/java/io/github/simbo1905/json/schema/JsonSchemaCheckIT.java +++ b/json-java21-schema/src/test/java/io/github/simbo1905/json/schema/JsonSchemaCheckIT.java @@ -5,11 +5,14 @@ import jdk.sandbox.java.util.json.Json; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assumptions; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAdder; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -25,6 +28,8 @@ public class JsonSchemaCheckIT { new File("target/json-schema-test-suite/tests/draft2020-12"); private static final ObjectMapper MAPPER = new ObjectMapper(); private static final boolean STRICT = Boolean.getBoolean("json.schema.strict"); + private static final String METRICS_FMT = System.getProperty("json.schema.metrics", "").trim(); + private static final SuiteMetrics METRICS = new SuiteMetrics(); @SuppressWarnings("resource") @TestFactory @@ -37,6 +42,19 @@ Stream runOfficialSuite() throws Exception { private Stream testsFromFile(Path file) { try { JsonNode root = MAPPER.readTree(file.toFile()); + + // Count groups and tests discovered + int groupCount = root.size(); + METRICS.groupsDiscovered.add(groupCount); + perFile(file).groups.add(groupCount); + + int testCount = 0; + for (JsonNode group : root) { + testCount += group.get("tests").size(); + } + METRICS.testsDiscovered.add(testCount); + perFile(file).tests.add(testCount); + return StreamSupport.stream(root.spliterator(), false) .flatMap(group -> { String groupDesc = group.get("description").asText(); @@ -55,22 +73,50 @@ private Stream testsFromFile(Path file) { try { actual = schema.validate( Json.parse(test.get("data").toString())).valid(); + + // Count validation attempt + METRICS.validationsRun.increment(); + perFile(file).run.increment(); } catch (Exception e) { String reason = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); System.err.println("[JsonSchemaCheckIT] Skipping test due to exception: " + groupDesc + " — " + reason + " (" + file.getFileName() + ")"); + + // Count exception skip + METRICS.skipTestException.increment(); + perFile(file).skipException.increment(); + if (STRICT) throw e; Assumptions.assumeTrue(false, "Skipped: " + reason); return; // not reached when strict } if (STRICT) { - assertEquals(expected, actual); + try { + assertEquals(expected, actual); + // Count pass in strict mode + METRICS.passed.increment(); + perFile(file).pass.increment(); + } catch (AssertionError e) { + // Count failure in strict mode + METRICS.failed.increment(); + perFile(file).fail.increment(); + throw e; + } } else if (expected != actual) { System.err.println("[JsonSchemaCheckIT] Mismatch (ignored): " + groupDesc + " — expected=" + expected + ", actual=" + actual + " (" + file.getFileName() + ")"); + + // Count lenient mismatch skip + METRICS.skipLenientMismatch.increment(); + perFile(file).skipMismatch.increment(); + Assumptions.assumeTrue(false, "Mismatch ignored"); + } else { + // Count pass in lenient mode + METRICS.passed.increment(); + perFile(file).pass.increment(); } })); } catch (Exception ex) { @@ -78,6 +124,11 @@ private Stream testsFromFile(Path file) { String reason = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage(); System.err.println("[JsonSchemaCheckIT] Skipping group due to unsupported schema: " + groupDesc + " — " + reason + " (" + file.getFileName() + ")"); + + // Count unsupported group skip + METRICS.skipUnsupportedGroup.increment(); + perFile(file).skipUnsupported.increment(); + return Stream.of(DynamicTest.dynamicTest( groupDesc + " – SKIPPED: " + reason, () -> { if (STRICT) throw ex; Assumptions.assumeTrue(false, "Unsupported schema: " + reason); } @@ -88,4 +139,146 @@ private Stream testsFromFile(Path file) { throw new RuntimeException("Failed to process " + file, ex); } } + + private static SuiteMetrics.FileCounters perFile(Path file) { + return METRICS.perFile.computeIfAbsent(file.getFileName().toString(), k -> new SuiteMetrics.FileCounters()); + } + + @AfterAll + static void printAndPersistMetrics() throws Exception { + var strict = STRICT; + var totalRun = METRICS.validationsRun.sum(); + var passed = METRICS.passed.sum(); + var failed = METRICS.failed.sum(); + var skippedU = METRICS.skipUnsupportedGroup.sum(); + var skippedE = METRICS.skipTestException.sum(); + var skippedM = METRICS.skipLenientMismatch.sum(); + + System.out.printf( + "JSON-SCHEMA SUITE (%s): groups=%d testsScanned=%d run=%d passed=%d failed=%d skipped={unsupported=%d, exception=%d, lenientMismatch=%d}%n", + strict ? "STRICT" : "LENIENT", + METRICS.groupsDiscovered.sum(), + METRICS.testsDiscovered.sum(), + totalRun, passed, failed, skippedU, skippedE, skippedM + ); + + if (!METRICS_FMT.isEmpty()) { + var outDir = java.nio.file.Path.of("target"); + java.nio.file.Files.createDirectories(outDir); + var ts = java.time.OffsetDateTime.now().toString(); + if ("json".equalsIgnoreCase(METRICS_FMT)) { + var json = buildJsonSummary(strict, ts); + java.nio.file.Files.writeString(outDir.resolve("json-schema-compat.json"), json); + } else if ("csv".equalsIgnoreCase(METRICS_FMT)) { + var csv = buildCsvSummary(strict, ts); + java.nio.file.Files.writeString(outDir.resolve("json-schema-compat.csv"), csv); + } + } + } + + private static String buildJsonSummary(boolean strict, String timestamp) { + var totals = new StringBuilder(); + totals.append("{\n"); + totals.append(" \"mode\": \"").append(strict ? "STRICT" : "LENIENT").append("\",\n"); + totals.append(" \"timestamp\": \"").append(timestamp).append("\",\n"); + totals.append(" \"totals\": {\n"); + totals.append(" \"groupsDiscovered\": ").append(METRICS.groupsDiscovered.sum()).append(",\n"); + totals.append(" \"testsDiscovered\": ").append(METRICS.testsDiscovered.sum()).append(",\n"); + totals.append(" \"validationsRun\": ").append(METRICS.validationsRun.sum()).append(",\n"); + totals.append(" \"passed\": ").append(METRICS.passed.sum()).append(",\n"); + totals.append(" \"failed\": ").append(METRICS.failed.sum()).append(",\n"); + totals.append(" \"skipped\": {\n"); + totals.append(" \"unsupportedSchemaGroup\": ").append(METRICS.skipUnsupportedGroup.sum()).append(",\n"); + totals.append(" \"testException\": ").append(METRICS.skipTestException.sum()).append(",\n"); + totals.append(" \"lenientMismatch\": ").append(METRICS.skipLenientMismatch.sum()).append("\n"); + totals.append(" }\n"); + totals.append(" },\n"); + totals.append(" \"perFile\": [\n"); + + var files = new java.util.ArrayList(METRICS.perFile.keySet()); + java.util.Collections.sort(files); + var first = true; + for (String file : files) { + var counters = METRICS.perFile.get(file); + if (!first) totals.append(",\n"); + first = false; + totals.append(" {\n"); + totals.append(" \"file\": \"").append(file).append("\",\n"); + totals.append(" \"groups\": ").append(counters.groups.sum()).append(",\n"); + totals.append(" \"tests\": ").append(counters.tests.sum()).append(",\n"); + totals.append(" \"run\": ").append(counters.run.sum()).append(",\n"); + totals.append(" \"pass\": ").append(counters.pass.sum()).append(",\n"); + totals.append(" \"fail\": ").append(counters.fail.sum()).append(",\n"); + totals.append(" \"skipUnsupported\": ").append(counters.skipUnsupported.sum()).append(",\n"); + totals.append(" \"skipException\": ").append(counters.skipException.sum()).append(",\n"); + totals.append(" \"skipMismatch\": ").append(counters.skipMismatch.sum()).append("\n"); + totals.append(" }"); + } + totals.append("\n ]\n"); + totals.append("}\n"); + return totals.toString(); + } + + private static String buildCsvSummary(boolean strict, String timestamp) { + var csv = new StringBuilder(); + csv.append("mode,timestamp,groupsDiscovered,testsDiscovered,validationsRun,passed,failed,skipUnsupportedGroup,skipTestException,skipLenientMismatch\n"); + csv.append(strict ? "STRICT" : "LENIENT").append(","); + csv.append(timestamp).append(","); + csv.append(METRICS.groupsDiscovered.sum()).append(","); + csv.append(METRICS.testsDiscovered.sum()).append(","); + csv.append(METRICS.validationsRun.sum()).append(","); + csv.append(METRICS.passed.sum()).append(","); + csv.append(METRICS.failed.sum()).append(","); + csv.append(METRICS.skipUnsupportedGroup.sum()).append(","); + csv.append(METRICS.skipTestException.sum()).append(","); + csv.append(METRICS.skipLenientMismatch.sum()).append("\n"); + + csv.append("\nperFile breakdown:\n"); + csv.append("file,groups,tests,run,pass,fail,skipUnsupported,skipException,skipMismatch\n"); + + var files = new java.util.ArrayList(METRICS.perFile.keySet()); + java.util.Collections.sort(files); + for (String file : files) { + var counters = METRICS.perFile.get(file); + csv.append(file).append(","); + csv.append(counters.groups.sum()).append(","); + csv.append(counters.tests.sum()).append(","); + csv.append(counters.run.sum()).append(","); + csv.append(counters.pass.sum()).append(","); + csv.append(counters.fail.sum()).append(","); + csv.append(counters.skipUnsupported.sum()).append(","); + csv.append(counters.skipException.sum()).append(","); + csv.append(counters.skipMismatch.sum()).append("\n"); + } + return csv.toString(); + } +} + +/** + * Thread-safe metrics container for the JSON Schema Test Suite run. + */ +final class SuiteMetrics { + final LongAdder groupsDiscovered = new LongAdder(); + final LongAdder testsDiscovered = new LongAdder(); + + final LongAdder validationsRun = new LongAdder(); // attempted validations + final LongAdder passed = new LongAdder(); + final LongAdder failed = new LongAdder(); + + final LongAdder skipUnsupportedGroup = new LongAdder(); + final LongAdder skipTestException = new LongAdder(); // lenient only + final LongAdder skipLenientMismatch = new LongAdder(); // lenient only + + final ConcurrentHashMap perFile = new ConcurrentHashMap<>(); + + static final class FileCounters { + final LongAdder groups = new LongAdder(); + final LongAdder tests = new LongAdder(); + final LongAdder run = new LongAdder(); + final LongAdder pass = new LongAdder(); + final LongAdder fail = new LongAdder(); + final LongAdder skipUnsupported = new LongAdder(); + final LongAdder skipException = new LongAdder(); + final LongAdder skipMismatch = new LongAdder(); + } }