From ac5e60f23d6d3e72b184ff7496230c5acf04e49b Mon Sep 17 00:00:00 2001 From: zyf265600 Date: Fri, 18 Jul 2025 11:12:28 -0400 Subject: [PATCH 01/11] perf: add tree processing cache to avoid redundant computations Cache NewClassTree processing in NullnessNoInitAnnotatedTypeFactory to prevent duplicate type annotation calculations. --- .../NullnessNoInitAnnotatedTypeFactory.java | 16 ++++++++++++++++ docs/examples/NewObject.java | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 docs/examples/NewObject.java diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index befbe2fb9937..3e979f57ec8f 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -113,6 +113,9 @@ public class NullnessNoInitAnnotatedTypeFactory private final ExecutableElement mapGet = TreeUtils.getMethod("java.util.Map", "get", 1, processingEnv); + // High-level optimization: cache entire addComputedTypeAnnotations processing flow + private final java.util.Set fullyProcessedTrees = new java.util.HashSet<>(); + // List is in alphabetical order. If you update it, also update // ../../../../../../../../docs/manual/nullness-checker.tex // and make a pull request for variables NONNULL_ANNOTATIONS and BASE_COPYABLE_ANNOTATIONS in @@ -483,6 +486,19 @@ protected void replacePolyQualifier(AnnotatedTypeMirror lhsType, Tree context) { } } + @Override + protected void addComputedTypeAnnotations(Tree tree, AnnotatedTypeMirror type) { + if (tree != null && tree instanceof NewClassTree) { + String treeKey = tree.toString(); + if (fullyProcessedTrees.contains(treeKey)) { + return; + } + fullyProcessedTrees.add(treeKey); + } + + super.addComputedTypeAnnotations(tree, type); + } + @Override protected NullnessNoInitAnalysis createFlowAnalysis() { return new NullnessNoInitAnalysis(checker, this); diff --git a/docs/examples/NewObject.java b/docs/examples/NewObject.java new file mode 100644 index 000000000000..d2125fdfe33d --- /dev/null +++ b/docs/examples/NewObject.java @@ -0,0 +1,5 @@ +public class NewObject { + void test() { + Object object = new Object(); + } +} From 6caaa38bc36497e877dd74ed3037566be651ac4c Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 12 Aug 2025 21:50:04 -0400 Subject: [PATCH 02/11] Add fast-path in NullnessPropagationTreeAnnotator.visitNewClass: return if type already has @NonNull. --- .../NullnessNoInitAnnotatedTypeFactory.java | 25 +++++-------------- docs/examples/NewObject.java | 5 ---- 2 files changed, 6 insertions(+), 24 deletions(-) delete mode 100644 docs/examples/NewObject.java diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index 3e979f57ec8f..fc76775d257e 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -113,9 +113,6 @@ public class NullnessNoInitAnnotatedTypeFactory private final ExecutableElement mapGet = TreeUtils.getMethod("java.util.Map", "get", 1, processingEnv); - // High-level optimization: cache entire addComputedTypeAnnotations processing flow - private final java.util.Set fullyProcessedTrees = new java.util.HashSet<>(); - // List is in alphabetical order. If you update it, also update // ../../../../../../../../docs/manual/nullness-checker.tex // and make a pull request for variables NONNULL_ANNOTATIONS and BASE_COPYABLE_ANNOTATIONS in @@ -486,19 +483,6 @@ protected void replacePolyQualifier(AnnotatedTypeMirror lhsType, Tree context) { } } - @Override - protected void addComputedTypeAnnotations(Tree tree, AnnotatedTypeMirror type) { - if (tree != null && tree instanceof NewClassTree) { - String treeKey = tree.toString(); - if (fullyProcessedTrees.contains(treeKey)) { - return; - } - fullyProcessedTrees.add(treeKey); - } - - super.addComputedTypeAnnotations(tree, type); - } - @Override protected NullnessNoInitAnalysis createFlowAnalysis() { return new NullnessNoInitAnalysis(checker, this); @@ -750,8 +734,9 @@ public Void visitUnary(UnaryTree tree, AnnotatedTypeMirror type) { // explicit nullable annotations are left intact for the visitor to inspect. @Override public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { - // The constructor return type should already be NONNULL, so in most cases this will do - // nothing. + if (type.hasEffectiveAnnotation(NONNULL)) { + return null; + } type.addMissingAnnotation(NONNULL); return null; } @@ -761,7 +746,9 @@ public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { @Override public Void visitNewArray(NewArrayTree tree, AnnotatedTypeMirror type) { super.visitNewArray(tree, type); - type.addMissingAnnotation(NONNULL); + if (!type.hasEffectiveAnnotation(NONNULL)) { + type.addMissingAnnotation(NONNULL); + } return null; } diff --git a/docs/examples/NewObject.java b/docs/examples/NewObject.java deleted file mode 100644 index d2125fdfe33d..000000000000 --- a/docs/examples/NewObject.java +++ /dev/null @@ -1,5 +0,0 @@ -public class NewObject { - void test() { - Object object = new Object(); - } -} From da837a3d3775bfe5bd5a7126047e3d617119ceb5 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Thu, 11 Sep 2025 00:27:02 -0400 Subject: [PATCH 03/11] performance comparison --- .../nullness/perf/NewClassPerf$Result.class | Bin 0 -> 1925 bytes .../jtreg/nullness/perf/NewClassPerf.class | Bin 0 -> 6220 bytes checker/jtreg/nullness/perf/NewClassPerf.java | 282 ++++++++++++++++++ .../NullnessNoInitAnnotatedTypeFactory.java | 9 +- 4 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 checker/jtreg/nullness/perf/NewClassPerf$Result.class create mode 100644 checker/jtreg/nullness/perf/NewClassPerf.class create mode 100644 checker/jtreg/nullness/perf/NewClassPerf.java diff --git a/checker/jtreg/nullness/perf/NewClassPerf$Result.class b/checker/jtreg/nullness/perf/NewClassPerf$Result.class new file mode 100644 index 0000000000000000000000000000000000000000..66bde90f194ef5a3a77ed76b7d9d2d5213a22295 GIT binary patch literal 1925 zcmbVN-BTM?6#w0jWJ|Jzln+~4ii)5Ksic0?uLUe5ATR-HgQ=~bH_1)9upgQIz`+L} zbwum<>YERJXF8*8sWXnxj{lO5=k5lxP^>fNA-nh7bAIP{&iUQDzyI~E0YePc~OoaEf6taPmFfD9k$!-!1E|mx6+n z1W7ZDWXoNTqHP$2quZ92tEO-oeG2+jWN?OI_#`)%SooG{H}s~?Yle{M4C7HuY0`3$ zB!Qt9gBVgUtl}x0C6?>rv!cOW_nL5;6W4|78-&yA>85VA+%=izI6@jDIIrM>icwr- z=M~d2L%Sm&iWcxHaErkLuD|ku8%Xo$1To?1GBdN@G77ds1;@MKJ zBnNRt#cOz-Qt9rZ+4gn=0p5}ddQ-()(8$%b9UAFiwDwRMRS7VwVh(RhH~n)0Ci$*N z^Sp`$6lvWO6g-?=>9i#xdQHLN;rIiyEIox1-chlP6>7afhAKDw5O+0sm4+^Vv{lht z7WaIIw#JVMY$ja)1F6l)DvEZ4eEZ9~CDwhjCY((vdnuRy4$oDC!b;RYQ#5pLF`TY= zyuQ6g+6ZA%729{}Vo8^nXQGEyw9LUJ?M-q*tSQ)*5DMuo4oGX4h=7DMpu7yYKsPi36{Cl zFoZi%w(V`dtwlOUI;E2vwREzDft$N+@&7i%luwS0ilmda$PJ^SdqT^gCWREC8s42+ zZnZywr3|R?ha&CHKDSEEUa>6U1h}iQVMjM{$ou&sXq#~8OEOMO1NP&ahYz& zSfJ93ydz1iWm?6sh8}t*pwas}y$M6VazyZc%^p~Z>bsy9F@kagtGYqq%@h{>(VkkeFFb;4vhTl-SkEOkRX^1x>rehv_ zZwRZlWD*OQjL~-t_!@oK!+Cr|q`$=^?odmjz1R$5-$ONc h-oQ}Joe*^nR_Bj9m literal 0 HcmV?d00001 diff --git a/checker/jtreg/nullness/perf/NewClassPerf.class b/checker/jtreg/nullness/perf/NewClassPerf.class new file mode 100644 index 0000000000000000000000000000000000000000..aeb8f8f305e24c92ecb43813dbc6c39db4b59c8c GIT binary patch literal 6220 zcmaJ_30xf4dH;W~Fv~Ks&?01GU1Lj@7qknO9ZNDR$pR#7BtS$UFgolEyMtJ1*;&ud z3K2(5(xyk+rgiJKvC=e6+(T&_$48w7S+UjAZj-wAmGr*v+ijXOw(@^-z#?$@@!OgA z-uJ%uegE%#^M$uxeigueDaTNQS`85$b*L9Koii_(oq4lxw)5EJIVi6XeIro5| zHqkZ~MN}Xi8e+?6M_eaYI2(hGH5zWwu@+5&W{+1PXLnBJ@>b`t=}wo(<*Y@f>BPX& zN&fEisg+Z0{?sxa=|;rStYN*57Hkm2R~{p{#kIR9OLo5OTIyWEhQvGax=lx`V$@Ku zof$Kqd#@G6Mu9$P7UqVm*$%~Dow-@Z7Hk!0jz!KFEJ0!wEml27!yRNuX1#f{R2sIN zsTd4w)3F_QQG;10=UTyWBX-%zkoRo?@8!Dpw~eWtck4)?P0;A(W~?#O$(aR$np}0u z$+m&&gSQ{IO69zpZop0?HKcTOpmVt~=Sr?MLrd9ZGHvlqKr0T@#x8ATnKRtI3-@Zc zPsh7(KTW;L2?UhV(Je}YG<%cJrtIeOodda&n~u}}_UPD)2L!jynCGotyO1&6@tix& zw~|YZxNUo&($c;FU(ZDY$B7}0eH!-bIDiL3zgsRatyWax=L^(mHLH!qA=slh^(Z#= zQ5+Ji@v$lyiJ4McR}_7MEdknw@jk|P4Z}L#jpNJAamX%D=IIdH1v6i^j!h{A4ppPz6QmXE zQ5`1~Yb_8bh6ar5cmxwXZc4c~E&9MJ`d7!EANc9&BLbkKI7RJx-cdUZXK2iu1G}f3 z%V#a87EM45r|}+@YfM2TL2r&CBUqoCDcX)}sGS|k)E({Vm?E`wOyMkLH;YB9kQJm> zk$t70!HMZ8a+Ie?{Bjv1qe9OJqReGi5)gk= zAUEUT)EUFfMe%_pIyJfLo-<0CZx*aoOnrumv=qgM7+?X*U_PM!HfCK#eY{*KnDkh~ ziQ+p1?E!H(O+bynA|9Adx!_~0PW(=Kh2g>d-GZ7+u?Bn(zE{JOI=&CzFSuhd)n}$n zr)0UEBg_+7)5-SwV`hxH6Qg~P?WLP8m5z`&AWCca0p{05US4%NM*liibaQqgh9AO5 zH9W23GCme&lf^9D$5O^q*3oi3<*6ylv9jYTd(-5Zz&96ZRpm*)vlv0lEs`&Z5XK&J z*z!Xdu^pXu>{(^m7_Q)_b^HuI!9H&!RTQYV?AR7xA;yt6&!FQ5H5YGaga(?sGaSnCB9`E*`^MSvTvLVT^D2|B5~T!r0k5hg^snYJ?LErvkI9Xmr~ z+YIkN65MY)XFGzEryO&}nzfzt;Y5exSs-!8Q1H_0%NWIB}h$dYQx{!Yi=tLCkdL@G9iD1w^*cntr9 zf7b9XI{p>^78cYc&tnYNnvX;qn1xrQ;_3tkf%Rz(5 zIv8@ICRGdMwNB!iH0!cnS}3^3YdHX~y@wvHs)PW31uZq?;BX{B4unmM=6 zX42-Rrk`D(Y-E;{O;jXts)ykvTL`OS-7#&?FyZYKLzg?`PT$c8MA33w!o4Ga+f&!i z#$ocVp3-DHg$hMg28_uzyviTBTbG0~^_oQoP|Hzir)-nCf+9&W5lBj~?Izn+nI=41 z-43KvmwRLv708vw8TXbdY>{Yw!t&5C`lA7w$ zC9P7(TFO>^or=nSVrZ8-3XHa>u(+_X+bP`&q8`ECRX|xq)|FFiSLKi{ht-*lu03GS zT28N7;t>x~gIS_Mf_`0&$WhjUj9qZeT!G4MtUgdx`4Q{!vQ@}fj6+)g_=%wrMoK?H z4zCtts+X>1ZjVvnhmIdRF+9SNJW1+aJIjDuJCG|_L*FgO)pOXG?4Ah33tQrO6blp0HSL_Z-oaELXT-pSRdX zsr$Qs1M4$`RPuB+ssW+T^kFt#Y=gEB*!KByF=%nB{k_Vx%QlYA6|Ggss}T)L;7S&~ zWawy4SDEGa7>UJW@-xNXb- zPfET%%(#O^y@oDZgWF|52ATSY)JTqTJX|$?H#FX5`8kIa$4Rl7BSt9d+UFrJ@I_>V zzg*^T3nOY&PLPcv{R+3Lp69u^R{IJX7SMPV>jYPEt6*Sf{PwHZRD;3fI(c#dcP48W zuwy70;cr*uMYN}0!aXlzH`uCmao8I1#s|Z31>KyD_xh{x!@={Rx>ry$QJboue*uFd z6A`|RsJ8`-1tceZlGex~je4yjT2g8^)=>P7SdUHAdo#bUY(+b9+l6g7$k8zF#u&ey zJc>5n)7tT2_MM+aQpUI|LSV+3$oNghheMu1Enxz_d<~~3;*ZYbvB6|~vV!b*GCu7; z&iRkLdek1+yzv?2lJNpZn=2^BOa5v(9MAgWxp4f1KfV->-|vq<7>>WiAAf5&{&qF~ zP_10K@pLl&VSn+v)T8}He1sNzY5_kuluB0c!xj8!>NRX5m_L36o4O(#H2VX+tFATD zT35kOwMMSLKyP{$pTp0wrGJfm>KlC4c67Z5zRC{!u6-P`Z0b*`4k=`_k-l$%@ZEPm(U{<bnWI~Jo@yJ9p{-+}oP4)3V=dTw3Oa4Z^_lEdi z^F7vn6@RPZ`yWKb(?4Ftzf<*Vcyl5(kN@e4v_?qq6-Dqhp?GhPw?-lrso|p{_4K2< zio`0?)K#zEHtg0~w99Bzv0q=2+gr5HETYy&9|A@hPh)7rI8l3qA6+MKCr-h{B=UTg zk;VIA;Ymj3M{yRPKn|bAIXuVJm*``khwUM{12u2qP7Mu2@CGmGQ6Eh;)bOSzjhdWh zYFh>^Z$d#f{37E#LRm(dW%Gh;J$g;HO|;L;T?6W6XGL~2OWVBc98|;Pf^=L)L$lm_ z1@+DHF3u07Dzckr^6|j)H!UwY$rk2Opv2!Yo#X-oXO^LIks&dMUVfEPLC}M`8;mXb zLW3sT_>bBsw(_5^HyD;x3z5YI5`P_QNzvHDS9ULK^a90pagUeLn`K|K99WQp73n?N z-Yk6;d01J(Py6bFKkxzO$qynTkMg}n9+UU*$^3_1V)AW0YGso2)86kn?>8?6uBglN PF1DC9Yfe!danbq>X4_1I literal 0 HcmV?d00001 diff --git a/checker/jtreg/nullness/perf/NewClassPerf.java b/checker/jtreg/nullness/perf/NewClassPerf.java new file mode 100644 index 000000000000..a8ff7cf0b20b --- /dev/null +++ b/checker/jtreg/nullness/perf/NewClassPerf.java @@ -0,0 +1,282 @@ +/* + * @test + * @summary Measure impact of skipping hasEffectiveAnnotation(NONNULL) fast-path on many `new` sites. + * + * @run NewClassPerf + */ + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class NewClassPerf { + + private static final int RUNS = Integer.getInteger("perf.runs", 10); + private static final int GROUPS = Integer.getInteger("perf.groups", 400); // ~2k new-exprs total (5 per group) + // Protocol: AB | BA | BOTH | SEPARATE. Default BOTH runs interleaved AB and BA to cancel order bias. + private static final String PROTOCOL = System.getProperty("perf.protocol", "BOTH"); + private static final int WARMUP = Integer.getInteger("perf.warmupPerVariant", 1); + + public static void main(String[] args) throws Exception { + Path workDir = Paths.get(".").toAbsolutePath().normalize(); + Path src = workDir.resolve("ManyNew.java"); + writeManyNewSource(src, GROUPS); + + switch (PROTOCOL.toUpperCase()) { + case "AB": { + Result[] ab = timeInterleaved(src, "AB"); + printResults("Interleaved AB", ab[0], ab[1]); + break; + } + case "BA": { + Result[] ba = timeInterleaved(src, "BA"); + printResults("Interleaved BA", ba[0], ba[1]); + break; + } + case "SEPARATE": { + Result[] sep = timeSeparate(src, WARMUP); + printResults("Separate (warmup=" + WARMUP + ")", sep[0], sep[1]); + break; + } + case "BOTH": + default: { + Result[] ab = timeInterleaved(src, "AB"); + Result[] ba = timeInterleaved(src, "BA"); + System.out.println("==== Interleaved AB ===="); + printResults("AB", ab[0], ab[1]); + System.out.println(); + System.out.println("==== Interleaved BA ===="); + printResults("BA", ba[0], ba[1]); + // Consistency check + double sign1 = Math.signum((ab[1].median() - ab[0].median())); + double sign2 = Math.signum((ba[1].median() - ba[0].median())); + System.out.println(); + if (sign1 == 0 || sign2 == 0 || Math.signum(sign1) != Math.signum(sign2)) { + System.out.println("Direction: ORDER-SENSITIVE (results differ between AB and BA)"); + } else { + System.out.println("Direction: CONSISTENT across AB and BA"); + } + break; + } + } + } + + private static void writeManyNewSource(Path file, int groups) throws IOException { + StringBuilder sb = new StringBuilder(1024 * 1024); + sb.append("import java.util.*;\n"); + sb.append("public class ManyNew {\n"); + // Avoid initialization checker errors; use @Nullable type variable upper bound. + sb.append(" static class Box { T t; Box(){ this.t = (T) (Object) new Object(); } }\n"); + sb.append(" void f() {\n"); + for (int i = 0; i < groups; i++) { + sb.append(" Object o").append(i).append(" = new Object();\n"); + sb.append(" ArrayList l").append(i).append(" = new ArrayList<>();\n"); + sb.append(" Box b").append(i).append(" = new Box<>();\n"); + sb.append(" int[] ai").append(i).append(" = new int[10];\n"); + sb.append(" String[] as").append(i).append(" = new String[10];\n"); + sb.append(" Runnable r").append(i) + .append(" = new Runnable(){ public void run(){} };\n"); + } + sb.append(" }\n"); + sb.append("}\n"); + try (BufferedWriter w = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) { + w.write(sb.toString()); + } + } + + private static Result timeVariant(Path src, boolean skipFastPath) throws Exception { + List timings = new ArrayList<>(); + for (int i = 0; i < RUNS; i++) { + timings.add(runOnceMs(src, skipFastPath)); + } + return new Result(timings); + } + + private static Result[] timeInterleaved(Path src, String order) throws Exception { + List aTimes = new ArrayList<>(); + List bTimes = new ArrayList<>(); + boolean firstA = !"BA".equalsIgnoreCase(order); + for (int i = 0; i < RUNS; i++) { + if (firstA) { + aTimes.add(runOnceMs(src, /*skipFastPath=*/false)); + bTimes.add(runOnceMs(src, /*skipFastPath=*/true)); + } else { + bTimes.add(runOnceMs(src, /*skipFastPath=*/true)); + aTimes.add(runOnceMs(src, /*skipFastPath=*/false)); + } + } + return new Result[] { new Result(aTimes), new Result(bTimes) }; + } + + private static long runOnceMs(Path src, boolean skipFastPath) throws Exception { + long start = System.nanoTime(); + int exit = runJavac(src, skipFastPath); + long end = System.nanoTime(); + if (exit != 0) { + throw new RuntimeException("javac failed with exit code " + exit); + } + return (end - start) / 1_000_000L; + } + + private static Result[] timeSeparate(Path src, int warmup) throws Exception { + // Variant A (fast-path enabled) + for (int i = 0; i < warmup; i++) { + runOnceMs(src, /*skipFastPath=*/false); + } + List aTimes = new ArrayList<>(); + for (int i = 0; i < RUNS; i++) { + aTimes.add(runOnceMs(src, /*skipFastPath=*/false)); + } + + // Variant B (fast-path disabled) + for (int i = 0; i < warmup; i++) { + runOnceMs(src, /*skipFastPath=*/true); + } + List bTimes = new ArrayList<>(); + for (int i = 0; i < RUNS; i++) { + bTimes.add(runOnceMs(src, /*skipFastPath=*/true)); + } + return new Result[] { new Result(aTimes), new Result(bTimes) }; + } + + private static void printResults(String label, Result a, Result b) { + // Print raw timings + System.out.println("Variant A (fast-path enabled) timings ms: " + a.timingsMs); + System.out.println("Variant B (fast-path disabled) timings ms: " + b.timingsMs); + + // Summary table and deltas + System.out.println(); + System.out.println("Results (ms) - " + label + ":"); + System.out.println("Variant | median | average"); + System.out.println(String.format("A | %7.2f | %7.2f", a.median(), a.average())); + System.out.println(String.format("B | %7.2f | %7.2f", b.median(), b.average())); + double medA = a.median(), medB = b.median(); + double avgA = a.average(), avgB = b.average(); + System.out.println(); + System.out.printf("Median delta (B vs A): %.3f%%%n", (medB - medA) / medA * 100.0); + System.out.printf("Average delta (B vs A): %.3f%%%n", (avgB - avgA) / avgA * 100.0); + } + + private static int runJavac(Path src, boolean skipFastPath) throws Exception { + String checkerJar = locateCheckerJar(); + + List cmd = new ArrayList<>(); + cmd.add(findJavacExecutable()); + // Add required --add-opens for running Checker Framework on JDK 9+ + String[] javacPkgs = new String[] { + "com.sun.tools.javac.api", + "com.sun.tools.javac.code", + "com.sun.tools.javac.comp", + "com.sun.tools.javac.file", + "com.sun.tools.javac.main", + "com.sun.tools.javac.parser", + "com.sun.tools.javac.processing", + "com.sun.tools.javac.tree", + "com.sun.tools.javac.util" + }; + for (String p : javacPkgs) { + cmd.add("-J--add-opens=jdk.compiler/" + p + "=ALL-UNNAMED"); + } + if (skipFastPath) { + cmd.add("-J-Dcf.skipNonnullFastPath=true"); + } + cmd.addAll(Arrays.asList( + "-classpath", checkerJar, + "-processor", "org.checkerframework.checker.nullness.NullnessChecker", + "-proc:only", + "-source", "8", + "-target", "8", + "-Xlint:-options", + src.getFileName().toString())); + + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(src.getParent().toFile()); + pb.redirectErrorStream(true); + Process p = pb.start(); + // Consume output to avoid buffer blockage. + byte[] out = p.getInputStream().readAllBytes(); + int code = p.waitFor(); + // In case of failure, print compiler output for debugging. + if (code != 0) { + System.out.write(out); + } + return code; + } + + private static String locateCheckerJar() { + try { + String testRoot = System.getProperty("test.root"); + if (testRoot != null) { + Path p = Paths.get(testRoot).toAbsolutePath().normalize().getParent().resolve("dist/checker.jar"); + if (Files.exists(p)) { + return p.toString(); + } + } + Path p1 = Paths.get("checker/dist/checker.jar").toAbsolutePath().normalize(); + if (Files.exists(p1)) { + return p1.toString(); + } + Path p2 = Paths.get("../../../dist/checker.jar").toAbsolutePath().normalize(); + if (Files.exists(p2)) { + return p2.toString(); + } + // Fallback: walk up from the current working dir (jtreg scratch) + Path cwd = Paths.get(".").toAbsolutePath().normalize(); + for (int i = 0; i < 8; i++) { + Path cand = cwd; + for (int j = 0; j < i; j++) cand = cand.getParent(); + if (cand == null) break; + Path jar = cand.resolve("checker/dist/checker.jar"); + if (Files.exists(jar)) return jar.toString(); + } + } catch (Throwable ignore) { + } + return "checker/dist/checker.jar"; + } + + private static String findJavacExecutable() { + String javaHome = System.getProperty("java.home"); + // Typical JDK layout: $JAVA_HOME/bin/javac + File jh = new File(javaHome); + File bin = new File(jh.getParentFile(), "bin"); + File javac = new File(bin, isWindows() ? "javac.exe" : "javac"); + if (javac.exists()) { + return javac.getAbsolutePath(); + } + // Fallback to PATH + return isWindows() ? "javac.exe" : "javac"; + } + + private static boolean isWindows() { + String os = System.getProperty("os.name", "").toLowerCase(); + return os.contains("win"); + } + + private static final class Result { + final List timingsMs; + Result(List t) { this.timingsMs = Collections.unmodifiableList(new ArrayList<>(t)); } + double average() { return timingsMs.stream().mapToLong(Long::longValue).average().orElse(0); } + double median() { + if (timingsMs.isEmpty()) return 0; + List copy = new ArrayList<>(timingsMs); + Collections.sort(copy); + int n = copy.size(); + if ((n & 1) == 1) { + return copy.get(n/2); + } else { + return (copy.get(n/2 - 1) + copy.get(n/2)) / 2.0; + } + } + } +} + + diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index fc76775d257e..fb469b0e42f7 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -79,6 +79,13 @@ public class NullnessNoInitAnnotatedTypeFactory NullnessNoInitTransfer, NullnessNoInitAnalysis> { + /** + * Runtime toggle: skip the {@code hasEffectiveAnnotation(NONNULL)} fast-path. Controlled via + * JVM system property {@code -Dcf.skipNonnullFastPath=true}. + */ + private static final boolean SKIP_NONNULL_FASTPATH = + Boolean.getBoolean("cf.skipNonnullFastPath"); + /** The @{@link NonNull} annotation. */ protected final AnnotationMirror NONNULL = AnnotationBuilder.fromClass(elements, NonNull.class); @@ -734,7 +741,7 @@ public Void visitUnary(UnaryTree tree, AnnotatedTypeMirror type) { // explicit nullable annotations are left intact for the visitor to inspect. @Override public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { - if (type.hasEffectiveAnnotation(NONNULL)) { + if (!SKIP_NONNULL_FASTPATH && type.hasEffectiveAnnotation(NONNULL)) { return null; } type.addMissingAnnotation(NONNULL); From 5bf54a6504f1423541fede53e5b281d31640c068 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Thu, 11 Sep 2025 01:28:44 -0400 Subject: [PATCH 04/11] remove redundant changes and files --- .../nullness/perf/NewClassPerf$Result.class | Bin 1925 -> 0 bytes checker/jtreg/nullness/perf/NewClassPerf.class | Bin 6220 -> 0 bytes checker/jtreg/nullness/perf/NewClassPerf.java | 2 +- .../NullnessNoInitAnnotatedTypeFactory.java | 4 +--- 4 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 checker/jtreg/nullness/perf/NewClassPerf$Result.class delete mode 100644 checker/jtreg/nullness/perf/NewClassPerf.class diff --git a/checker/jtreg/nullness/perf/NewClassPerf$Result.class b/checker/jtreg/nullness/perf/NewClassPerf$Result.class deleted file mode 100644 index 66bde90f194ef5a3a77ed76b7d9d2d5213a22295..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1925 zcmbVN-BTM?6#w0jWJ|Jzln+~4ii)5Ksic0?uLUe5ATR-HgQ=~bH_1)9upgQIz`+L} zbwum<>YERJXF8*8sWXnxj{lO5=k5lxP^>fNA-nh7bAIP{&iUQDzyI~E0YePc~OoaEf6taPmFfD9k$!-!1E|mx6+n z1W7ZDWXoNTqHP$2quZ92tEO-oeG2+jWN?OI_#`)%SooG{H}s~?Yle{M4C7HuY0`3$ zB!Qt9gBVgUtl}x0C6?>rv!cOW_nL5;6W4|78-&yA>85VA+%=izI6@jDIIrM>icwr- z=M~d2L%Sm&iWcxHaErkLuD|ku8%Xo$1To?1GBdN@G77ds1;@MKJ zBnNRt#cOz-Qt9rZ+4gn=0p5}ddQ-()(8$%b9UAFiwDwRMRS7VwVh(RhH~n)0Ci$*N z^Sp`$6lvWO6g-?=>9i#xdQHLN;rIiyEIox1-chlP6>7afhAKDw5O+0sm4+^Vv{lht z7WaIIw#JVMY$ja)1F6l)DvEZ4eEZ9~CDwhjCY((vdnuRy4$oDC!b;RYQ#5pLF`TY= zyuQ6g+6ZA%729{}Vo8^nXQGEyw9LUJ?M-q*tSQ)*5DMuo4oGX4h=7DMpu7yYKsPi36{Cl zFoZi%w(V`dtwlOUI;E2vwREzDft$N+@&7i%luwS0ilmda$PJ^SdqT^gCWREC8s42+ zZnZywr3|R?ha&CHKDSEEUa>6U1h}iQVMjM{$ou&sXq#~8OEOMO1NP&ahYz& zSfJ93ydz1iWm?6sh8}t*pwas}y$M6VazyZc%^p~Z>bsy9F@kagtGYqq%@h{>(VkkeFFb;4vhTl-SkEOkRX^1x>rehv_ zZwRZlWD*OQjL~-t_!@oK!+Cr|q`$=^?odmjz1R$5-$ONc h-oQ}Joe*^nR_Bj9m diff --git a/checker/jtreg/nullness/perf/NewClassPerf.class b/checker/jtreg/nullness/perf/NewClassPerf.class deleted file mode 100644 index aeb8f8f305e24c92ecb43813dbc6c39db4b59c8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6220 zcmaJ_30xf4dH;W~Fv~Ks&?01GU1Lj@7qknO9ZNDR$pR#7BtS$UFgolEyMtJ1*;&ud z3K2(5(xyk+rgiJKvC=e6+(T&_$48w7S+UjAZj-wAmGr*v+ijXOw(@^-z#?$@@!OgA z-uJ%uegE%#^M$uxeigueDaTNQS`85$b*L9Koii_(oq4lxw)5EJIVi6XeIro5| zHqkZ~MN}Xi8e+?6M_eaYI2(hGH5zWwu@+5&W{+1PXLnBJ@>b`t=}wo(<*Y@f>BPX& zN&fEisg+Z0{?sxa=|;rStYN*57Hkm2R~{p{#kIR9OLo5OTIyWEhQvGax=lx`V$@Ku zof$Kqd#@G6Mu9$P7UqVm*$%~Dow-@Z7Hk!0jz!KFEJ0!wEml27!yRNuX1#f{R2sIN zsTd4w)3F_QQG;10=UTyWBX-%zkoRo?@8!Dpw~eWtck4)?P0;A(W~?#O$(aR$np}0u z$+m&&gSQ{IO69zpZop0?HKcTOpmVt~=Sr?MLrd9ZGHvlqKr0T@#x8ATnKRtI3-@Zc zPsh7(KTW;L2?UhV(Je}YG<%cJrtIeOodda&n~u}}_UPD)2L!jynCGotyO1&6@tix& zw~|YZxNUo&($c;FU(ZDY$B7}0eH!-bIDiL3zgsRatyWax=L^(mHLH!qA=slh^(Z#= zQ5+Ji@v$lyiJ4McR}_7MEdknw@jk|P4Z}L#jpNJAamX%D=IIdH1v6i^j!h{A4ppPz6QmXE zQ5`1~Yb_8bh6ar5cmxwXZc4c~E&9MJ`d7!EANc9&BLbkKI7RJx-cdUZXK2iu1G}f3 z%V#a87EM45r|}+@YfM2TL2r&CBUqoCDcX)}sGS|k)E({Vm?E`wOyMkLH;YB9kQJm> zk$t70!HMZ8a+Ie?{Bjv1qe9OJqReGi5)gk= zAUEUT)EUFfMe%_pIyJfLo-<0CZx*aoOnrumv=qgM7+?X*U_PM!HfCK#eY{*KnDkh~ ziQ+p1?E!H(O+bynA|9Adx!_~0PW(=Kh2g>d-GZ7+u?Bn(zE{JOI=&CzFSuhd)n}$n zr)0UEBg_+7)5-SwV`hxH6Qg~P?WLP8m5z`&AWCca0p{05US4%NM*liibaQqgh9AO5 zH9W23GCme&lf^9D$5O^q*3oi3<*6ylv9jYTd(-5Zz&96ZRpm*)vlv0lEs`&Z5XK&J z*z!Xdu^pXu>{(^m7_Q)_b^HuI!9H&!RTQYV?AR7xA;yt6&!FQ5H5YGaga(?sGaSnCB9`E*`^MSvTvLVT^D2|B5~T!r0k5hg^snYJ?LErvkI9Xmr~ z+YIkN65MY)XFGzEryO&}nzfzt;Y5exSs-!8Q1H_0%NWIB}h$dYQx{!Yi=tLCkdL@G9iD1w^*cntr9 zf7b9XI{p>^78cYc&tnYNnvX;qn1xrQ;_3tkf%Rz(5 zIv8@ICRGdMwNB!iH0!cnS}3^3YdHX~y@wvHs)PW31uZq?;BX{B4unmM=6 zX42-Rrk`D(Y-E;{O;jXts)ykvTL`OS-7#&?FyZYKLzg?`PT$c8MA33w!o4Ga+f&!i z#$ocVp3-DHg$hMg28_uzyviTBTbG0~^_oQoP|Hzir)-nCf+9&W5lBj~?Izn+nI=41 z-43KvmwRLv708vw8TXbdY>{Yw!t&5C`lA7w$ zC9P7(TFO>^or=nSVrZ8-3XHa>u(+_X+bP`&q8`ECRX|xq)|FFiSLKi{ht-*lu03GS zT28N7;t>x~gIS_Mf_`0&$WhjUj9qZeT!G4MtUgdx`4Q{!vQ@}fj6+)g_=%wrMoK?H z4zCtts+X>1ZjVvnhmIdRF+9SNJW1+aJIjDuJCG|_L*FgO)pOXG?4Ah33tQrO6blp0HSL_Z-oaELXT-pSRdX zsr$Qs1M4$`RPuB+ssW+T^kFt#Y=gEB*!KByF=%nB{k_Vx%QlYA6|Ggss}T)L;7S&~ zWawy4SDEGa7>UJW@-xNXb- zPfET%%(#O^y@oDZgWF|52ATSY)JTqTJX|$?H#FX5`8kIa$4Rl7BSt9d+UFrJ@I_>V zzg*^T3nOY&PLPcv{R+3Lp69u^R{IJX7SMPV>jYPEt6*Sf{PwHZRD;3fI(c#dcP48W zuwy70;cr*uMYN}0!aXlzH`uCmao8I1#s|Z31>KyD_xh{x!@={Rx>ry$QJboue*uFd z6A`|RsJ8`-1tceZlGex~je4yjT2g8^)=>P7SdUHAdo#bUY(+b9+l6g7$k8zF#u&ey zJc>5n)7tT2_MM+aQpUI|LSV+3$oNghheMu1Enxz_d<~~3;*ZYbvB6|~vV!b*GCu7; z&iRkLdek1+yzv?2lJNpZn=2^BOa5v(9MAgWxp4f1KfV->-|vq<7>>WiAAf5&{&qF~ zP_10K@pLl&VSn+v)T8}He1sNzY5_kuluB0c!xj8!>NRX5m_L36o4O(#H2VX+tFATD zT35kOwMMSLKyP{$pTp0wrGJfm>KlC4c67Z5zRC{!u6-P`Z0b*`4k=`_k-l$%@ZEPm(U{<bnWI~Jo@yJ9p{-+}oP4)3V=dTw3Oa4Z^_lEdi z^F7vn6@RPZ`yWKb(?4Ftzf<*Vcyl5(kN@e4v_?qq6-Dqhp?GhPw?-lrso|p{_4K2< zio`0?)K#zEHtg0~w99Bzv0q=2+gr5HETYy&9|A@hPh)7rI8l3qA6+MKCr-h{B=UTg zk;VIA;Ymj3M{yRPKn|bAIXuVJm*``khwUM{12u2qP7Mu2@CGmGQ6Eh;)bOSzjhdWh zYFh>^Z$d#f{37E#LRm(dW%Gh;J$g;HO|;L;T?6W6XGL~2OWVBc98|;Pf^=L)L$lm_ z1@+DHF3u07Dzckr^6|j)H!UwY$rk2Opv2!Yo#X-oXO^LIks&dMUVfEPLC}M`8;mXb zLW3sT_>bBsw(_5^HyD;x3z5YI5`P_QNzvHDS9ULK^a90pagUeLn`K|K99WQp73n?N z-Yk6;d01J(Py6bFKkxzO$qynTkMg}n9+UU*$^3_1V)AW0YGso2)86kn?>8?6uBglN PF1DC9Yfe!danbq>X4_1I diff --git a/checker/jtreg/nullness/perf/NewClassPerf.java b/checker/jtreg/nullness/perf/NewClassPerf.java index a8ff7cf0b20b..f0f257676656 100644 --- a/checker/jtreg/nullness/perf/NewClassPerf.java +++ b/checker/jtreg/nullness/perf/NewClassPerf.java @@ -2,7 +2,7 @@ * @test * @summary Measure impact of skipping hasEffectiveAnnotation(NONNULL) fast-path on many `new` sites. * - * @run NewClassPerf + * @run main/timeout=600 NewClassPerf */ import java.io.BufferedWriter; diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index fb469b0e42f7..ce6cf38ed09c 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -753,9 +753,7 @@ public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { @Override public Void visitNewArray(NewArrayTree tree, AnnotatedTypeMirror type) { super.visitNewArray(tree, type); - if (!type.hasEffectiveAnnotation(NONNULL)) { - type.addMissingAnnotation(NONNULL); - } + type.addMissingAnnotation(NONNULL); return null; } From 588ed4712279067d61cfe23f215ba8df91ef1b08 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Sun, 28 Sep 2025 21:40:20 -0400 Subject: [PATCH 05/11] refine code with Formatter --- checker/jtreg/nullness/perf/NewClassPerf.java | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/checker/jtreg/nullness/perf/NewClassPerf.java b/checker/jtreg/nullness/perf/NewClassPerf.java index f0f257676656..6b79b2c6ff76 100644 --- a/checker/jtreg/nullness/perf/NewClassPerf.java +++ b/checker/jtreg/nullness/perf/NewClassPerf.java @@ -16,7 +16,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Formatter; import java.util.List; +import java.util.Locale; public class NewClassPerf { @@ -72,22 +74,23 @@ public static void main(String[] args) throws Exception { private static void writeManyNewSource(Path file, int groups) throws IOException { StringBuilder sb = new StringBuilder(1024 * 1024); - sb.append("import java.util.*;\n"); - sb.append("public class ManyNew {\n"); - // Avoid initialization checker errors; use @Nullable type variable upper bound. - sb.append(" static class Box { T t; Box(){ this.t = (T) (Object) new Object(); } }\n"); - sb.append(" void f() {\n"); - for (int i = 0; i < groups; i++) { - sb.append(" Object o").append(i).append(" = new Object();\n"); - sb.append(" ArrayList l").append(i).append(" = new ArrayList<>();\n"); - sb.append(" Box b").append(i).append(" = new Box<>();\n"); - sb.append(" int[] ai").append(i).append(" = new int[10];\n"); - sb.append(" String[] as").append(i).append(" = new String[10];\n"); - sb.append(" Runnable r").append(i) - .append(" = new Runnable(){ public void run(){} };\n"); + try (Formatter fmt = new Formatter(sb, Locale.ROOT)) { + fmt.format("import java.util.*;%n"); + fmt.format("public class ManyNew {%n"); + // Avoid initialization checker errors; use @Nullable type variable upper bound. + fmt.format(" static class Box { T t; Box(){ this.t = (T) (Object) new Object(); } }%n"); + fmt.format(" void f() {%n"); + for (int i = 0; i < groups; i++) { + fmt.format(" Object o%d = new Object();%n", i); + fmt.format(" ArrayList l%d = new ArrayList<>();%n", i); + fmt.format(" Box b%d = new Box<>();%n", i); + fmt.format(" int[] ai%d = new int[10];%n", i); + fmt.format(" String[] as%d = new String[10];%n", i); + fmt.format(" Runnable r%d = new Runnable(){ public void run(){} };%n", i); + } + fmt.format(" }%n"); + fmt.format("}%n"); } - sb.append(" }\n"); - sb.append("}\n"); try (BufferedWriter w = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) { w.write(sb.toString()); } From a5fb809f082867b1df4c5d1ef8bf9c13ecb11aa8 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Mon, 6 Oct 2025 17:10:06 -0400 Subject: [PATCH 06/11] Extract a reusable test harness with Driver and CodeGenerator abstractions (Fixes #1411) --- checker/harness/harness-core/build.gradle | 19 + .../harness/core/CodeGenerator.java | 124 ++++ .../checkerframework/harness/core/Driver.java | 251 ++++++++ .../core/ExternalProcessJavacDriver.java | 351 +++++++++++ .../harness/core/HarnessIO.java | 578 ++++++++++++++++++ .../harness/core/InProcessJavacDriver.java | 232 +++++++ .../harness/core/JtregDriver.java | 423 +++++++++++++ .../harness/harness-driver-cli/build.gradle | 38 ++ .../checkerframework/harness/cli/Main.java | 458 ++++++++++++++ .../harness/harness-generators/build.gradle | 20 + .../nullness/NewAndArrayGenerator.java | 147 +++++ checker/harness/jtreg/JtregPerfHarness.java | 161 +++++ checker/harness/jtreg/TEST.ROOT | 6 + checker/harness/settings.gradle | 2 + checker/jtreg/nullness/perf/NewClassPerf.java | 169 ++--- .../NullnessNoInitAnnotatedTypeFactory.java | 12 +- 16 files changed, 2908 insertions(+), 83 deletions(-) create mode 100644 checker/harness/harness-core/build.gradle create mode 100644 checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/CodeGenerator.java create mode 100644 checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/Driver.java create mode 100644 checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java create mode 100644 checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java create mode 100644 checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/InProcessJavacDriver.java create mode 100644 checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java create mode 100644 checker/harness/harness-driver-cli/build.gradle create mode 100644 checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java create mode 100644 checker/harness/harness-generators/build.gradle create mode 100644 checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java create mode 100644 checker/harness/jtreg/JtregPerfHarness.java create mode 100644 checker/harness/jtreg/TEST.ROOT create mode 100644 checker/harness/settings.gradle diff --git a/checker/harness/harness-core/build.gradle b/checker/harness/harness-core/build.gradle new file mode 100644 index 000000000000..8740d6344930 --- /dev/null +++ b/checker/harness/harness-core/build.gradle @@ -0,0 +1,19 @@ +plugins { + id 'java' +} + +group = 'io.github.eisop' + +repositories { + mavenCentral() +} + +dependencies { + // JSON library for IO utilities. + implementation 'com.google.code.gson:gson:2.10.1' +} + +java { + withJavadocJar() + withSourcesJar() +} diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/CodeGenerator.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/CodeGenerator.java new file mode 100644 index 000000000000..8ef4d6a91e38 --- /dev/null +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/CodeGenerator.java @@ -0,0 +1,124 @@ +package org.checkerframework.harness.core; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Pluggable source generator API. + * + *

Implementations write compilable Java sources under {@link GenerationRequest#outputDir()} and + * return created files plus metadata. Output should be reproducible for the same {@code + * seed}/{@code sampleCount}/{@code extra} so A/B runs are comparable. + */ +public interface CodeGenerator { + /** + * Returns the generator's short, stable identifier. + * + * @return non-null name used in metadata and reports + */ + String name(); + + /** + * Generates Java sources for the given request. + * + * @param req generation parameters (output directory, seed, sample count, extras) + * @return generation result containing the sources directory, created files, and metadata + * @throws Exception if generation fails (I/O or content production errors) + */ + GenerationResult generate(GenerationRequest req) throws Exception; + + /** Input describing where to write sources and how many/what to generate. */ + final class GenerationRequest { + private final Path outputDir; + private final long seed; + private final int sampleCount; + private final Map extra; + + /** + * Creates a new generation request. + * + * @param outputDir base directory under which sources will be written + * @param seed pseudo-random seed for deterministic content + * @param sampleCount number of source files to emit (generator-defined semantics) + * @param extra optional string key/value parameters for generator-specific tuning + */ + public GenerationRequest( + Path outputDir, long seed, int sampleCount, Map extra) { + this.outputDir = outputDir; + this.seed = seed; + this.sampleCount = sampleCount; + this.extra = extra; + } + + /** + * @return output directory where sources should be written + */ + public Path outputDir() { + return outputDir; + } + + /** + * @return deterministic seed driving content variability + */ + public long seed() { + return seed; + } + + /** + * @return requested number of samples/files + */ + public int sampleCount() { + return sampleCount; + } + + /** + * @return optional generator-specific parameters; may be null + */ + public Map extra() { + return extra; + } + } + + /** Output describing where sources were written, which files were created, and metadata. */ + final class GenerationResult { + private final Path sourcesDir; + private final List sourceFiles; + private final Map metadata; + + /** + * Creates a new immutable generation result. + * + * @param sourcesDir directory containing all generated sources + * @param sourceFiles deterministic, sorted list of created source file paths + * @param metadata generator-defined metadata for reporting + */ + public GenerationResult( + Path sourcesDir, List sourceFiles, Map metadata) { + this.sourcesDir = sourcesDir; + this.sourceFiles = sourceFiles; + this.metadata = metadata; + } + + /** + * @return directory containing generated sources + */ + public Path sourcesDir() { + return sourcesDir; + } + + /** + * @return list of generated source files + */ + public List sourceFiles() { + return sourceFiles; + } + + /** + * @return metadata map for diagnostics/reporting + */ + public Map metadata() { + return metadata; + } + } +} diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/Driver.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/Driver.java new file mode 100644 index 000000000000..0a9bd51faa93 --- /dev/null +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/Driver.java @@ -0,0 +1,251 @@ +package org.checkerframework.harness.core; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Driver API: executes a single harness run (generate sources, then invoke {@code javac} with the + * requested annotation processors) and returns timing, diagnostics, and metadata. + * + *

Designed for A/B comparisons: implementations should produce stable, deterministically-sorted + * diagnostics so results are comparable across runs. + */ +public interface Driver { + /** + * Executes one run per the given specification: calls the {@link CodeGenerator} to write + * sources to disk, then compiles them with the configured processors and flags. + * + *

Implementations may report compilation failures via diagnostics rather than throwing; only + * unrecoverable generation/IO errors should surface as exceptions. + * + * @param spec the run description (generator, generation request, compiler config, flags, + * label) + * @return the outcome of the run (elapsed time, diagnostics, metadata, work directory) + * @throws Exception on unrecoverable errors during generation or setup + */ + RunResult runOnce(RunSpec spec) throws Exception; + + /** Input describing how to perform a single run. */ + final class RunSpec { + private final CodeGenerator generator; + private final CodeGenerator.GenerationRequest genReq; + private final CompilerConfig compiler; + private final List javacFlags; + private final String label; + + /** + * Creates a new run specification. + * + * @param generator code generator implementation + * @param genReq generation request (output dir/seed/samples/extras) + * @param compiler compiler configuration (classpath, processors, release, out dir) + * @param javacFlags extra javac/processor flags + * @param label human-readable label such as "baseline" or "update" + */ + public RunSpec( + CodeGenerator generator, + CodeGenerator.GenerationRequest genReq, + CompilerConfig compiler, + List javacFlags, + String label) { + this.generator = generator; + this.genReq = genReq; + this.compiler = compiler; + this.javacFlags = javacFlags; + this.label = label; + } + + public CodeGenerator generator() { + return generator; + } + + public CodeGenerator.GenerationRequest genReq() { + return genReq; + } + + public CompilerConfig compiler() { + return compiler; + } + + public List javacFlags() { + return javacFlags; + } + + public String label() { + return label; + } + } + + /** Outcome of a run. */ + final class RunResult { + private final String label; + private final long wallMillis; + private final boolean success; + private final List diagnostics; + private final Map metrics; + private final Path workDir; + private final Map meta; + + /** + * Creates a new run result. + * + * @param label run label (baseline/update) + * @param wallMillis wall-clock duration in milliseconds + * @param success true if compilation succeeded + * @param diagnostics deterministically-sorted diagnostics + * @param metrics optional metrics map (reserved for future use) + * @param workDir working directory containing generated sources + * @param meta metadata such as timestamp, generator, flags + */ + public RunResult( + String label, + long wallMillis, + boolean success, + List diagnostics, + Map metrics, + Path workDir, + Map meta) { + this.label = label; + this.wallMillis = wallMillis; + this.success = success; + this.diagnostics = diagnostics; + this.metrics = metrics; + this.workDir = workDir; + this.meta = meta; + } + + public String label() { + return label; + } + + public long wallMillis() { + return wallMillis; + } + + public boolean success() { + return success; + } + + public List diagnostics() { + return diagnostics; + } + + public Map metrics() { + return metrics; + } + + public Path workDir() { + return workDir; + } + + public Map meta() { + return meta; + } + } + + /** Stable diagnostic record suitable for equality and sorting. */ + final class DiagnosticEntry { + private final String file; + private final int line; + private final int column; + private final String kind; + private final String message; + + /** + * Creates a diagnostic entry. + * + * @param file source path (relative when possible) + * @param line 1-based line number, 0 if unknown + * @param column 1-based column number, 0 if unknown + * @param kind diagnostic kind (ERROR/WARNING/NOTE) + * @param message diagnostic message text + */ + public DiagnosticEntry(String file, int line, int column, String kind, String message) { + this.file = file; + this.line = line; + this.column = column; + this.kind = kind; + this.message = message; + } + + public String file() { + return file; + } + + public int line() { + return line; + } + + public int column() { + return column; + } + + public String kind() { + return kind; + } + + public String message() { + return message; + } + } + + /** Compiler configuration for invoking javac and its processors. */ + final class CompilerConfig { + private final Path javacExecutable; + private final List classpath; + private final List processors; + private final List processorPath; + private final List sourceOpts; + private final Path outDir; + + /** + * Creates a new compiler configuration. + * + * @param javacExecutable explicit path to javac executable (nullable for system default) + * @param classpath regular compilation classpath + * @param processors FQNs of annotation processors + * @param processorPath locations where processors are loaded from + * @param sourceOpts language/compilation options (e.g., --release, -proc:only) + * @param outDir output directory passed as -d + */ + public CompilerConfig( + Path javacExecutable, + List classpath, + List processors, + List processorPath, + List sourceOpts, + Path outDir) { + this.javacExecutable = javacExecutable; + this.classpath = classpath; + this.processors = processors; + this.processorPath = processorPath; + this.sourceOpts = sourceOpts; + this.outDir = outDir; + } + + public Path javacExecutable() { + return javacExecutable; + } + + public List classpath() { + return classpath; + } + + public List processors() { + return processors; + } + + public List processorPath() { + return processorPath; + } + + public List sourceOpts() { + return sourceOpts; + } + + public Path outDir() { + return outDir; + } + } +} diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java new file mode 100644 index 000000000000..1573ac3ea9df --- /dev/null +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java @@ -0,0 +1,351 @@ +package org.checkerframework.harness.core; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * External-process driver that launches {@code javac} via {@link ProcessBuilder}. + * + *

Flow: 1) Generate sources using the provided {@link CodeGenerator}. 2) Build the {@code javac} + * command line (add-opens, classpath, processors, flags, output dir). 3) Launch a separate {@code + * javac} process; measure wall time. 4) If exit code != 0, attach the entire stdout as a single + * ERROR diagnostic (best-effort visibility). 5) Assemble a {@link Driver.RunResult} (with {@code + * success} from exit code), persist {@code result.json}, return. + * + *

Failure policy: generation/IO errors are thrown; compilation failures are surfaced via + * diagnostics while still returning a {@link Driver.RunResult} so A/B reports can be produced. + */ +public final class ExternalProcessJavacDriver implements Driver { + + /** + * Executes a single compile by invoking an external javac process. + * + * @param spec run specification: generator request, compiler config, extra javac flags, and + * label + * @return a {@link Driver.RunResult} with process wall time, exit-code based success, parsed + * diagnostics, and meta + * @throws Exception if source generation or process setup fails (non-zero compilation exit + * codes do not throw) + */ + @Override + public RunResult runOnce(RunSpec spec) throws Exception { + Objects.requireNonNull(spec); + CodeGenerator.GenerationResult genRes = spec.generator().generate(spec.genReq()); + + List cmd = new ArrayList(); + + // Diff vs InProcessJavacDriver: this driver executes a separate JVM/javac process (better + // isolation), + // with success determined by exit code, instead of using ToolProvider. Diagnostics are + // coarse unless parsed. + // javac executable + String javac = findJavacExecutable(); + cmd.add(javac); + + // Add required --add-opens similar to other engines + String[] pkgs = + new String[] { + "com.sun.tools.javac.api", + "com.sun.tools.javac.code", + "com.sun.tools.javac.comp", + "com.sun.tools.javac.file", + "com.sun.tools.javac.main", + "com.sun.tools.javac.parser", + "com.sun.tools.javac.processing", + "com.sun.tools.javac.tree", + "com.sun.tools.javac.util" + }; + for (String p : pkgs) { + cmd.add("-J--add-opens=jdk.compiler/" + p + "=ALL-UNNAMED"); + } + + // Force English diagnostics for stable parsing (parity with Locale.ROOT in in-process) + cmd.add("-J-Duser.language=en"); + cmd.add("-J-Duser.country=US"); + + if (spec.javacFlags() != null) { + cmd.addAll(spec.javacFlags()); + } + + // Build core args from config: source/release, proc mode, lints (parity with in-process + // engine) + List args = new ArrayList(); + if (spec.compiler().sourceOpts() != null) { + args.addAll(spec.compiler().sourceOpts()); + } + + // Processor path + if (spec.compiler().processorPath() != null && !spec.compiler().processorPath().isEmpty()) { + args.add("-processorpath"); + args.add(joinPaths(spec.compiler().processorPath())); + } + + // Classpath + if (spec.compiler().classpath() != null && !spec.compiler().classpath().isEmpty()) { + args.add("-classpath"); + args.add(joinPaths(spec.compiler().classpath())); + } else if (spec.compiler().processorPath() != null + && !spec.compiler().processorPath().isEmpty()) { + // Parity with InProcessJavacDriver: if no explicit classpath is given, fall back to + // using the processorPath so that processor dependencies (e.g., annotation types in + // checker-qual.jar) are visible to the compiler and processors at runtime. + args.add("-classpath"); + args.add(joinPaths(spec.compiler().processorPath())); + } + + if (spec.compiler().processors() != null && !spec.compiler().processors().isEmpty()) { + args.add("-processor"); + args.add(joinComma(spec.compiler().processors())); + } + + if (spec.compiler().outDir() != null) { + args.addAll(Arrays.asList("-d", spec.compiler().outDir().toString())); + } + + // Keep a copy of option-only args for reporting + List optionArgsForMeta = new ArrayList(args); + + // Append sources: pass paths relative to sourcesDir to avoid duplicate prefixes + for (Path p : genRes.sourceFiles()) { + String rel; + try { + Path base = genRes.sourcesDir(); + rel = (base != null) ? base.relativize(p).toString() : p.toString(); + } catch (Throwable ignore) { + rel = p.getFileName().toString(); + } + args.add(rel); + } + + cmd.addAll(args); + + ProcessBuilder pb = new ProcessBuilder(cmd); + if (genRes.sourcesDir() != null) { + pb.directory(genRes.sourcesDir().toFile()); + } + pb.redirectErrorStream(true); + + // Measure only javac execution (process run) to align with in-process timing + long start = System.nanoTime(); + Process pr = pb.start(); + int code = pr.waitFor(); + long end = System.nanoTime(); + // Read stdout after timing window to avoid skewing wallMillis with IO/diagnostic parsing + byte[] out = pr.getInputStream().readAllBytes(); + String stdout = new String(out, StandardCharsets.UTF_8); + + List diags = parseJavacDiagnostics(stdout); + if (diags.isEmpty() && code != 0) { + // Fallback: attach whole stdout as a single ERROR diagnostic for visibility + diags.add(new Driver.DiagnosticEntry("", 0, 0, "ERROR", stdout)); + } + + // Normalize file path to relative path against sourcesDir (match in-process behavior) + java.util.List norm = + new java.util.ArrayList(diags.size()); + for (Driver.DiagnosticEntry d : diags) { + String f = d.file(); + String rel = f; + try { + if (f != null && !f.isEmpty() && !f.startsWith("<")) { + java.nio.file.Path p = Paths.get(f); + java.nio.file.Path base = genRes.sourcesDir(); + if (base != null) { + try { + rel = base.relativize(p).toString(); + } catch (Throwable ignore) { + rel = p.getFileName().toString(); + } + } else { + rel = p.getFileName().toString(); + } + } + } catch (Throwable ignore) { + } + norm.add( + new Driver.DiagnosticEntry( + rel == null ? "" : rel, d.line(), d.column(), d.kind(), d.message())); + } + // Deterministic sort: file → line → column → kind → message (same as in-process) + norm.sort( + (a, b) -> { + int c; + c = a.file().compareTo(b.file()); + if (c != 0) return c; + c = Integer.compare(a.line(), b.line()); + if (c != 0) return c; + c = Integer.compare(a.column(), b.column()); + if (c != 0) return c; + c = a.kind().compareTo(b.kind()); + if (c != 0) return c; + return a.message().compareTo(b.message()); + }); + + Map meta = new java.util.HashMap(); + meta.put("timestamp", Instant.now().toString()); + meta.put("generator", spec.generator().name()); + meta.put("label", spec.label()); + List reportArgs = new ArrayList(optionArgsForMeta); + int di = reportArgs.indexOf("-d"); + if (di >= 0) { + reportArgs.remove(di); + if (di < reportArgs.size()) reportArgs.remove(di); + } + meta.put("flags", String.join(" ", reportArgs)); + + RunResult result = + new RunResult( + spec.label(), + (end - start) / 1_000_000L, + code == 0, + java.util.Collections.unmodifiableList(norm), + java.util.Collections.emptyMap(), + genRes.sourcesDir(), + meta); + + Path outFile = genRes.sourcesDir().resolve("result.json"); + HarnessIO.writeJson(outFile, result); + return result; + } + + /** + * Best-effort parse of javac stdout into structured diagnostics. Supports formats with and + * without column numbers: file:line:column: kind: message file:line: kind: message Lines not + * matching a new record are appended to the previous message. + */ + private static List parseJavacDiagnostics(String stdout) { + List out = new ArrayList(); + if (stdout == null || stdout.isEmpty()) return out; + + Pattern pWithCol = + Pattern.compile("^(.+?):(\\d+):(\\d+):\\s*(error|warning|note):\\s*(.*)$"); + Pattern pNoCol = Pattern.compile("^(.+?):(\\d+):\\s*(error|warning|note):\\s*(.*)$"); + + String[] lines = stdout.split("\\r?\\n"); + String curFile = null, curKind = null; + int curLine = 0, curCol = 0; + StringBuilder curMsg = null; + + for (String line : lines) { + if (line == null) line = ""; + Matcher m = pWithCol.matcher(line); + Matcher m2 = pNoCol.matcher(line); + boolean matched = false; + if (m.matches()) { + // flush previous + if (curFile != null) { + out.add( + new Driver.DiagnosticEntry( + curFile, + curLine, + curCol, + curKind, + curMsg == null ? "" : curMsg.toString().trim())); + } + curFile = m.group(1); + curLine = safeInt(m.group(2)); + curCol = safeInt(m.group(3)); + curKind = toUpperKind(m.group(4)); + curMsg = new StringBuilder(m.group(5) == null ? "" : m.group(5)); + matched = true; + } else if (m2.matches()) { + if (curFile != null) { + out.add( + new Driver.DiagnosticEntry( + curFile, + curLine, + curCol, + curKind, + curMsg == null ? "" : curMsg.toString().trim())); + } + curFile = m2.group(1); + curLine = safeInt(m2.group(2)); + curCol = 0; + curKind = toUpperKind(m2.group(3)); + curMsg = new StringBuilder(m2.group(4) == null ? "" : m2.group(4)); + matched = true; + } + + if (!matched) { + // continuation for previous diagnostic + if (curFile != null) { + if (curMsg.length() > 0) curMsg.append(' '); + curMsg.append(line.trim()); + } + } + } + + if (curFile != null) { + out.add( + new Driver.DiagnosticEntry( + curFile, + curLine, + curCol, + curKind, + curMsg == null ? "" : curMsg.toString().trim())); + } + + // Stable sort is done by the report layer; maintain insertion order here + return out; + } + + private static int safeInt(String s) { + try { + return Integer.parseInt(s); + } catch (Throwable t) { + return 0; + } + } + + private static String toUpperKind(String k) { + if (k == null) return ""; + String kk = k.trim().toUpperCase(Locale.ROOT); + // Normalize common terms + if ("ERROR".equals(kk) || "WARNING".equals(kk) || "NOTE".equals(kk)) return kk; + return kk; + } + + private static String joinPaths(List paths) { + String sep = File.pathSeparator; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < paths.size(); i++) { + if (i > 0) sb.append(sep); + sb.append(paths.get(i).toString()); + } + return sb.toString(); + } + + private static String joinComma(List items) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < items.size(); i++) { + if (i > 0) sb.append(','); + sb.append(items.get(i)); + } + return sb.toString(); + } + + private static String findJavacExecutable() { + String javaHome = System.getProperty("java.home"); + File jh = new File(javaHome); + File bin = new File(jh.getParentFile(), "bin"); + File javac = new File(bin, isWindows() ? "javac.exe" : "javac"); + if (javac.exists()) return javac.getAbsolutePath(); + return isWindows() ? "javac.exe" : "javac"; + } + + private static boolean isWindows() { + String os = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + return os.contains("win"); + } +} diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java new file mode 100644 index 000000000000..ab403693c42c --- /dev/null +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java @@ -0,0 +1,578 @@ +package org.checkerframework.harness.core; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * IO utilities for persisting harness results and generating human-readable reports. + * + *

Responsibilities: - Serialize/deserialize single {@link Driver.RunResult} snapshots (JSON). - + * Emit concise A/B Markdown reports for a baseline/update pair. - Emit aggregated series reports + * used by protocol orchestration (SINGLE/CROSS). + */ +public final class HarnessIO { + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private HarnessIO() {} + + /** + * Writes the given object as JSON. If {@code data} is a {@link Driver.RunResult}, it is + * converted to a lean serializable DTO to avoid leaking implementation types. + * + * @param file destination path; parent directories are created as needed + * @param data a POJO or {@link Driver.RunResult} + * @throws IOException if writing fails + */ + public static void writeJson(Path file, Object data) throws IOException { + Files.createDirectories(file.getParent()); + try (BufferedWriter w = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) { + Object toWrite = data; + if (data instanceof Driver.RunResult) { + toWrite = toSerializable((Driver.RunResult) data); + } + w.write(GSON.toJson(toWrite)); + } + } + + /** + * Writes a unified Markdown report for SINGLE/CROSS protocols. + * + * @param file destination Markdown file + * @param baseline representative baseline run result (used for flags/diagnostics display) + * @param update representative update run result (used for flags/diagnostics display) + * @param context optional key/value context (protocol, runs, engine, generator params) + * @param seriesA timing samples for baseline variant (milliseconds) + * @param runsA number of requested runs for baseline + * @param successA number of successful runs for baseline + * @param seriesB timing samples for update variant (milliseconds) + * @param runsB number of requested runs for update + * @param successB number of successful runs for update + * @throws IOException if writing fails + */ + public static void writeUnifiedReport( + Path file, + Driver.RunResult baseline, + Driver.RunResult update, + Map context, + java.util.List seriesA, + int runsA, + int successA, + java.util.List seriesB, + int runsB, + int successB) + throws IOException { + Files.createDirectories(file.getParent()); + List out = new ArrayList<>(); + + out.add("## Test Results"); + out.add( + "Generated: " + + ZonedDateTime.now(ZoneId.systemDefault()) + .format(DateTimeFormatter.RFC_1123_DATE_TIME)); + out.add(""); + + // Description + out.add("### Description"); + out.add("| Key | Value |"); + out.add("| --- | --- |"); + out.add("| Generator | " + escapeMd(getStringMeta(baseline, "generator")) + " |"); + out.add("| Label A | " + escapeMd(baseline.label()) + " |"); + out.add("| Label B | " + escapeMd(update.label()) + " |"); + out.add( + "| Baseline result.json | " + + code(baseline.workDir().resolve("result.json").toString()) + + " |"); + out.add( + "| Update result.json | " + + code(update.workDir().resolve("result.json").toString()) + + " |"); + if (context != null) { + String proto = context.getOrDefault("protocol", ""); + String runs = context.getOrDefault("runs", ""); + String engine = context.getOrDefault("engine", ""); + String sampleCount = context.getOrDefault("sampleCount", ""); + String seed = context.getOrDefault("seed", ""); + String groupsPerFile = context.getOrDefault("groupsPerFile", ""); + if (!proto.isEmpty()) out.add("| Protocol | " + escapeMd(proto) + " |"); + if (!runs.isEmpty()) out.add("| Runs requested | " + escapeMd(runs) + " |"); + if (!engine.isEmpty()) out.add("| Engine | " + escapeMd(engine) + " |"); + if (!sampleCount.isEmpty()) out.add("| sampleCount | " + escapeMd(sampleCount) + " |"); + if (!seed.isEmpty()) out.add("| seed | " + escapeMd(seed) + " |"); + if (!groupsPerFile.isEmpty()) + out.add("| groupsPerFile | " + escapeMd(groupsPerFile) + " |"); + } + out.add(""); + + // Environment + out.add("### Environment"); + out.add("| Key | Value |"); + out.add("| --- | --- |"); + out.add("| JDK Version | " + code(System.getProperty("java.version", "")) + " |"); + out.add("| JDK Home | " + code(System.getProperty("java.home", "")) + " |"); + out.add( + "| OS | " + + escapeMd(System.getProperty("os.name", "")) + + " " + + escapeMd(System.getProperty("os.version", "")) + + " (" + + escapeMd(System.getProperty("os.arch", "")) + + ") |"); + out.add("| CPU Cores | " + Runtime.getRuntime().availableProcessors() + " |"); + out.add(""); + + // Diagnostics (placed first per request) + out.add("### Diagnostics"); + // Short legend for common kinds + out.add("NOTE: informational message; does not affect compilation result"); + out.add("WARNING: potential issue; compilation still succeeds"); + out.add("ERROR: compilation error; typically causes build failure"); + boolean eq = equalDiagnostics(baseline, update); + // By-kind summary + java.util.Map kindA = countDiagnosticsByKind(baseline); + java.util.Map kindB = countDiagnosticsByKind(update); + java.util.TreeSet allKinds = new java.util.TreeSet(); + allKinds.addAll(kindA.keySet()); + allKinds.addAll(kindB.keySet()); + if (!allKinds.isEmpty()) { + out.add(""); + out.add("| Kind | Baseline | Update | Delta |"); + out.add("| --- | ---:| ---:| ---:|"); + for (String k : allKinds) { + int va = kindA.getOrDefault(k, Integer.valueOf(0)).intValue(); + int vb = kindB.getOrDefault(k, Integer.valueOf(0)).intValue(); + String delta = + (va == 0) + ? (vb == 0 ? "0" : String.valueOf(vb)) + : String.format( + java.util.Locale.ROOT, "%.3f%%", (vb - va) * 100.0 / va); + out.add("| " + escapeMd(k) + " | " + va + " | " + vb + " | " + delta + " |"); + } + } + if (!eq) { + out.add(""); + out.add("Top differences (first 5 of each):"); + out.add("| Index | Baseline | Update |"); + out.add("| ---:| --- | --- |"); + int n = + Math.min( + 5, + Math.max(baseline.diagnostics().size(), update.diagnostics().size())); + for (int i = 0; i < n; i++) { + String ba = + i < baseline.diagnostics().size() + ? escapeMd(formatDiag(baseline.diagnostics().get(i))) + : ""; + String ub = + i < update.diagnostics().size() + ? escapeMd(formatDiag(update.diagnostics().get(i))) + : ""; + out.add("| " + i + " | " + ba + " | " + ub + " |"); + } + } + out.add(""); + out.add("Match status of diagnostics are: " + (eq ? "**IDENTICAL**" : "**DIFFERENT**")); + out.add(""); + + // Summary (aggregated) + out.add("### Summary (aggregated over successful samples)"); + out.add( + "| Variant | median (ms) | average (ms) | min (ms) | max (ms) | samples | success | succ% |"); + out.add("| --- | ---:| ---:| ---:| ---:| ---:| ---:| ---:|"); + out.add(formatSeriesRow("A", seriesA, runsA, successA)); + out.add(formatSeriesRow("B", seriesB, runsB, successB)); + out.add(""); + + // Baseline vs Update comparison (aggregated) + out.add("### Performance Comparison"); + if (seriesA != null && !seriesA.isEmpty() && seriesB != null && !seriesB.isEmpty()) { + double medA = medianOf(seriesA); + double medB = medianOf(seriesB); + double avgA = averageOf(seriesA); + double avgB = averageOf(seriesB); + out.add("| Metric | Baseline (ms) | Update (ms) | Delta |"); + out.add("| --- | ---:| ---:| ---:|"); + out.add( + "| median | " + + fmt2(medA) + + " ms | " + + fmt2(medB) + + " ms | " + + pctD(medA, medB) + + " |"); + out.add( + "| average | " + + fmt2(avgA) + + " ms | " + + fmt2(avgB) + + " ms | " + + pctD(avgA, avgB) + + " |"); + } else { + out.add("n/a (no samples)"); + } + out.add(""); + + out.add("### Reproduction Commands"); + String proto = (context == null) ? "" : context.getOrDefault("protocol", ""); + String runs = (context == null) ? "" : context.getOrDefault("runs", ""); + String engine = (context == null) ? "" : context.getOrDefault("engine", ""); + String sampleCount = (context == null) ? "" : context.getOrDefault("sampleCount", ""); + String seed = (context == null) ? "" : context.getOrDefault("seed", ""); + String gpf = (context == null) ? "" : context.getOrDefault("groupsPerFile", ""); + String baselineFlags = (context == null) ? "" : context.getOrDefault("baselineFlags", ""); + String updateFlags = (context == null) ? "" : context.getOrDefault("updateFlags", ""); + String proc = getStringMeta(baseline, "processor"); + String ppath = extractPathFromFlags(getStringMeta(baseline, "flags"), "-processorpath"); + if (ppath.isEmpty()) + ppath = extractPathFromFlags(getStringMeta(baseline, "flags"), "-processorPath"); + if (proc.isEmpty()) proc = extractProcessorFromFlags(getStringMeta(baseline, "flags")); + String release = extractReleaseFromFlags(getStringMeta(baseline, "flags")); + String baseCmd = + buildReproCommand( + proto, + runs, + engine, + sampleCount, + seed, + gpf, + proc, + ppath, + release, + baselineFlags, + false); + String upCmd = + buildReproCommand( + proto, + runs, + engine, + sampleCount, + seed, + gpf, + proc, + ppath, + release, + updateFlags, + true); + out.add("- Baseline:"); + out.add("\n```bash"); + out.add(baseCmd); + out.add("```\n"); + out.add("- Update:"); + out.add("\n```bash"); + out.add(upCmd); + out.add("```\n"); + out.add(""); + + // Flags (moved later per request) + out.add("### Flags"); + out.add("- Baseline javac args:"); + out.add("\n```"); + for (String f : splitArgsForDisplay(getStringMeta(baseline, "flags"))) out.add(f); + out.add("```\n"); + out.add("- Update javac args:"); + out.add("\n```"); + for (String f : splitArgsForDisplay(getStringMeta(update, "flags"))) out.add(f); + out.add("```\n"); + + Files.write(file, out, StandardCharsets.UTF_8); + } + + private static List splitArgsForDisplay(String s) { + List out = new ArrayList<>(); + if (s == null) return out; + String trimmed = s.trim(); + if (trimmed.isEmpty()) return out; + for (String tok : trimmed.split("\\s+")) { + if (!tok.isEmpty()) out.add(tok); + } + return out; + } + + private static String formatSeriesRow( + String label, java.util.List series, int runs, int success) { + int samples = (series == null) ? 0 : series.size(); + String medStr = + (samples == 0) + ? "n/a" + : String.format( + Locale.ROOT, + "%.2f", + new Object[] {Double.valueOf(medianOf(series))}); + String avgStr = + (samples == 0) + ? "n/a" + : String.format( + Locale.ROOT, + "%.2f", + new Object[] {Double.valueOf(averageOf(series))}); + String minStr = (samples == 0) ? "n/a" : String.valueOf(minOf(series)); + String maxStr = (samples == 0) ? "n/a" : String.valueOf(maxOf(series)); + String pctStr = + (runs <= 0) + ? "n/a" + : String.format( + Locale.ROOT, + "%.2f%%", + new Object[] {Double.valueOf(success * 100.0 / runs)}); + return "| " + label + " | " + medStr + " | " + avgStr + " | " + minStr + " | " + maxStr + + " | " + samples + " | " + success + " | " + pctStr + " |"; + } + + private static double averageOf(java.util.List series) { + if (series == null || series.isEmpty()) return 0.0; + long sum = 0L; + for (Long v : series) { + if (v != null) sum += v.longValue(); + } + return ((double) sum) / series.size(); + } + + private static String fmt2(double v) { + return String.format(Locale.ROOT, "%.2f", new Object[] {Double.valueOf(v)}); + } + + private static String pctD(double base, double upd) { + if (base <= 0.0) return "n/a"; + double d = (upd - base) / base * 100.0; + return String.format(Locale.ROOT, "%.3f%%", new Object[] {Double.valueOf(d)}); + } + + private static double medianOf(java.util.List series) { + if (series == null || series.isEmpty()) return 0.0; + java.util.ArrayList copy = new java.util.ArrayList(series); + java.util.Collections.sort(copy); + int n = copy.size(); + if ((n & 1) == 1) return copy.get(n / 2); + return (copy.get(n / 2 - 1) + copy.get(n / 2)) / 2.0; + } + + private static long minOf(java.util.List series) { + long m = Long.MAX_VALUE; + for (Long v : series) { + if (v != null && v.longValue() < m) m = v.longValue(); + } + return m == Long.MAX_VALUE ? 0L : m; + } + + private static long maxOf(java.util.List series) { + long m = Long.MIN_VALUE; + for (Long v : series) { + if (v != null && v.longValue() > m) m = v.longValue(); + } + return m == Long.MIN_VALUE ? 0L : m; + } + + /** Return true if two diagnostic lists are identical under stable ordering. */ + private static boolean equalDiagnostics(Driver.RunResult a, Driver.RunResult b) { + if (a.diagnostics().size() != b.diagnostics().size()) return false; + for (int i = 0; i < a.diagnostics().size(); i++) { + Driver.DiagnosticEntry da = a.diagnostics().get(i); + Driver.DiagnosticEntry db = b.diagnostics().get(i); + if (!da.file().equals(db.file())) return false; + if (da.line() != db.line()) return false; + if (da.column() != db.column()) return false; + if (!da.kind().equals(db.kind())) return false; + if (!da.message().equals(db.message())) return false; + } + return true; + } + + private static String formatDiag(Driver.DiagnosticEntry d) { + return d.file() + + ":" + + d.line() + + ":" + + d.column() + + " " + + d.kind() + + " " + + summarize(d.message()); + } + + private static String summarize(String s) { + if (s == null) return ""; + String trimmed = s.replace('\n', ' ').replace("\r", " "); + if (trimmed.length() > 160) { + return trimmed.substring(0, 160) + "..."; + } + return trimmed; + } + + private static String pct(long base, long upd) { + if (base <= 0) return "n/a"; + double d = ((double) (upd - base)) / base * 100.0; + return String.format(Locale.ROOT, "%.3f%%", new Object[] {Double.valueOf(d)}); + } + + /** Convert simple values/maps to compact strings for tables (maps are key-sorted). */ + private static String valueToString(Object o) { + if (o == null) return ""; + if (o instanceof Map) { + // Stable order by key string + java.util.List> entries = + new java.util.ArrayList>(((Map) o).entrySet()); + java.util.Collections.sort( + entries, + (e1, e2) -> String.valueOf(e1.getKey()).compareTo(String.valueOf(e2.getKey()))); + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Map.Entry e : entries) { + if (!first) sb.append(','); + first = false; + sb.append(String.valueOf(e.getKey())) + .append(':') + .append(String.valueOf(e.getValue())); + } + return sb.toString(); + } + return String.valueOf(o); + } + + private static String escapeMd(String s) { + if (s == null) return ""; + String r = s.replace("|", "\\|"); + r = r.replace("\n", " ").replace("\r", " "); + return r; + } + + private static String code(String s) { + if (s == null || s.isEmpty()) return ""; + // Use backticks to improve readability in tables + String cleaned = s.replace("`", "'"); + return "`" + cleaned + "`"; + } + + private static String getStringMeta(Driver.RunResult r, String key) { + if (r == null || r.meta() == null) return ""; + Object v = r.meta().get(key); + return v == null ? "" : String.valueOf(v); + } + + private static SerializableRunResult toSerializable(Driver.RunResult r) { + return new SerializableRunResult( + r.label(), + r.wallMillis(), + r.success(), + r.diagnostics(), + r.metrics(), + r.workDir() == null ? null : r.workDir().toString(), + r.meta()); + } + + private static String extractPathFromFlags(String flags, String key) { + if (flags == null || flags.isEmpty()) return ""; + String[] toks = flags.trim().split("\\s+"); + for (int i = 0; i < toks.length - 1; i++) { + if (toks[i].equals(key) && !toks[i + 1].startsWith("-")) { + return toks[i + 1]; + } + } + return ""; + } + + private static String extractProcessorFromFlags(String flags) { + if (flags == null || flags.isEmpty()) return ""; + String[] toks = flags.trim().split("\\s+"); + for (int i = 0; i < toks.length - 1; i++) { + if (toks[i].equals("-processor") && !toks[i + 1].startsWith("-")) { + return toks[i + 1]; + } + } + return ""; + } + + private static String extractReleaseFromFlags(String flags) { + if (flags == null || flags.isEmpty()) return ""; + String[] toks = flags.trim().split("\\s+"); + for (int i = 0; i < toks.length - 1; i++) { + if (toks[i].equals("--release") && !toks[i + 1].startsWith("-")) { + return toks[i + 1]; + } + } + return ""; + } + + private static String buildReproCommand( + String protocol, + String runs, + String engine, + String sampleCount, + String seed, + String groupsPerFile, + String processor, + String processorPath, + String release, + String variantFlags, + boolean isUpdate) { + StringBuilder sb = new StringBuilder(); + sb.append("./gradlew :harness-driver-cli:run --no-daemon --console=plain --args=\""); + sb.append("--generator NewAndArray"); + if (!sampleCount.isEmpty()) sb.append(" --sampleCount ").append(sampleCount); + if (!seed.isEmpty()) sb.append(" --seed ").append(seed); + if (!processor.isEmpty()) sb.append(" --processor ").append(processor); + if (!processorPath.isEmpty()) sb.append(" --processor-path ").append(processorPath); + if (!release.isEmpty()) sb.append(" --release ").append(release); + if (!protocol.isEmpty()) sb.append(" --protocol ").append(protocol); + if (!runs.isEmpty()) sb.append(" --runs ").append(runs); + if (!engine.isEmpty()) sb.append(" --engine ").append(engine); + if (!groupsPerFile.isEmpty()) sb.append(" --extra.groupsPerFile ").append(groupsPerFile); + if (variantFlags != null && !variantFlags.isEmpty()) { + sb.append(isUpdate ? " --update-flags " : " --baseline-flags "); + sb.append(variantFlags); + } + sb.append("\""); + return sb.toString(); + } + + private static java.util.Map countDiagnosticsByKind(Driver.RunResult r) { + java.util.Map m = new java.util.HashMap(); + if (r != null && r.diagnostics() != null) { + for (Driver.DiagnosticEntry d : r.diagnostics()) { + String k = d.kind(); + Integer prev = m.get(k); + m.put(k, Integer.valueOf(prev == null ? 1 : (prev.intValue() + 1))); + } + } + return m; + } + + static final class SerializableRunResult { + final String label; + final long wallMillis; + final List diagnostics; + final java.util.Map metrics; + final String workDir; + final java.util.Map meta; + final boolean success; + + SerializableRunResult( + String label, + long wallMillis, + boolean success, + List diagnostics, + java.util.Map metrics, + String workDir, + java.util.Map meta) { + this.label = label; + this.wallMillis = wallMillis; + this.success = success; + this.diagnostics = diagnostics; + this.metrics = metrics; + this.workDir = workDir; + this.meta = meta; + } + } +} diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/InProcessJavacDriver.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/InProcessJavacDriver.java new file mode 100644 index 000000000000..faf1006472f1 --- /dev/null +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/InProcessJavacDriver.java @@ -0,0 +1,232 @@ +package org.checkerframework.harness.core; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.JavaCompiler; +import javax.tools.JavaCompiler.CompilationTask; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; + +/** + * In-process driver that invokes {@code javac} via {@link javax.tools.ToolProvider}. + * + *

Flow: 1) Generate sources using the provided {@link CodeGenerator}. 2) Build the {@code javac} + * argument list from {@link Driver.CompilerConfig} and {@code javacFlags}. 3) Compile with a {@link + * javax.tools.DiagnosticCollector}; measure wall time; collect diagnostics. 4) Deterministically + * sort diagnostics (file → line → column → kind → message). 5) Assemble a {@link Driver.RunResult} + * (including {@code success}), persist {@code result.json} next to the generated sources, and + * return. + * + *

Failure policy: generation/IO errors are thrown; plain compilation failures are surfaced via + * diagnostics while still returning a {@link Driver.RunResult} so A/B reports can be produced. + */ +public final class InProcessJavacDriver implements Driver { + + /** + * Executes one generate+compile run and returns timing, diagnostics, and metadata. + * + * @param spec run specification: generator request, compiler config, extra javac flags, and + * label + * @return a {@link Driver.RunResult} containing wall-clock time, success, diagnostics, and + * metadata + * @throws Exception if source generation or setup fails; plain compilation failures are + * reported via diagnostics + */ + @Override + public RunResult runOnce(RunSpec spec) throws Exception { + Objects.requireNonNull(spec); + CodeGenerator.GenerationResult genRes = spec.generator().generate(spec.genReq()); + + // Diff vs ExternalProcessJavacDriver: this driver invokes javac in-process via ToolProvider + // (lower overhead, direct DiagnosticCollector). The external driver launches a separate + // javac process (better environment isolation, success determined by exit code). + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + + if (compiler == null) { + throw new IllegalStateException("No system Java compiler. Are you running on a JRE?"); + } + + DiagnosticCollector diags = new DiagnosticCollector<>(); + List args = new ArrayList<>(); + + // Source/target/release + if (spec.compiler().sourceOpts() != null) { + args.addAll(spec.compiler().sourceOpts()); + } + + // Classpath: if empty, fall back to processorPath so processors' dependent annotations are + // visible + boolean hasCp = + spec.compiler().classpath() != null && !spec.compiler().classpath().isEmpty(); + if (hasCp) { + args.add("-classpath"); + args.add( + spec.compiler().classpath().stream() + .map(Path::toString) + .collect(Collectors.joining(java.io.File.pathSeparator))); + } else if (spec.compiler().processorPath() != null + && !spec.compiler().processorPath().isEmpty()) { + args.add("-classpath"); + args.add( + spec.compiler().processorPath().stream() + .map(Path::toString) + .collect(Collectors.joining(java.io.File.pathSeparator))); + } + + // Processor path + if (spec.compiler().processorPath() != null && !spec.compiler().processorPath().isEmpty()) { + args.add("-processorpath"); + args.add( + spec.compiler().processorPath().stream() + .map(Path::toString) + .collect(Collectors.joining(java.io.File.pathSeparator))); + } + + // Processors + if (spec.compiler().processors() != null && !spec.compiler().processors().isEmpty()) { + args.add("-processor"); + args.add(String.join(",", spec.compiler().processors())); + } + + // Annotation processor and javac flags. + // For in-process runs, translate -Dk=v into temporary System properties (javac doesn't + // accept -D). + java.util.Map prevSysProps = new java.util.HashMap(); + if (spec.javacFlags() != null) { + for (String f : spec.javacFlags()) { + if (f == null) continue; + // Accept both -Dk=v and -J-Dk=v (the latter is common with external javac) + if (f.startsWith("-J") && f.length() > 2) { + f = f.substring(2); + } + if (f.startsWith("-D")) { + int eq = f.indexOf('='); + if (eq > 2) { + String key = f.substring(2, eq); + String val = f.substring(eq + 1); + String old = System.getProperty(key); + prevSysProps.put(key, old == null ? null : old); + System.setProperty(key, val); + continue; // do not pass -D to javac + } + } + args.add(f); + } + } + + // Output dir + if (spec.compiler().outDir() != null) { + Files.createDirectories(spec.compiler().outDir()); + args.addAll(Arrays.asList("-d", spec.compiler().outDir().toString())); + } + + // Compile sources + StandardJavaFileManager fm = + compiler.getStandardFileManager(diags, Locale.ROOT, StandardCharsets.UTF_8); + Iterable units = + fm.getJavaFileObjectsFromPaths(genRes.sourceFiles()); + List finalArgs = Collections.unmodifiableList(args); + + long start = System.nanoTime(); + CompilationTask task = + compiler.getTask( + new PrintWriter(new StringWriter()), fm, diags, finalArgs, null, units); + boolean ok = Boolean.TRUE.equals(task.call()); + long end = System.nanoTime(); + + // Restore previous system properties to avoid cross-run leakage + for (java.util.Map.Entry e : prevSysProps.entrySet()) { + if (e.getValue() == null) { + System.clearProperty(e.getKey()); + } else { + System.setProperty(e.getKey(), e.getValue()); + } + } + + // Convert compiler diagnostics to stable, comparable records + List diagEntries = new ArrayList<>(); + for (Diagnostic d : diags.getDiagnostics()) { + JavaFileObject src = d.getSource(); + String file = ""; + if (src != null) { + try { + java.nio.file.Path p = Paths.get(src.toUri()); + java.nio.file.Path base = genRes.sourcesDir(); + if (base != null) { + try { + file = base.relativize(p).toString(); + } catch (Throwable ignore) { + file = p.getFileName().toString(); + } + } else { + file = p.getFileName().toString(); + } + } catch (Throwable ignore) { + file = ""; + } + } + int line = (int) d.getLineNumber(); + int col = (int) d.getColumnNumber(); + String kind = d.getKind().name(); + String msg = d.getMessage(Locale.ROOT); + diagEntries.add(new Driver.DiagnosticEntry(file, line, col, kind, msg)); + } + // Sort ensures deterministic ordering across runs for set-equality comparisons + diagEntries.sort( + (a, b) -> { + int c; + c = a.file().compareTo(b.file()); + if (c != 0) return c; + c = Integer.compare(a.line(), b.line()); + if (c != 0) return c; + c = Integer.compare(a.column(), b.column()); + if (c != 0) return c; + c = a.kind().compareTo(b.kind()); + if (c != 0) return c; + return a.message().compareTo(b.message()); + }); + // diagnostics collected and deterministically sorted + + // If compilation failed, keep going; diagnostics will reflect issues. Caller can inspect + // results. + + // Minimal metadata captured for reporting + java.util.Map meta = new java.util.HashMap(); + meta.put("timestamp", Instant.now().toString()); + meta.put("generator", spec.generator().name()); + meta.put("label", spec.label()); + meta.put("flags", String.join(" ", finalArgs)); + + RunResult result = + new RunResult( + spec.label(), + (end - start) / 1_000_000L, + ok, + Collections.unmodifiableList(diagEntries), + java.util.Collections.emptyMap(), + genRes.sourcesDir(), + meta); + + // Persist a JSON snapshot next to the generated sources (read by the CLI reporter) + Path out = genRes.sourcesDir().resolve("result.json"); + HarnessIO.writeJson(out, result); + + return result; + } +} diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java new file mode 100644 index 000000000000..cb8e32603c6f --- /dev/null +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java @@ -0,0 +1,423 @@ +package org.checkerframework.harness.core; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +/** + * Executes a single jtreg test and returns timing, diagnostics, and metadata. + * + *

Behavior: - Measures only the jtreg process runtime (stdout is read after timing). - Persists + * the full stdout next to the generated sources. - Reports a single ERROR diagnostic on non-zero + * exit; detailed aggregation is handled by the CLI. - VM/javac options are forwarded via jtreg + * command-line flags. - Source generation is kept to comply with {@link Driver} contract; jtreg + * tests may generate their own sources and do not consume the generated ones here. + * + *

Protocol orchestration (SINGLE/CROSS) is implemented by the CLI, not by this driver. + */ +public final class JtregDriver implements Driver { + + private final Path jtregBin; // ${root}/jtreg/bin/jtreg + private final String testPath; // e.g. checker/harness/jtreg/JtregPerfHarness.java + + public JtregDriver(Path jtregBin, String testPath) { + this.jtregBin = jtregBin; + this.testPath = testPath; + } + + /** + * Runs a single jtreg test (typically {@code JtregPerfHarness}) and reports timing and + * diagnostics. + * + * @param spec run specification; generated sources are used as a working directory and artifact + * sink + * @return a {@link Driver.RunResult} whose {@code wallMillis} is jtreg runtime or parsed median + * when available + * @throws Exception if source generation or process setup fails; jtreg failures are captured as + * diagnostics + */ + @Override + public RunResult runOnce(RunSpec spec) throws Exception { + Objects.requireNonNull(spec); + // Still generate sources per interface (not used by jtreg), to keep a workDir + CodeGenerator.GenerationResult genRes = spec.generator().generate(spec.genReq()); + + List cmd = new ArrayList(); + cmd.add(jtregExecutable()); + // Use samevm for faster execution (parity with NewClassPerf.java); print summary to console + cmd.addAll(Arrays.asList("-samevm", "-verbose:summary")); + + // Jtreg should run from the project root (where checker/ is), not harness-driver-cli + File projectRoot = findProjectRoot(); + + // Build -vmoptions string with all test VM properties + StringBuilder vmOpts = new StringBuilder(); + + if (spec.javacFlags() != null) { + for (String f : spec.javacFlags()) { + if (f == null) continue; + String opt = f.trim(); + if (opt.isEmpty()) continue; + // All flags go to test VM (JtregPerfHarness constructs its own javac command) + if (vmOpts.length() > 0) vmOpts.append(' '); + vmOpts.append(opt); + } + } + + // Ensure the test VM can find the generated sources directory (must be absolute) + if (genRes.sourcesDir() != null) { + Path absSrcDir = + genRes.sourcesDir().isAbsolute() + ? genRes.sourcesDir().normalize() + : projectRoot.toPath().resolve(genRes.sourcesDir()).normalize(); + if (vmOpts.length() > 0) vmOpts.append(' '); + vmOpts.append("-Dharness.srcdir=").append(absSrcDir.toString()); + } + + // Inject processorpath for JtregPerfHarness + if (spec.compiler().processorPath() != null && !spec.compiler().processorPath().isEmpty()) { + String cp = joinPaths(spec.compiler().processorPath()); + if (vmOpts.length() > 0) vmOpts.append(' '); + vmOpts.append("-Dharness.processorpath=").append(cp); + } + + // Commit aggregated options to command + if (vmOpts.length() > 0) { + cmd.add("-vmoptions:" + vmOpts.toString()); + } + + // Work/report dirs under the harness workDir + Path workDir = genRes.sourcesDir().resolve("jtreg-work"); + Path reportDir = genRes.sourcesDir().resolve("jtreg-report"); + // Clean previous jtreg artifacts to avoid stale .jtr affecting parsing across runs + try { + deleteRecursively(workDir); + } catch (Throwable ignore) { + } + try { + deleteRecursively(reportDir); + } catch (Throwable ignore) { + } + try { + Files.createDirectories(workDir); + Files.createDirectories(reportDir); + } catch (Throwable ignore) { + } + cmd.add("-w"); + cmd.add(workDir.toString()); + cmd.add("-r"); + cmd.add(reportDir.toString()); + + // Convert test path to absolute from project root, since jtreg runs there + String absTestPath = testPath; + try { + Path tp = Paths.get(testPath); + if (!tp.isAbsolute()) { + tp = projectRoot.toPath().resolve(testPath).normalize(); + } + absTestPath = tp.toString(); + } catch (Throwable ignore) { + } + cmd.add(absTestPath); + + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(projectRoot); + pb.redirectErrorStream(true); + Path stdoutPath = genRes.sourcesDir().resolve("jtreg-stdout.txt"); + File stdoutFile = stdoutPath.toFile(); + try { + stdoutFile.getParentFile().mkdirs(); + } catch (Throwable ignore) { + } + pb.redirectOutput(stdoutFile); + + // Measure only jtreg execution; avoid IO deadlock by redirecting output to file + long start = System.nanoTime(); + Process p = pb.start(); + // Allow override via -Dharness.timeoutSec, default 600s + int timeoutSec = 600; + try { + String ts = System.getProperty("harness.timeoutSec", "600"); + timeoutSec = Integer.parseInt(ts.trim()); + } catch (Throwable ignore) { + } + boolean finished = p.waitFor(timeoutSec, java.util.concurrent.TimeUnit.SECONDS); + int code; + if (!finished) { + // Timeout: kill process tree if possible + try { + killProcessTree(p); + } catch (Throwable ignore) { + } + code = Integer.MIN_VALUE; // special code for timeout + } else { + code = p.exitValue(); + } + long end = System.nanoTime(); + String stdout = ""; + try { + if (stdoutFile.exists()) { + stdout = Files.readString(stdoutPath, StandardCharsets.UTF_8); + } + } catch (Throwable ignore) { + } + + List diags = new ArrayList(); + if (code != 0) { + String msg = + (code == Integer.MIN_VALUE) + ? ("jtreg timed out after " + timeoutSec + "s.\n" + stdout) + : stdout; + diags.add(new Driver.DiagnosticEntry("", 0, 0, "ERROR", msg)); + } + + // Find HARNESS_RESULT line in .jtr file and parse median timing + long median = findMedianFromJtr(workDir); + + if (median == -1 && code == 0) { + // Succeeded but couldn't parse results, which is a failure of the harness itself. + diags.add( + new Driver.DiagnosticEntry( + "", + 0, + 0, + "ERROR", + "Failed to parse HARNESS_RESULT from test output.")); + code = 1; // Mark as failure + } + + Map meta = new java.util.HashMap(); + meta.put("timestamp", Instant.now().toString()); + meta.put("generator", spec.generator().name()); + meta.put("label", spec.label()); + meta.put( + "flags", + String.join( + " ", + spec.javacFlags() == null + ? java.util.Collections.emptyList() + : spec.javacFlags())); + meta.put("jtregExit", Integer.valueOf(code)); + meta.put("stdoutPath", stdoutPath.toString()); + if (code == Integer.MIN_VALUE) meta.put("timeoutSec", Integer.valueOf(timeoutSec)); + // Minimal metrics; optionally enrich from .jtr when available + Map metrics = new java.util.HashMap(); + try { + Long jtrElapsed = findElapsedFromJtr(workDir); + if (jtrElapsed != null) metrics.put("jtrElapsedMillis", jtrElapsed); + String jtrStatus = findResultStatusFromJtr(workDir); + if (jtrStatus != null) meta.put("jtrResult", jtrStatus); + } catch (Throwable ignore) { + } + RunResult result = + new RunResult( + spec.label(), + median != -1 + ? median + : (end - start) / 1_000_000L, // Use median if available + code == 0, + java.util.Collections.unmodifiableList(diags), + metrics, + genRes.sourcesDir(), + meta); + HarnessIO.writeJson(genRes.sourcesDir().resolve("result.json"), result); + return result; + } + + private static void deleteRecursively(Path p) throws java.io.IOException { + if (p == null || !Files.exists(p)) return; + java.util.List paths = new java.util.ArrayList<>(); + Files.walk(p).forEach(paths::add); + // delete children first + for (int i = paths.size() - 1; i >= 0; i--) { + try { + Files.deleteIfExists(paths.get(i)); + } catch (Throwable ignore) { + } + } + } + + /** Extract median timing, elapsed time, and result status from .jtr file in a single pass. */ + private static long findMedianFromJtr(Path workDir) { + if (workDir == null) return -1; + try { + java.util.Optional jtr = + java.nio.file.Files.walk(workDir) + .filter(p -> p.getFileName().toString().endsWith(".jtr")) + .findFirst(); + if (!jtr.isPresent()) return -1; + + String content = Files.readString(jtr.get(), StandardCharsets.UTF_8); + for (String line : content.split("\\R")) { + if (line.startsWith("HARNESS_RESULT:")) { + String data = line.substring("HARNESS_RESULT:".length()).trim(); + for (String part : data.split(",")) { + String[] kv = part.trim().split("="); + if (kv.length == 2 && "median".equals(kv[0])) { + return (long) Double.parseDouble(kv[1]); + } + } + } + } + } catch (Throwable ignore) { + } + return -1; + } + + private static Long findElapsedFromJtr(Path workDir) { + if (workDir == null) return null; + try { + java.util.Optional jtr = + java.nio.file.Files.walk(workDir) + .filter(p -> p.getFileName().toString().endsWith(".jtr")) + .findFirst(); + if (!jtr.isPresent()) return null; + + for (String line : Files.readAllLines(jtr.get(), StandardCharsets.UTF_8)) { + int idx = line.indexOf("elapsed="); + if (idx >= 0) { + int end = line.indexOf(' ', idx); + String num = + (end > idx) ? line.substring(idx + 8, end) : line.substring(idx + 8); + try { + return Long.valueOf(Long.parseLong(num.trim())); + } catch (NumberFormatException ignore) { + } + } + } + } catch (Throwable ignore) { + } + return null; + } + + private static String findResultStatusFromJtr(Path workDir) { + if (workDir == null) return null; + try { + java.util.Optional jtr = + java.nio.file.Files.walk(workDir) + .filter(p -> p.getFileName().toString().endsWith(".jtr")) + .findFirst(); + if (!jtr.isPresent()) return null; + + for (String line : Files.readAllLines(jtr.get(), StandardCharsets.UTF_8)) { + if (line.startsWith("result:")) return line.substring("result:".length()).trim(); + if (line.startsWith("test result:")) + return line.substring("test result:".length()).trim(); + } + } catch (Throwable ignore) { + } + return null; + } + + private String jtregExecutable() { + Path bin = jtregBin; + if (bin != null) { + File f = bin.toFile(); + if (f.isDirectory()) { + File jt = new File(f, isWindows() ? "jtreg.exe" : "jtreg"); + if (jt.exists()) return jt.getPath(); + } else if (f.exists()) { + return f.getPath(); + } + } + // Fallback: search upwards for repo-level jtreg/bin/jtreg + Path cwd = Paths.get(".").toAbsolutePath().normalize(); + for (int i = 0; i < 6; i++) { + Path base = cwd; + for (int j = 0; j < i; j++) base = base.getParent(); + if (base == null) break; + File jt = + base.resolve("jtreg") + .resolve("bin") + .resolve(isWindows() ? "jtreg.exe" : "jtreg") + .toFile(); + if (jt.exists()) return jt.getPath(); + } + return "jtreg"; + } + + private static boolean isWindows() { + String os = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + return os.contains("win"); + } + + private static String joinPaths(List paths) { + String sep = File.pathSeparator; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < paths.size(); i++) { + if (i > 0) sb.append(sep); + sb.append(paths.get(i).toString()); + } + return sb.toString(); + } + + private static File findProjectRoot() { + // When running via gradlew from checker/harness, user.dir is typically harness-driver-cli + // Walk up from user.dir until we find a directory containing: + // - checker/ subdirectory AND + // - build.gradle or settings.gradle at the same level + String userDir = System.getProperty("user.dir", "."); + File candidate = new File(userDir).getAbsoluteFile(); + + for (int i = 0; i < 8; i++) { + File checkerDir = new File(candidate, "checker"); + File checkerBuildGradle = new File(checkerDir, "build.gradle"); + File rootBuildGradle = new File(candidate, "build.gradle"); + File settingsGradle = new File(candidate, "settings.gradle"); + + // Check if this directory contains checker/ with its own build.gradle AND a root gradle + // file + if (checkerDir.exists() + && checkerDir.isDirectory() + && checkerBuildGradle.exists() + && (rootBuildGradle.exists() || settingsGradle.exists())) { + return candidate; + } + + // Move up one level + File parent = candidate.getParentFile(); + if (parent == null) break; + candidate = parent; + } + + // Fallback: return original user.dir + return new File(userDir).getAbsoluteFile(); + } + + private static void killProcessTree(Process p) { + if (p == null) return; + try { + java.lang.ProcessHandle h = p.toHandle(); + // Best-effort: kill descendants first, then the root + try { + h.descendants() + .forEach( + ph -> { + try { + ph.destroyForcibly(); + } catch (Throwable ignore) { + } + }); + } catch (Throwable ignore) { + } + try { + h.destroyForcibly(); + } catch (Throwable ignore) { + } + } catch (Throwable ignore) { + try { + p.destroyForcibly(); + } catch (Throwable ignore2) { + } + } + } +} diff --git a/checker/harness/harness-driver-cli/build.gradle b/checker/harness/harness-driver-cli/build.gradle new file mode 100644 index 000000000000..cdf80909b2f9 --- /dev/null +++ b/checker/harness/harness-driver-cli/build.gradle @@ -0,0 +1,38 @@ +plugins { + id 'application' + id 'java' +} + +group = 'io.github.eisop' + +repositories { + mavenCentral() +} + +dependencies { + implementation project(':harness-core') + implementation project(':harness-generators') +} + +application { + mainClass = 'org.checkerframework.harness.cli.Main' + // Open required javac internals for in-process ToolProvider (JDK 9+) + applicationDefaultJvmArgs = [ + '--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', + '--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED' + ] +} + +java { + withJavadocJar() + withSourcesJar() +} + + diff --git a/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java b/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java new file mode 100644 index 000000000000..19ce23059553 --- /dev/null +++ b/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java @@ -0,0 +1,458 @@ +package org.checkerframework.harness.cli; + +import org.checkerframework.harness.core.CodeGenerator; +import org.checkerframework.harness.core.Driver; +import org.checkerframework.harness.core.HarnessIO; +import org.checkerframework.harness.core.InProcessJavacDriver; +import org.checkerframework.harness.generators.nullness.NewAndArrayGenerator; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * CLI entry point that orchestrates SINGLE/CROSS protocols, constructs a RunSpec for the chosen + * engine, delegates execution to a Driver, and emits JSON/Markdown reports. + */ +public final class Main { + public static void main(String[] args) throws Exception { + Map opts = parseArgs(args); + if (opts.containsKey("--help") || !opts.containsKey("--generator")) { + printHelp(); + return; + } + + String generatorName = opts.getOrDefault("--generator", "NewAndArray"); + int sampleCount = Integer.parseInt(opts.getOrDefault("--sampleCount", "100")); + long seed = Long.parseLong(opts.getOrDefault("--seed", "1")); + String baselineFlags = opts.getOrDefault("--baseline-flags", "").trim(); + String updateFlags = opts.getOrDefault("--update-flags", "").trim(); + String processor = + opts.getOrDefault( + "--processor", "org.checkerframework.checker.nullness.NullnessChecker"); + String processorPath = opts.getOrDefault("--processor-path", ""); + String release = opts.getOrDefault("--release", "17"); + String protocol = + opts.getOrDefault("--protocol", "SINGLE").toUpperCase(Locale.ROOT); // SINGLE|CROSS + int runs = Integer.parseInt(opts.getOrDefault("--runs", "5")); + if (runs < 1) { + System.err.println("WARNING: --runs < 1; falling back to 1."); + runs = 1; + } + String engine = opts.getOrDefault("--engine", "inproc"); // inproc|external|jtreg + String jtregBin = opts.getOrDefault("--jtreg", Paths.get("jtreg", "bin").toString()); + String jtregTest = + opts.getOrDefault("--jtreg-test", "checker/harness/jtreg/JtregPerfHarness.java"); + + CodeGenerator generator = selectGenerator(generatorName); + + Path resultDir = Paths.get("checker/harness/result").toAbsolutePath().normalize(); + Path baselineDir = resultDir.resolve("baseline"); + Path updateDir = resultDir.resolve("update"); + Files.createDirectories(baselineDir); + Files.createDirectories(updateDir); + + Driver driver; + if ("external".equalsIgnoreCase(engine)) { + driver = new org.checkerframework.harness.core.ExternalProcessJavacDriver(); + } else if ("jtreg".equalsIgnoreCase(engine)) { + driver = + new org.checkerframework.harness.core.JtregDriver( + Paths.get(jtregBin), jtregTest); + } else { + driver = new InProcessJavacDriver(); + } + + // Common compiler config: shared across engines. For jtreg, pass + // -Dharness.release= + // via flags so the test-side harness can translate it to --release. + List processors = java.util.Arrays.asList(processor); + List pp = + processorPath.isEmpty() + ? new ArrayList() + : java.util.Arrays.asList(Paths.get(processorPath)); + List sourceOpts = + java.util.Arrays.asList("--release", release, "-proc:only", "-Xlint:-options"); + + // Add perf-related flags for jtreg engine. Strip -J prefix if present (not needed for test + // VM). + if ("jtreg".equalsIgnoreCase(engine)) { + List baselineJavacFlags = new ArrayList<>(); + for (String f : splitFlags(baselineFlags)) { + // Strip -J prefix: -J-Dcf.xxx -> -Dcf.xxx (test VM doesn't use javac's -J syntax) + baselineJavacFlags.add(f.startsWith("-J") ? f.substring(2) : f); + } + baselineJavacFlags.add("-Dperf.runs=" + runs); + baselineJavacFlags.add("-Dcf.skipNonnullFastPath=false"); + baselineJavacFlags.add("-Dharness.release=" + release); + baselineJavacFlags.add("-Dharness.processor=" + processor); + + List updateJavacFlags = new ArrayList<>(); + for (String f : splitFlags(updateFlags)) { + updateJavacFlags.add(f.startsWith("-J") ? f.substring(2) : f); + } + updateJavacFlags.add("-Dperf.runs=" + runs); + updateJavacFlags.add("-Dcf.skipNonnullFastPath=true"); + updateJavacFlags.add("-Dharness.release=" + release); + updateJavacFlags.add("-Dharness.processor=" + processor); + + baselineFlags = String.join(" ", baselineJavacFlags); + updateFlags = String.join(" ", updateJavacFlags); + } + + Driver.CompilerConfig compilerCfgBaseline = + new Driver.CompilerConfig( + null, + List.of(), + processors, + pp, + sourceOpts, + baselineDir.resolve("sampleCount")); + + Driver.CompilerConfig compilerCfgUpdate = + new Driver.CompilerConfig( + null, + List.of(), + processors, + pp, + sourceOpts, + updateDir.resolve("sampleCount")); + + CodeGenerator.GenerationRequest genReqBaseline = + new CodeGenerator.GenerationRequest( + baselineDir.resolve("src"), seed, sampleCount, extractExtra(opts)); + CodeGenerator.GenerationRequest genReqUpdate = + new CodeGenerator.GenerationRequest( + updateDir.resolve("src"), seed, sampleCount, extractExtra(opts)); + + java.util.List baselineFlagList = splitFlags(baselineFlags); + java.util.List updateFlagList = splitFlags(updateFlags); + + // Warn if protocol requires update variant but user didn't provide + // --update-flags + if (!"SINGLE".equals(protocol) && updateFlagList.isEmpty()) { + System.err.println( + "WARNING: --update-flags is empty; proceeding without update-specific flags."); + } + + // If engine=jtreg and --release is provided but -Dharness.release is missing, + // show a warning and suggest adding it + if ("jtreg".equalsIgnoreCase(engine)) { + boolean hasHarnessRelease = + baselineFlagList.stream().anyMatch(s -> s.startsWith("-Dharness.release=")) + || updateFlagList.stream() + .anyMatch(s -> s.startsWith("-Dharness.release=")); + if (!release.isEmpty() && !hasHarnessRelease) { + System.err.println( + "WARNING: engine=jtreg with --release=" + + release + + ": add -Dharness.release=" + + release + + " to --baseline-flags/--update-flags for consistent language level inside tests."); + } + } + + Driver.RunSpec specBaseline = + new Driver.RunSpec( + generator, + genReqBaseline, + compilerCfgBaseline, + baselineFlagList, + "baseline"); + Driver.RunSpec specUpdate = + new Driver.RunSpec( + generator, genReqUpdate, compilerCfgUpdate, updateFlagList, "update"); + + if ("jtreg".equalsIgnoreCase(engine)) { + // For jtreg, the looping is handled inside the test. Run each variant just once. + Driver.RunResult lastA = driver.runOnce(specBaseline); + Driver.RunResult lastB = driver.runOnce(specUpdate); + + // The driver returns a result with the median time. We pass it as a single-element + // list. + List aTimings = lastA.success() ? List.of(lastA.wallMillis()) : List.of(); + List bTimings = lastB.success() ? List.of(lastB.wallMillis()) : List.of(); + + if (lastA != null && lastB != null) { + Map ctx = new HashMap(); + ctx.put("protocol", protocol + " (jtreg-internal)"); + ctx.put("runs", String.valueOf(runs)); + ctx.put("engine", engine); + ctx.put("sampleCount", String.valueOf(sampleCount)); + ctx.put("seed", String.valueOf(seed)); + String gpf = extractExtra(opts).getOrDefault("groupsPerFile", ""); + if (!gpf.isEmpty()) ctx.put("groupsPerFile", gpf); + ctx.put("baselineFlags", String.join(" ", baselineFlagList)); + ctx.put("updateFlags", String.join(" ", updateFlagList)); + HarnessIO.writeUnifiedReport( + resultDir.resolve("report.md"), + lastA, + lastB, + ctx, + aTimings, + lastA.success() ? 1 : 0, + 1, + bTimings, + lastB.success() ? 1 : 0, + 1); + printErrorsToConsole(lastA, lastB); + } + } else if ("SINGLE".equals(protocol)) { + // Warmup once per variant (not timed) to mitigate JIT/IO-cache cold-start effects and + // reduce drift in measured runs. This keeps SINGLE results more stable without changing + // its simple execution model. + try { + driver.runOnce(specBaseline); + } catch (Throwable ignore) { + } + try { + driver.runOnce(specUpdate); + } catch (Throwable ignore) { + } + + // Run baseline 'runs' times, then update 'runs' times; collect timings and emit summary + // statistics. + ArrayList aTimings = new ArrayList(); + ArrayList bTimings = new ArrayList(); + int aSuccess = 0; + int bSuccess = 0; + Driver.RunResult lastA = null; + Driver.RunResult lastB = null; + for (int i = 0; i < runs; i++) { + lastA = driver.runOnce(specBaseline); + if (lastA.success()) { + aTimings.add(Long.valueOf(lastA.wallMillis())); + aSuccess++; + } + } + for (int i = 0; i < runs; i++) { + lastB = driver.runOnce(specUpdate); + if (lastB.success()) { + bTimings.add(Long.valueOf(lastB.wallMillis())); + bSuccess++; + } + } + // Persist last results for compatibility + if (lastA != null) HarnessIO.writeJson(baselineDir.resolve("result.json"), lastA); + if (lastB != null) HarnessIO.writeJson(updateDir.resolve("result.json"), lastB); + // Unified report (aggregated metrics always shown) + if (lastA != null && lastB != null) { + Map ctx = new HashMap(); + ctx.put("protocol", protocol); + ctx.put("runs", String.valueOf(runs)); + ctx.put("engine", engine); + ctx.put("sampleCount", String.valueOf(sampleCount)); + ctx.put("seed", String.valueOf(seed)); + String gpf = extractExtra(opts).getOrDefault("groupsPerFile", ""); + if (!gpf.isEmpty()) ctx.put("groupsPerFile", gpf); + ctx.put("baselineFlags", String.join(" ", baselineFlagList)); + ctx.put("updateFlags", String.join(" ", updateFlagList)); + HarnessIO.writeUnifiedReport( + resultDir.resolve("report.md"), + lastA, + lastB, + ctx, + aTimings, + runs, + aSuccess, + bTimings, + runs, + bSuccess); + printErrorsToConsole(lastA, lastB); + } + } else { + if ("CROSS".equals(protocol)) { + // CROSS (ABBA): per iteration run AB then BA; compute per-iteration pairwise + // averages (A_i, B_i). + ArrayList aPairwise = new ArrayList(); + ArrayList bPairwise = new ArrayList(); + int aSuccess = 0; + int bSuccess = 0; + + boolean warmedA = false; + boolean warmedB = false; + + // Representative single-run results for reporting (flags/diagnostics only). + // Note: Aggregated timing statistics (median/average/min/max) are computed from + // the full series (pairwise A_i/B_i over all iterations). The representative + // results below do NOT participate in those aggregates; they are only used to + // populate readable, single-run fields in the report such as Flags and Diagnostics. + // We choose the "last BA" results from the final iteration to avoid extra runs + // and because any single iteration would be equivalent for this purpose. + Driver.RunResult reprA = null; + Driver.RunResult reprB = null; + for (int i = 0; i < runs; i++) { + // Round 1: AB + if (!warmedA) { + driver.runOnce(specBaseline); + warmedA = true; + } + Driver.RunResult resAB_A = driver.runOnce(specBaseline); + long tAB_A = resAB_A.wallMillis(); + if (!warmedB) { + driver.runOnce(specUpdate); + warmedB = true; + } + Driver.RunResult resAB_B = driver.runOnce(specUpdate); + long tAB_B = resAB_B.wallMillis(); + + // Round 2: BA + Driver.RunResult resBA_B = driver.runOnce(specUpdate); + long tBA_B = resBA_B.wallMillis(); + Driver.RunResult resBA_A = driver.runOnce(specBaseline); + long tBA_A = resBA_A.wallMillis(); + + long aPair = (tAB_A + tBA_A) / 2L; // A_i + long bPair = (tAB_B + tBA_B) / 2L; // B_i + aPairwise.add(aPair); + bPairwise.add(bPair); + // Count success per iteration as both directions succeeded (exit status) + // Here success is approximated via presence of timings (drivers already report + // success) + aSuccess++; + bSuccess++; + // Record the most recent results as representatives for flags/diagnostics only + reprA = resBA_A; // last A in this iteration + reprB = resBA_B; // last B in this iteration + } + + // Unified report aggregates pairwise averages across iterations (series) and + // uses reprA/reprB solely for metadata and diagnostics presentation. + Map ctx = new HashMap(); + ctx.put("protocol", protocol); + ctx.put("runs", String.valueOf(runs)); + ctx.put("engine", engine); + ctx.put("sampleCount", String.valueOf(sampleCount)); + ctx.put("seed", String.valueOf(seed)); + String gpf = extractExtra(opts).getOrDefault("groupsPerFile", ""); + if (!gpf.isEmpty()) ctx.put("groupsPerFile", gpf); + ctx.put("baselineFlags", String.join(" ", baselineFlagList)); + ctx.put("updateFlags", String.join(" ", updateFlagList)); + HarnessIO.writeUnifiedReport( + resultDir.resolve("report.md"), + reprA, + reprB, + ctx, + aPairwise, + runs, + aSuccess, + bPairwise, + runs, + bSuccess); + printErrorsToConsole(reprA, reprB); + } else { + throw new IllegalArgumentException( + "Unknown protocol: " + protocol + ". Supported: SINGLE|CROSS"); + } + } + + System.out.println( + "Report generated at: " + resultDir.resolve("report.md").toAbsolutePath()); + } + + private static CodeGenerator selectGenerator(String name) { + if ("NewAndArray".equals(name)) { + return new NewAndArrayGenerator(); + } + throw new IllegalArgumentException( + "Unknown generator: " + name + ". Supported: NewAndArray"); + } + + private static Map parseArgs(String[] args) { + Map m = new HashMap<>(); + String key = null; + for (String a : args) { + if (a.startsWith("--")) { + key = a; + m.putIfAbsent(key, ""); + } else if (key != null) { + String prev = m.getOrDefault(key, ""); + m.put(key, prev.isEmpty() ? a : prev + " " + a); + } + } + return m; + } + + private static List splitFlags(String flags) { + if (flags.isEmpty()) return new ArrayList(); + List out = new ArrayList(); + for (String s : flags.trim().split("\\s+")) { + if (!s.isEmpty()) out.add(s); + } + return out; + } + + private static Map extractExtra(Map opts) { + Map extra = new HashMap<>(); + for (Map.Entry e : opts.entrySet()) { + String k = e.getKey(); + if (k.startsWith("--extra.")) { + extra.put(k.substring("--extra.".length()), e.getValue()); + } + } + return extra; + } + + private static void printHelp() { + System.out.println("Usage: --generator --sampleCount --seed "); + System.out.println(" --baseline-flags --update-flags "); + System.out.println( + " --processor --processor-path --release "); + System.out.println( + " --protocol --runs [--engine ]"); + System.out.println(); + System.out.println("Examples:"); + System.out.println(" --generator NewAndArray --sampleCount 200 --seed 42 \\"); + System.out.println( + " --baseline-flags -AfastNewClass=false --update-flags -AfastNewClass=true \\"); + System.out.println( + " --processor org.checkerframework.checker.nullness.NullnessChecker \\"); + System.out.println(" --processor-path --release 17"); + System.out.println(); + System.out.println("Protocols:"); + System.out.println( + " SINGLE: warm up each variant once (not timed), then run baseline 'runs' times and update 'runs' times; emit single.md with median/average; report.md shows the last pair."); + System.out.println( + " CROSS : per iteration run AB then BA; compute pairwise A_i=(T_AB^A+T_BA^A)/2, B_i=(T_AB^B+T_BA^B)/2, then summarize across i."); + System.out.println( + " Each variant is warmed once on its first appearance (not timed)."); + System.out.println(); + } + + private static void printErrorsToConsole(Driver.RunResult a, Driver.RunResult b) { + java.util.concurrent.atomic.AtomicBoolean any = + new java.util.concurrent.atomic.AtomicBoolean(false); + StringBuilder sb = new StringBuilder(); + java.util.function.Consumer dump = + r -> { + int count = 0; + for (Driver.DiagnosticEntry d : r.diagnostics()) { + if ("ERROR".equals(d.kind())) { + if (count == 0) { + sb.append("\n--- " + r.label() + " ERROR diagnostics ---\n"); + } + any.set(true); + count++; + sb.append(d.file()) + .append(":") + .append(d.line()) + .append(":") + .append(d.column()) + .append(" ") + .append(d.kind()) + .append(" ") + .append(d.message()) + .append("\n"); + } + } + }; + dump.accept(a); + dump.accept(b); + if (any.get()) System.out.print(sb.toString()); + } +} diff --git a/checker/harness/harness-generators/build.gradle b/checker/harness/harness-generators/build.gradle new file mode 100644 index 000000000000..62c9d6d7a802 --- /dev/null +++ b/checker/harness/harness-generators/build.gradle @@ -0,0 +1,20 @@ +plugins { + id 'java' +} + +group = 'io.github.eisop' + +repositories { + mavenCentral() +} + +dependencies { + implementation project(':harness-core') +} + +java { + withJavadocJar() + withSourcesJar() +} + + diff --git a/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java b/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java new file mode 100644 index 000000000000..7b136e5c995d --- /dev/null +++ b/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java @@ -0,0 +1,147 @@ +package org.checkerframework.harness.generators.nullness; + +import org.checkerframework.harness.core.CodeGenerator; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Formatter; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; + +/** + * Generates Java sources that exercise object and array allocations: - new Object() - new + * ArrayList<>() - new int[...], new String[...] - anonymous class: {@code new Runnable(){ ... }} + * + *

Behavior: - Emits one public class per file; file count equals {@code max(1, + * req.sampleCount())}. - Per-file work is controlled by {@code extra["groupsPerFile"]} (default + * 400); each group emits six allocation expressions. - Output is deterministic for the same seed, + * sample count, and extras. + * + *

Optional extras: - groupsPerFile: int, default 400 + */ +public final class NewAndArrayGenerator implements CodeGenerator { + + private static final String DEFAULT_PACKAGE = "bench.nullness.newarray"; + private static final String DEFAULT_CLASS_PREFIX = "ManyNew"; + + @Override + public String name() { + return "NewAndArray"; + } + + @Override + public GenerationResult generate(GenerationRequest req) throws Exception { + final String packageName = DEFAULT_PACKAGE; + final String classPrefix = DEFAULT_CLASS_PREFIX; + final int groupsPerFile = + parsePositiveInt( + req.extra() == null ? null : req.extra().get("groupsPerFile"), 400); + + final Path sourcesDir = req.outputDir().resolve(name() + "-sources"); + Files.createDirectories(sourcesDir); + + List created = new ArrayList<>(Math.max(1, req.sampleCount())); + final int fileCount = Math.max(1, req.sampleCount()); + final int pad = numDigits(fileCount); + + // Seeded RNG: ensures deterministic but seed-dependent output content + final Random rng = new Random(req.seed()); + for (int i = 0; i < fileCount; i++) { + String simpleClassName = classPrefix + "_" + leftPad(i + 1, pad); + Path file = sourcesDir.resolve(simpleClassName + ".java"); + // Derive a per-file RNG to avoid cross-file dependence on loop iteration counts + Random fileRng = new Random(rng.nextLong()); + writeOneFile(file, packageName, simpleClassName, groupsPerFile, fileRng, req.seed()); + created.add(file); + } + + Map meta = new LinkedHashMap<>(); + meta.put("generator", name()); + meta.put("files", fileCount); + meta.put("groupsPerFile", groupsPerFile); + meta.put("packageName", packageName); + meta.put("classPrefix", classPrefix); + meta.put("seed", req.seed()); + + return new GenerationResult(sourcesDir, created, meta); + } + + private static void writeOneFile( + Path file, String packageName, String className, int groups, Random rng, long seed) + throws IOException { + StringBuilder sb = new StringBuilder(256 + groups * 200); + try (Formatter fmt = new Formatter(sb, Locale.ROOT)) { + // Generated file header for traceability + fmt.format( + "// Generated by NewAndArrayGenerator; groupsPerFile=%d; seed=%d%n", + groups, seed); + fmt.format("// Package: %s | Class: %s%n%n", packageName, className); + if (!packageName.isEmpty()) { + fmt.format("package %s;%n%n", packageName); + } + fmt.format("import java.util.*;%n%n"); + fmt.format("public class %s {%n", className); + + // Small generic box to include a constructor allocation and a field write. + // Uses Object upper bound to avoid initialization-checker interaction; compiles + // cleanly with the Nullness Checker alone. + fmt.format( + " static class Box { T t; Box(){ this.t = (T)(Object)new Object(); } }%n"); + + fmt.format(" void f() {%n"); + for (int i = 0; i < groups; i++) { + // Seed-influenced small variations that do not change structure/line count + int arrLen1 = 8 + (Math.abs(rng.nextInt()) % 17); // [8,24] + int arrLen2 = 8 + (Math.abs(rng.nextInt()) % 17); // [8,24] + // Emit six allocation forms per group + fmt.format(" Object o%d = new Object();%n", i); + fmt.format(" ArrayList l%d = new ArrayList<>();%n", i); + fmt.format(" Box b%d = new Box<>();%n", i); + fmt.format(" int[] ai%d = new int[%d];%n", i, arrLen1); + fmt.format(" String[] as%d = new String[%d];%n", i, arrLen2); + fmt.format(" Runnable r%d = new Runnable(){ public void run(){} };%n", i); + } + fmt.format(" }%n"); // end method + fmt.format("}%n"); // end class + } + + Files.createDirectories(file.getParent()); + try (BufferedWriter w = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) { + w.write(sb.toString()); + } + } + + private static int parsePositiveInt(String s, int def) { + if (s == null) return def; + try { + int v = Integer.parseInt(s.trim()); + return v > 0 ? v : def; + } catch (NumberFormatException e) { + return def; + } + } + + private static int numDigits(int n) { + if (n <= 0) return 1; + return (int) Math.floor(Math.log10(n)) + 1; + } + + private static String leftPad(int value, int width) { + String pattern = repeat('0', width); + return new DecimalFormat(pattern).format(value); + } + + private static String repeat(char ch, int count) { + StringBuilder sb = new StringBuilder(count); + for (int i = 0; i < count; i++) sb.append(ch); + return sb.toString(); + } +} diff --git a/checker/harness/jtreg/JtregPerfHarness.java b/checker/harness/jtreg/JtregPerfHarness.java new file mode 100644 index 000000000000..246890b01f87 --- /dev/null +++ b/checker/harness/jtreg/JtregPerfHarness.java @@ -0,0 +1,161 @@ +/* + * @test + * @summary jtreg-side wrapper that performs a single compilation timing. Cross/SINGLE orchestration is done by the CLI. + * @run main/timeout=600 JtregPerfHarness + */ + +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; + +public class JtregPerfHarness { + + public static void main(String[] args) throws Exception { + Path srcDir = + Paths.get(System.getProperty("harness.srcdir", ".")).toAbsolutePath().normalize(); + boolean skipFastPath = Boolean.getBoolean("cf.skipNonnullFastPath"); + + List sources = new ArrayList<>(); + Files.walk(srcDir) + .forEach( + p -> { + if (p.toString().endsWith(".java")) sources.add(p); + }); + if (sources.isEmpty()) + throw new IllegalStateException("No .java files found in harness.srcdir: " + srcDir); + + long one = runOnce(sources, skipFastPath); + // Emit a single-sample HARNESS_RESULT for the driver to parse + System.out.println( + "HARNESS_RESULT: median=" + one + ", average=" + one + ", samples=[" + one + "]"); + } + + // Execute one compile and return wall time (ms). + private static long runOnce(List sources, boolean skipFastPath) throws Exception { + long start = System.nanoTime(); + ExecResult r = compileWithCF(sources, skipFastPath); + long end = System.nanoTime(); + if (r.exitCode != 0) + throw new RuntimeException( + "javac failed with exit code " + r.exitCode + ". Output:\n" + r.stdout); + return (end - start) / 1_000_000L; + } + + // Build and execute a javac command. Returns exit code and stdout. + private static ExecResult compileWithCF(List sources, boolean skipFastPath) + throws Exception { + String javac = findJavac(); + List cmd = new ArrayList<>(); + cmd.add(javac); + // Add required --add-opens for JDK 9+ + String[] pkgs = + new String[] { + "com.sun.tools.javac.api", "com.sun.tools.javac.code", + "com.sun.tools.javac.comp", + "com.sun.tools.javac.file", "com.sun.tools.javac.main", + "com.sun.tools.javac.parser", + "com.sun.tools.javac.processing", "com.sun.tools.javac.tree", + "com.sun.tools.javac.util" + }; + for (String p : pkgs) { + cmd.add("-J--add-opens=jdk.compiler/" + p + "=ALL-UNNAMED"); + } + if (skipFastPath) { + cmd.add("-J-Dcf.skipNonnullFastPath=true"); + } + + String release = System.getProperty("harness.release", "17").trim(); + cmd.add("--release"); + cmd.add(release); + + String processor = + System.getProperty( + "harness.processor", + "org.checkerframework.checker.nullness.NullnessChecker"); + cmd.add("-processor"); + cmd.add(processor); + + // Use processorpath from system property injected by JtregDriver. + String ppath = System.getProperty("harness.processorpath", "").trim(); + if (ppath.isEmpty()) { + throw new IllegalStateException("-Dharness.processorpath was not set"); + } + cmd.add("-processorpath"); + cmd.add(ppath); + cmd.add("-classpath"); + cmd.add(ppath); + + cmd.add("-proc:only"); + cmd.add("-Xlint:-options"); + + // Add all source files. Since we run from the source dir, just use file names. + for (Path p : sources) { + cmd.add(p.getFileName().toString()); + } + + Path srcDir = + Paths.get(System.getProperty("harness.srcdir", ".")).toAbsolutePath().normalize(); + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(srcDir.toFile()); + pb.redirectErrorStream(true); + Process p = pb.start(); + byte[] out = p.getInputStream().readAllBytes(); + int code = p.waitFor(); + return new ExecResult(code, new String(out, StandardCharsets.UTF_8)); + } + + private static final class ExecResult { + final int exitCode; + final String stdout; + + ExecResult(int exitCode, String stdout) { + this.exitCode = exitCode; + this.stdout = stdout; + } + } + + private static String findJavac() { + String javaHome = System.getProperty("java.home"); + java.io.File jh = new java.io.File(javaHome); + java.io.File bin = new java.io.File(jh.getParentFile(), "bin"); + java.io.File exe = new java.io.File(bin, isWindows() ? "javac.exe" : "javac"); + if (exe.exists()) return exe.getAbsolutePath(); + return isWindows() ? "javac.exe" : "javac"; + } + + private static boolean isWindows() { + String os = System.getProperty("os.name", "").toLowerCase(java.util.Locale.ROOT); + return os.contains("win"); + } + + private static final class Result { + final List timingsMs; + + Result(List t) { + this.timingsMs = Collections.unmodifiableList(new ArrayList<>(t)); + } + + double average() { + return timingsMs.stream().mapToLong(Long::longValue).average().orElse(0); + } + + double median() { + if (timingsMs.isEmpty()) return 0; + List copy = new ArrayList<>(timingsMs); + Collections.sort(copy); + int n = copy.size(); + return (n & 1) == 1 ? copy.get(n / 2) : (copy.get(n / 2 - 1) + copy.get(n / 2)) / 2.0; + } + + // Prints a machine-readable report to stdout for JtregDriver to parse. + void printReport() { + // HARNESS_RESULT: median=123.0, average=124.5, samples=[120,123,129] + StringBuilder sb = new StringBuilder(); + sb.append("HARNESS_RESULT: "); + sb.append("median=").append(median()).append(", "); + sb.append("average=").append(average()).append(", "); + sb.append("samples=").append(timingsMs); + System.out.println(sb.toString()); + } + } +} diff --git a/checker/harness/jtreg/TEST.ROOT b/checker/harness/jtreg/TEST.ROOT new file mode 100644 index 000000000000..4b4049f57b18 --- /dev/null +++ b/checker/harness/jtreg/TEST.ROOT @@ -0,0 +1,6 @@ +# This file identifies the root of the harness test-suite hierarchy. +# Test-suite configuration for performance harness tests. + +# The list of keywords supported in this test suite +keys=perf harness + diff --git a/checker/harness/settings.gradle b/checker/harness/settings.gradle new file mode 100644 index 000000000000..8ea5d284be05 --- /dev/null +++ b/checker/harness/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'harness' +include 'harness-core', 'harness-generators', 'harness-driver-cli' diff --git a/checker/jtreg/nullness/perf/NewClassPerf.java b/checker/jtreg/nullness/perf/NewClassPerf.java index 6b79b2c6ff76..397d95a8d8cf 100644 --- a/checker/jtreg/nullness/perf/NewClassPerf.java +++ b/checker/jtreg/nullness/perf/NewClassPerf.java @@ -7,7 +7,6 @@ import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -23,8 +22,10 @@ public class NewClassPerf { private static final int RUNS = Integer.getInteger("perf.runs", 10); - private static final int GROUPS = Integer.getInteger("perf.groups", 400); // ~2k new-exprs total (5 per group) - // Protocol: AB | BA | BOTH | SEPARATE. Default BOTH runs interleaved AB and BA to cancel order bias. + private static final int GROUPS = + Integer.getInteger("perf.groups", 400); // ~2k new-exprs total (5 per group) + // Protocol: AB | BA | BOTH | SEPARATE. Default BOTH runs interleaved AB and BA to cancel order + // bias. private static final String PROTOCOL = System.getProperty("perf.protocol", "BOTH"); private static final int WARMUP = Integer.getInteger("perf.warmupPerVariant", 1); @@ -34,41 +35,46 @@ public static void main(String[] args) throws Exception { writeManyNewSource(src, GROUPS); switch (PROTOCOL.toUpperCase()) { - case "AB": { - Result[] ab = timeInterleaved(src, "AB"); - printResults("Interleaved AB", ab[0], ab[1]); - break; - } - case "BA": { - Result[] ba = timeInterleaved(src, "BA"); - printResults("Interleaved BA", ba[0], ba[1]); - break; - } - case "SEPARATE": { - Result[] sep = timeSeparate(src, WARMUP); - printResults("Separate (warmup=" + WARMUP + ")", sep[0], sep[1]); - break; - } + case "AB": + { + Result[] ab = timeInterleaved(src, "AB"); + printResults("Interleaved AB", ab[0], ab[1]); + break; + } + case "BA": + { + Result[] ba = timeInterleaved(src, "BA"); + printResults("Interleaved BA", ba[0], ba[1]); + break; + } + case "SEPARATE": + { + Result[] sep = timeSeparate(src, WARMUP); + printResults("Separate (warmup=" + WARMUP + ")", sep[0], sep[1]); + break; + } case "BOTH": - default: { - Result[] ab = timeInterleaved(src, "AB"); - Result[] ba = timeInterleaved(src, "BA"); - System.out.println("==== Interleaved AB ===="); - printResults("AB", ab[0], ab[1]); - System.out.println(); - System.out.println("==== Interleaved BA ===="); - printResults("BA", ba[0], ba[1]); - // Consistency check - double sign1 = Math.signum((ab[1].median() - ab[0].median())); - double sign2 = Math.signum((ba[1].median() - ba[0].median())); - System.out.println(); - if (sign1 == 0 || sign2 == 0 || Math.signum(sign1) != Math.signum(sign2)) { - System.out.println("Direction: ORDER-SENSITIVE (results differ between AB and BA)"); - } else { - System.out.println("Direction: CONSISTENT across AB and BA"); + default: + { + Result[] ab = timeInterleaved(src, "AB"); + Result[] ba = timeInterleaved(src, "BA"); + System.out.println("==== Interleaved AB ===="); + printResults("AB", ab[0], ab[1]); + System.out.println(); + System.out.println("==== Interleaved BA ===="); + printResults("BA", ba[0], ba[1]); + // Consistency check + double sign1 = Math.signum((ab[1].median() - ab[0].median())); + double sign2 = Math.signum((ba[1].median() - ba[0].median())); + System.out.println(); + if (sign1 == 0 || sign2 == 0 || Math.signum(sign1) != Math.signum(sign2)) { + System.out.println( + "Direction: ORDER-SENSITIVE (results differ between AB and BA)"); + } else { + System.out.println("Direction: CONSISTENT across AB and BA"); + } + break; } - break; - } } } @@ -78,7 +84,8 @@ private static void writeManyNewSource(Path file, int groups) throws IOException fmt.format("import java.util.*;%n"); fmt.format("public class ManyNew {%n"); // Avoid initialization checker errors; use @Nullable type variable upper bound. - fmt.format(" static class Box { T t; Box(){ this.t = (T) (Object) new Object(); } }%n"); + fmt.format( + " static class Box { T t; Box(){ this.t = (T) (Object) new Object(); } }%n"); fmt.format(" void f() {%n"); for (int i = 0; i < groups; i++) { fmt.format(" Object o%d = new Object();%n", i); @@ -110,14 +117,14 @@ private static Result[] timeInterleaved(Path src, String order) throws Exception boolean firstA = !"BA".equalsIgnoreCase(order); for (int i = 0; i < RUNS; i++) { if (firstA) { - aTimes.add(runOnceMs(src, /*skipFastPath=*/false)); - bTimes.add(runOnceMs(src, /*skipFastPath=*/true)); + aTimes.add(runOnceMs(src, /* skipFastPath= */ false)); + bTimes.add(runOnceMs(src, /* skipFastPath= */ true)); } else { - bTimes.add(runOnceMs(src, /*skipFastPath=*/true)); - aTimes.add(runOnceMs(src, /*skipFastPath=*/false)); + bTimes.add(runOnceMs(src, /* skipFastPath= */ true)); + aTimes.add(runOnceMs(src, /* skipFastPath= */ false)); } } - return new Result[] { new Result(aTimes), new Result(bTimes) }; + return new Result[] {new Result(aTimes), new Result(bTimes)}; } private static long runOnceMs(Path src, boolean skipFastPath) throws Exception { @@ -133,22 +140,22 @@ private static long runOnceMs(Path src, boolean skipFastPath) throws Exception { private static Result[] timeSeparate(Path src, int warmup) throws Exception { // Variant A (fast-path enabled) for (int i = 0; i < warmup; i++) { - runOnceMs(src, /*skipFastPath=*/false); + runOnceMs(src, /* skipFastPath= */ false); } List aTimes = new ArrayList<>(); for (int i = 0; i < RUNS; i++) { - aTimes.add(runOnceMs(src, /*skipFastPath=*/false)); + aTimes.add(runOnceMs(src, /* skipFastPath= */ false)); } // Variant B (fast-path disabled) for (int i = 0; i < warmup; i++) { - runOnceMs(src, /*skipFastPath=*/true); + runOnceMs(src, /* skipFastPath= */ true); } List bTimes = new ArrayList<>(); for (int i = 0; i < RUNS; i++) { - bTimes.add(runOnceMs(src, /*skipFastPath=*/true)); + bTimes.add(runOnceMs(src, /* skipFastPath= */ true)); } - return new Result[] { new Result(aTimes), new Result(bTimes) }; + return new Result[] {new Result(aTimes), new Result(bTimes)}; } private static void printResults(String label, Result a, Result b) { @@ -175,31 +182,37 @@ private static int runJavac(Path src, boolean skipFastPath) throws Exception { List cmd = new ArrayList<>(); cmd.add(findJavacExecutable()); // Add required --add-opens for running Checker Framework on JDK 9+ - String[] javacPkgs = new String[] { - "com.sun.tools.javac.api", - "com.sun.tools.javac.code", - "com.sun.tools.javac.comp", - "com.sun.tools.javac.file", - "com.sun.tools.javac.main", - "com.sun.tools.javac.parser", - "com.sun.tools.javac.processing", - "com.sun.tools.javac.tree", - "com.sun.tools.javac.util" - }; + String[] javacPkgs = + new String[] { + "com.sun.tools.javac.api", + "com.sun.tools.javac.code", + "com.sun.tools.javac.comp", + "com.sun.tools.javac.file", + "com.sun.tools.javac.main", + "com.sun.tools.javac.parser", + "com.sun.tools.javac.processing", + "com.sun.tools.javac.tree", + "com.sun.tools.javac.util" + }; for (String p : javacPkgs) { cmd.add("-J--add-opens=jdk.compiler/" + p + "=ALL-UNNAMED"); } if (skipFastPath) { cmd.add("-J-Dcf.skipNonnullFastPath=true"); } - cmd.addAll(Arrays.asList( - "-classpath", checkerJar, - "-processor", "org.checkerframework.checker.nullness.NullnessChecker", - "-proc:only", - "-source", "8", - "-target", "8", - "-Xlint:-options", - src.getFileName().toString())); + cmd.addAll( + Arrays.asList( + "-classpath", + checkerJar, + "-processor", + "org.checkerframework.checker.nullness.NullnessChecker", + "-proc:only", + "-source", + "8", + "-target", + "8", + "-Xlint:-options", + src.getFileName().toString())); ProcessBuilder pb = new ProcessBuilder(cmd); pb.directory(src.getParent().toFile()); @@ -219,7 +232,12 @@ private static String locateCheckerJar() { try { String testRoot = System.getProperty("test.root"); if (testRoot != null) { - Path p = Paths.get(testRoot).toAbsolutePath().normalize().getParent().resolve("dist/checker.jar"); + Path p = + Paths.get(testRoot) + .toAbsolutePath() + .normalize() + .getParent() + .resolve("dist/checker.jar"); if (Files.exists(p)) { return p.toString(); } @@ -266,20 +284,25 @@ private static boolean isWindows() { private static final class Result { final List timingsMs; - Result(List t) { this.timingsMs = Collections.unmodifiableList(new ArrayList<>(t)); } - double average() { return timingsMs.stream().mapToLong(Long::longValue).average().orElse(0); } + + Result(List t) { + this.timingsMs = Collections.unmodifiableList(new ArrayList<>(t)); + } + + double average() { + return timingsMs.stream().mapToLong(Long::longValue).average().orElse(0); + } + double median() { if (timingsMs.isEmpty()) return 0; List copy = new ArrayList<>(timingsMs); Collections.sort(copy); int n = copy.size(); if ((n & 1) == 1) { - return copy.get(n/2); + return copy.get(n / 2); } else { - return (copy.get(n/2 - 1) + copy.get(n/2)) / 2.0; + return (copy.get(n / 2 - 1) + copy.get(n / 2)) / 2.0; } } } } - - diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index ce6cf38ed09c..befbe2fb9937 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -79,13 +79,6 @@ public class NullnessNoInitAnnotatedTypeFactory NullnessNoInitTransfer, NullnessNoInitAnalysis> { - /** - * Runtime toggle: skip the {@code hasEffectiveAnnotation(NONNULL)} fast-path. Controlled via - * JVM system property {@code -Dcf.skipNonnullFastPath=true}. - */ - private static final boolean SKIP_NONNULL_FASTPATH = - Boolean.getBoolean("cf.skipNonnullFastPath"); - /** The @{@link NonNull} annotation. */ protected final AnnotationMirror NONNULL = AnnotationBuilder.fromClass(elements, NonNull.class); @@ -741,9 +734,8 @@ public Void visitUnary(UnaryTree tree, AnnotatedTypeMirror type) { // explicit nullable annotations are left intact for the visitor to inspect. @Override public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { - if (!SKIP_NONNULL_FASTPATH && type.hasEffectiveAnnotation(NONNULL)) { - return null; - } + // The constructor return type should already be NONNULL, so in most cases this will do + // nothing. type.addMissingAnnotation(NONNULL); return null; } From d212da1fa78f014fe4d1624b0af9429436779336 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Mon, 6 Oct 2025 21:16:28 -0400 Subject: [PATCH 07/11] bug fix report refine --- .../core/ExternalProcessJavacDriver.java | 31 ++++++- .../harness/core/HarnessIO.java | 88 +++++++++++++------ .../harness/core/JtregDriver.java | 28 +++++- .../checkerframework/harness/cli/Main.java | 88 ++++++++++++++++--- .../nullness/NewAndArrayGenerator.java | 7 +- checker/harness/jtreg/JtregPerfHarness.java | 13 ++- .../NullnessNoInitAnnotatedTypeFactory.java | 22 ++++- 7 files changed, 221 insertions(+), 56 deletions(-) diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java index 1573ac3ea9df..05ff4207c051 100644 --- a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/ExternalProcessJavacDriver.java @@ -88,20 +88,20 @@ public RunResult runOnce(RunSpec spec) throws Exception { // Processor path if (spec.compiler().processorPath() != null && !spec.compiler().processorPath().isEmpty()) { args.add("-processorpath"); - args.add(joinPaths(spec.compiler().processorPath())); + args.add(joinPathsResolved(spec.compiler().processorPath())); } // Classpath if (spec.compiler().classpath() != null && !spec.compiler().classpath().isEmpty()) { args.add("-classpath"); - args.add(joinPaths(spec.compiler().classpath())); + args.add(joinPathsResolved(spec.compiler().classpath())); } else if (spec.compiler().processorPath() != null && !spec.compiler().processorPath().isEmpty()) { // Parity with InProcessJavacDriver: if no explicit classpath is given, fall back to // using the processorPath so that processor dependencies (e.g., annotation types in // checker-qual.jar) are visible to the compiler and processors at runtime. args.add("-classpath"); - args.add(joinPaths(spec.compiler().processorPath())); + args.add(joinPathsResolved(spec.compiler().processorPath())); } if (spec.compiler().processors() != null && !spec.compiler().processors().isEmpty()) { @@ -326,6 +326,31 @@ private static String joinPaths(List paths) { return sb.toString(); } + /** + * Joins paths while resolving relative paths to absolute paths based on current working + * directory. This ensures paths work correctly even when the javac process changes its working + * directory. + */ + private static String joinPathsResolved(List paths) { + String sep = File.pathSeparator; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < paths.size(); i++) { + if (i > 0) sb.append(sep); + Path p = paths.get(i); + // Convert relative paths to absolute paths based on current working directory + // This prevents issues when ProcessBuilder changes the working directory + if (!p.isAbsolute()) { + try { + p = p.toAbsolutePath().normalize(); + } catch (Throwable ignore) { + // Fallback to original path if resolution fails + } + } + sb.append(p.toString()); + } + return sb.toString(); + } + private static String joinComma(List items) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < items.size(); i++) { diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java index ab403693c42c..6ffd4a2f1433 100644 --- a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/HarnessIO.java @@ -234,14 +234,21 @@ public static void writeUnifiedReport( String gpf = (context == null) ? "" : context.getOrDefault("groupsPerFile", ""); String baselineFlags = (context == null) ? "" : context.getOrDefault("baselineFlags", ""); String updateFlags = (context == null) ? "" : context.getOrDefault("updateFlags", ""); - String proc = getStringMeta(baseline, "processor"); - String ppath = extractPathFromFlags(getStringMeta(baseline, "flags"), "-processorpath"); - if (ppath.isEmpty()) - ppath = extractPathFromFlags(getStringMeta(baseline, "flags"), "-processorPath"); + String proc = (context == null) ? "" : context.getOrDefault("processor", ""); + String ppath = (context == null) ? "" : context.getOrDefault("processorPath", ""); + String release = (context == null) ? "" : context.getOrDefault("release", ""); + String jtreg = (context == null) ? "" : context.getOrDefault("jtreg", ""); + String jtregTest = (context == null) ? "" : context.getOrDefault("jtregTest", ""); + if (proc.isEmpty()) proc = getStringMeta(baseline, "processor"); + if (ppath.isEmpty()) { + ppath = extractPathFromFlags(getStringMeta(baseline, "flags"), "-processorpath"); + if (ppath.isEmpty()) + ppath = extractPathFromFlags(getStringMeta(baseline, "flags"), "-processorPath"); + } if (proc.isEmpty()) proc = extractProcessorFromFlags(getStringMeta(baseline, "flags")); - String release = extractReleaseFromFlags(getStringMeta(baseline, "flags")); - String baseCmd = - buildReproCommand( + if (release.isEmpty()) release = extractReleaseFromFlags(getStringMeta(baseline, "flags")); + String unifiedCmd = + buildUnifiedReproCommand( proto, runs, engine, @@ -251,28 +258,12 @@ public static void writeUnifiedReport( proc, ppath, release, + jtreg, + jtregTest, baselineFlags, - false); - String upCmd = - buildReproCommand( - proto, - runs, - engine, - sampleCount, - seed, - gpf, - proc, - ppath, - release, - updateFlags, - true); - out.add("- Baseline:"); + updateFlags); out.add("\n```bash"); - out.add(baseCmd); - out.add("```\n"); - out.add("- Update:"); - out.add("\n```bash"); - out.add(upCmd); + out.add(unifiedCmd); out.add("```\n"); out.add(""); @@ -515,6 +506,8 @@ private static String buildReproCommand( String processor, String processorPath, String release, + String jtreg, + String jtregTest, String variantFlags, boolean isUpdate) { StringBuilder sb = new StringBuilder(); @@ -528,6 +521,8 @@ private static String buildReproCommand( if (!protocol.isEmpty()) sb.append(" --protocol ").append(protocol); if (!runs.isEmpty()) sb.append(" --runs ").append(runs); if (!engine.isEmpty()) sb.append(" --engine ").append(engine); + if (!jtreg.isEmpty()) sb.append(" --jtreg ").append(jtreg); + if (!jtregTest.isEmpty()) sb.append(" --jtreg-test ").append(jtregTest); if (!groupsPerFile.isEmpty()) sb.append(" --extra.groupsPerFile ").append(groupsPerFile); if (variantFlags != null && !variantFlags.isEmpty()) { sb.append(isUpdate ? " --update-flags " : " --baseline-flags "); @@ -537,6 +532,45 @@ private static String buildReproCommand( return sb.toString(); } + /** Build a unified reproduction command that includes both baseline and update flags. */ + private static String buildUnifiedReproCommand( + String protocol, + String runs, + String engine, + String sampleCount, + String seed, + String groupsPerFile, + String processor, + String processorPath, + String release, + String jtreg, + String jtregTest, + String baselineFlags, + String updateFlags) { + StringBuilder sb = new StringBuilder(); + sb.append("../../gradlew :harness-driver-cli:run --no-daemon --console=plain --args=\""); + sb.append("--generator NewAndArray"); + if (!sampleCount.isEmpty()) sb.append(" --sampleCount ").append(sampleCount); + if (!seed.isEmpty()) sb.append(" --seed ").append(seed); + if (!processor.isEmpty()) sb.append(" --processor ").append(processor); + if (!processorPath.isEmpty()) sb.append(" --processor-path ").append(processorPath); + if (!release.isEmpty()) sb.append(" --release ").append(release); + if (!protocol.isEmpty()) sb.append(" --protocol ").append(protocol); + if (!runs.isEmpty()) sb.append(" --runs ").append(runs); + if (!engine.isEmpty()) sb.append(" --engine ").append(engine); + if (!jtreg.isEmpty()) sb.append(" --jtreg ").append(jtreg); + if (!jtregTest.isEmpty()) sb.append(" --jtreg-test ").append(jtregTest); + if (!groupsPerFile.isEmpty()) sb.append(" --extra.groupsPerFile ").append(groupsPerFile); + if (baselineFlags != null && !baselineFlags.isEmpty()) { + sb.append(" --baseline-flags ").append(baselineFlags); + } + if (updateFlags != null && !updateFlags.isEmpty()) { + sb.append(" --update-flags ").append(updateFlags); + } + sb.append("\""); + return sb.toString(); + } + private static java.util.Map countDiagnosticsByKind(Driver.RunResult r) { java.util.Map m = new java.util.HashMap(); if (r != null && r.diagnostics() != null) { diff --git a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java index cb8e32603c6f..87300fc8699d 100644 --- a/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java +++ b/checker/harness/harness-core/src/main/java/org/checkerframework/harness/core/JtregDriver.java @@ -83,9 +83,9 @@ public RunResult runOnce(RunSpec spec) throws Exception { vmOpts.append("-Dharness.srcdir=").append(absSrcDir.toString()); } - // Inject processorpath for JtregPerfHarness + // Inject processorpath for JtregPerfHarness (resolve relative paths to absolute) if (spec.compiler().processorPath() != null && !spec.compiler().processorPath().isEmpty()) { - String cp = joinPaths(spec.compiler().processorPath()); + String cp = joinPathsResolved(spec.compiler().processorPath()); if (vmOpts.length() > 0) vmOpts.append(' '); vmOpts.append("-Dharness.processorpath=").append(cp); } @@ -360,6 +360,30 @@ private static String joinPaths(List paths) { return sb.toString(); } + /** + * Joins paths while resolving relative paths to absolute paths based on current working + * directory. This ensures paths work correctly even when jtreg changes its working directory. + */ + private static String joinPathsResolved(List paths) { + String sep = File.pathSeparator; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < paths.size(); i++) { + if (i > 0) sb.append(sep); + Path p = paths.get(i); + // Convert relative paths to absolute paths based on current working directory + // This prevents issues when jtreg changes the working directory + if (!p.isAbsolute()) { + try { + p = p.toAbsolutePath().normalize(); + } catch (Throwable ignore) { + // Fallback to original path if resolution fails + } + } + sb.append(p.toString()); + } + return sb.toString(); + } + private static File findProjectRoot() { // When running via gradlew from checker/harness, user.dir is typically harness-driver-cli // Walk up from user.dir until we find a directory containing: diff --git a/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java b/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java index 19ce23059553..18dccb5d7c94 100644 --- a/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java +++ b/checker/harness/harness-driver-cli/src/main/java/org/checkerframework/harness/cli/Main.java @@ -28,18 +28,18 @@ public static void main(String[] args) throws Exception { } String generatorName = opts.getOrDefault("--generator", "NewAndArray"); - int sampleCount = Integer.parseInt(opts.getOrDefault("--sampleCount", "100")); - long seed = Long.parseLong(opts.getOrDefault("--seed", "1")); + int sampleCount = Integer.parseInt(opts.getOrDefault("--sampleCount", "100").trim()); + long seed = Long.parseLong(opts.getOrDefault("--seed", "1").trim()); String baselineFlags = opts.getOrDefault("--baseline-flags", "").trim(); String updateFlags = opts.getOrDefault("--update-flags", "").trim(); String processor = opts.getOrDefault( "--processor", "org.checkerframework.checker.nullness.NullnessChecker"); String processorPath = opts.getOrDefault("--processor-path", ""); - String release = opts.getOrDefault("--release", "17"); + String release = opts.getOrDefault("--release", "17").trim(); String protocol = opts.getOrDefault("--protocol", "SINGLE").toUpperCase(Locale.ROOT); // SINGLE|CROSS - int runs = Integer.parseInt(opts.getOrDefault("--runs", "5")); + int runs = Integer.parseInt(opts.getOrDefault("--runs", "5").trim()); if (runs < 1) { System.err.println("WARNING: --runs < 1; falling back to 1."); runs = 1; @@ -72,10 +72,16 @@ public static void main(String[] args) throws Exception { // -Dharness.release= // via flags so the test-side harness can translate it to --release. List processors = java.util.Arrays.asList(processor); - List pp = - processorPath.isEmpty() - ? new ArrayList() - : java.util.Arrays.asList(Paths.get(processorPath)); + List pp = new ArrayList(); + if (!processorPath.isEmpty()) { + // Split by platform-specific path separator (: on Unix, ; on Windows) + String[] pathParts = processorPath.split(java.io.File.pathSeparator); + for (String part : pathParts) { + if (!part.trim().isEmpty()) { + pp.add(Paths.get(part.trim())); + } + } + } List sourceOpts = java.util.Arrays.asList("--release", release, "-proc:only", "-Xlint:-options"); @@ -180,7 +186,7 @@ public static void main(String[] args) throws Exception { if (lastA != null && lastB != null) { Map ctx = new HashMap(); - ctx.put("protocol", protocol + " (jtreg-internal)"); + ctx.put("protocol", protocol); // Keep original protocol for reproduction ctx.put("runs", String.valueOf(runs)); ctx.put("engine", engine); ctx.put("sampleCount", String.valueOf(sampleCount)); @@ -189,6 +195,13 @@ public static void main(String[] args) throws Exception { if (!gpf.isEmpty()) ctx.put("groupsPerFile", gpf); ctx.put("baselineFlags", String.join(" ", baselineFlagList)); ctx.put("updateFlags", String.join(" ", updateFlagList)); + ctx.put("processor", processor); + if (!processorPath.isEmpty()) ctx.put("processorPath", processorPath); + if (!release.isEmpty()) ctx.put("release", release); + if ("jtreg".equalsIgnoreCase(engine)) { + ctx.put("jtreg", jtregBin); + ctx.put("jtregTest", jtregTest); + } HarnessIO.writeUnifiedReport( resultDir.resolve("report.md"), lastA, @@ -333,6 +346,13 @@ public static void main(String[] args) throws Exception { if (!gpf.isEmpty()) ctx.put("groupsPerFile", gpf); ctx.put("baselineFlags", String.join(" ", baselineFlagList)); ctx.put("updateFlags", String.join(" ", updateFlagList)); + ctx.put("processor", processor); + if (!processorPath.isEmpty()) ctx.put("processorPath", processorPath); + if (!release.isEmpty()) ctx.put("release", release); + if ("jtreg".equalsIgnoreCase(engine)) { + ctx.put("jtreg", jtregBin); + ctx.put("jtregTest", jtregTest); + } HarnessIO.writeUnifiedReport( resultDir.resolve("report.md"), reprA, @@ -401,18 +421,48 @@ private static Map extractExtra(Map opts) { private static void printHelp() { System.out.println("Usage: --generator --sampleCount --seed "); System.out.println(" --baseline-flags --update-flags "); - System.out.println( - " --processor --processor-path --release "); + System.out.println(" --processor --processor-path --release "); System.out.println( " --protocol --runs [--engine ]"); + System.out.println(" [--jtreg ] [--jtreg-test ]"); + System.out.println(" [--extra. ]"); + System.out.println(); + System.out.println("Engines:"); + System.out.println( + " inproc : Fast, in-process javac via ToolProvider (development/debugging)"); + System.out.println(" external: Separate javac process, better isolation (CI/testing)"); + System.out.println(" jtreg : Full jtreg test suite, most comprehensive (benchmarking)"); System.out.println(); System.out.println("Examples:"); - System.out.println(" --generator NewAndArray --sampleCount 200 --seed 42 \\"); + System.out.println(" # Basic nullness checker performance test:"); + System.out.println(" --generator NewAndArray --sampleCount 10 --seed 42 \\"); System.out.println( - " --baseline-flags -AfastNewClass=false --update-flags -AfastNewClass=true \\"); + " --processor org.checkerframework.checker.nullness.NullnessChecker \\"); + System.out.println( + " --processor-path ../../../checker/dist/checker.jar:../../../checker-qual/build/libs/checker-qual-*.jar \\"); + System.out.println(" --release 17 --protocol SINGLE --runs 5 --engine external \\"); + System.out.println(" --baseline-flags -J-Dcf.skipNonnullFastPath=false \\"); + System.out.println(" --update-flags -J-Dcf.skipNonnullFastPath=true"); + System.out.println(); + System.out.println(" # JTReg comprehensive test:"); + System.out.println(" --generator NewAndArray --sampleCount 3 --seed 42 \\"); System.out.println( " --processor org.checkerframework.checker.nullness.NullnessChecker \\"); - System.out.println(" --processor-path --release 17"); + System.out.println( + " --processor-path ../../../checker/dist/checker.jar:../../../checker-qual/build/libs/checker-qual-*.jar \\"); + System.out.println(" --release 17 --protocol SINGLE --runs 3 --engine jtreg \\"); + System.out.println( + " --jtreg ../../jtreg/bin --jtreg-test checker/harness/jtreg/JtregPerfHarness.java \\"); + System.out.println( + " --baseline-flags -Dharness.release=17 -Dcf.skipNonnullFastPath=false \\"); + System.out.println(" --update-flags -Dharness.release=17 -Dcf.skipNonnullFastPath=true"); + System.out.println(); + System.out.println("Processor Path:"); + System.out.println( + " Supports both relative and absolute paths. Multiple paths separated by ':'."); + System.out.println(" Relative paths are resolved from current working directory."); + System.out.println( + " Example: ../../../checker/dist/checker.jar:../../../checker-qual/build/libs/checker-qual-*.jar"); System.out.println(); System.out.println("Protocols:"); System.out.println( @@ -422,6 +472,16 @@ private static void printHelp() { System.out.println( " Each variant is warmed once on its first appearance (not timed)."); System.out.println(); + System.out.println("Flags:"); + System.out.println(" Use -J prefix for JVM flags: -J-Dcf.skipNonnullFastPath=false"); + System.out.println( + " Use -D prefix for system properties (jtreg engine): -Dharness.release=17"); + System.out.println(" Engine-specific behavior handled automatically."); + System.out.println(); + System.out.println("Output:"); + System.out.println( + " Generates unified report.md with performance comparison, diagnostics, and reproduction commands."); + System.out.println(); } private static void printErrorsToConsole(Driver.RunResult a, Driver.RunResult b) { diff --git a/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java b/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java index 7b136e5c995d..fb460355c9af 100644 --- a/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java +++ b/checker/harness/harness-generators/src/main/java/org/checkerframework/harness/generators/nullness/NewAndArrayGenerator.java @@ -22,10 +22,10 @@ * *

Behavior: - Emits one public class per file; file count equals {@code max(1, * req.sampleCount())}. - Per-file work is controlled by {@code extra["groupsPerFile"]} (default - * 400); each group emits six allocation expressions. - Output is deterministic for the same seed, + * 20); each group emits six allocation expressions. - Output is deterministic for the same seed, * sample count, and extras. * - *

Optional extras: - groupsPerFile: int, default 400 + *

Optional extras: - groupsPerFile: int, default 20 */ public final class NewAndArrayGenerator implements CodeGenerator { @@ -42,8 +42,7 @@ public GenerationResult generate(GenerationRequest req) throws Exception { final String packageName = DEFAULT_PACKAGE; final String classPrefix = DEFAULT_CLASS_PREFIX; final int groupsPerFile = - parsePositiveInt( - req.extra() == null ? null : req.extra().get("groupsPerFile"), 400); + parsePositiveInt(req.extra() == null ? null : req.extra().get("groupsPerFile"), 20); final Path sourcesDir = req.outputDir().resolve(name() + "-sources"); Files.createDirectories(sourcesDir); diff --git a/checker/harness/jtreg/JtregPerfHarness.java b/checker/harness/jtreg/JtregPerfHarness.java index 246890b01f87..bf760fe041b6 100644 --- a/checker/harness/jtreg/JtregPerfHarness.java +++ b/checker/harness/jtreg/JtregPerfHarness.java @@ -14,6 +14,7 @@ public static void main(String[] args) throws Exception { Path srcDir = Paths.get(System.getProperty("harness.srcdir", ".")).toAbsolutePath().normalize(); boolean skipFastPath = Boolean.getBoolean("cf.skipNonnullFastPath"); + int runs = Integer.parseInt(System.getProperty("perf.runs", "1")); List sources = new ArrayList<>(); Files.walk(srcDir) @@ -24,10 +25,14 @@ public static void main(String[] args) throws Exception { if (sources.isEmpty()) throw new IllegalStateException("No .java files found in harness.srcdir: " + srcDir); - long one = runOnce(sources, skipFastPath); - // Emit a single-sample HARNESS_RESULT for the driver to parse - System.out.println( - "HARNESS_RESULT: median=" + one + ", average=" + one + ", samples=[" + one + "]"); + List timings = new ArrayList<>(); + for (int i = 0; i < runs; i++) { + long time = runOnce(sources, skipFastPath); + timings.add(time); + } + + Result result = new Result(timings); + result.printReport(); } // Execute one compile and return wall time (ms). diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index befbe2fb9937..9adbcb198515 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -79,6 +79,21 @@ public class NullnessNoInitAnnotatedTypeFactory NullnessNoInitTransfer, NullnessNoInitAnalysis> { + /** + * Runtime toggle: skip the {@code hasEffectiveAnnotation(NONNULL)} fast-path. Controlled via + * JVM system property {@code -Dcf.skipNonnullFastPath=true}. + */ + private static final boolean SKIP_NONNULL_FASTPATH = + Boolean.getBoolean("cf.skipNonnullFastPath"); + + private static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + /** The @{@link NonNull} annotation. */ protected final AnnotationMirror NONNULL = AnnotationBuilder.fromClass(elements, NonNull.class); @@ -734,8 +749,11 @@ public Void visitUnary(UnaryTree tree, AnnotatedTypeMirror type) { // explicit nullable annotations are left intact for the visitor to inspect. @Override public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { - // The constructor return type should already be NONNULL, so in most cases this will do - // nothing. + // TEMP: make the delay solely controlled by the system property, independent of + // whether the type already carries @NonNull, so the switch always takes effect. + if (!SKIP_NONNULL_FASTPATH) { + sleep(2); + } type.addMissingAnnotation(NONNULL); return null; } From f5e46bba9cb33058631a01302a522771ca8c8324 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Mon, 6 Oct 2025 21:22:42 -0400 Subject: [PATCH 08/11] remove test code --- .../NullnessNoInitAnnotatedTypeFactory.java | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java index 9adbcb198515..befbe2fb9937 100644 --- a/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java +++ b/checker/src/main/java/org/checkerframework/checker/nullness/NullnessNoInitAnnotatedTypeFactory.java @@ -79,21 +79,6 @@ public class NullnessNoInitAnnotatedTypeFactory NullnessNoInitTransfer, NullnessNoInitAnalysis> { - /** - * Runtime toggle: skip the {@code hasEffectiveAnnotation(NONNULL)} fast-path. Controlled via - * JVM system property {@code -Dcf.skipNonnullFastPath=true}. - */ - private static final boolean SKIP_NONNULL_FASTPATH = - Boolean.getBoolean("cf.skipNonnullFastPath"); - - private static void sleep(long millis) { - try { - Thread.sleep(millis); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - /** The @{@link NonNull} annotation. */ protected final AnnotationMirror NONNULL = AnnotationBuilder.fromClass(elements, NonNull.class); @@ -749,11 +734,8 @@ public Void visitUnary(UnaryTree tree, AnnotatedTypeMirror type) { // explicit nullable annotations are left intact for the visitor to inspect. @Override public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { - // TEMP: make the delay solely controlled by the system property, independent of - // whether the type already carries @NonNull, so the switch always takes effect. - if (!SKIP_NONNULL_FASTPATH) { - sleep(2); - } + // The constructor return type should already be NONNULL, so in most cases this will do + // nothing. type.addMissingAnnotation(NONNULL); return null; } From d596794a938344bea3238d1e17df810037bc26f7 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 7 Oct 2025 10:05:47 -0400 Subject: [PATCH 09/11] add Readme and setup jtreg setup script --- checker/harness/README.md | 297 +++++++++++++++++++++++++++++++++ checker/harness/setup-jtreg.sh | 141 ++++++++++++++++ 2 files changed, 438 insertions(+) create mode 100644 checker/harness/README.md create mode 100755 checker/harness/setup-jtreg.sh diff --git a/checker/harness/README.md b/checker/harness/README.md new file mode 100644 index 000000000000..ca5c7f784ca4 --- /dev/null +++ b/checker/harness/README.md @@ -0,0 +1,297 @@ +## Summary + +Adds a reusable test harness for A/B performance testing of annotation processors. + +Provides pluggable `Driver` and `CodeGenerator` interfaces with three execution modes (in-process, external-process, jtreg), two test protocols (SINGLE, CROSS), and automatic Markdown reports. + +## Motivation + +The Checker Framework's type-checking is performance-sensitive. Optimizations can significantly impact analysis time, but lacked systematic benchmarking infrastructure. + +### Previous limitations + +- **No A/B testing framework**: CF Requires manual timing and spreadsheet analysis +- **Ad-hoc test scripts**: Existing perf tests (e.g., `NewClassPerf.java`) duplicated timing logic and couldn't produce comparable reports +- **Order bias**: Simple sequential runs ("A 10x, then B 10x") suffer from JIT warmup drift + +### What this enables + +- Test fast-path optimizations with one command → get median/average/min/max + % delta +- CROSS protocol (ABBA) mitigates order effects for reliable comparisons +- Unified markdown reports with diagnostics, reproduction commands, and environment snapshot +- Reusable across checkers (Nullness, Initialization, etc.) and workload generators + +Developers can now validate optimizations with `./gradlew :harness-driver-cli:run --args="..."` instead of writing custom test harnesses. + +## Components + +### Core APIs + +- **`CodeGenerator`** - interface: Writes test source files. Includes `NewAndArrayGenerator` for nullness checking tests (deterministic output via seed). +- **`Driver`** - interface: Runs one test cycle (generate sources → compile → collect results). Three implementations: + - `InProcessJavacDriver`: Calls javac via `ToolProvider` (fast, shared JVM) + - `ExternalProcessJavacDriver`: Runs javac as separate process (clean isolation) + - `JtregDriver`: Wraps existing jtreg tests + +### CLI Tool + +**`Main`**: Single command-line entry point with flexible flag handling: + +- **Protocols**: + - **SINGLE**: Run baseline N times → run update N times → compare medians + - **CROSS (ABBA)**: Alternate AB/BA to cancel out JIT/cache order effects + +- **Flag system**: Isolate the exact change being tested + - `--baseline-flags`: Options for baseline configuration + - `--update-flags`: Options for update configuration + + **Example**: Testing a fast-path optimization in the nullness checker + + Baseline code (always adds annotation): + ```java + // NullnessNoInitAnnotatedTypeFactory:750 + public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { + type.addMissingAnnotation(NONNULL); + return null; + } + ``` + + Optimized code (skip if already present): + ```java + private static final boolean SKIP_NONNULL_FASTPATH = + Boolean.getBoolean("cf.skipNonnullFastPath"); + + public Void visitNewClass(NewClassTree tree, AnnotatedTypeMirror type) { + if (!SKIP_NONNULL_FASTPATH && type.hasEffectiveAnnotation(NONNULL)) { + return null; // Early exit saves redundant work + } + type.addMissingAnnotation(NONNULL); + return null; + } + ``` + + Benchmark command: + ```bash + --baseline-flags -Dcf.skipNonnullFastPath=true # Disable fast-path (slow) + --update-flags -Dcf.skipNonnullFastPath=false # Enable fast-path (fast) + ``` + + The harness compiles identical sources under both configs and reports speedup. + +- **Flag forwarding** (handles three execution contexts): + + Annotation processors read runtime configuration from `System.getProperty("cf.*")`. Each engine forwards flags differently based on its JVM boundary: + + - **In-process** (`InProcessJavacDriver`): + - Javac runs in the same JVM as the harness + - `-Dcf.skipNonnullFastPath=false` → `System.setProperty("cf.skipNonnullFastPath", "false")` + - Properties are set before compilation, restored after to prevent cross-run pollution + - Accepts both `-D` and `-J-D` forms (strips `-J` prefix) + + - **External** (`ExternalProcessJavacDriver`): + - Javac runs in a separate JVM subprocess + - `-J-Dcf.skipNonnullFastPath=false` → passed as JVM argument to `ProcessBuilder` + - The `-J` prefix tells javac to forward the flag to its internal JVM + + - **Jtreg** (`JtregDriver`): + - Test runs in jtreg's test VM, which then spawns javac + - `-Dcf.skipNonnullFastPath=false` → forwarded via jtreg's `-vmoption` flag + - Jtreg passes this to the test VM; `JtregPerfHarness` reads it and constructs javac args + +- **Auto-warmup**: Each variant runs once (untimed) before measurements to stabilize JIT + +## How to Use + +Run from the repository root. Build artifacts first (needed by --processor-path): +- checker/dist/checker.jar +- checker-qual/build/libs/checker-qual-*.jar + +```bash +./gradlew :checker:shadowJar -x test --no-daemon --console=plain +./gradlew :checker:assembleForJavac -x test --no-daemon --console=plain +./gradlew :checker-qual:jar -x test --no-daemon --console=plain +``` + +**Run these whenever code under `checker/` or `checker-qual/` changes (or after a clean clone) to keep artifacts up to date.** + +### 1) Navigate to the harness subproject + +The harness is a standalone Gradle subproject with its own `settings.gradle`. You must run commands from within its directory: + +```bash +cd checker/harness +``` + +All subsequent commands assume you are in `checker/harness/`. The harness uses the root project's `gradlew` wrapper via `../../gradlew`. + +### 2) Run harness commands + +**In-process (fast dev loop):** +```bash +../../gradlew :harness-driver-cli:run --no-daemon --console=plain --args="\ + --generator NewAndArray \ + --sampleCount 3 \ + --seed 42 \ + --processor org.checkerframework.checker.nullness.NullnessChecker \ + --processor-path ../../../checker/dist/checker.jar:../../../checker-qual/build/libs/checker-qual-*.jar \ + --release 17 \ + --protocol SINGLE \ + --runs 3 \ + --engine inproc \ + --extra.groupsPerFile 10 \ + --baseline-flags -Dcf.skipNonnullFastPath=false \ + --update-flags -Dcf.skipNonnullFastPath=true" +``` + +**External process (clean isolation):** +```bash +../../gradlew :harness-driver-cli:run --no-daemon --console=plain --args="\ + --generator NewAndArray \ + --sampleCount 3 \ + --seed 42 \ + --processor org.checkerframework.checker.nullness.NullnessChecker \ + --processor-path ../../../checker/dist/checker.jar:../../../checker-qual/build/libs/checker-qual-*.jar \ + --release 17 \ + --protocol SINGLE \ + --runs 3 \ + --engine external \ + --extra.groupsPerFile 10 \ + --baseline-flags -J-Dcf.skipNonnullFastPath=false \ + --update-flags -J-Dcf.skipNonnullFastPath=true" +``` +Note: Use `-J-D` prefix for system properties with external engine. + +**JTReg (Compatibility with Existing Tests)** + +Before running, verify that **JTReg** is available under `checker/harness`: + +```bash +# Check JTReg path and version +ls -la ../../../jtreg/bin/jtreg +../../../jtreg/bin/jtreg -version || ../../../jtreg/bin/jtreg --version +``` + +If JTReg is **not installed**, install it using one of the following methods (run from `checker/harness`): + +**Automated (Recommended)** + +```bash +bash setup-jtreg.sh +``` + +**Manual (Fallback)** + +```bash +cd ../../.. +curl -L -o jtreg.tar.gz https://builds.shipilev.net/jtreg/jtreg-7.5+b01.tar.gz +tar -xzf jtreg.tar.gz && mv jtreg-* jtreg && rm jtreg.tar.gz +chmod +x jtreg/bin/jtreg jtreg/bin/jtdiff +``` + +**Verification Steps (run from `checker/harness`)** + +```bash +# 1) Confirm files exist and are executable +ls -la ../../../jtreg/bin/jtreg +test -x ../../../jtreg/bin/jtreg && echo "ok: executable" + +# 2) Print version to confirm it runs +../../../jtreg/bin/jtreg -version 2>/dev/null || ../../../jtreg/bin/jtreg --version + +# 3) (macOS only) Clear quarantine attributes if needed +xattr -d com.apple.quarantine ../../../jtreg/bin/jtreg 2>/dev/null || true +``` +Once JTReg is installed, run the following command from `checker/harness`: + +```bash +../../gradlew :harness-driver-cli:run --no-daemon --console=plain --args="\ + --generator NewAndArray \ + --sampleCount 3 \ + --seed 42 \ + --processor org.checkerframework.checker.nullness.NullnessChecker \ + --processor-path ../../../checker/dist/checker.jar:../../../checker-qual/build/libs/checker-qual-*.jar \ + --release 17 \ + --protocol SINGLE \ + --runs 3 \ + --engine jtreg \ + --jtreg ../../jtreg/bin \ + --jtreg-test checker/harness/jtreg/JtregPerfHarness.java \ + --extra.groupsPerFile 10 \ + --baseline-flags -Dharness.release=17 -Dcf.skipNonnullFastPath=false \ + --update-flags -Dharness.release=17 -Dcf.skipNonnullFastPath=true" +``` + +Note: Jtreg requires `-Dharness.release` to match `--release`, and uses `-D` (not `-J-D`). + +Output: `checker/harness/result/report.md` (config, env, diagnostics, timing stats, comparison, reproduction commands). + +### 3) What each flag means (quick reference) + +| Flag | Purpose | +| --- | --- | +| `--generator ` | Pick a source generator that creates the .java workload. Today: `NewAndArray`. | +| `--sampleCount N`, `--seed S` | Control how many files to generate and make them reproducible. Same seed → same sources. | +| `--processor`, `--processor-path` | Which annotation processor to run, and where to find it + its deps. Include `checker/dist/checker.jar` AND `checker-qual/build/libs/checker-qual-*.jar`. | +| `--release <8\|11\|17\|21>` | Compile target platform (language level and stdlib). Must match your test plan. | +| `--protocol SINGLE\|CROSS` | SINGLE: run A then B (each N times). CROSS: do AB then BA per iteration to reduce order bias; we report per-iteration averages `Aᵢ`, `Bᵢ`. | +| `--runs N` | Number of timed iterations. SINGLE → N samples for A and N for B. CROSS → N iterations; each iteration yields one `A_i` and one `B_i`. First occurrence of A and B is auto‑warmed (not timed). | +| `--engine inproc\|external\|jtreg` | Where compilation runs: in the same JVM (fast), in a forked `javac` process (clean isolation), or via jtreg (compatible with existing jtreg setups). | +| `--baseline-flags ...` | Flags for the “A” variant (e.g., optimization OFF). Typically controls processor behavior via system properties or `-A...` options. | +| `--update-flags ...` | Flags for the “B” variant (e.g., optimization ON). Keep everything else identical to baseline except what you intentionally change. | + +Extra generator knobs: +- `--extra.groupsPerFile `: increases per‑file work (default 400). More groups → longer compile → less noise. + +Engine‑specific system property forwarding (for `cf.*` and other `-D` keys) + +- inproc: pass `-Dkey=value` (also accepts `-J-Dkey=value`; we strip `-J` automatically). + - value: the exact string assigned to the JVM system property; processors read it via + `System.getProperty("key")` (or `Boolean.getBoolean("key")` for booleans). + - Booleans: `true` / `false` (lowercase recommended). + - Numbers: plain decimal like `123`. + - Strings: use quotes if value contains spaces, e.g. `-Dfoo="a b"`. + - Parsing rule: only the first `=` splits key/value; everything after it is the value. + - You may repeat `-D` to set multiple properties. + +- external: pass `-J-Dkey=value` (because `-J` forwards to the forked JVM that hosts `javac`). + - Same value rules as above; quoting is handled by your shell then by `javac`’s JVM. + +- jtreg: pass `-Dkey=value` (we forward via `-vmoption` to the test JVM). + - Additionally set `-Dharness.release=` so the jtreg test uses the same language + level as `--release`. + +Examples +- Toggle fast‑path: + - baseline: `-Dcf.skipNonnullFastPath=false` + - update: `-Dcf.skipNonnullFastPath=true` +- jtreg with matching release: + - `-Dharness.release=17 -Dcf.skipNonnullFastPath=true` + +### 3) Pick the right engine + +| Engine | Use when | Pros | Notes | +| -------- | --------------------- | ------------------------------------- | ------------------------------------- | +| inproc | Fast local iteration | Lowest overhead; direct diagnostics | Same JVM as CLI; best for development | +| external | Release/CI validation | Real javac process; clean environment | Slightly slower startup | +| jtreg | Reuse jtreg infra | Works with existing jtreg tests | Highest overhead; requires jtreg | + +## Reporting + +- Unified `report.md` including: + - Test configuration: protocol, runs, engine, generator parameters + - Environment: JDK version/home, OS, CPU cores + - Diagnostics: ERROR/WARNING counts with details and match status + - Timing statistics: median, average, min, max, sample count, success rate + - Baseline vs. update comparison with absolute and percentage deltas + - Copy-paste reproduction commands for both variants + - Full compiler flags (source opts, processor path/classpath, processors) + +For jtreg engine, samples count always equals to 1 is correct because Main.java executes only one driver.runOnce() call per variant, while the --runs x (x > 1) parameter is handled internally by JtregPerfHarness which performs multiple compilations and returns aggregated statistics. + +## Future Work + +- Expand test signals: capture GC events, heap usage, and compare across multiple JDKs. +- Broaden coverage: add more generators to exercise diverse code shapes and workloads. +- Improve UX: detect malformed CLI inputs and surface actionable warnings/errors. + +Fixes #1411 \ No newline at end of file diff --git a/checker/harness/setup-jtreg.sh b/checker/harness/setup-jtreg.sh new file mode 100755 index 000000000000..d33ebb33e2df --- /dev/null +++ b/checker/harness/setup-jtreg.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# JTReg Test Harness Setup Script +# +# This script downloads and configures the JTReg test harness required for +# Checker Framework performance benchmarks. JTReg is the official OpenJDK +# regression test harness used for testing Java compiler implementations. +# +# Usage: Run from the checker/harness directory: +# bash setup-jtreg.sh +# +# The script will install JTReg in the appropriate location relative to the +# project structure to ensure compatibility with the performance test suite. + +set -e + +# JTReg version configuration +readonly JTREG_VERSION="7.5" +readonly JTREG_BUILD="b01" +readonly JTREG_DOWNLOAD_URL="https://builds.shipilev.net/jtreg/jtreg-${JTREG_VERSION}+${JTREG_BUILD}.tar.gz" + +# Path resolution: from checker/harness to project structure +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly CF_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" # checker-framework root +readonly INSTALL_PARENT="$(dirname "$CF_ROOT")" # parent of checker-framework +readonly JTREG_INSTALL_PATH="${INSTALL_PARENT}/jtreg" + +echo "==========================================" +echo "JTReg Test Harness Setup v${JTREG_VERSION}" +echo "==========================================" +echo "Project root: ${CF_ROOT}" +echo "Install target: ${JTREG_INSTALL_PATH}" +echo "" + +# Validate project structure +if [[ ! -d "${CF_ROOT}/checker" ]]; then + echo "ERROR: Invalid project structure. Expected checker-framework root at: ${CF_ROOT}" + echo "Please run this script from the checker/harness directory." + exit 1 +fi + +# Check for existing installation +if [[ -f "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then + echo "✓ JTReg is already installed at: ${JTREG_INSTALL_PATH}" + + # Display version information if available + if [[ -f "${JTREG_INSTALL_PATH}/release" ]]; then + echo "" + echo "Current installation:" + cat "${JTREG_INSTALL_PATH}/release" | sed 's/^/ /' + fi + + echo "" + echo "To reinstall, remove the existing directory:" + echo " rm -rf \"${JTREG_INSTALL_PATH}\"" + exit 0 +fi + +# Create secure temporary workspace +readonly TMP_WORKSPACE=$(mktemp -d) +trap "rm -rf '${TMP_WORKSPACE}'" EXIT + +echo "→ Downloading JTReg ${JTREG_VERSION}+${JTREG_BUILD}..." +cd "${TMP_WORKSPACE}" + +# Download with fallback mechanisms +if command -v curl >/dev/null 2>&1; then + if ! curl -L --fail -o jtreg.tar.gz "${JTREG_DOWNLOAD_URL}"; then + echo "ERROR: Download failed using curl" + exit 1 + fi +elif command -v wget >/dev/null 2>&1; then + if ! wget -O jtreg.tar.gz "${JTREG_DOWNLOAD_URL}"; then + echo "ERROR: Download failed using wget" + exit 1 + fi +else + echo "ERROR: Neither curl nor wget is available" + echo "" + echo "Manual installation required:" + echo "1. Download: ${JTREG_DOWNLOAD_URL}" + echo "2. Extract to: ${JTREG_INSTALL_PATH}" + echo "3. Ensure executables have proper permissions" + exit 1 +fi + +echo "→ Extracting JTReg archive..." +if ! tar -xzf jtreg.tar.gz; then + echo "ERROR: Failed to extract JTReg archive" + exit 1 +fi + +# Locate extracted directory (handles version-specific naming) +readonly EXTRACTED_JTREG=$(find . -maxdepth 1 -type d -name "jtreg*" | head -1) +if [[ -z "${EXTRACTED_JTREG}" ]]; then + echo "ERROR: JTReg directory not found after extraction" + exit 1 +fi + +echo "→ Installing JTReg to target location..." +if ! mv "${EXTRACTED_JTREG}" "${JTREG_INSTALL_PATH}"; then + echo "ERROR: Failed to move JTReg to installation directory" + exit 1 +fi + +# Configure executable permissions +echo "→ Configuring JTReg executables..." +chmod +x "${JTREG_INSTALL_PATH}/bin/jtreg" 2>/dev/null || true +chmod +x "${JTREG_INSTALL_PATH}/bin/jtdiff" 2>/dev/null || true + +# Installation verification +if [[ -f "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then + echo "" + echo "==========================================" + echo "✓ JTReg installation completed successfully" + echo "==========================================" + echo "Installation path: ${JTREG_INSTALL_PATH}" + + # Display version information + if [[ -f "${JTREG_INSTALL_PATH}/release" ]]; then + echo "" + echo "JTReg version information:" + cat "${JTREG_INSTALL_PATH}/release" | sed 's/^/ /' + fi + + echo "" + echo "The performance test suite is now ready for execution." + echo "Example usage from checker/harness directory:" + echo "" + echo " ../../gradlew :harness-driver-cli:run --args=\"\\" + echo " --engine jtreg \\" + echo " --jtreg ../../../jtreg/bin \\" + echo " --generator NewAndArray \\" + echo " --processor org.checkerframework.checker.nullness.NullnessChecker \\" + echo " ... \"" + echo "" +else + echo "ERROR: JTReg installation verification failed" + echo "Expected executable not found: ${JTREG_INSTALL_PATH}/bin/jtreg" + exit 1 +fi From d358cc85a535b26aa1cdc6e541532b8dc8f95b81 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 7 Oct 2025 10:24:08 -0400 Subject: [PATCH 10/11] update jtreg setup script --- checker/harness/README.md | 13 +- checker/harness/setup-jtreg.sh | 224 ++++++++++++++++++--------------- 2 files changed, 121 insertions(+), 116 deletions(-) diff --git a/checker/harness/README.md b/checker/harness/README.md index ca5c7f784ca4..d7752cf086fc 100644 --- a/checker/harness/README.md +++ b/checker/harness/README.md @@ -179,15 +179,6 @@ If JTReg is **not installed**, install it using one of the following methods (ru bash setup-jtreg.sh ``` -**Manual (Fallback)** - -```bash -cd ../../.. -curl -L -o jtreg.tar.gz https://builds.shipilev.net/jtreg/jtreg-7.5+b01.tar.gz -tar -xzf jtreg.tar.gz && mv jtreg-* jtreg && rm jtreg.tar.gz -chmod +x jtreg/bin/jtreg jtreg/bin/jtdiff -``` - **Verification Steps (run from `checker/harness`)** ```bash @@ -292,6 +283,4 @@ For jtreg engine, samples count always equals to 1 is correct because Main.java - Expand test signals: capture GC events, heap usage, and compare across multiple JDKs. - Broaden coverage: add more generators to exercise diverse code shapes and workloads. -- Improve UX: detect malformed CLI inputs and surface actionable warnings/errors. - -Fixes #1411 \ No newline at end of file +- Improve UX: detect malformed CLI inputs and surface actionable warnings/errors. \ No newline at end of file diff --git a/checker/harness/setup-jtreg.sh b/checker/harness/setup-jtreg.sh index d33ebb33e2df..a5f807f369de 100755 --- a/checker/harness/setup-jtreg.sh +++ b/checker/harness/setup-jtreg.sh @@ -1,141 +1,157 @@ -#!/bin/bash - +#!/usr/bin/env bash # JTReg Test Harness Setup Script # -# This script downloads and configures the JTReg test harness required for -# Checker Framework performance benchmarks. JTReg is the official OpenJDK -# regression test harness used for testing Java compiler implementations. -# -# Usage: Run from the checker/harness directory: +# Usage (run from checker/harness): # bash setup-jtreg.sh # -# The script will install JTReg in the appropriate location relative to the -# project structure to ensure compatibility with the performance test suite. - -set -e - -# JTReg version configuration -readonly JTREG_VERSION="7.5" -readonly JTREG_BUILD="b01" -readonly JTREG_DOWNLOAD_URL="https://builds.shipilev.net/jtreg/jtreg-${JTREG_VERSION}+${JTREG_BUILD}.tar.gz" - -# Path resolution: from checker/harness to project structure -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly CF_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" # checker-framework root -readonly INSTALL_PARENT="$(dirname "$CF_ROOT")" # parent of checker-framework -readonly JTREG_INSTALL_PATH="${INSTALL_PARENT}/jtreg" +# Notes: +# - Installs JTReg alongside checker-framework at ../jtreg so that +# relative paths like ../../../jtreg/bin/jtreg work in scripts/docs. +# - Downloads the official GitHub release (the tag’s “+” must be URL-encoded). +# - Override defaults via environment variables: JTREG_VERSION, JTREG_BUILD +# e.g., JTREG_VERSION=7.4 JTREG_BUILD=1 bash setup-jtreg.sh + +set -euo pipefail + +# -------- Version / URLs -------- +JTREG_VERSION="${JTREG_VERSION:-7.4}" +JTREG_BUILD="${JTREG_BUILD:-1}" + +# GitHub release tags are like jtreg-7.4+1 (the + must be %2B in URLs). +JTREG_TAG_ENC="jtreg-${JTREG_VERSION}%2B${JTREG_BUILD}" +JTREG_ARCHIVE="jtreg-${JTREG_VERSION}+${JTREG_BUILD}.tar.gz" +JTREG_URL_GH="https://github.com/openjdk/jtreg/releases/download/${JTREG_TAG_ENC}/${JTREG_ARCHIVE}" + +# Optional alternate source (may not exist for all versions; keep for manual use) +# JTREG_URL_ALT="https://builds.shipilev.net/jtreg/jtreg-${JTREG_VERSION}+b${JTREG_BUILD}.tar.gz" + +# -------- Path resolution (run from checker/harness) -------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Repository root (contains checker/ etc.) +CF_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" +# Install to the parent of checker-framework: /jtreg +INSTALL_PARENT="$(dirname "$CF_ROOT")" +JTREG_INSTALL_PATH="${INSTALL_PARENT}/jtreg" echo "==========================================" -echo "JTReg Test Harness Setup v${JTREG_VERSION}" +echo "JTReg Test Harness Setup v${JTREG_VERSION}+${JTREG_BUILD}" echo "==========================================" echo "Project root: ${CF_ROOT}" echo "Install target: ${JTREG_INSTALL_PATH}" echo "" -# Validate project structure +# -------- Sanity check: repository layout -------- if [[ ! -d "${CF_ROOT}/checker" ]]; then - echo "ERROR: Invalid project structure. Expected checker-framework root at: ${CF_ROOT}" - echo "Please run this script from the checker/harness directory." - exit 1 + echo "ERROR: Invalid repository layout." + echo "Run this script from checker/harness within checker-framework." + exit 1 fi -# Check for existing installation -if [[ -f "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then - echo "✓ JTReg is already installed at: ${JTREG_INSTALL_PATH}" - - # Display version information if available - if [[ -f "${JTREG_INSTALL_PATH}/release" ]]; then - echo "" - echo "Current installation:" - cat "${JTREG_INSTALL_PATH}/release" | sed 's/^/ /' - fi - - echo "" - echo "To reinstall, remove the existing directory:" - echo " rm -rf \"${JTREG_INSTALL_PATH}\"" - exit 0 +# -------- Existing installation -------- +if [[ -x "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then + echo "✓ JTReg already installed at: ${JTREG_INSTALL_PATH}" + echo "" + if [[ -f "${JTREG_INSTALL_PATH}/release" ]]; then + echo "Current installation info:" + sed 's/^/ /' "${JTREG_INSTALL_PATH}/release" || true + fi + echo "" + echo "To reinstall, remove the existing directory first:" + echo " rm -rf \"${JTREG_INSTALL_PATH}\"" + exit 0 fi -# Create secure temporary workspace -readonly TMP_WORKSPACE=$(mktemp -d) -trap "rm -rf '${TMP_WORKSPACE}'" EXIT +# -------- Temp workspace -------- +TMP_WORKSPACE="$(mktemp -d)" +trap 'rm -rf "${TMP_WORKSPACE}"' EXIT -echo "→ Downloading JTReg ${JTREG_VERSION}+${JTREG_BUILD}..." +echo "→ Downloading JTReg ${JTREG_VERSION}+${JTREG_BUILD} ..." +echo " URL: ${JTREG_URL_GH}" cd "${TMP_WORKSPACE}" -# Download with fallback mechanisms +download_ok=false if command -v curl >/dev/null 2>&1; then - if ! curl -L --fail -o jtreg.tar.gz "${JTREG_DOWNLOAD_URL}"; then - echo "ERROR: Download failed using curl" - exit 1 - fi + # -L follow redirects; --fail causes 4xx/5xx to return nonzero exit + if curl -L --fail -o jtreg.tar.gz "${JTREG_URL_GH}"; then + download_ok=true + fi elif command -v wget >/dev/null 2>&1; then - if ! wget -O jtreg.tar.gz "${JTREG_DOWNLOAD_URL}"; then - echo "ERROR: Download failed using wget" - exit 1 - fi + if wget -O jtreg.tar.gz "${JTREG_URL_GH}"; then + download_ok=true + fi else - echo "ERROR: Neither curl nor wget is available" - echo "" - echo "Manual installation required:" - echo "1. Download: ${JTREG_DOWNLOAD_URL}" - echo "2. Extract to: ${JTREG_INSTALL_PATH}" - echo "3. Ensure executables have proper permissions" - exit 1 + echo "ERROR: Neither curl nor wget is available." + echo "Manual steps:" + echo " 1) Download: ${JTREG_URL_GH}" + echo " 2) Extract to: ${JTREG_INSTALL_PATH}" + exit 1 fi -echo "→ Extracting JTReg archive..." -if ! tar -xzf jtreg.tar.gz; then - echo "ERROR: Failed to extract JTReg archive" - exit 1 +if [[ "${download_ok}" != true ]]; then + echo "ERROR: Download from GitHub failed." + # To enable the alternate source, uncomment below and set JTREG_URL_ALT as needed. + # echo "Trying alternate URL: ${JTREG_URL_ALT}" + # if curl -L --fail -o jtreg.tar.gz "${JTREG_URL_ALT}"; then + # download_ok=true + # fi + # if [[ "${download_ok}" != true ]]; then + # echo "ERROR: Alternate download also failed." + # exit 1 + # fi + exit 1 fi -# Locate extracted directory (handles version-specific naming) -readonly EXTRACTED_JTREG=$(find . -maxdepth 1 -type d -name "jtreg*" | head -1) +echo "→ Extracting archive ..." +tar -xzf jtreg.tar.gz + +# Robust directory match (exclude .) +EXTRACTED_JTREG="$(find . -mindepth 1 -maxdepth 1 -type d -name "jtreg*" | head -1 || true)" if [[ -z "${EXTRACTED_JTREG}" ]]; then - echo "ERROR: JTReg directory not found after extraction" - exit 1 + echo "ERROR: Extracted JTReg directory not found." + exit 1 fi -echo "→ Installing JTReg to target location..." -if ! mv "${EXTRACTED_JTREG}" "${JTREG_INSTALL_PATH}"; then - echo "ERROR: Failed to move JTReg to installation directory" - exit 1 -fi +echo "→ Installing to ${JTREG_INSTALL_PATH} ..." +mkdir -p "$(dirname "${JTREG_INSTALL_PATH}")" +mv "${EXTRACTED_JTREG}" "${JTREG_INSTALL_PATH}" -# Configure executable permissions -echo "→ Configuring JTReg executables..." +echo "→ Setting executable permissions ..." chmod +x "${JTREG_INSTALL_PATH}/bin/jtreg" 2>/dev/null || true chmod +x "${JTREG_INSTALL_PATH}/bin/jtdiff" 2>/dev/null || true -# Installation verification -if [[ -f "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then - echo "" - echo "==========================================" - echo "✓ JTReg installation completed successfully" - echo "==========================================" - echo "Installation path: ${JTREG_INSTALL_PATH}" - - # Display version information - if [[ -f "${JTREG_INSTALL_PATH}/release" ]]; then - echo "" - echo "JTReg version information:" - cat "${JTREG_INSTALL_PATH}/release" | sed 's/^/ /' - fi +# Optional: clear macOS Gatekeeper quarantine flags +if [[ "$(uname -s)" == "Darwin" ]]; then + xattr -dr com.apple.quarantine "${JTREG_INSTALL_PATH}" 2>/dev/null || true +fi +# -------- Verify -------- +if [[ -x "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then + echo "" + echo "==========================================" + echo "✓ JTReg installation completed" + echo "==========================================" + echo "Installation path: ${JTREG_INSTALL_PATH}" + echo "" + if [[ -f "${JTREG_INSTALL_PATH}/release" ]]; then + echo "JTReg release file:" + sed 's/^/ /' "${JTREG_INSTALL_PATH}/release" || true echo "" - echo "The performance test suite is now ready for execution." - echo "Example usage from checker/harness directory:" - echo "" - echo " ../../gradlew :harness-driver-cli:run --args=\"\\" - echo " --engine jtreg \\" - echo " --jtreg ../../../jtreg/bin \\" - echo " --generator NewAndArray \\" - echo " --processor org.checkerframework.checker.nullness.NullnessChecker \\" - echo " ... \"" - echo "" + fi + echo "Version check:" + "${JTREG_INSTALL_PATH}/bin/jtreg" -version 2>/dev/null || "${JTREG_INSTALL_PATH}/bin/jtreg" --version || true + echo "" + echo "Example (run from checker/harness):" + cat <<'EOF' + ../../gradlew :harness-driver-cli:run --no-daemon --console=plain --args="\ + --engine jtreg \ + --jtreg ../../jtreg/bin \ + --jtreg-test checker/harness/jtreg/JtregPerfHarness.java \ + --generator NewAndArray \ + --processor org.checkerframework.checker.nullness.NullnessChecker \ + ..." +EOF else - echo "ERROR: JTReg installation verification failed" - echo "Expected executable not found: ${JTREG_INSTALL_PATH}/bin/jtreg" - exit 1 + echo "ERROR: JTReg installation verification failed." + echo "Missing executable: ${JTREG_INSTALL_PATH}/bin/jtreg" + exit 1 fi From 898aec1969bd6b416fc4174f4a4aba07ab5fb1cd Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 7 Oct 2025 10:36:55 -0400 Subject: [PATCH 11/11] update jtreg setup script --- checker/harness/setup-jtreg.sh | 76 ++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/checker/harness/setup-jtreg.sh b/checker/harness/setup-jtreg.sh index a5f807f369de..45a992111531 100755 --- a/checker/harness/setup-jtreg.sh +++ b/checker/harness/setup-jtreg.sh @@ -7,7 +7,7 @@ # Notes: # - Installs JTReg alongside checker-framework at ../jtreg so that # relative paths like ../../../jtreg/bin/jtreg work in scripts/docs. -# - Downloads the official GitHub release (the tag’s “+” must be URL-encoded). +# - Downloads prebuilt archives from Shipilev builds. # - Override defaults via environment variables: JTREG_VERSION, JTREG_BUILD # e.g., JTREG_VERSION=7.4 JTREG_BUILD=1 bash setup-jtreg.sh @@ -17,13 +17,11 @@ set -euo pipefail JTREG_VERSION="${JTREG_VERSION:-7.4}" JTREG_BUILD="${JTREG_BUILD:-1}" -# GitHub release tags are like jtreg-7.4+1 (the + must be %2B in URLs). -JTREG_TAG_ENC="jtreg-${JTREG_VERSION}%2B${JTREG_BUILD}" -JTREG_ARCHIVE="jtreg-${JTREG_VERSION}+${JTREG_BUILD}.tar.gz" -JTREG_URL_GH="https://github.com/openjdk/jtreg/releases/download/${JTREG_TAG_ENC}/${JTREG_ARCHIVE}" - -# Optional alternate source (may not exist for all versions; keep for manual use) -# JTREG_URL_ALT="https://builds.shipilev.net/jtreg/jtreg-${JTREG_VERSION}+b${JTREG_BUILD}.tar.gz" +# Shipilev provides versioned zips like jtreg-7.4+1.zip +JTREG_ARCHIVE="jtreg-${JTREG_VERSION}+${JTREG_BUILD}.zip" +JTREG_URL_PRIMARY="https://builds.shipilev.net/jtreg/${JTREG_ARCHIVE}" +# Fallback (rolling zip; may change over time—use only if primary fails) +JTREG_URL_FALLBACK="https://builds.shipilev.net/jtreg/jtreg.zip" # -------- Path resolution (run from checker/harness) -------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -61,48 +59,62 @@ if [[ -x "${JTREG_INSTALL_PATH}/bin/jtreg" ]]; then exit 0 fi +# -------- Tooling checks -------- +need_cmd() { + command -v "$1" >/dev/null 2>&1 +} +if ! need_cmd unzip; then + echo "ERROR: unzip is required but not found on PATH." + echo "Install unzip and retry." + exit 1 +fi +if ! need_cmd curl && ! need_cmd wget; then + echo "ERROR: Neither curl nor wget is available." + echo "Manual steps:" + echo " 1) Download: ${JTREG_URL_PRIMARY} (or ${JTREG_URL_FALLBACK})" + echo " 2) Extract to: ${JTREG_INSTALL_PATH}" + exit 1 +fi + # -------- Temp workspace -------- TMP_WORKSPACE="$(mktemp -d)" trap 'rm -rf "${TMP_WORKSPACE}"' EXIT echo "→ Downloading JTReg ${JTREG_VERSION}+${JTREG_BUILD} ..." -echo " URL: ${JTREG_URL_GH}" +echo " URL: ${JTREG_URL_PRIMARY}" cd "${TMP_WORKSPACE}" download_ok=false -if command -v curl >/dev/null 2>&1; then - # -L follow redirects; --fail causes 4xx/5xx to return nonzero exit - if curl -L --fail -o jtreg.tar.gz "${JTREG_URL_GH}"; then - download_ok=true +outfile="jtreg.zip" + +fetch() { + local url="$1" + if need_cmd curl; then + curl -L --fail -o "${outfile}" "${url}" + else + wget -O "${outfile}" "${url}" fi -elif command -v wget >/dev/null 2>&1; then - if wget -O jtreg.tar.gz "${JTREG_URL_GH}"; then +} + +if fetch "${JTREG_URL_PRIMARY}"; then + download_ok=true +else + echo "WARN: Primary download failed; trying fallback ..." + if fetch "${JTREG_URL_FALLBACK}"; then download_ok=true fi -else - echo "ERROR: Neither curl nor wget is available." - echo "Manual steps:" - echo " 1) Download: ${JTREG_URL_GH}" - echo " 2) Extract to: ${JTREG_INSTALL_PATH}" - exit 1 fi if [[ "${download_ok}" != true ]]; then - echo "ERROR: Download from GitHub failed." - # To enable the alternate source, uncomment below and set JTREG_URL_ALT as needed. - # echo "Trying alternate URL: ${JTREG_URL_ALT}" - # if curl -L --fail -o jtreg.tar.gz "${JTREG_URL_ALT}"; then - # download_ok=true - # fi - # if [[ "${download_ok}" != true ]]; then - # echo "ERROR: Alternate download also failed." - # exit 1 - # fi + echo "ERROR: Could not download JTReg (all URLs failed)." + echo "Manual steps:" + echo " 1) Download: ${JTREG_URL_PRIMARY} (or ${JTREG_URL_FALLBACK})" + echo " 2) Extract to: ${JTREG_INSTALL_PATH}" exit 1 fi echo "→ Extracting archive ..." -tar -xzf jtreg.tar.gz +unzip -q "${outfile}" # Robust directory match (exclude .) EXTRACTED_JTREG="$(find . -mindepth 1 -maxdepth 1 -type d -name "jtreg*" | head -1 || true)"