From 523229a605e245ad9e8b11dea03b15b8a64f9d70 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:33:19 +0100 Subject: [PATCH 01/10] Make the compiler's output depend only on its input Compiling the same source twice gave different module bytes, for two reasons: - Each module was stamped with the wall-clock time of its compile. Nothing reads that timestamp, but it made every compile's output new. The compiler now stamps the epoch, as FileStructure already allows for reproducible output. - A variable named _ got a name numbered by a counter shared by every compile in the JVM, so a compile's names depended on what else the JVM had compiled before it. A method now numbers within itself, as a property outside a method already did. The timestamp is what made the build's compiles cascade: any recompile of a module, even for a comment, gave new bytes, so every module compiled against it was out of date, and so were their tests and the javatools tests that use the XDK. Now a change that does not change a module's bytes stops at that module. A trailing comment in lib_ecstasy costs one compile (7s) instead of a rebuild of all 22 XDK modules and their tests (96s). The counter matters for compiles that share a JVM: the embedding API, the language server and compiles of several modules in one process. --- .../ast/MethodDeclarationStatement.java | 13 ++- .../ast/TypeCompositionStatement.java | 2 +- .../org/xvm/api/DeterministicOutputTest.java | 109 ++++++++++++++++++ .../ast/CompilerCounterAtomicTest.java | 25 ++-- 4 files changed, 132 insertions(+), 17 deletions(-) create mode 100644 javatools/src/test/java/org/xvm/api/DeterministicOutputTest.java diff --git a/javatools/src/main/java/org/xvm/compiler/ast/MethodDeclarationStatement.java b/javatools/src/main/java/org/xvm/compiler/ast/MethodDeclarationStatement.java index a5a424319d..1cd55c5f32 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/MethodDeclarationStatement.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/MethodDeclarationStatement.java @@ -6,7 +6,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.xvm.asm.Annotation; @@ -284,9 +283,14 @@ protected AstNode getCodeContainer() { return null; } + /** + * Numbers within this method, so that the names they make, such as those of variables named + * {@code _}, depend only on the method's own source: a counter shared by every compile in the JVM + * would make a module's bytes depend on what else the JVM has compiled. + */ @Override protected int getCodeContainerCounter() { - return COUNTER.getAndIncrement(); + return m_counter++; } // ----- compile phases ------------------------------------------------------------------------ @@ -1257,7 +1261,10 @@ public String getDumpDesc() { // complementary statement for the constructor points to the finalizer and vice versa private transient MethodDeclarationStatement m_stmtComplement; - private static final AtomicInteger COUNTER = new AtomicInteger(); + /** + * The next number of {@link #getCodeContainerCounter()}. + */ + private transient int m_counter; private static final Field[] CHILD_FIELDS = fieldsForNames(MethodDeclarationStatement.class, "condition", "annotations", "typeParams", "returns", "redundant", "params", "body"); diff --git a/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java b/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java index 51168b30b2..d7322af23c 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java @@ -431,7 +431,7 @@ protected void registerStructures(StageMgr mgr, ErrorListener errs) { } // create the FileStructure and "this" ModuleStructure - FileStructure struct = new FileStructure(sModule); + FileStructure struct = new FileStructure(sModule, null); component = struct.getModule(); pool = struct.getConstantPool(); diff --git a/javatools/src/test/java/org/xvm/api/DeterministicOutputTest.java b/javatools/src/test/java/org/xvm/api/DeterministicOutputTest.java new file mode 100644 index 0000000000..19e05ce174 --- /dev/null +++ b/javatools/src/test/java/org/xvm/api/DeterministicOutputTest.java @@ -0,0 +1,109 @@ +package org.xvm.api; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; + +import java.net.URISyntaxException; + +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.xvm.asm.DirRepository; +import org.xvm.asm.ErrorList; +import org.xvm.asm.ModuleStructure; + +/** + * The compiler's output depends only on its input: compiling the same source twice in one JVM gives the + * same module bytes. A variable named {@code _} gets a name of its own, numbered within its method, and a + * number shared by every compile in the JVM would make each compile's bytes differ, so that every module + * compiled after it would be out of date and miss the build cache. + */ +class DeterministicOutputTest { + private static final String SOURCE = """ + module Deterministic { + void run() { + try { + check(1); + } catch (Exception _) {} + if (Int count := find()) { + check(count); + } + } + + void other() { + try { + check(2); + } catch (IllegalState _) {} + } + + conditional Int find() { + return True, 3; + } + + void check(Int n) {} + } + """; + + @TempDir + Path input; + + @Test + void compilingTheSameSourceTwiceGivesTheSameBytes() throws URISyntaxException { + var modules = DeterministicOutputTest.class.getResource("/xdk-modules"); + assertNotNull(modules, "the build puts the compiled XDK modules in the test resources"); + var support = new EmbeddingSupport().configure(new DirRepository(new File(modules.toURI()), true), null); + + byte[] first = bytes(compile(support)); + byte[] second = bytes(compile(support)); + + assertArrayEquals(first, second, () -> "the same source compiled to different module bytes: " + + firstDifference(first, second)); + } + + private ModuleStructure compile(EmbeddingSupport support) { + var errs = new ErrorList(25); + var module = support.compile(SOURCE, new DirRepository(input.toFile(), true), errs); + assertNotNull(module, () -> "compile failed: " + errs.getErrors()); + assertFalse(errs.hasSeriousErrors(), () -> "compile failed: " + errs.getErrors()); + return module; + } + + /** + * The printable text around the first byte where the two differ, from each, such as the name that + * differs. + */ + private static String firstDifference(byte[] first, byte[] second) { + int at = 0; + while (at < Math.min(first.length, second.length) && first[at] == second[at]) { + ++at; + } + return "at " + at + ", \"" + text(first, at) + "\" against \"" + text(second, at) + "\""; + } + + private static String text(byte[] bytes, int at) { + var text = new StringBuilder(); + for (int i = Math.max(0, at - 24), end = Math.min(bytes.length, at + 24); i < end; ++i) { + char ch = (char) (bytes[i] & 0xFF); + text.append(ch >= ' ' && ch < 0x7F ? ch : '.'); + } + return text.toString(); + } + + private static byte[] bytes(ModuleStructure module) { + var out = new ByteArrayOutputStream(); + try { + module.getFileStructure().writeTo(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return out.toByteArray(); + } +} diff --git a/javatools/src/test/java/org/xvm/compiler/ast/CompilerCounterAtomicTest.java b/javatools/src/test/java/org/xvm/compiler/ast/CompilerCounterAtomicTest.java index ac445b9515..5d7d858c03 100644 --- a/javatools/src/test/java/org/xvm/compiler/ast/CompilerCounterAtomicTest.java +++ b/javatools/src/test/java/org/xvm/compiler/ast/CompilerCounterAtomicTest.java @@ -30,14 +30,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Guards compiler-wide counters that are shared when multiple compilations run in one JVM. + * Guards compiler-wide counters that are shared when multiple compilations run in one JVM, and the + * counter of a method, which counts within that method only. */ public class CompilerCounterAtomicTest { @Test public void staticCompilerCountersAreAtomic() throws Exception { assertAtomicCounter(ConditionalStatement.class, "LABEL_COUNTER"); - assertAtomicCounter(MethodDeclarationStatement.class, "COUNTER"); assertAtomicCounter(ElseExpression.class, "COUNTER"); assertAtomicCounter(ElvisExpression.class, "COUNTER"); } @@ -93,17 +93,16 @@ public void conditionalStatementLabelIdsAreUniqueUnderParallelAllocation() } @Test - public void methodDeclarationCodeContainerCountersRemainSequential() - throws Exception { - withCounter(MethodDeclarationStatement.class, "COUNTER", 200, () -> { - TestMethodDeclarationStatement stmt1 = new TestMethodDeclarationStatement("method1"); - TestMethodDeclarationStatement stmt2 = new TestMethodDeclarationStatement("method2"); - - assertEquals(200, stmt1.codeContainerCounter()); - assertEquals(201, stmt2.codeContainerCounter()); - assertEquals(202, stmt1.codeContainerCounter()); - assertEquals(203, getAtomicCounter(MethodDeclarationStatement.class, "COUNTER").get()); - }); + public void methodDeclarationsCountWithinEachMethod() { + TestMethodDeclarationStatement stmt1 = new TestMethodDeclarationStatement("method1"); + TestMethodDeclarationStatement stmt2 = new TestMethodDeclarationStatement("method2"); + + // each method starts at 0, whatever other methods have counted, so that a method's names, + // such as those of its variables named _, depend only on its own source + assertEquals(0, stmt1.codeContainerCounter()); + assertEquals(0, stmt2.codeContainerCounter()); + assertEquals(1, stmt1.codeContainerCounter()); + assertEquals(1, stmt2.codeContainerCounter()); } @Test From ca56fcc3f34a47d730ae7c3d5b0af0ebe8da3a63 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:52:45 +0100 Subject: [PATCH 02/10] Stop a commit that changes no source from rebuilding the XDK javatools' build-info.properties records the commit and branch it was built from, for xtc --version. That file is in the javatools jar, which every task that compiles or runs modules takes as an input, so each commit, branch switch or rebase made every XDK compile, every XDK test and the javatools tests out of date, and no build cache entry from another commit could be used: 106s for a build after a commit that changed nothing. Nothing that compiles or runs modules reads the commit or the branch, and the version test compares the output with the same file. So the XTC plugin's tasks now take the javatools jar as a classpath, which it is, and the plugin's projects and javatools compare classpaths without those two properties, with Gradle's runtime classpath normalization. A different commit now rebuilds only the jar and the distribution, which should carry it: 11s. The version numbers in the same file are still compared, and a change to any class or other resource in the jar still reruns everything. --- javatools/build.gradle.kts | 12 +++++++++++ .../xtclang/plugin/XtcProjectDelegate.java | 20 +++++++++++++++++++ .../xtclang/plugin/tasks/XtcCompileTask.java | 7 +++---- .../xtclang/plugin/tasks/XtcLauncherTask.java | 12 +++++++---- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/javatools/build.gradle.kts b/javatools/build.gradle.kts index 36368859a9..c795cbf6bc 100644 --- a/javatools/build.gradle.kts +++ b/javatools/build.gradle.kts @@ -234,6 +234,18 @@ val jar = tasks.named("jar") { } } +// The commit and branch in build-info.properties only feed `xtc --version`, and its test compares the output with +// the same file, so the tests do not depend on them: compare the classpath without them, so that a commit that +// changes no source does not rerun the tests. The XTC plugin does the same for the tasks that compile and run modules. +normalization { + runtimeClasspath { + properties("build-info.properties") { + ignoreProperty("git.commit") + ignoreProperty("git.status") + } + } +} + // ModuleInfoTest parses real module sources of sibling XDK projects. Declare them as inputs so // an edit to those sources re-runs the tests instead of reusing an up-to-date or cached result. tasks.test { diff --git a/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java b/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java index 4b70f89b63..ac3f66bfc5 100644 --- a/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java +++ b/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java @@ -238,6 +238,11 @@ protected static E ensureExtension(final Project project, final String name, */ private static final Set HIDDEN_TASK_NAMES = Set.of("jar", "classes"); + /** + * The javatools jar's build information, which also records the commit and branch it was built from. + */ + private static final String JAVATOOLS_BUILD_INFO = "build-info.properties"; + private static boolean isHiddenTaskName(final String taskName) { return HIDDEN_TASK_NAMES.contains(taskName) || taskName.endsWith("java"); } @@ -300,11 +305,26 @@ private void createXtcComponents(final Project project) { project.getComponents().add(component); } + /** + * Compare the javatools jar without the commit and branch that its build-info.properties records + * for {@code xtc --version}. They change with every commit and branch switch, while nothing that + * compiles or runs modules reads them: tracked, they made every XTC task out of date and every + * build cache entry unusable after a commit that changed no source. The tasks declare the jar as + * a classpath, which this runtime classpath normalization applies to. + */ + private static void ignoreJavaToolsCommit(final Project project) { + project.getNormalization().getRuntimeClasspath().properties(JAVATOOLS_BUILD_INFO, properties -> { + properties.ignoreProperty("git.commit"); + properties.ignoreProperty("git.status"); + }); + } + /** * This method, "apply", is a delegate target call for an XTC project delegating plugin */ public void apply(final Project project) { applyJavaPlugin(tasks, project); + ignoreJavaToolsCommit(project); createXtcComponents(project); // Add xtc extension. diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java index 412f7e070c..0c279573f6 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java @@ -24,6 +24,7 @@ import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; @@ -253,8 +254,7 @@ public void outputFilename(final Provider<@NotNull String> from, final Provider< @Override @Optional - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) + @Classpath public FileCollection getInputXtcJavaToolsConfig() { return shouldTrackLauncherRuntimeInputs() ? super.getInputXtcJavaToolsConfig() @@ -263,8 +263,7 @@ public FileCollection getInputXtcJavaToolsConfig() { @Override @Optional - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) + @Classpath public FileCollection getInputLauncherRuntimeCandidates() { return shouldTrackLauncherRuntimeInputs() ? super.getInputLauncherRuntimeCandidates() diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java index 93dcbc2362..288247e391 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcLauncherTask.java @@ -10,6 +10,7 @@ import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; @@ -186,14 +187,17 @@ protected E getExtension() { return ext; } - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) + /** + * The javatools jar, as a classpath: Gradle then compares its contents through the project's + * runtime classpath normalization, which ignores the commit and branch it records for + * {@code xtc --version} (see {@link XtcProjectDelegate}). + */ + @Classpath public FileCollection getInputXtcJavaToolsConfig() { return javaToolsConfig.get(); } - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) + @Classpath public FileCollection getInputLauncherRuntimeCandidates() { return objects.fileCollection() .from(javaToolsConfig.get()) From ef5b37f60e85db066ce4fbdfaf5228ad955c48b9 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:20:00 +0100 Subject: [PATCH 03/10] Stop a commit from rerunning the run and test tasks of XDK consumers XtcRunTask took the unpacked XDK directory as an input of its own, besides the XDK's modules and its jars, which XtcLauncherTask already takes. The directory holds only those, so it tracked nothing more, but it compared javatools.jar as a plain file, with the commit and branch that its build-info.properties records, which the classpath comparison ignores. So every commit reran the tests of a project that consumes the XDK, such as manualTests (9s). The task still depends on the task that unpacks the XDK, which it names explicitly. --- .../main/java/org/xtclang/plugin/tasks/XtcRunTask.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java index 43917e77c5..c026b5fe39 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcRunTask.java @@ -24,15 +24,12 @@ import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; -import org.gradle.api.file.Directory; import org.gradle.api.file.FileCollection; import org.gradle.api.logging.LogLevel; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; -import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Nested; @@ -215,13 +212,6 @@ private String resolveJavaExecutable() { return executable; } - // XTC modules needed to resolve module path (the contents of the XDK required to build and run this project) - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - Provider<@NotNull Directory> getInputXdkModules() { - return xdkContentsDir; - } - // XTC modules needed to resolve module path (the ones in the output of the project source set, that the compileXtc tasks create) @Optional @InputFiles // should really be enough with an "input directories" but that doesn't exist in Gradle. From 4cab09cf4186adaa7d5e2dbb40a5394b51a5f764 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:20:00 +0100 Subject: [PATCH 04/10] Run ModuleInfoTest on its own, with the XDK sources it reads The javatools tests took the sources of lib_net, lib_json and lib_web as inputs, because ModuleInfoTest parses them, so any edit to those libraries, even a comment, reran all of the tests (62s) for the sake of one class that takes 0.2s. ModuleInfoTest now runs in a task of its own, moduleInfoTest, which check runs, with those sources as its inputs. A comment in lib_json now costs a build of 8s instead of 70s. --- javatools/build.gradle.kts | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/javatools/build.gradle.kts b/javatools/build.gradle.kts index c795cbf6bc..a12dfe6915 100644 --- a/javatools/build.gradle.kts +++ b/javatools/build.gradle.kts @@ -246,14 +246,32 @@ normalization { } } -// ModuleInfoTest parses real module sources of sibling XDK projects. Declare them as inputs so -// an edit to those sources re-runs the tests instead of reusing an up-to-date or cached result. -tasks.test { +// ModuleInfoTest parses real module sources of sibling XDK projects, so it runs on its own, with those sources as +// inputs: an edit to them reruns it instead of reusing an up-to-date or cached result, without rerunning the other +// tests, which do not read them. +val moduleInfoTest = tasks.register("moduleInfoTest") { + description = "Run ModuleInfoTest, which parses the module sources of lib_net, lib_json and lib_web" + group = VERIFICATION_GROUP + + val testSourceSet = sourceSets.test.get() + testClassesDirs = testSourceSet.output.classesDirs + classpath = testSourceSet.runtimeClasspath + useJUnitPlatform() + + include("**/ModuleInfoTest.class") inputs.files(listOf("lib_net", "lib_json", "lib_web").map { File(compositeRoot, "$it/src/main/x") }) .withPropertyName("siblingModuleSources") .withPathSensitivity(PathSensitivity.RELATIVE) } +tasks.test { + exclude("**/ModuleInfoTest.class") +} + +tasks.check { + dependsOn(moduleInfoTest) +} + val versionOutputTest = tasks.register("versionOutputTest") { description = "Run tests that verify version output contains git and API information" group = VERIFICATION_GROUP From ea03507aa9105375a626b82ac01b1d45fe7e0ca2 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:22:18 +0100 Subject: [PATCH 05/10] Define the API of a module: what a module that imports it compiles against ModuleApi writes a module file without its methods' code, the source text of their bodies, its timestamp and its source directory, followed by whether each method is abstract, native or has code. The compiler reads nothing else from the modules it compiles against, so a module compiles to the same bytes against two versions of an import with the same API: a build can then skip compiling the importers after a change inside a method body. ModuleApiTest compiles a library in two versions that differ only inside a method body, and checks that their APIs and an importer's bytes are the same; and that a changed signature, an interface method that gains a default body, a changed constant and a new method each change the API. ModuleApiStripTest checks that no method of the XDK's modules keeps code or source text in the API. --- .../src/main/java/org/xvm/asm/ModuleApi.java | 118 +++++++++++++++ .../test/java/org/xvm/api/ModuleApiTest.java | 142 ++++++++++++++++++ .../java/org/xvm/asm/ModuleApiStripTest.java | 77 ++++++++++ 3 files changed, 337 insertions(+) create mode 100644 javatools/src/main/java/org/xvm/asm/ModuleApi.java create mode 100644 javatools/src/test/java/org/xvm/api/ModuleApiTest.java create mode 100644 javatools/src/test/java/org/xvm/asm/ModuleApiStripTest.java diff --git a/javatools/src/main/java/org/xvm/asm/ModuleApi.java b/javatools/src/main/java/org/xvm/asm/ModuleApi.java new file mode 100644 index 0000000000..5defb09d65 --- /dev/null +++ b/javatools/src/main/java/org/xvm/asm/ModuleApi.java @@ -0,0 +1,118 @@ +package org.xvm.asm; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import java.nio.file.Files; +import java.nio.file.Path; + +import java.util.Objects; + +import java.util.stream.Stream; + +/** + * What a module that imports a compiled module compiles against: the module file without its methods' code, the + * source text of their bodies, the time it was built and the directory it was compiled from, together with whether + * each method is abstract, native or has code. + * + *

The compiler reads no method's code or source text from the modules it compiles against, so two module files + * with the same API compile every module that imports them to the same bytes. A build can then skip compiling the + * importers again after a change that leaves the API the same, such as one inside a method body. + */ +public final class ModuleApi { + private static final int ABSTRACT = 1; + private static final int NATIVE = 2; + private static final int CODE = 4; + + private ModuleApi() { + } + + /** + * @param module a module file, as the compiler writes it + * + * @return the API of the module + * + * @throws IOException if the module file cannot be read + */ + public static byte[] of(InputStream module) + throws IOException { + var file = new FileStructure(module, false, false); + try (var ignored = ConstantPool.withPool(file.getConstantPool())) { + byte[] methods = strip(file); + var out = new ByteArrayOutputStream(); + file.writeTo(out); + out.write(methods); + return out.toByteArray(); + } + } + + /** + * Write the API of a module file to another file. + * + * @param module a module file, as the compiler writes it + * @param api the file to write the API to + * + * @throws IOException if the module file cannot be read or the API cannot be written + */ + public static void write(Path module, Path api) + throws IOException { + byte[] bytes; + try (InputStream in = Files.newInputStream(module)) { + bytes = of(in); + } + try (OutputStream out = Files.newOutputStream(api)) { + out.write(bytes); + } + } + + /** + * Remove what the API leaves out from each module in the file. + * + * @return for each method, whether it is abstract, native or has code, in the order the file holds the methods, + * which is the same for two files whose other contents are the same + */ + static byte[] strip(FileStructure file) { + var flags = new ByteArrayOutputStream(); + for (Component component : tree(file).toList()) { + switch (component) { + case ModuleStructure module -> { + module.setTimestamp(null); + module.setSourceDir(null); + } + case MethodStructure method -> flags.write(strip(method)); + default -> { + } + } + } + return flags.toByteArray(); + } + + /** + * Drop the code of a method and the source text of its body. + * + * @return whether the method is abstract, native or has code + */ + private static int strip(MethodStructure method) { + boolean fAbstract = method.isAbstract(); + int flags = (fAbstract ? ABSTRACT : 0) + | (method.isNative() ? NATIVE : 0) + | (method.hasCode() ? CODE : 0); + // marking a method abstract drops its code; it is then marked as it was + method.setAbstract(true); + method.setAbstract(fAbstract); + method.configureSource(null, 0); + return flags; + } + + /** + * @return the component and every component that it holds, each with its conditional variants, in the order the + * file holds them + */ + private static Stream tree(Component component) { + return Stream.concat(Stream.of(component), component.children().stream() + .flatMap(child -> Stream.iterate(child, Objects::nonNull, Component::getNextSibling)) + .flatMap(ModuleApi::tree)); + } +} diff --git a/javatools/src/test/java/org/xvm/api/ModuleApiTest.java b/javatools/src/test/java/org/xvm/api/ModuleApiTest.java new file mode 100644 index 0000000000..78f007817b --- /dev/null +++ b/javatools/src/test/java/org/xvm/api/ModuleApiTest.java @@ -0,0 +1,142 @@ +package org.xvm.api; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; + +import java.net.URISyntaxException; + +import java.nio.file.Files; +import java.nio.file.Path; + +import java.util.Arrays; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import org.xvm.asm.DirRepository; +import org.xvm.asm.ErrorList; +import org.xvm.asm.ModuleApi; +import org.xvm.asm.ModuleStructure; + +/** + * A module that imports another compiles to the same bytes against two versions of it whose {@link ModuleApi} is the + * same, and the changes it can see change the API: a build may skip compiling importers when the API stays the same. + */ +class ModuleApiTest { + private static final String LIBRARY = """ + module Library { + String describe(Int n) { + return n > 0 ? "positive" : "other"; + } + + interface Shape { + Int area(); + } + + static Int Limit = 10; + } + """; + + private static final String IMPORTER = """ + module Importer { + package library import Library; + + String run() { + return library.describe(library.Limit); + } + } + """; + + @TempDir + Path work; + + private EmbeddingSupport support; + + @BeforeEach + void configure() throws URISyntaxException { + var modules = ModuleApiTest.class.getResource("/xdk-modules"); + assertNotNull(modules, "the build puts the compiled XDK modules in the test resources"); + support = new EmbeddingSupport().configure(new DirRepository(new File(modules.toURI()), true), null); + } + + @Test + void aChangeInsideAMethodBodyKeepsTheApiAndTheImportersBytes() throws IOException { + String changed = LIBRARY.replace(""" + return n > 0 ? "positive" : "other"; + """, """ + // a comment, another statement and other strings + if (n == 42) { + return "the answer"; + } + return n > 0 ? "more than nothing" : "nothing at all"; + """); + byte[] first = bytes(compile(LIBRARY, null)); + byte[] second = bytes(compile(changed, null)); + assertFalse(Arrays.equals(first, second), "the change is in the library's bytes"); + + assertArrayEquals(ModuleApi.of(new ByteArrayInputStream(first)), + ModuleApi.of(new ByteArrayInputStream(second))); + assertArrayEquals(bytes(compileAgainst(first)), bytes(compileAgainst(second)), + "the importer compiles to the same bytes against both"); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("visibleChanges") + void aChangeAnImporterCanSeeChangesTheApi(String change, String target, String replacement) throws IOException { + String changed = LIBRARY.replace(target, replacement); + assertFalse(changed.equals(LIBRARY), "the change applies"); + + byte[] before = ModuleApi.of(new ByteArrayInputStream(bytes(compile(LIBRARY, null)))); + byte[] after = ModuleApi.of(new ByteArrayInputStream(bytes(compile(changed, null)))); + assertFalse(Arrays.equals(before, after), change + " changes the API"); + } + + static Stream visibleChanges() { + return Stream.of( + Arguments.of("a changed signature", "String describe(Int n)", "String describe(Int n, Int base = 0)"), + Arguments.of("an interface method that gains a default body", "Int area();", + "Int area() {\n return 0;\n }"), + Arguments.of("a changed constant", "static Int Limit = 10;", "static Int Limit = 11;"), + Arguments.of("a new method", "static Int Limit = 10;", + "static Int Limit = 10;\n\n void reset() {}")); + } + + /** + * @return the importer, compiled against the specified library module file + */ + private ModuleStructure compileAgainst(byte[] library) throws IOException { + Path dir = Files.createTempDirectory(work, "library"); + Files.write(dir.resolve("Library.xtc"), library); + return compile(IMPORTER, dir); + } + + private ModuleStructure compile(String source, Path modules) { + var errs = new ErrorList(25); + var module = support.compile(source, modules == null ? null : new DirRepository(modules.toFile(), true), errs); + assertNotNull(module, () -> "compile failed: " + errs.getErrors()); + assertFalse(errs.hasSeriousErrors(), () -> "compile failed: " + errs.getErrors()); + return module; + } + + private static byte[] bytes(ModuleStructure module) { + var out = new ByteArrayOutputStream(); + try { + module.getFileStructure().writeTo(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return out.toByteArray(); + } +} diff --git a/javatools/src/test/java/org/xvm/asm/ModuleApiStripTest.java b/javatools/src/test/java/org/xvm/asm/ModuleApiStripTest.java new file mode 100644 index 0000000000..1c48b2fb64 --- /dev/null +++ b/javatools/src/test/java/org/xvm/asm/ModuleApiStripTest.java @@ -0,0 +1,77 @@ +package org.xvm.asm; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +import java.net.URISyntaxException; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.xvm.asm.constants.MethodConstant; + +/** + * {@link ModuleApi} reaches every method of the XDK's modules: after it strips one, no method that the module file + * names has code or source text left, and no module has a build time or a source directory. + */ +class ModuleApiStripTest { + @Test + void strippingTheXdkModulesLeavesNoCodeSourceTimestampOrDirectory() throws IOException, URISyntaxException { + var resource = ModuleApiStripTest.class.getResource("/xdk-modules"); + assertNotNull(resource, "the build puts the compiled XDK modules in the test resources"); + File[] modules = new File(resource.toURI()).listFiles((dir, name) -> name.endsWith(".xtc")); + assertTrue(modules != null && modules.length > 10, () -> "found " + Arrays.toString(modules)); + + for (File module : modules) { + FileStructure stripped = read(strip(module)); + try (var ignored = ConstantPool.withPool(stripped.getConstantPool())) { + List methods = Arrays.stream(stripped.getConstantPool().getConstants()) + .filter(MethodConstant.class::isInstance) + .map(constant -> ((MethodConstant) constant).getComponent()) + .filter(MethodStructure.class::isInstance) + .map(MethodStructure.class::cast) + .toList(); + assertFalse(methods.isEmpty(), module.getName()); + for (MethodStructure method : methods) { + // named without its signature, whose description needs the modules it imports + assertFalse(method.hasCode(), () -> module.getName() + ": " + method.getName()); + assertNull(method.getSourceText(), () -> module.getName() + ": " + method.getName()); + } + for (ModuleStructure each : stripped.children()) { + assertNull(each.getTimestamp(), () -> module.getName() + ": " + each.getName()); + assertNull(each.getSourceDir(), () -> module.getName() + ": " + each.getName()); + } + } + } + } + + /** + * @return the module file without what its API leaves out + */ + private static byte[] strip(File module) throws IOException { + FileStructure file; + try (var in = new FileInputStream(module)) { + file = new FileStructure(in, false, false); + } + try (var ignored = ConstantPool.withPool(file.getConstantPool())) { + ModuleApi.strip(file); + var out = new ByteArrayOutputStream(); + file.writeTo(out); + return out.toByteArray(); + } + } + + private static FileStructure read(byte[] bytes) throws IOException { + return new FileStructure(new ByteArrayInputStream(bytes), false, false); + } +} From 8ea0d537591a4a1c7f5a43ef61a6c8ffd256fa50 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:22:18 +0100 Subject: [PATCH 06/10] Compile a module again only when the API of a module it imports changes A compile task took the whole files of the modules it imports as input, so any change to ecstasy, such as one string inside a method body, compiled all 21 modules that import it again, though each came out byte for byte the same: 117 s of the 170 s of task time in that build. An artifact transform now turns each imported module into its API (ModuleApi), and a compile task takes those as its input in place of the modules. The compile still reads the modules themselves, and the tasks that run or ship them, tests and the distribution among them, still take the whole modules as input. The transform loads javatools for each run and closes the class loader after it, so that no class loader keeps a jar open that the build rewrites; with a javatools that has no ModuleApi it copies the modules, which keeps the old behavior. The same body change in ecstasy now compiles 1 module instead of 22 and reruns the tests: 37 s instead of 59 s. --- .../xtclang/plugin/XtcJavaToolsRuntime.java | 2 +- .../xtclang/plugin/XtcModuleApiTransform.java | 131 ++++++++++++++++++ .../xtclang/plugin/XtcProjectDelegate.java | 31 +++++ .../xtclang/plugin/tasks/XtcCompileTask.java | 20 ++- 4 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java diff --git a/plugin/src/main/java/org/xtclang/plugin/XtcJavaToolsRuntime.java b/plugin/src/main/java/org/xtclang/plugin/XtcJavaToolsRuntime.java index a6d25c55a9..c091b0ddf7 100644 --- a/plugin/src/main/java/org/xtclang/plugin/XtcJavaToolsRuntime.java +++ b/plugin/src/main/java/org/xtclang/plugin/XtcJavaToolsRuntime.java @@ -114,7 +114,7 @@ public static File resolveJavaTools( return resolveRuntime(projectVersion, javaToolsConfig, xdkFileTree, logger).launcherJar(); } - private static XtcLauncherRuntime resolveRuntimeFromFiles( + static XtcLauncherRuntime resolveRuntimeFromFiles( final String source, final List files, final String artifactVersion) { diff --git a/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java b/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java new file mode 100644 index 0000000000..b9710b4167 --- /dev/null +++ b/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java @@ -0,0 +1,131 @@ +package org.xtclang.plugin; + +import static org.xtclang.plugin.XtcPluginConstants.XTC_MODULE_FILE_EXTENSION; +import static org.xtclang.plugin.XtcPluginUtils.failure; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; + +import org.gradle.api.artifacts.transform.CacheableTransform; +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; + +/** + * Turns the XTC modules that a compile depends on into their APIs, as javatools' {@code org.xvm.asm.ModuleApi} + * defines them: what a module that imports them compiles against. A compile task takes the APIs as its input in + * place of the modules, so a change to a module that leaves its API the same, such as one inside a method body, does + * not compile the modules that import it again. + * + *

The input is a directory of modules, as a project's compile task produces it, or a module file; the output is a + * directory with the API of each module under the module's file name. With a javatools that has no + * {@code ModuleApi}, the output is a copy of each module, which a compile then depends on as a whole, as before. + * + *

Javatools is loaded for each transform and the class loader closed after it: one kept open across builds would + * read a jar that the build has since rewritten in place. + */ +@CacheableTransform +public abstract class XtcModuleApiTransform implements TransformAction { + /** + * The artifact type of a module's API. + */ + public static final String ARTIFACT_TYPE = "xtc-api"; + + private static final String MODULE_API_CLASS = "org.xvm.asm.ModuleApi"; + + private static final Logger LOG = Logging.getLogger(XtcModuleApiTransform.class); + + public interface Parameters extends TransformParameters { + /** + * @return the jars to find javatools.jar among, as the compile task finds it + */ + @Classpath + ConfigurableFileCollection getJavaTools(); + } + + @InputArtifact + @PathSensitive(PathSensitivity.NAME_ONLY) + public abstract Provider getModules(); + + @Override + public void transform(final TransformOutputs outputs) { + final File input = getModules().get().getAsFile(); + final File output = outputs.dir(input.getName()); + final List modules = modules(input); + if (modules.isEmpty()) { + return; + } + + final var runtime = XtcJavaToolsRuntime.resolveRuntimeFromFiles("module API", + getParameters().getJavaTools().getFiles().stream().filter(File::isFile).toList(), null); + if (runtime == null) { + throw failure("Found no javatools.jar to read the APIs of the modules in '{}' with.", input); + } + try (var loader = new URLClassLoader(urls(runtime.classpath()), ClassLoader.getPlatformClassLoader())) { + final Method write = moduleApiWriter(loader); + for (final Path module : modules) { + final Path api = output.toPath().resolve(module.getFileName().toString()); + if (write == null) { + Files.copy(module, api); + } else { + write.invoke(null, module, api); + } + } + } catch (final IOException | IllegalAccessException e) { + throw failure(e, "Failed to write the APIs of the modules in '{}'.", input); + } catch (final InvocationTargetException e) { + throw failure(e.getCause(), "Failed to write the APIs of the modules in '{}'.", input); + } + } + + /** + * @return the module files of the input, in name order; none if it does not exist + */ + private static List modules(final File input) { + if (input.isFile()) { + return List.of(input.toPath()); + } + final File[] files = input.listFiles((dir, name) -> name.endsWith('.' + XTC_MODULE_FILE_EXTENSION)); + return files == null ? List.of() : Stream.of(files).sorted().map(File::toPath).toList(); + } + + /** + * @return {@code ModuleApi.write(Path, Path)}, or null if the javatools has none + */ + private static Method moduleApiWriter(final ClassLoader loader) { + try { + return loader.loadClass(MODULE_API_CLASS).getMethod("write", Path.class, Path.class); + } catch (final ClassNotFoundException | NoSuchMethodException e) { + LOG.info("[plugin] This javatools has no {}: compiles depend on whole modules.", MODULE_API_CLASS); + return null; + } + } + + private static URL[] urls(final List classpath) { + return classpath.stream().map(file -> { + try { + return file.toURI().toURL(); + } catch (final MalformedURLException e) { + throw failure(e, "Invalid javatools classpath entry '{}'.", file); + } + }).toArray(URL[]::new); + } +} diff --git a/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java b/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java index ac3f66bfc5..2a3ed90f22 100644 --- a/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java +++ b/plugin/src/main/java/org/xtclang/plugin/XtcProjectDelegate.java @@ -31,6 +31,7 @@ import static org.xtclang.plugin.XtcPluginConstants.XTC_EXTENSION_NAME_TEST; import static org.xtclang.plugin.XtcPluginConstants.XTC_LANGUAGE_NAME; import static org.xtclang.plugin.XtcPluginConstants.XTC_TEST_TASK_NAME; +import static org.xtclang.plugin.XtcPluginConstants.XTC_MODULE_FILE_EXTENSION; import static org.xtclang.plugin.XtcPluginConstants.XTC_SOURCE_FILE_EXTENSION; import static org.xtclang.plugin.XtcPluginConstants.XTC_SOURCE_SET_DIRECTORY_ROOT_NAME; import static org.xtclang.plugin.XtcPluginUtils.capitalize; @@ -56,6 +57,7 @@ import org.gradle.api.attributes.LibraryElements; import org.gradle.api.component.AdhocComponentWithVariants; import org.gradle.api.file.Directory; +import org.gradle.api.file.FileCollection; import org.gradle.api.file.ProjectLayout; import org.gradle.api.logging.Logger; import org.gradle.api.model.ObjectFactory; @@ -340,6 +342,7 @@ public void apply(final Project project) { checkProjectIsVersioned(project); createDefaultSourceSets(project); createXtcDependencyConfigs(project); + registerModuleApiTransform(project); createDefaultRunTask(project); createDefaultTestTask(project); @@ -402,6 +405,34 @@ public static String incomingXtcModuleDependencies(final String sourceSetName) { return MAIN_SOURCE_SET_NAME.equals(sourceSetName) ? XTC_CONFIG_NAME_INCOMING : XTC_CONFIG_NAME_INCOMING + capitalize(sourceSetName); } + /** + * The APIs of the modules of an incoming module configuration, which a compile depends on in place of the + * modules (see {@link XtcModuleApiTransform}). + */ + public static FileCollection moduleApis(final Configuration config) { + return config.getIncoming().artifactView(view -> view.attributes(attributes -> + attributes.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, XtcModuleApiTransform.ARTIFACT_TYPE))).getFiles(); + } + + /** + * Turn module directories and module files into the modules' APIs when a compile asks for them, with the + * javatools that the compile uses. + */ + private void registerModuleApiTransform(final Project project) { + final var xdkJars = project.files(getXdkContentsDir(project)) + .builtBy(tasks.named(XDK_EXTRACT_TASK_NAME)) + .getAsFileTree() + .matching(pattern -> pattern.include("**/*.jar")); + final var javaTools = project.files(configs.named(XDK_CONFIG_NAME_JAVATOOLS_INCOMING), xdkJars); + for (final String from : List.of(ArtifactTypeDefinition.DIRECTORY_TYPE, XTC_MODULE_FILE_EXTENSION)) { + project.getDependencies().registerTransform(XtcModuleApiTransform.class, spec -> { + spec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, from); + spec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, XtcModuleApiTransform.ARTIFACT_TYPE); + spec.parameters(parameters -> parameters.getJavaTools().from(javaTools)); + }); + } + } + @SuppressWarnings("unused") static String outgoingXtcModules(final SourceSet sourceSet) { return SourceSet.isMain(sourceSet) ? XTC_CONFIG_NAME_OUTGOING : XTC_CONFIG_NAME_OUTGOING + capitalize(sourceSet.getName()); diff --git a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java index 0c279573f6..ca065fe5b2 100644 --- a/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java +++ b/plugin/src/main/java/org/xtclang/plugin/tasks/XtcCompileTask.java @@ -67,6 +67,9 @@ public abstract class XtcCompileTask extends XtcSourceTask implements XtcCompile // Source-set-specific module dependencies (avoids circular dependency with own output) private final ConfigurableFileCollection compileModuleDependencies; + // The APIs of the module dependencies, which is what the compile reads from them + private final ConfigurableFileCollection compileModuleDependencyApis; + /** * Create an XTC Compile task. This goes through the Gradle build script, and task creation through * the project ObjectFactory. @@ -103,15 +106,18 @@ public XtcCompileTask(final ObjectFactory objects, final Project project, final // The test configuration also includes main output. // This avoids the circular dependency where a task's output is also its input this.compileModuleDependencies = objects.fileCollection(); + this.compileModuleDependencyApis = objects.fileCollection(); final var configurations = project.getConfigurations(); final var mainConfig = configurations.findByName(XtcProjectDelegate.incomingXtcModuleDependencies(SourceSet.MAIN_SOURCE_SET_NAME)); if (mainConfig != null) { compileModuleDependencies.from(mainConfig); + compileModuleDependencyApis.from(XtcProjectDelegate.moduleApis(mainConfig)); } if (!SourceSet.MAIN_SOURCE_SET_NAME.equals(sourceSet.getName())) { final var sourceConfig = configurations.findByName(XtcProjectDelegate.incomingXtcModuleDependencies(sourceSet)); if (sourceConfig != null) { compileModuleDependencies.from(sourceConfig); + compileModuleDependencyApis.from(XtcProjectDelegate.moduleApis(sourceConfig)); } } @@ -138,11 +144,21 @@ public String getCompileSourceSetName() { * Test compile: xtcModule + xtcModuleTest (includes main output) */ @Override + @Internal("What the compile reads from these is in getXtcModuleDependencyApis()") + public FileCollection getXtcModuleDependencies() { + return compileModuleDependencies; + } + + /** + * The APIs of the module dependencies ({@link org.xtclang.plugin.XtcModuleApiTransform}): the compile reads + * nothing else from the modules it compiles against, so a change to one of them that leaves its API the same, + * such as one inside a method body, leaves this task up to date. + */ @Optional @InputFiles @PathSensitive(PathSensitivity.RELATIVE) - public FileCollection getXtcModuleDependencies() { - return compileModuleDependencies; + public FileCollection getXtcModuleDependencyApis() { + return compileModuleDependencyApis; } // TODO Why do we even have these internals? From 739d0f363c029f3e88f780d3ae9304b127b57e2c Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:02:40 +0100 Subject: [PATCH 07/10] Fall back to whole modules where the module API transform cannot read one ConfigurationCacheCompatibilityTest failed on CI: its project depends on a file that is not a module and has no javatools, and the transform failed the build where the compile had only compared the file. Wherever the transform cannot compute an API, it now copies the file, so that a compile depends on all of it, as before: with no javatools, with a javatools that has no ModuleApi, for a file it cannot read as a module, or in a JVM too old to load javatools, which the compile itself runs in a JVM of its own. The API files have the extension .api: they are only compared, never loaded, and are not valid module files. --- .../xtclang/plugin/XtcModuleApiTransform.java | 77 +++++++++++++++---- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java b/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java index b9710b4167..a0107ae5e5 100644 --- a/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java +++ b/plugin/src/main/java/org/xtclang/plugin/XtcModuleApiTransform.java @@ -12,6 +12,7 @@ import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.List; import java.util.stream.Stream; @@ -36,8 +37,10 @@ * not compile the modules that import it again. * *

The input is a directory of modules, as a project's compile task produces it, or a module file; the output is a - * directory with the API of each module under the module's file name. With a javatools that has no - * {@code ModuleApi}, the output is a copy of each module, which a compile then depends on as a whole, as before. + * directory with the API of each module, named after the module with the extension {@code .api}: the API files + * are only compared, never loaded, and are not valid module files. Without a javatools that has + * {@code ModuleApi}, or for a file that cannot be read as a module, the output is a copy of the file, which a compile + * then depends on as a whole, as before. * *

Javatools is loaded for each transform and the class loader closed after it: one kept open across builds would * read a jar that the build has since rewritten in place. @@ -51,6 +54,8 @@ public abstract class XtcModuleApiTransform implements TransformAction modules(final File input) { } /** - * @return {@code ModuleApi.write(Path, Path)}, or null if the javatools has none + * @return {@code ModuleApi.write(Path, Path)}, or null if the javatools has none, or cannot run in this JVM, such as + * one older than the javatools was built for: the compiles themselves run javatools in their own JVM */ private static Method moduleApiWriter(final ClassLoader loader) { try { return loader.loadClass(MODULE_API_CLASS).getMethod("write", Path.class, Path.class); - } catch (final ClassNotFoundException | NoSuchMethodException e) { - LOG.info("[plugin] This javatools has no {}: compiles depend on whole modules.", MODULE_API_CLASS); + } catch (final ClassNotFoundException | NoSuchMethodException | LinkageError e) { + LOG.info("[plugin] Cannot use {} from this javatools ({}): compiles depend on whole modules.", MODULE_API_CLASS, e.toString()); return null; } } From d5350b7882408828d4e8ec898de3c02d9ff405cc Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:37:08 +0100 Subject: [PATCH 08/10] Stamp compiled modules with Instant.EPOCH, and say why --- .../java/org/xvm/compiler/ast/TypeCompositionStatement.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java b/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java index d7322af23c..126c71c51d 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java @@ -4,6 +4,8 @@ import java.lang.reflect.Field; +import java.time.Instant; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -430,8 +432,8 @@ protected void registerStructures(StageMgr mgr, ErrorListener errs) { return; } - // create the FileStructure and "this" ModuleStructure - FileStructure struct = new FileStructure(sModule, null); + // create the FileStructure and "this" ModuleStructure; the epoch keeps the output reproducible + FileStructure struct = new FileStructure(sModule, Instant.EPOCH); component = struct.getModule(); pool = struct.getConstantPool(); From 2974cee5a7fa0e1fab1124f6227ccc5290ec4a72 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:30:19 +0100 Subject: [PATCH 09/10] Let xtc build stamp a time with --timestamp; keep the epoch by default The compiler stamps every module with the epoch, so that the same source compiles to the same bytes. To record a time instead, xtc build --timestamp stamps the modules it compiles with the time of the compile ("now") or with an ISO-8601 instant, such as a release commit's time. The Gradle plugin passes it from the compiler extension's timestamp setting, which is unset by default. A module that is up to date is not compiled again and keeps its stamp; --rebuild restamps it. A module's description shows the timestamp only when a time was set, instead of always showing the epoch, which says nothing. --- doc/xtc-cli.md | 1 + .../java/org/xvm/asm/ModuleStructure.java | 14 ++- .../src/main/java/org/xvm/tool/Compiler.java | 15 +++- .../java/org/xvm/tool/LauncherOptions.java | 37 +++++++- .../org/xvm/tool/CompilerTimestampTest.java | 86 +++++++++++++++++++ .../test/java/org/xvm/tool/OptionsTest.java | 24 ++++++ .../xtclang/plugin/XtcCompilerExtension.java | 7 ++ .../internal/DefaultXtcCompilerExtension.java | 7 ++ .../plugin/launchers/DirectStrategy.java | 3 +- .../launchers/ForkedCommandLineBuilder.java | 5 ++ .../plugin/runtime/DirectCompileRequest.java | 3 +- .../impl/IsolatedLauncherOptionsBuilder.java | 5 ++ .../xtclang/plugin/tasks/XtcCompileTask.java | 9 ++ .../launchers/LauncherArgumentsTest.java | 21 +++++ 14 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 javatools/src/test/java/org/xvm/tool/CompilerTimestampTest.java diff --git a/doc/xtc-cli.md b/doc/xtc-cli.md index e47688fb4f..01d103f011 100644 --- a/doc/xtc-cli.md +++ b/doc/xtc-cli.md @@ -222,6 +222,7 @@ xtc build [options] | `--rebuild` | Force rebuild | | `--strict` | Treat warnings as errors | | `--nowarn` | Suppress warnings | +| `--timestamp