Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/xtc-cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ xtc build [options] <source_files>
| `--rebuild` | Force rebuild |
| `--strict` | Treat warnings as errors |
| `--nowarn` | Suppress warnings |
| `--timestamp <time>` | Stamp the compiled modules with a time: `now`, or an ISO-8601 instant such as `2026-01-01T00:00:00Z`. Without it they carry the epoch, so that the same source compiles to the same bytes |

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggest:

Optionally stamp the compiled modules with a time. For <time>, use the literal now or an ISO-8601 instant like 2027-06-13T00:00:00Z.

| `-v, --verbose` | Verbose output |

### Examples
Expand Down
36 changes: 33 additions & 3 deletions javatools/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -234,14 +234,44 @@ val jar = tasks.named<Jar>("jar") {
}
}

// ModuleInfoTest parses real module sources of sibling XDK projects. Declare them as inputs so
// an edit to those sources re-runs the tests instead of reusing an up-to-date or cached result.
tasks.test {
// The commit and branch in build-info.properties only feed `xtc --version`, and its test compares the output with
// the same file, so the tests do not depend on them: compare the classpath without them, so that a commit that
// changes no source does not rerun the tests. The XTC plugin does the same for the tasks that compile and run modules.
normalization {
runtimeClasspath {
properties("build-info.properties") {
ignoreProperty("git.commit")
ignoreProperty("git.status")
}
}
}

// ModuleInfoTest parses real module sources of sibling XDK projects, so it runs on its own, with those sources as
// inputs: an edit to them reruns it instead of reusing an up-to-date or cached result, without rerunning the other
// tests, which do not read them.
val moduleInfoTest = tasks.register<Test>("moduleInfoTest") {
description = "Run ModuleInfoTest, which parses the module sources of lib_net, lib_json and lib_web"
group = VERIFICATION_GROUP

val testSourceSet = sourceSets.test.get()
testClassesDirs = testSourceSet.output.classesDirs
classpath = testSourceSet.runtimeClasspath
useJUnitPlatform()

include("**/ModuleInfoTest.class")
inputs.files(listOf("lib_net", "lib_json", "lib_web").map { File(compositeRoot, "$it/src/main/x") })
.withPropertyName("siblingModuleSources")
.withPathSensitivity(PathSensitivity.RELATIVE)
}

tasks.test {
exclude("**/ModuleInfoTest.class")
}

tasks.check {
dependsOn(moduleInfoTest)
}

val versionOutputTest = tasks.register<Test>("versionOutputTest") {
description = "Run tests that verify version output contains git and API information"
group = VERIFICATION_GROUP
Expand Down
118 changes: 118 additions & 0 deletions javatools/src/main/java/org/xvm/asm/ModuleApi.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package org.xvm.asm;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import java.nio.file.Files;
import java.nio.file.Path;

import java.util.Objects;

import java.util.stream.Stream;

/**
* What a module that imports a compiled module compiles against: the module file without its methods' code, the
* source text of their bodies, the time it was built and the directory it was compiled from, together with whether
* each method is abstract, native or has code.
*
* <p>The compiler reads no method's code or source text from the modules it compiles against, so two module files
* with the same API compile every module that imports them to the same bytes. A build can then skip compiling the
* importers again after a change that leaves the API the same, such as one inside a method body.
*/
public final class ModuleApi {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand why this class exists. Wouldn't it make more sense to just put the logic onto FileStructure, ModuleStructure, etc.?

private static final int ABSTRACT = 1;
private static final int NATIVE = 2;
private static final int CODE = 4;

private ModuleApi() {
}

/**
* @param module a module file, as the compiler writes it
*
* @return the API of the module
*
* @throws IOException if the module file cannot be read
*/
public static byte[] of(InputStream module)
throws IOException {
var file = new FileStructure(module, false, false);
try (var ignored = ConstantPool.withPool(file.getConstantPool())) {
byte[] methods = strip(file);
var out = new ByteArrayOutputStream();
file.writeTo(out);
out.write(methods);
return out.toByteArray();
}
}

/**
* Write the API of a module file to another file.
*
* @param module a module file, as the compiler writes it
* @param api the file to write the API to
*
* @throws IOException if the module file cannot be read or the API cannot be written
*/
public static void write(Path module, Path api)
throws IOException {
byte[] bytes;
try (InputStream in = Files.newInputStream(module)) {
bytes = of(in);
}
try (OutputStream out = Files.newOutputStream(api)) {
out.write(bytes);
}
}

/**
* Remove what the API leaves out from each module in the file.
*
* @return for each method, whether it is abstract, native or has code, in the order the file holds the methods,
* which is the same for two files whose other contents are the same
*/
static byte[] strip(FileStructure file) {
var flags = new ByteArrayOutputStream();
for (Component component : tree(file).toList()) {
switch (component) {
case ModuleStructure module -> {
module.setTimestamp(null);
module.setSourceDir(null);
}
case MethodStructure method -> flags.write(strip(method));
default -> {
}
}
}
return flags.toByteArray();
}

/**
* Drop the code of a method and the source text of its body.
*
* @return whether the method is abstract, native or has code
*/
private static int strip(MethodStructure method) {
boolean fAbstract = method.isAbstract();
int flags = (fAbstract ? ABSTRACT : 0)
| (method.isNative() ? NATIVE : 0)
| (method.hasCode() ? CODE : 0);
// marking a method abstract drops its code; it is then marked as it was
method.setAbstract(true);
method.setAbstract(fAbstract);
method.configureSource(null, 0);
return flags;
}

/**
* @return the component and every component that it holds, each with its conditional variants, in the order the
* file holds them
*/
private static Stream<Component> tree(Component component) {
return Stream.concat(Stream.of(component), component.children().stream()
.flatMap(child -> Stream.<Component>iterate(child, Objects::nonNull, Component::getNextSibling))
.flatMap(ModuleApi::tree));
}
}
14 changes: 10 additions & 4 deletions javatools/src/main/java/org/xvm/asm/ModuleStructure.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import java.security.DigestOutputStream;
import java.security.MessageDigest;

import java.time.Instant;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -255,7 +257,8 @@ public void setSourceDir(LiteralConstant constDir) {
}

/**
* Determine the date/time that the module was created.
* Determine the date/time that the module was created. The compiler records the epoch, so that the

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't use epoch; use null. That means "does not have a date/time".

* same source compiles to the same bytes, unless it was asked for a time ({@code --timestamp}).
*
* @return the literal constant indicating the compilation timestamp, or null
*/
Expand Down Expand Up @@ -944,9 +947,12 @@ public String getDescription() {
}

sb.append(", source-dir=")

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mostly good, except that the FileStructure already has both the time and the dir being optional (i.e. null is fine for both), so let's just change our default handling so that it's null unless they tell us to do something.

.append(m_constDir == null ? "none" : m_constDir.getValueString())
.append(", timestamp=")
.append(m_constDir == null ? "none" : m_constTimestamp.getValueString());
.append(m_constDir == null ? "none" : m_constDir.getValueString());
if (m_constTimestamp != null && !Instant.EPOCH.toString().equals(m_constTimestamp.getValue())) {
// only a time that was asked for; the epoch says nothing
sb.append(", timestamp=")
.append(m_constTimestamp.getValueString());
}

return sb.toString();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import org.xvm.asm.Annotation;
Expand Down Expand Up @@ -286,7 +285,7 @@ protected AstNode getCodeContainer() {

@Override
protected int getCodeContainerCounter() {
return COUNTER.getAndIncrement();
return m_counter++;
}

// ----- compile phases ------------------------------------------------------------------------
Expand Down Expand Up @@ -1257,7 +1256,10 @@ public String getDumpDesc() {
// complementary statement for the constructor points to the finalizer and vice versa
private transient MethodDeclarationStatement m_stmtComplement;

private static final AtomicInteger COUNTER = new AtomicInteger();
/**
* A unique counter with the context of this method.
*/
protected transient int m_counter;

private static final Field[] CHILD_FIELDS = fieldsForNames(MethodDeclarationStatement.class,
"condition", "annotations", "typeParams", "returns", "redundant", "params", "body");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

import java.lang.reflect.Field;

import java.time.Instant;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -430,8 +432,8 @@ protected void registerStructures(StageMgr mgr, ErrorListener errs) {
return;
}

// create the FileStructure and "this" ModuleStructure
FileStructure struct = new FileStructure(sModule);
// create the FileStructure and "this" ModuleStructure; the epoch keeps the output reproducible
FileStructure struct = new FileStructure(sModule, Instant.EPOCH);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hate having this constructor. Just get rid of the constructor that takes a time, and default the time to null, and we'll tell the FileStructure what time we want it to have if that's what we want.

component = struct.getModule();
pool = struct.getConstantPool();

Expand Down
15 changes: 13 additions & 2 deletions javatools/src/main/java/org/xvm/tool/Compiler.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
import java.io.File;
import java.io.IOException;

import java.time.Instant;
import java.time.format.DateTimeParseException;

import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -208,6 +211,12 @@ protected int process() {
binFile));
}
}
try {
opts.getTimestamp();
} catch (DateTimeParseException e) {
log(ERROR, "The --timestamp {} is neither \"now\" nor an ISO-8601 instant, such as "
+ "2026-01-01T00:00:00Z", quoted(e.getParsedString()));
}
checkErrors("target selection");

boolean fRebuild = opts.isForcedRebuild();
Expand Down Expand Up @@ -542,13 +551,15 @@ private boolean addVersion(ModuleInfo info, Version ver) {
*/
@SuppressWarnings("UnusedReturnValue")
protected int emitModules(List<Node> allNodes, ModuleRepository repoOutput) {
CompilerOptions opts = options();
var version = opts.getVersion();
CompilerOptions opts = options();
var version = opts.getVersion();
Optional<Instant> timestamp = opts.getTimestamp();
for (Node nodeModule : allNodes) {
ModuleStructure module = (ModuleStructure) nodeModule.type().getComponent();

assert !module.isFingerprint();
version.ifPresent(module::setVersion);
timestamp.ifPresent(time -> module.setTimestamp(module.getConstantPool().ensureTimeConstant(time)));

if (repoOutput != null) {
try {
Expand Down
37 changes: 36 additions & 1 deletion javatools/src/main/java/org/xvm/tool/LauncherOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import java.nio.file.Path;

import java.time.Instant;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -67,7 +69,11 @@ public abstract class LauncherOptions {
.addOption(builder("o").argName("file").hasArg()
.desc("File or directory to write output to").get())
.addOption(builder().longOpt("set-version").argName("version").hasArg()
.desc("Specify the version to stamp onto the compiled module(s)").get()));
.desc("Specify the version to stamp onto the compiled module(s)").get())
.addOption(builder().longOpt("timestamp").argName("time").hasArg()
.desc("Stamp the compiled module(s) with a time: \"now\", or an ISO-8601 instant such as "
+ "2026-01-01T00:00:00Z; without it they carry the epoch, so that the same source compiles "
+ "to the same bytes").get()));

/**
* Base Apache Commons CLI Options schema for the runners.
Expand Down Expand Up @@ -614,6 +620,10 @@ public T addModulePath(final String path) {
* Simple wrapper around Apache Commons CLI CommandLine with typed getters.
*/
public static class CompilerOptions extends LauncherOptions {
/**
* The value of {@code --timestamp} that stamps the time of the compile.
*/
public static final String TIMESTAMP_NOW = "now";

CompilerOptions(final CommandLine commandLine) {
super(commandLine, COMPILER_OPTIONS, "build");
Expand Down Expand Up @@ -659,6 +669,19 @@ public Optional<Version> getVersion() {
return optionValue("set-version").map(Version::new);
}

/**
* @return the time to stamp the compiled modules with, if one was asked for: for "now", the time
* of this call
*
* @throws java.time.format.DateTimeParseException if the value is neither "now" nor an ISO-8601
* instant
*/
public Optional<Instant> getTimestamp() {
return optionValue("timestamp").map(value -> TIMESTAMP_NOW.equalsIgnoreCase(value)
? Instant.now()
: Instant.parse(value));
}

public boolean isForcedRebuild() {
return hasOption("rebuild");
}
Expand Down Expand Up @@ -696,6 +719,7 @@ public String[] toCommandLine() {
getResourceLocations().stream().map(File::getPath).forEach(path -> args.addAll(List.of("-r", path)));
getOutputLocation().ifPresent(output -> args.addAll(List.of("-o", output.getPath())));
getVersion().ifPresent(version -> args.addAll(List.of("--set-version", version.toString())));
optionValue("timestamp").ifPresent(timestamp -> args.addAll(List.of("--timestamp", timestamp)));
getInputLocations().stream().map(File::getPath).forEach(args::add);
return args.toArray(String[]::new);
}
Expand Down Expand Up @@ -865,6 +889,17 @@ public Builder setModuleVersion(final Version version) {
return setModuleVersion(version.toString());
}

/**
* Stamp the compiled modules with a time.
*
* @param timestamp "now", or an ISO-8601 instant such as 2026-01-01T00:00:00Z
*/
public Builder setTimestamp(final String timestamp) {
removeeArgsAndValues("--timestamp");
args.addAll(List.of("--timestamp", timestamp));
return this;
}

/**
* Add an input source file to compile.
*
Expand Down
Loading
Loading