diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 8837faa8049..875fd1730d7 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -77,6 +77,11 @@ on: type: boolean required: false default: false + lsp-heavy-tests: + description: 'Run every LSP test and case: the ones that start processes, the extended retention workload and every parameterized case (also enable include-lang)' + type: boolean + required: false + default: false include-lang: description: 'Include lang projects in build and run tree-sitter validation' type: boolean @@ -90,6 +95,12 @@ env: ORG_GRADLE_PROJECT_includeBuildAttachManualTests: ${{ github.event_name != 'workflow_dispatch' || !(github.event.inputs.skip-tests == true || github.event.inputs.skip-tests == 'true') }} ORG_GRADLE_PROJECT_xtcPluginOverrideVerboseLogging: true RUN_INTEGRATION_TESTS: true + # Keep compiler regressions and a complete retention smoke test in every applicable PR run: the LSP + # tests' default tier, which leaves out the tests that start processes and the longer repetition + # workload, and runs each parameterized test's first case only. Pushes run every test and case, and + # a dispatch with lsp-heavy-tests runs them before merge. + ORG_GRADLE_PROJECT_lspTests: ${{ (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.lsp-heavy-tests == true || github.event.inputs.lsp-heavy-tests == 'true'))) && 'all' || 'default' }} + ORG_GRADLE_PROJECT_lspTestForks: 2 # Lang build inclusion - overrides gradle.properties default (true) to exclude lang # from push/PR CI builds. Only included via workflow_dispatch with include-lang=true. @@ -696,7 +707,7 @@ jobs: # Gradle stops at the first failing task and names it in the output. # - core: tree-sitter grammar compiles (tree-sitter generate), parser handles the # XDK corpus, native libraries build for all platforms (cached by content - # hash); LSP adapter tests with the tree-sitter adapter; DSL tests, which also + # hash); both LSP adapters against one compiler-default artifact; DSL tests, which also # read the XDK corpus # - VS Code extension: headless integration tests, then bundle and package the VSIX, and, # only when the repository variable VSCODE_MARKETPLACE_PUBLISH is true, the @@ -729,14 +740,17 @@ jobs: echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "🌳 LANG VALIDATION: core $VALIDATE_CORE, VS Code extension $VALIDATE_VSCODE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - ./gradlew $GRADLE_OPTIONS $LANG_OPTS -Plsp.adapter=treesitter $LANG_TASKS + # Keep the same packaged default as the main build and stdio checks. Switching + # defaults rewrites build-info and the JAR, invalidating every test's classpath and + # rerunning the entire suite. Tests select Tree-sitter explicitly where needed. + ./gradlew $GRADLE_OPTIONS $LANG_OPTS -Plsp.adapter=compiler $LANG_TASKS echo "" if [ "$VALIDATE_CORE" = "true" ]; then echo "✅ Grammar: VALID" echo "✅ Corpus parsing: PASSED" echo "✅ Native libraries: BUILT" - echo "✅ Tree-sitter adapter tests: PASSED" + echo "✅ Compiler and Tree-sitter adapter tests: PASSED" echo "✅ DSL tests: PASSED" fi if [ "$VALIDATE_VSCODE" = "true" ]; then @@ -750,6 +764,18 @@ jobs: echo "🎉 ALL LANG VALIDATION COMPLETE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + - name: Verify the packaged compiler LSP over stdio + if: | + success() && matrix.os == 'ubuntu-latest' && + steps.publish-flags.outputs.validate-core == 'true' + shell: bash + env: + GRADLE_OPTS: ${{ steps.versions.outputs.gradle-jvm-opts }} + run: | + ./gradlew ${{ steps.versions.outputs.gradle-options }} \ + -PincludeBuildLang=true -PincludeBuildAttachLang=true -Plsp.adapter=compiler \ + :lang:lsp-server:compilerStdioTest + - name: Require compiler-consumer test results if: | success() && matrix.os == 'ubuntu-latest' && @@ -759,12 +785,58 @@ jobs: python3 - <<'PYTEST' from pathlib import Path from xml.etree import ElementTree - result = ElementTree.parse(Path("lang/lsp-server/build/test-results/test/TEST-org.xvm.lsp.adapter.CompilerConsumerTest.xml")).getroot() - counts = {key: int(result.attrib[key]) for key in ("tests", "failures", "errors", "skipped")} - assert counts["tests"] > 0 and all(counts[key] == 0 for key in ("failures", "errors", "skipped")), counts - print("adapter.CompilerConsumerTest", counts) + results = Path("lang/lsp-server/build/test-results/test") + suites = [ + "adapter.CompilerConsumerTest", + "adapter.XdkAdapterTest", + "adapter.XdkNavigationTest", + "adapter.XdkDiagnosticTest", + "adapter.XdkAdapterLifecycleTest", + "adapter.SemanticModelTest", + "adapter.CompilerProjectTest", + "adapter.TypeInfoDiagnosticsTest", + "adapter.TypeInfoFinalCompositionTest", + "adapter.EmbeddingDiagnosticsTest", + "adapter.XdkModuleSessionTest", + "adapter.XdkRecoveryTest", + "adapter.XdkPartialAnalysisTest", + "adapter.CompilerCallSiteTest", + "server.XdkModuleServerTest", + "server.XdkLanguageServerTest", + "server.AdapterBackendTest", + ] + import os, re + # the default tier runs only a parameterized test's first case and skips the others, which + # JUnit names [2] and on (the report keeps no reason); any other skip is a test that did not + # run, which these suites must not have + later = re.compile(r"\[(\d+)\]") + tiered = os.environ.get("ORG_GRADLE_PROJECT_lspTests", "default") != "all" + def counts(path): + result = ElementTree.parse(path).getroot() + found = {key: int(result.attrib[key]) for key in ("tests", "failures", "errors")} + skipped = [case.get("name") for case in result.iter("testcase") if case.find("skipped") is not None] + cases = [name for name in skipped if tiered and (m := later.match(name)) and int(m.group(1)) > 1] + found["skipped"] = len(skipped) - len(cases) + found["later cases"] = len(cases) + return found + for suite in suites: + found = counts(results / f"TEST-org.xvm.lsp.{suite}.xml") + assert found["tests"] > 0 and all(found[key] == 0 for key in ("failures", "errors", "skipped")), (suite, found) + print(suite, found) + found = counts("lang/lsp-server/build/test-results/compilerStdioTest/TEST-org.xvm.lsp.server.XdkStdioTest.xml") + assert found["tests"] > 0 and all(found[key] == 0 for key in ("failures", "errors", "skipped")), ("server.XdkStdioTest", found) + print("server.XdkStdioTest", found) PYTEST + - name: Upload LSP test results and timings + if: always() && matrix.os == 'ubuntu-latest' + uses: actions/upload-artifact@v7 + with: + name: lsp-test-results-${{ env.ARTIFACT_SHA }} + path: lang/lsp-server/build/test-results/**/*.xml + if-no-files-found: ignore + retention-days: 7 + # The checks that lang validation ran passed; later runs of this pull request on the # same inputs skip them. - name: Record lang core checks pass diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index d57b4ba8b5c..f702b3e0eef 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -642,10 +642,13 @@ jobs: cp "${VSIX_FILES[0]}" "$ASSET_NAME" sha256sum "$ASSET_NAME" > "$CHECKSUM_NAME" VERSION=$(unzip -p "$ASSET_NAME" extension/package.json | jq -r '.version') + # The oldest VS Code the extension installs on, from its manifest (e.g. ^1.140.0). + ENGINE=$(unzip -p "$ASSET_NAME" extension/package.json | jq -r '.engines.vscode') echo "asset-name=$ASSET_NAME" >> $GITHUB_OUTPUT echo "checksum-name=$CHECKSUM_NAME" >> $GITHUB_OUTPUT echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "engine=${ENGINE#^}" >> $GITHUB_OUTPUT echo "📦 Prepared release assets (extension version $VERSION):" ls -lh "$ASSET_NAME" "$CHECKSUM_NAME" @@ -665,6 +668,7 @@ jobs: RELEASE_TAG="vscode-extension-snapshots" VERSION="${{ steps.prepare.outputs.version }}" + ENGINE="${{ steps.prepare.outputs.engine }}" COMMIT="${{ steps.commit.outputs.commit }}" ASSET_NAME="${{ steps.prepare.outputs.asset-name }}" CHECKSUM_NAME="${{ steps.prepare.outputs.checksum-name }}" @@ -676,7 +680,7 @@ jobs: Build commit: \`${COMMIT}\` Build date: ${BUILD_DATE} - Requires VS Code 1.101 or newer. Install from the Extensions view via ... -> Install from VSIX..., + Requires VS Code ${ENGINE} or newer. Install from the Extensions view via ... -> Install from VSIX..., or from a terminal: \`code --install-extension ${ASSET_NAME}\` Download the current GitHub snapshot VSIX: \`${ASSET_NAME}\`." diff --git a/.gitignore b/.gitignore index 7b956e7b7bc..93e9e6394ae 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ *.dSYM *.xtc +# Python bytecode (the tour scripts) +__pycache__/ +*.pyc + # Packages *.7z *.dmg diff --git a/AGENTS.md b/AGENTS.md index 396ced895d0..1a70e793732 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,9 +40,12 @@ Non-`clean` combinations are fine: `./gradlew build installDist`, `./gradlew tes - `settings.gradle.kts` is an aggregator root; root `build` is an aggregate lifecycle over the included builds. - Always included: `javatools`, `javatools_jitbridge`, `javatools_utils`, `javatools_unicode`, `plugin`, `xdk`, `docker`. - Optional builds are gated by `includeBuildLang` / `includeBuildManualTests` (visibility) and attached to the - root lifecycle by `includeBuildAttachLang` / `includeBuildAttachManualTests`. All default to `false`. + root lifecycle by `includeBuildAttachLang` / `includeBuildAttachManualTests`. `gradle.properties` turns both + lang flags on, since the language server compiles through javatools; CI's push and pull request builds turn + `includeBuildLang` off (`commit.yml`). The manual-tests flags default to `false`. - `manualTests` is a fake third-party consumer build that verifies published/composite plugin+XDK behavior. -- Running any `:lang:*` task from root needs BOTH flags (either one alone fails with "project ':lang' not found"): +- Running any `:lang:*` task from root needs BOTH flags on (either one alone fails with "project ':lang' not + found"); pass them where they may be off, such as in CI: ```bash ./gradlew :lang: -PincludeBuildLang=true -PincludeBuildAttachLang=true @@ -105,6 +108,9 @@ Non-`clean` combinations are fine: `./gradlew build installDist`, `./gradlew tes - When fixing bugs in existing code, do not perform broad modernization rewrites unless they are needed for the fix or make the touched code materially safer or clearer. - Keep the style of the code you add to an existing class consistent with its current style. +- Match inline comment capitalization and punctuation to the surrounding method/class. In Java code + that uses lowercase comment openings without trailing periods, follow that convention; keep normal + sentence formatting for Javadoc. ## General diff --git a/README.md b/README.md index c11515b6687..bfd7a2eb4cb 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ This repository contains comprehensive documentation organized hierarchically: ### Language Documentation - **[Ecstasy Language Wiki](https://github.com/xtclang/xvm/wiki)** - Language specification, tutorials, and examples - **[Ecstasy Language Website](http://xtclang.org/)** - Official language website and resources -- **[Core Documentation](doc/DOCUMENTATION.md)** - Language specification files, BNF grammar, and VM instruction set +- **[Core Documentation](doc/documentation.md)** - Language specification files, BNF grammar, and VM instruction set --- diff --git a/build-logic/common-plugins/src/main/kotlin/ArchiveChecks.kt b/build-logic/common-plugins/src/main/kotlin/ArchiveChecks.kt new file mode 100644 index 00000000000..39ade0fee13 --- /dev/null +++ b/build-logic/common-plugins/src/main/kotlin/ArchiveChecks.kt @@ -0,0 +1,72 @@ +import org.gradle.api.GradleException +import java.io.File +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.security.MessageDigest +import java.util.zip.ZipFile + +/** + * Fails, and deletes [archive], unless it reads as a whole zip or jar: every entry at its offset, with all of its + * data. Two builds of one checkout that write or copy the same jar at once can leave one that lists its entries but + * cannot read them. A later build would take such a file as up to date, since the fingerprint it recorded was taken + * from the finished file, so the check deletes it and fails instead, and the next build makes it again. + * + * @param what what the archive is, as the failure names it + */ +fun requireWholeArchive( + archive: File, + what: String, +) { + val broken = + try { + ZipFile(archive).use { zip -> + zip.entries().asSequence().filterNot { it.isDirectory }.forEach { entry -> + zip.getInputStream(entry).use { it.transferTo(OutputStream.nullOutputStream()) } + } + } + null + } catch (e: IOException) { + e + } + if (broken != null) { + archive.delete() + throw GradleException( + "$what, $archive, is not a whole archive (${broken.message}): another build was probably writing it at the " + + "same time. It is deleted; run the build again.", + broken, + ) + } +} + +/** + * Fails, and deletes [archive], unless its [entry] has the same bytes as [expected], as a copy of [expected] that an + * archive was made with must. See [requireWholeArchive] for why such a copy can differ. + * + * @param what what the archive is, as the failure names it + */ +fun requireArchivedCopy( + archive: File, + entry: String, + expected: File, + what: String, +) { + val archived = ZipFile(archive).use { zip -> zip.getEntry(entry)?.let { found -> zip.getInputStream(found).use(::sha256) } } + if (archived == null || !archived.contentEquals(expected.inputStream().use(::sha256))) { + archive.delete() + throw GradleException( + "$what, $archive, does not hold $expected as $entry: another build was probably writing it at the same " + + "time. It is deleted; run the build again.", + ) + } +} + +private fun sha256(input: InputStream): ByteArray { + val digest = MessageDigest.getInstance("SHA-256") + val buffer = ByteArray(1 shl 16) + while (true) { + val read = input.read(buffer) + if (read < 0) return digest.digest() + digest.update(buffer, 0, read) + } +} diff --git a/build-logic/common-plugins/src/main/kotlin/CopyFileTask.kt b/build-logic/common-plugins/src/main/kotlin/CopyFileTask.kt index 5f352fc171f..0492b14fa0a 100644 --- a/build-logic/common-plugins/src/main/kotlin/CopyFileTask.kt +++ b/build-logic/common-plugins/src/main/kotlin/CopyFileTask.kt @@ -1,4 +1,5 @@ import org.gradle.api.DefaultTask +import org.gradle.api.GradleException import org.gradle.api.file.RegularFileProperty import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.OutputFile @@ -6,8 +7,14 @@ import org.gradle.api.tasks.PathSensitive import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskAction import org.gradle.work.DisableCachingByDefault +import java.nio.file.Files -/** Copies one file without claiming ownership of its destination directory. */ +/** + * Copies one file without claiming ownership of its destination directory. The copy must have the source's bytes, + * and a jar or zip must read as a whole archive ([requireWholeArchive]); otherwise it is deleted and the task fails, + * since a source that another build was writing while it was copied leaves a copy that a later build would take as + * up to date. + */ @DisableCachingByDefault(because = "Copying a single file is cheaper than caching it") abstract class CopyFileTask : DefaultTask() { @get:InputFile @@ -19,6 +26,18 @@ abstract class CopyFileTask : DefaultTask() { @TaskAction fun copy() { - sourceFile.get().asFile.copyTo(outputFile.get().asFile, overwrite = true) + val source = sourceFile.get().asFile + val target = outputFile.get().asFile + source.copyTo(target, overwrite = true) + if (Files.mismatch(source.toPath(), target.toPath()) != -1L) { + target.delete() + throw GradleException( + "$source changed while it was copied to $target: another build was probably writing it. The copy is " + + "deleted; run the build again.", + ) + } + if (target.extension in setOf("jar", "zip")) { + requireWholeArchive(target, "The copy of ${source.name}") + } } } diff --git a/build-logic/common-plugins/src/main/kotlin/InstallIntellijPluginTask.kt b/build-logic/common-plugins/src/main/kotlin/InstallIntellijPluginTask.kt new file mode 100644 index 00000000000..e247e27650c --- /dev/null +++ b/build-logic/common-plugins/src/main/kotlin/InstallIntellijPluginTask.kt @@ -0,0 +1,173 @@ +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ArchiveOperations +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.FileSystemOperations +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.UntrackedTask +import org.gradle.process.ExecOperations +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.zip.ZipFile +import javax.inject.Inject + +/** + * Installs a plugin zip, and the Marketplace plugins it depends on, into an installed IntelliJ IDE, replacing an + * earlier install of it. The IDE must not be running. It is driven through the IDE's own `installPlugins` command, + * with the launcher the IDE's product-info.json names for this OS, so nothing about the IDE's layout or its + * directories is assumed. + * + * No IDE command names the plugins directory, but the IDE logs it at startup, so a bare `installPlugins` with only + * its log redirected reports it, and refuses if the IDE is running; [pluginsDirectory] overrides what it reports. + * `installPlugins` then installs the dependencies. The plugin itself is unpacked into that directory, since + * `installPlugins` skips a plugin that is installed in any version, and prefers a newer one from any channel the IDE + * subscribes to over the zip it is given. + */ +@UntrackedTask(because = "It installs into an IDE outside the build") +abstract class InstallIntellijPluginTask @Inject constructor( + private val execOperations: ExecOperations, + private val fileSystemOperations: FileSystemOperations, + private val archiveOperations: ArchiveOperations, +) : DefaultTask() { + /** The IDE's installation: the .app on macOS, its installation directory elsewhere. */ + @get:Input + @get:Optional + abstract val ideHome: Property + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + abstract val pluginZip: RegularFileProperty + + /** Marketplace plugins the plugin depends on, which `installPlugins` installs only when named. */ + @get:Input + abstract val dependencyIds: ListProperty + + @get:Input + abstract val sinceBuild: Property + + @get:Input + @get:Optional + abstract val pluginsDirectory: Property + + @get:Internal + abstract val workDirectory: DirectoryProperty + + @TaskAction + fun install() { + val home = File( + ideHome.orNull ?: throw GradleException( + "Name the IntelliJ IDE to install into with org.xtclang.idea.home, for example in " + + "~/.gradle/gradle.properties:\n org.xtclang.idea.home=/Applications/IntelliJ IDEA.app", + ), + ) + val ide = IntellijInstallation.of(home) + if (!IntellijInstallation.isAtLeast(ide.build, sinceBuild.get())) { + throw GradleException("${ide.name} ${ide.build} at $home is older than the plugin's since-build ${sinceBuild.get()}") + } + + // The probe also refuses a running IDE, so it runs even when the plugins directory is named, before anything is deleted + val work = workDirectory.get().asFile.apply { deleteRecursively(); mkdirs() } + val logDir = work.resolve("probe-log") + run(ide, emptyList(), "-Didea.log.path=\"${logDir.absolutePath}\"") + val pluginsDir = File( + pluginsDirectory.orNull ?: IntellijInstallation.pluginsPathFromLog(logDir.resolve("idea.log")) + ?: throw GradleException("${ide.name} did not log its plugins directory in $logDir; name it with org.xtclang.idea.pluginsDir"), + ) + + // Dependencies first, so that a failure leaves the earlier install in place; installPlugins skips installed ones + if (dependencyIds.get().isNotEmpty()) { + logger.info(run(ide, dependencyIds.get() + "--give-consent-to-use-third-party-plugins")) + } + + // The plugin itself is unpacked here: installPlugins would take a newer version of it from any channel the IDE + // subscribes to. This does what the IDE does with a plugin zip, replacing the directory at its root whole + val zip = pluginZip.get().asFile + val installed = pluginsDir.resolve(IntellijInstallation.rootEntry(zip)) + fileSystemOperations.delete { delete(installed) } + fileSystemOperations.copy { + from(archiveOperations.zipTree(zip)) + into(pluginsDir) + } + logger.lifecycle("Installed ${zip.name} into ${ide.name} ${ide.build}: $installed") + } + + /** Runs `installPlugins` with [args], and returns its output; fails if the IDE is running or the command fails. */ + private fun run(ide: IntellijInstallation, args: List, jvmOption: String? = null): String { + val output = ByteArrayOutputStream() + val result = execOperations.exec { + executable = ide.launcher.absolutePath + args(listOf("installPlugins") + args) + if (jvmOption != null) { + environment("JAVA_TOOL_OPTIONS", listOfNotNull(System.getenv("JAVA_TOOL_OPTIONS"), jvmOption).joinToString(" ")) + } + standardOutput = output + errorOutput = output + isIgnoreExitValue = true + } + val text = output.toString(Charsets.UTF_8).trim() + if ("Only one instance" in text) { + throw GradleException("${ide.name} is running; quit it first, then install again") + } + if (result.exitValue != 0) { + throw GradleException("${ide.launcher} installPlugins failed (exit ${result.exitValue}):\n$text") + } + return text + } +} + +/** An installed IntelliJ IDE, as its product-info.json describes it. */ +internal data class IntellijInstallation(val name: String, val build: String, val launcher: File) { + companion object { + fun of(home: File): IntellijInstallation { + val productInfo = home.walk().maxDepth(3).firstOrNull { it.name == "product-info.json" } + ?: throw GradleException("No product-info.json in $home; org.xtclang.idea.home must name an IntelliJ installation") + return of(productInfo, System.getProperty("os.name"), System.getProperty("os.arch")) + } + + fun of(productInfo: File, osName: String, osArch: String): IntellijInstallation { + val json = JsonSlurper().parse(productInfo) as Map<*, *> + val os = when { + osName.startsWith("Mac") -> "macOS" + osName.startsWith("Windows") -> "Windows" + else -> "Linux" + } + val arch = if (osArch == "x86_64") "amd64" else osArch + val launches = (json["launch"] as List<*>).map { it as Map<*, *> }.filter { it["os"] == os } + val launch = launches.firstOrNull { it["arch"] == arch } ?: launches.firstOrNull() + ?: throw GradleException("$productInfo names no launcher for $os") + val launcher = productInfo.parentFile.resolve(launch["launcherPath"] as String).normalize() + return IntellijInstallation(json["name"] as String, json["buildNumber"] as String, launcher) + } + + /** Whether build number [build] is [since] or later, compared component by component. */ + fun isAtLeast(build: String, since: String): Boolean { + val actual = build.split('.').map { it.toIntOrNull() ?: 0 } + val required = since.split('.').map { it.toIntOrNull() ?: 0 } + val difference = required.indices.map { actual.getOrElse(it) { 0 } - required[it] }.firstOrNull { it != 0 } + return (difference ?: 0) >= 0 + } + + private val pluginsPathLine = Regex("""^\s+idea\.plugins\.path=(.+?)(?: -> .*)?$""") + + /** The plugins directory the IDE logged at its last startup; it adds " -> exception" when the directory is missing. */ + fun pluginsPathFromLog(log: File): String? = + log.takeIf(File::isFile)?.readLines() + ?.mapNotNull { pluginsPathLine.find(it)?.groupValues?.get(1) } + ?.lastOrNull() + + /** The directory a plugin zip unpacks into, which the IDE replaces whole when it installs the zip. */ + fun rootEntry(zip: File): String = + ZipFile(zip).use { it.entries().asSequence().map { entry -> entry.name.substringBefore('/') }.first() } + .also { require(it.isNotEmpty() && it != "." && it != "..") { "$zip has no plugin directory at its root" } } + } +} diff --git a/build-logic/common-plugins/src/main/kotlin/IntellijRunIdeSupport.kt b/build-logic/common-plugins/src/main/kotlin/IntellijRunIdeSupport.kt index f33e1fe8a37..ae6fc8cf0d5 100644 --- a/build-logic/common-plugins/src/main/kotlin/IntellijRunIdeSupport.kt +++ b/build-logic/common-plugins/src/main/kotlin/IntellijRunIdeSupport.kt @@ -1,20 +1,26 @@ import org.gradle.api.DefaultTask import org.gradle.api.file.DirectoryProperty -import org.gradle.api.file.RegularFileProperty import org.gradle.api.provider.ListProperty import org.gradle.api.provider.Property import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputDirectory -import org.gradle.api.tasks.InputFile -import org.gradle.api.tasks.Optional import org.gradle.api.tasks.PathSensitive import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskAction import org.gradle.api.tasks.UntrackedTask -import java.io.File -import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.nio.file.Files +import java.nio.file.LinkOption.NOFOLLOW_LINKS +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.nio.file.StandardOpenOption.READ +import java.nio.file.attribute.BasicFileAttributes import kotlin.concurrent.thread +/** + * Reports the versions, caches and log paths used by the development IDE. Reads live sandbox state at + * execution time so the report also explains what a reused sandbox contains. + */ @UntrackedTask(because = "Runtime reporting task reads live sandbox state and should never be state-tracked") abstract class RunIdeEnvironmentReportTask : DefaultTask() { @get:Input @@ -36,6 +42,14 @@ abstract class RunIdeEnvironmentReportTask : DefaultTask() { @get:PathSensitive(PathSensitivity.RELATIVE) abstract val sandboxDir: DirectoryProperty + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sandboxLogDir: DirectoryProperty + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sandboxSystemDir: DirectoryProperty + @get:InputDirectory @get:PathSensitive(PathSensitivity.RELATIVE) abstract val mavenLocalRoot: DirectoryProperty @@ -43,16 +57,14 @@ abstract class RunIdeEnvironmentReportTask : DefaultTask() { @get:Input abstract val pluginNames: ListProperty - @get:InputFile - @get:Optional - @get:PathSensitive(PathSensitivity.RELATIVE) - abstract val lspLogFile: RegularFileProperty + @get:Input + abstract val lspLogDirectory: Property @TaskAction fun report() { val sandbox = sandboxDir.get().asFile - val ideaLog = sandbox.resolve("log/idea.log") - val systemDir = sandbox.resolve("system") + val ideaLog = sandboxLogDir.get().file("idea.log").asFile + val systemDir = sandboxSystemDir.get().asFile val sandboxIsReused = systemDir.exists() && systemDir.listFiles().orEmpty().isNotEmpty() val xtcArtifacts = mavenLocalRoot.get().asFile.resolve("org/xtclang") @@ -84,18 +96,19 @@ abstract class RunIdeEnvironmentReportTask : DefaultTask() { logger.lifecycle("[runIde] Nuke sandbox (keeps IDE download): ./gradlew :lang:intellij-plugin:clean") logger.lifecycle("[runIde] Nuke cached IDE + metadata: rm -rf lang/.intellijPlatform/localPlatformArtifacts") - lspLogFile.orNull?.asFile?.let { logFile -> - logger.lifecycle("[runIde] LSP log: ${logFile.absolutePath} (tailing to console)") - } + logger.lifecycle("[runIde] LSP logs: ${lspLogDirectory.get()}/server-*/server.log (tailing new output to console)") } } +/** + * Forwards new language-server log lines to the Gradle console while the development IDE runs. A named daemon + * thread follows session creation and log rollover; restarting this task replaces the previous tailer. + */ @UntrackedTask(because = "Runtime log tail task manages background thread state and should never be state-tracked") abstract class StartLogTailTask : DefaultTask() { - @get:InputFile - @get:Optional - @get:PathSensitive(PathSensitivity.NONE) - abstract val logFile: RegularFileProperty + // The directory may not exist until the first server starts; its live contents are not inputs. + @get:Input + abstract val logDirectory: Property @get:Input abstract val threadName: Property @@ -105,34 +118,31 @@ abstract class StartLogTailTask : DefaultTask() { @TaskAction fun startTail() { - val file = logFile.orNull?.asFile ?: return + val root = Path.of(logDirectory.get()) val name = threadName.get() - Thread.getAllStackTraces().keys.firstOrNull { it.name == name }?.interrupt() - - val startSize = if (file.exists()) file.length() else 0L val prefix = linePrefix.get() + val initial = ServerLogTails.poll(root, emptyMap(), skipExisting = true).positions thread(isDaemon = true, name = name) { - runCatching { - while (!file.exists() && !Thread.currentThread().isInterrupted) { + // Owned exclusively by this tail thread; no compiler or IDE callbacks access the cursors. + var positions = initial + try { + while (!Thread.currentThread().isInterrupted) { + val batch = ServerLogTails.poll(root, positions) + positions = batch.positions + batch.lines.forEach { logger.lifecycle("$prefix$it") } Thread.sleep(500) } - RandomAccessFile(file, "r").use { raf -> - raf.seek(startSize) - while (!Thread.currentThread().isInterrupted) { - val line = raf.readLine() - if (line != null) { - logger.lifecycle("$prefix$line") - } else { - Thread.sleep(200) - } - } - } + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } catch (failure: Exception) { + logger.warn("Ecstasy log tail stopped: ${failure.message}") } } } } +/** Stops the named development-IDE log tailer when the IDE run finishes. */ @UntrackedTask(because = "Runtime log tail stop task manages background thread state and should never be state-tracked") abstract class StopLogTailTask : DefaultTask() { @get:Input @@ -143,3 +153,53 @@ abstract class StopLogTailTask : DefaultTask() { Thread.getAllStackTraces().keys.firstOrNull { it.name == threadName.get() }?.interrupt() } } + +/** Detached file cursors follow new sessions and rollover without keeping old file handles open. */ +internal object ServerLogTails { + private val sessionName = Regex("server-[0-9]+-[0-9]+") + + /** File identity and byte offset used to distinguish appended text from truncation or rollover. */ + data class Position(val key: Any?, val offset: Long) + + /** New log lines and the detached file cursors to use on the next poll. */ + data class Batch(val positions: Map, val lines: List) + + fun poll(root: Path, previous: Map, skipExisting: Boolean = false): Batch { + val paths = if (Files.isDirectory(root, NOFOLLOW_LINKS)) { + Files.list(root).use { sessions -> + sessions + .filter { Files.isDirectory(it, NOFOLLOW_LINKS) && sessionName.matches(it.fileName.toString()) } + .map { it.resolve("server.log") } + .toList() + } + } else emptyList() + val rows = paths.mapNotNull { path -> + try { + val attributes = Files.readAttributes(path, BasicFileAttributes::class.java, NOFOLLOW_LINKS) + if (!attributes.isRegularFile) return@mapNotNull null + val prior = previous[path]?.takeIf { it.key == attributes.fileKey() && it.offset <= attributes.size() } + if (skipExisting) { + return@mapNotNull Triple(path, Position(attributes.fileKey(), attributes.size()), emptyList()) + } + val offset = prior?.offset ?: 0L + Files.newByteChannel(path, setOf(READ, NOFOLLOW_LINKS)).use { channel -> + channel.position(offset) + val bytes = ByteBuffer.allocate(minOf((channel.size() - offset).coerceAtLeast(0), 64 * 1024L).toInt()) + while (bytes.hasRemaining() && channel.read(bytes) > 0) { /* bounded poll */ } + // Hold partial lines for the next poll; split oversized lines at a UTF-8 boundary. + val end = (0 until bytes.position()).lastOrNull { bytes[it] == '\n'.code.toByte() }?.plus(1) + ?: if (bytes.position() < 64 * 1024) { + 0 + } else { + (0 until bytes.position()).lastOrNull { bytes[it].toInt() and 0xC0 != 0x80 } ?: 0 + } + val text = String(bytes.array(), 0, end, Charsets.UTF_8) + Triple(path, Position(attributes.fileKey(), offset + end), text.lineSequence().filter(String::isNotEmpty).toList()) + } + } catch (_: NoSuchFileException) { + null // A retired session or rolled file may disappear between discovery and reading. + } + } + return Batch(rows.associate { it.first to it.second }, rows.flatMap { it.third }) + } +} diff --git a/build-logic/common-plugins/src/main/kotlin/MarketplaceChannelVersion.kt b/build-logic/common-plugins/src/main/kotlin/MarketplaceChannelVersion.kt new file mode 100644 index 00000000000..70e54dd8816 --- /dev/null +++ b/build-logic/common-plugins/src/main/kotlin/MarketplaceChannelVersion.kt @@ -0,0 +1,79 @@ +import org.gradle.api.GradleException +import org.gradle.api.provider.Property +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters +import org.w3c.dom.Element +import org.xml.sax.InputSource +import java.io.IOException +import java.io.StringReader +import java.net.URI +import java.net.URLEncoder +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.charset.StandardCharsets.UTF_8 +import java.time.Duration +import javax.xml.parsers.DocumentBuilderFactory + +/** + * The newest version of a JetBrains Marketplace plugin in one of its channels, such as LSP4IJ's `nightly`, + * that an IDE line accepts. It is read from the Marketplace's plugin repository listing, the one IDEs + * read for a custom plugin repository. As a value source it is read again on each build, so a newer + * version invalidates the configuration cache. + */ +abstract class MarketplaceChannelVersion : ValueSource { + interface Parameters : ValueSourceParameters { + val pluginId: Property + val channel: Property + + /** The IDE line the version must accept, such as `IU-262`. */ + val build: Property + } + + override fun obtain(): String { + val pluginId = parameters.pluginId.get() + val channel = parameters.channel.get() + val build = parameters.build.get() + val query = listOf("channel" to channel, "pluginId" to pluginId, "build" to build) + .joinToString("&") { (name, value) -> "$name=${URLEncoder.encode(value, UTF_8)}" } + val uri = URI("https://plugins.jetbrains.com/plugins/list?$query") + val response = try { + HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + .send(HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(30)).build(), HttpResponse.BodyHandlers.ofString()) + } catch (e: IOException) { + throw GradleException("Cannot read the newest $pluginId of the Marketplace's $channel channel from $uri: $e", e) + } + if (response.statusCode() != 200) { + throw GradleException("Cannot read the Marketplace's $channel channel: $uri answered ${response.statusCode()}") + } + return MarketplaceListing.newestVersion(response.body()) + ?: throw GradleException("The Marketplace's $channel channel has no $pluginId for $build ($uri)") + } +} + +/** + * Reads a Marketplace plugin repository listing: the version of its newest plugin, by the date of its upload, so + * the listing's order doesn't matter. An empty listing, which the Marketplace answers when no version fits, has none. + */ +internal object MarketplaceListing { + fun newestVersion(listing: String): String? { + if (listing.isBlank()) { + return null + } + val factory = DocumentBuilderFactory.newInstance().apply { + setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + } + val plugins = factory.newDocumentBuilder().parse(InputSource(StringReader(listing))).getElementsByTagName("idea-plugin") + return (0 until plugins.length) + .map { plugins.item(it) as Element } + .maxByOrNull { it.getAttribute("date").toLongOrNull() ?: Long.MIN_VALUE } + ?.getElementsByTagName("version") + ?.item(0) + ?.textContent + ?.trim() + ?.takeIf(String::isNotEmpty) + } +} diff --git a/build-logic/common-plugins/src/main/kotlin/PruneTestReportsTask.kt b/build-logic/common-plugins/src/main/kotlin/PruneTestReportsTask.kt new file mode 100644 index 00000000000..233ae91206d --- /dev/null +++ b/build-logic/common-plugins/src/main/kotlin/PruneTestReportsTask.kt @@ -0,0 +1,91 @@ +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.TaskAction +import org.gradle.work.DisableCachingByDefault +import java.nio.channels.FileChannel +import java.nio.channels.OverlappingFileLockException +import java.nio.file.Files +import java.nio.file.LinkOption.NOFOLLOW_LINKS +import java.nio.file.Path +import java.nio.file.StandardOpenOption.CREATE +import java.nio.file.StandardOpenOption.WRITE + +/** Prunes disposable payloads only after a runner confirms that its IDE has exited. */ +@DisableCachingByDefault(because = "Retention acts on completed test runs, not reproducible task outputs") +abstract class PruneTestReportsTask : DefaultTask() { + @get:Internal + abstract val reportsDirectory: DirectoryProperty + + @get:Input + abstract val retainedRuns: Property + + init { + retainedRuns.convention(5) + } + + @TaskAction + fun prune() { + val count = TestReportRetention.prune(reportsDirectory.get().asFile.toPath(), retainedRuns.get()) + logger.lifecycle("Test report retention: pruned payloads from {} old runs; summaries retained", count) + } +} + +/** + * Prunes disposable IDE payloads from completed playbook runs while retaining their reports. Cleanup is + * serialized by a file lock, honors explicit artifact retention, and refuses to traverse symlinks or delete a + * checkout. + */ +internal object TestReportRetention { + fun prune(root: Path, keep: Int = 5): Int { + require(keep >= 1) { "Keep at least one complete run" } + if (!Files.isDirectory(root, NOFOLLOW_LINKS)) return 0 + return FileChannel.open(root.resolve(".retention.lock"), CREATE, WRITE, NOFOLLOW_LINKS).use { channel -> + val lock = try { + channel.tryLock() + } catch (_: OverlappingFileLockException) { + null + } + lock?.use { pruneClosed(root, keep) } ?: 0 + } + } + + private fun pruneClosed(root: Path, keep: Int): Int { + val runs = children(root) + .filter { it.fileName.toString().startsWith("run-") && Files.isDirectory(it, NOFOLLOW_LINKS) } + .filter { Files.isRegularFile(it.resolve(".completed"), NOFOLLOW_LINKS) } + .sortedByDescending { Files.getLastModifiedTime(it.resolve(".completed")) } + val old = runs.drop(keep).filterNot { Files.exists(it.resolve(".keep-artifacts"), NOFOLLOW_LINKS) }.toSet() + runs.filterNot { Files.exists(it.resolve(".keep-artifacts"), NOFOLLOW_LINKS) }.forEach { run -> + // Starter stores each sandbox separately from the run's result files. + val sandboxes = children(root.resolve("out/ide-tests/tests")) + .filter { Files.isDirectory(it, NOFOLLOW_LINKS) } + .map { it.resolve("XtcCompilerPlaybook-${run.fileName}") } + .filter { Files.isDirectory(it, NOFOLLOW_LINKS) } + sandboxes.forEach { sandbox -> + val payloads = if (run in old) children(sandbox) + else listOf("system", "plugins", "config", "temp").map(sandbox::resolve) + payloads.forEach(::removePayload) + } + if (run in old) { + children(run).filter { + Files.isDirectory(it, NOFOLLOW_LINKS) || Files.isSymbolicLink(it) || it.fileName.toString().endsWith(".log") + }.forEach(::removePayload) + } + } + return old.size + } + + private fun children(path: Path): List = + if (Files.isDirectory(path, NOFOLLOW_LINKS)) Files.list(path).use { it.toList() } else emptyList() + + private fun removePayload(path: Path) { + if (!Files.exists(path, NOFOLLOW_LINKS)) return + // walk does not follow symlinks; never touch a source worktree accidentally placed here. + val paths = Files.walk(path).use { it.toList() } + if (paths.any { it.fileName.toString() == ".git" }) return + paths.sortedDescending().forEach(Files::deleteIfExists) + } +} diff --git a/build-logic/common-plugins/src/main/kotlin/VerifyPluginInternalApis.kt b/build-logic/common-plugins/src/main/kotlin/VerifyPluginInternalApis.kt new file mode 100644 index 00000000000..19cdc92b12e --- /dev/null +++ b/build-logic/common-plugins/src/main/kotlin/VerifyPluginInternalApis.kt @@ -0,0 +1,82 @@ +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction +import org.gradle.work.DisableCachingByDefault +import java.nio.file.Path + +/** + * Checks the verifier's internal-API reports against the exact LSP4IJ integration dependencies. + * Binary compatibility remains the verifier's responsibility. Unknown APIs, dependency upgrades and + * incomplete reports fail here, so accepting existing workarounds cannot hide new platform internals. + */ +@DisableCachingByDefault(because = "Verifies external tool reports without producing an artifact") +abstract class VerifyPluginInternalApis : DefaultTask() { + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val reportsDirectory: DirectoryProperty + + @get:InputFile + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val acceptedApis: RegularFileProperty + + @get:Input + abstract val lsp4ijVersion: Property + + @TaskAction + fun verify() { + val count = InternalApiPolicy.verify( + reportsDirectory.get().asFile.toPath(), acceptedApis.get().asFile.readLines(), lsp4ijVersion.get(), + ) + logger.lifecycle("Plugin internal API check: {} known LSP4IJ usages; no unreviewed internal APIs", count) + } +} + +/** Matches full API signatures, never package wildcards; unfamiliar report lines fail closed. */ +internal object InternalApiPolicy { + private val API = Regex("Internal (?:class|interface|method|constructor) com\\.redhat\\.devtools\\.lsp4ij\\.[^*]+") + private val PARAMETERS = Regex("\\(([^()]*)\\)") + private val PARAMETER_NAME = Regex("^([\\w.$]+(?:\\[\\])*) [\\w$]+$") + private val COUNT = Regex("(\\d+) usages? of internal API") + private val DEPENDENCY = Regex("com\\.redhat\\.devtools\\.lsp4ij:([^\\s]+)") + + fun verify(reports: Path, policy: List, version: String): Int { + val entries = policy.map(String::trim).filter { it.isNotEmpty() && !it.startsWith('#') } + require(entries.firstOrNull() == "lsp4ij=$version") { + "Review the internal API policy for LSP4IJ $version before changing the dependency version" + } + val accepted = entries.drop(1).toSet() + require(accepted.isNotEmpty() && accepted.all { API.matches(it) }) { + "Internal API exceptions must name exact LSP4IJ classes or member signatures" + } + val verdicts = reports.toFile().walkTopDown().filter { it.name == "verification-verdict.txt" }.toList() + check(verdicts.isNotEmpty()) { "No Plugin Verifier verdicts found in $reports" } + return verdicts.sumOf { verdict -> + val text = verdict.readText() + val count = COUNT.find(text)?.groupValues?.get(1)?.toInt() ?: 0 + val report = verdict.resolveSibling("internal-api-usages.txt") + val usages = if (report.isFile) report.readLines().filter(String::isNotBlank) else emptyList() + check(usages.size == count) { "Incomplete internal API report beside $verdict: expected $count, found ${usages.size}" } + val dependencies = verdict.resolveSibling("dependencies.txt") + check(dependencies.isFile && DEPENDENCY.findAll(dependencies.readText()).any { it.groupValues[1] == version }) { + "Verifier dependency report does not contain the reviewed LSP4IJ $version beside $verdict" + } + val unknown = usages.filter { signature(it.substringBefore(" is ")) !in accepted } + check(unknown.isEmpty()) { "Unreviewed internal APIs in $report:\n${unknown.joinToString("\n")}" } + usages.size + } + } + + /** Debug information changes parameter names between reports; only their types form the API. */ + private fun signature(api: String): String = PARAMETERS.replace(api) { parameters -> + parameters.groupValues[1].split(',').joinToString(", ", "(", ")") { parameter -> + PARAMETER_NAME.replace(parameter.trim(), "$1") + } + } +} diff --git a/build-logic/common-plugins/src/test/kotlin/ArchiveChecksTest.kt b/build-logic/common-plugins/src/test/kotlin/ArchiveChecksTest.kt new file mode 100644 index 00000000000..28cc0099624 --- /dev/null +++ b/build-logic/common-plugins/src/test/kotlin/ArchiveChecksTest.kt @@ -0,0 +1,69 @@ +import org.gradle.api.GradleException +import org.junit.jupiter.api.Assertions.assertDoesNotThrow +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class ArchiveChecksTest { + @TempDir + lateinit var directory: Path + + @Test + fun `a whole jar passes and stays`() { + val jar = jar("whole.jar", mapOf("a/A.class" to "a".repeat(10_000), "b/B.class" to "b".repeat(10_000))) + assertDoesNotThrow { requireWholeArchive(jar.toFile(), "The jar") } + assertTrue(Files.exists(jar)) + } + + @Test + fun `a jar that another build was still writing fails, and is deleted so that it is made again`() { + val whole = Files.readAllBytes(jar("whole.jar", mapOf("a/A.class" to "a".repeat(10_000), "b/B.class" to "b".repeat(10_000)))) + // the first half of the jar, and the end of it: its entries are listed, but not where they are said to be + val torn = directory.resolve("torn.jar") + Files.write(torn, whole.copyOfRange(0, whole.size / 2) + whole.copyOfRange(whole.size - 200, whole.size)) + val failure = assertThrows(GradleException::class.java) { requireWholeArchive(torn.toFile(), "The jar") } + assertTrue("another build was probably writing it" in failure.message.orEmpty(), failure.message) + assertFalse(Files.exists(torn)) + } + + @Test + fun `an archive must hold the file it was made with, byte for byte`() { + val server = jar("server.jar", mapOf("Server.class" to "server")) + val other = jar("other.jar", mapOf("Other.class" to "other")) + val plugin = jar("plugin.zip", mapOf("bin/server.jar" to String(Files.readAllBytes(server), Charsets.ISO_8859_1))) + assertDoesNotThrow { requireArchivedCopy(plugin.toFile(), "bin/server.jar", server.toFile(), "The zip") } + + assertThrows(GradleException::class.java) { requireArchivedCopy(plugin.toFile(), "bin/server.jar", other.toFile(), "The zip") } + assertFalse(Files.exists(plugin)) + } + + @Test + fun `an archive without the entry fails`() { + val server = jar("server.jar", mapOf("Server.class" to "server")) + val plugin = jar("plugin.zip", mapOf("lib/plugin.jar" to "plugin")) + assertThrows(GradleException::class.java) { requireArchivedCopy(plugin.toFile(), "bin/server.jar", server.toFile(), "The zip") } + assertFalse(Files.exists(plugin)) + } + + /** A jar of [entries], each a name and its text, stored as bytes one to one. */ + private fun jar( + name: String, + entries: Map, + ): Path { + val jar = directory.resolve(name) + ZipOutputStream(Files.newOutputStream(jar)).use { zip -> + entries.forEach { (entry, text) -> + zip.putNextEntry(ZipEntry(entry)) + zip.write(text.toByteArray(Charsets.ISO_8859_1)) + zip.closeEntry() + } + } + return jar + } +} diff --git a/build-logic/common-plugins/src/test/kotlin/IntellijInstallationTest.kt b/build-logic/common-plugins/src/test/kotlin/IntellijInstallationTest.kt new file mode 100644 index 00000000000..5c1d4ce4c74 --- /dev/null +++ b/build-logic/common-plugins/src/test/kotlin/IntellijInstallationTest.kt @@ -0,0 +1,92 @@ +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class IntellijInstallationTest { + @TempDir + lateinit var directory: Path + + // the launch entries of IntelliJ IDEA 2026.3's multi-platform distribution, plus a macOS bundle's + private val productInfo by lazy { + directory.resolve("Contents/Resources").toFile().apply { mkdirs() }.resolve("product-info.json").apply { + writeText( + """ + {"name": "IntelliJ IDEA", "buildNumber": "263.7307.200", "launch": [ + {"os": "Windows", "arch": "amd64", "launcherPath": "bin/idea.bat"}, + {"os": "Linux", "arch": "amd64", "launcherPath": "bin/idea.sh"}, + {"os": "Linux", "arch": "aarch64", "launcherPath": "bin/idea-arm.sh"}, + {"os": "macOS", "arch": "aarch64", "launcherPath": "../MacOS/idea"} + ]} + """.trimIndent(), + ) + } + } + + @Test + fun `the launcher is the one product-info names for this OS and architecture`() { + val resources = productInfo.parentFile + assertEquals(resources.resolve("../MacOS/idea").normalize(), IntellijInstallation.of(productInfo, "Mac OS X", "aarch64").launcher) + assertEquals(resources.resolve("bin/idea.bat"), IntellijInstallation.of(productInfo, "Windows 11", "amd64").launcher) + assertEquals(resources.resolve("bin/idea-arm.sh"), IntellijInstallation.of(productInfo, "Linux", "aarch64").launcher) + // Java says x86_64 where product-info says amd64 + assertEquals(resources.resolve("bin/idea.sh"), IntellijInstallation.of(productInfo, "Linux", "x86_64").launcher) + assertEquals("IntelliJ IDEA", IntellijInstallation.of(productInfo, "Linux", "amd64").name) + assertEquals("263.7307.200", IntellijInstallation.of(productInfo, "Linux", "amd64").build) + } + + @Test + fun `an installation is found from its home, a macOS bundle included`() { + productInfo + assertEquals("263.7307.200", IntellijInstallation.of(directory.toFile()).build) + } + + @Test + fun `a build is compared with a since-build component by component`() { + assertTrue(IntellijInstallation.isAtLeast("263.7307.200", "262")) + assertTrue(IntellijInstallation.isAtLeast("262.1", "262")) + assertTrue(IntellijInstallation.isAtLeast("262", "262.0")) + assertFalse(IntellijInstallation.isAtLeast("261.9999", "262")) + assertFalse(IntellijInstallation.isAtLeast("262.10", "262.11")) + } + + @Test + fun `the plugins directory is the last one the IDE logged, without the note on a missing directory`() { + val log = directory.resolve("idea.log").toFile() + log.writeText( + """ + |2026-11-10 14:14:35,053 [ 103] INFO - #c.i.p.i.b.AppStarter - JVM options: [...] + | idea.config.path=/old/config + | idea.plugins.path=/old/plugins -> java.nio.file.NoSuchFileException: /old/plugins + |2026-11-10 14:17:53,461 [ 26] INFO - #c.i.p.i.b.AppStarter - JVM options: [...] + | idea.plugins.path=/Users/someone/Library/Application Support/JetBrains/IntelliJIdea2026.3/plugins + | idea.log.path=/tmp/log + """.trimMargin(), + ) + assertEquals( + "/Users/someone/Library/Application Support/JetBrains/IntelliJIdea2026.3/plugins", + IntellijInstallation.pluginsPathFromLog(log), + ) + log.writeText(" idea.plugins.path=/missing/plugins -> java.nio.file.NoSuchFileException: /missing/plugins\n") + assertEquals("/missing/plugins", IntellijInstallation.pluginsPathFromLog(log)) + assertNull(IntellijInstallation.pluginsPathFromLog(directory.resolve("absent.log").toFile())) + } + + @Test + fun `a plugin zip unpacks into the directory at its root`() { + val zip = directory.resolve("plugin.zip").toFile() + ZipOutputStream(zip.outputStream()).use { out -> + listOf("intellij-plugin/lib/plugin.jar", "intellij-plugin/bin/xtc-lsp").forEach { + out.putNextEntry(ZipEntry(it)) + out.closeEntry() + } + } + assertEquals("intellij-plugin", IntellijInstallation.rootEntry(zip)) + } +} diff --git a/build-logic/common-plugins/src/test/kotlin/InternalApiPolicyTest.kt b/build-logic/common-plugins/src/test/kotlin/InternalApiPolicyTest.kt new file mode 100644 index 00000000000..415feba257f --- /dev/null +++ b/build-logic/common-plugins/src/test/kotlin/InternalApiPolicyTest.kt @@ -0,0 +1,112 @@ +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +/** + * Protects the verifier gate against broad exceptions, dependency drift and incomplete reports. + * Uses report-format fixtures so new platform internals cannot silently inherit LSP4IJ exceptions. + */ +class InternalApiPolicyTest { + @TempDir lateinit var reports: Path + + private val api = "Internal method com.redhat.devtools.lsp4ij.LanguageServiceAccessor.getStartedServers() : java.util.Set" + private val policy get() = listOf("# Exact reviewed dependency", "lsp4ij=0.21.0", api) + + @Test + fun `accepts repeated uses of an exact signature across IDE reports`() { + report("IDE-A", listOf(usage(api), usage(api))) + report("IDE-B", listOf(usage(api))) + assertEquals(3, InternalApiPolicy.verify(reports, policy, "0.21.0")) + } + + @Test + fun `rejects new LSP4IJ members and IntelliJ internal APIs`() { + val unknown = "Internal method com.redhat.devtools.lsp4ij.LanguageServiceAccessor.newMethod() : void" + val platform = "Internal method com.intellij.ide.plugins.PluginManager.findEnabledPlugin() : void" + report("IDE", listOf(usage(api), usage(unknown), usage(platform))) + val error = assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + assertTrue(error.message!!.contains(unknown)) + assertTrue(error.message!!.contains(platform)) + } + + @Test + fun `a changed member signature requires review`() { + report("IDE", listOf(usage(api.replace("java.util.Set", "java.util.List")))) + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + } + + @Test + fun `debug parameter names do not change the API signature`() { + val member = "Internal method com.redhat.devtools.lsp4ij.LanguageServiceAccessor.hasAny" + val types = "(com.intellij.psi.PsiFile, java.util.function.Predicate) : boolean" + report("IDE-A", listOf(usage("$member(com.intellij.psi.PsiFile arg0, java.util.function.Predicate arg1) : boolean"))) + report("IDE-B", listOf(usage("$member(com.intellij.psi.PsiFile file, java.util.function.Predicate filter) : boolean"))) + assertEquals(2, InternalApiPolicy.verify(reports, listOf("lsp4ij=0.21.0", member + types), "0.21.0")) + } + + @Test + fun `parameter normalization preserves types order and arity`() { + val member = "Internal method com.redhat.devtools.lsp4ij.LanguageServiceAccessor.hasAny" + val policy = listOf("lsp4ij=0.21.0", "$member(com.intellij.psi.PsiFile, java.util.function.Predicate) : boolean") + for (parameters in listOf( + "com.intellij.psi.PsiElement file, java.util.function.Predicate filter", + "java.util.function.Predicate filter, com.intellij.psi.PsiFile file", + "com.intellij.psi.PsiFile file", + )) { + report("IDE", listOf(usage("$member($parameters) : boolean"))) + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + } + } + + @Test + fun `rejects both configured and verifier-resolved dependency upgrades`() { + report("IDE", listOf(usage(api)), dependency = "0.22.0") + assertThrows(IllegalArgumentException::class.java) { InternalApiPolicy.verify(reports, policy, "0.22.0") } + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + } + + @Test + fun `missing or truncated reports cannot pass`() { + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + val directory = report("IDE", listOf(usage(api)), count = 2) + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + Files.delete(directory.resolve("internal-api-usages.txt")) + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + } + + @Test + fun `a clean verdict does not require an empty usages file`() { + val directory = report("IDE", emptyList()) + Files.delete(directory.resolve("internal-api-usages.txt")) + assertEquals(0, InternalApiPolicy.verify(reports, policy, "0.21.0")) + } + + @Test + fun `unrecognized report text fails rather than being skipped`() { + report("IDE", listOf("Unrecognized internal API finding")) + assertThrows(IllegalStateException::class.java) { InternalApiPolicy.verify(reports, policy, "0.21.0") } + } + + @Test + fun `exception file cannot include platform APIs or package wildcards`() { + for (entry in listOf("Internal class com.intellij.InternalClass", "Internal class com.redhat.devtools.lsp4ij.*")) { + assertThrows(IllegalArgumentException::class.java) { + InternalApiPolicy.verify(reports, listOf("lsp4ij=0.21.0", entry), "0.21.0") + } + } + } + + private fun usage(symbol: String): String = "$symbol is invoked in org.xtclang.idea.lsp.Adapter.run() : void. This method is marked internal." + + private fun report(ide: String, usages: List, count: Int = usages.size, dependency: String = "0.21.0"): Path { + val directory = Files.createDirectories(reports.resolve("$ide/plugins/org.xtclang.idea/0.4.4-SNAPSHOT")) + Files.writeString(directory.resolve("verification-verdict.txt"), if (count == 0) "Compatible." else "Compatible. $count usages of internal API") + Files.writeString(directory.resolve("dependencies.txt"), "+--- com.redhat.devtools.lsp4ij:$dependency\n") + Files.writeString(directory.resolve("internal-api-usages.txt"), usages.joinToString("\n")) + return directory + } +} diff --git a/build-logic/common-plugins/src/test/kotlin/MarketplaceListingTest.kt b/build-logic/common-plugins/src/test/kotlin/MarketplaceListingTest.kt new file mode 100644 index 00000000000..6064846d356 --- /dev/null +++ b/build-logic/common-plugins/src/test/kotlin/MarketplaceListingTest.kt @@ -0,0 +1,42 @@ +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.xml.sax.SAXParseException + +/** + * Reads the newest version from the Marketplace's plugin repository listing, as `plugins/list` answers it for a + * channel, such as LSP4IJ's nightly, and an IDE line: the listings here are shaped as it answered on 2026-10-27. + */ +class MarketplaceListingTest { + @Test + fun `takes the version of the newest upload, wherever the listing names it`() { + val listing = + """ + "Plugin Development" + + + LSP4IJcom.redhat.devtools.lsp4ij0.21.1-20261027-145307 + + LSP4IJcom.redhat.devtools.lsp4ij + LSP4IJ

not this one]]>
+ 0.21.1-20261027-160212
+
+ """.trimIndent() + assertEquals("0.21.1-20261027-160212", MarketplaceListing.newestVersion(listing)) + } + + @Test + fun `an empty listing names no version`() { + assertNull(MarketplaceListing.newestVersion("")) + assertNull(MarketplaceListing.newestVersion("")) + } + + @Test + fun `a listing with a document type is refused`() { + val listing = + """]>""" + + "&v;" + assertThrows(SAXParseException::class.java) { MarketplaceListing.newestVersion(listing) } + } +} diff --git a/build-logic/common-plugins/src/test/kotlin/ServerLogTailsTest.kt b/build-logic/common-plugins/src/test/kotlin/ServerLogTailsTest.kt new file mode 100644 index 00000000000..e496c590ab1 --- /dev/null +++ b/build-logic/common-plugins/src/test/kotlin/ServerLogTailsTest.kt @@ -0,0 +1,56 @@ +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption.APPEND + +/** + * Protects the development IDE log tailer from losing or replaying output as servers start and logs roll + * over. Uses real temporary files to check partial UTF-8 lines, bounded reads and exclusion of unrelated + * files or symlinks without launching an IDE. + */ +class ServerLogTailsTest { + @TempDir lateinit var root: Path + + @Test + fun `oversized lines keep moving without splitting UTF-8 characters`() { + val log = Files.createDirectory(root.resolve("server-1-1")).resolve("server.log") + val text = "é".repeat(40_000) + Files.writeString(log, "$text\n") + val first = ServerLogTails.poll(root, emptyMap()) + val second = ServerLogTails.poll(root, first.positions) + assertEquals(text, (first.lines + second.lines).joinToString("")) + } + + @Test + fun `tail skips old output follows new sessions and handles rollover`() { + val first = Files.createDirectory(root.resolve("server-1-1")).resolve("server.log") + Files.writeString(first, "old output\n") + val initial = ServerLogTails.poll(root, emptyMap(), skipExisting = true) + assertEquals(emptyList(), initial.lines) + Files.writeString(first, "héllo\npartial", APPEND) + val appended = ServerLogTails.poll(root, initial.positions) + assertEquals(listOf("héllo"), appended.lines) + assertEquals(emptyList(), ServerLogTails.poll(root, appended.positions).lines) + Files.writeString(first, " line\n", APPEND) + val completed = ServerLogTails.poll(root, appended.positions) + assertEquals(listOf("partial line"), completed.lines) + Files.move(first, first.resolveSibling("server.2026-10-06.0.log")) + Files.writeString(first, "rolled\n") + val next = Files.createDirectory(root.resolve("server-2-2")).resolve("server.log") + Files.writeString(next, "started\n") + assertEquals(setOf("rolled", "started"), ServerLogTails.poll(root, completed.positions).lines.toSet()) + } + + @Test + fun `tail tolerates absent files and excludes foreign directories and symlinks`() { + assertEquals(emptyMap(), ServerLogTails.poll(root.resolve("absent"), emptyMap()).positions) + val foreign = Files.createDirectory(root.resolve("foreign")).resolve("server.log") + Files.writeString(foreign, "unrelated\n") + Files.createDirectory(root.resolve("server-1-1")) + Files.createSymbolicLink(root.resolve("server-2-2"), foreign.parent) + Files.createSymbolicLink(root.resolve("server-1-1/server.log"), foreign) + assertEquals(emptyList(), ServerLogTails.poll(root, emptyMap()).lines) + } +} diff --git a/build-logic/common-plugins/src/test/kotlin/TestReportRetentionTest.kt b/build-logic/common-plugins/src/test/kotlin/TestReportRetentionTest.kt new file mode 100644 index 00000000000..7ff0cc3419f --- /dev/null +++ b/build-logic/common-plugins/src/test/kotlin/TestReportRetentionTest.kt @@ -0,0 +1,75 @@ +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.FileTime + +/** + * Verifies that playbook cleanup reclaims disposable IDE caches without deleting reports, pinned artifacts, + * unfinished runs or Git checkouts. Real directory and symlink fixtures protect the destructive cleanup + * boundary. + */ +class TestReportRetentionTest { + @TempDir lateinit var root: Path + + private fun run(name: String, time: Long, completed: Boolean = true): Path { + val run = Files.createDirectories(root.resolve(name)) + Files.writeString(run.resolve("results.json"), "{\"failures\": 1}") + Files.writeString(Files.createDirectory(run.resolve("workspace")).resolve("Sample.x"), "module Sample {}") + if (completed) { + Files.writeString(run.resolve(".completed"), "IDE exited") + Files.setLastModifiedTime(run.resolve(".completed"), FileTime.fromMillis(time)) + } + return run + } + + @Test + fun `retention preserves receipts recent payloads pins and unfinished runs`() { + val old = run("run-old", 1) + val pinned = run("run-pinned", 2) + Files.createFile(pinned.resolve(".keep-artifacts")) + val recent = run("run-recent", 3) + val active = run("run-active", 4, completed = false) + Files.writeString(old.resolve("client-trace.log"), "trace") + TestReportRetention.prune(root, keep = 1) + assertTrue(Files.exists(old.resolve("results.json"))) + assertFalse(Files.exists(old.resolve("workspace"))) + assertFalse(Files.exists(old.resolve("client-trace.log"))) + listOf(pinned, recent, active).forEach { assertTrue(Files.exists(it.resolve("workspace/Sample.x"))) } + } + + @Test + fun `completed IntelliJ sandboxes release caches but retain recent diagnostic logs`() { + val old = run("run-old", 1) + val recent = run("run-recent", 2) + fun sandbox(run: Path): Path = root.resolve("out/ide-tests/tests/IU-1/XtcCompilerPlaybook-${run.fileName}") + listOf(old, recent).forEach { run -> + listOf("system", "config", "plugins", "temp", "log", "reports").forEach { name -> + Files.writeString(Files.createDirectories(sandbox(run).resolve(name)).resolve("data"), "generated") + } + } + TestReportRetention.prune(root, keep = 1) + assertFalse(Files.exists(sandbox(old).resolve("log"))) + assertFalse(Files.exists(sandbox(recent).resolve("system"))) + assertTrue(Files.exists(sandbox(recent).resolve("log/data"))) + assertTrue(Files.exists(sandbox(recent).resolve("reports/data"))) + } + + @Test + fun `cleanup never follows payload symlinks or removes Git worktrees`() { + val old = run("run-old", 1) + run("run-new", 2) + val outside = Files.createDirectories(root.resolve("source")) + Files.writeString(outside.resolve("precious.x"), "module Precious {}") + Files.createSymbolicLink(old.resolve("workspace/external"), outside) + val worktree = Files.createDirectories(old.resolve("unexpected-worktree")) + Files.writeString(worktree.resolve(".git"), "gitdir: preserve") + TestReportRetention.prune(root, keep = 1) + TestReportRetention.prune(root, keep = 1) + assertTrue(Files.exists(outside.resolve("precious.x"))) + assertTrue(Files.exists(worktree.resolve(".git"))) + assertFalse(Files.exists(old.resolve("workspace"))) + } +} diff --git a/build.gradle.kts b/build.gradle.kts index 2c86361a3f4..457f23c591a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -219,3 +219,47 @@ dockerTaskAliases.forEach { (alias, target) -> dependsOn(dockerSubproject.task(":$target")) } } + +/* + * The compiler playbooks in a Linux container, on a virtual display, so that no IDE window opens on the desktop and + * nothing typed there reaches the test IDE: lang/intellij-plugin/container/playbook.sh, with the container command + * that org.xtclang.docker.command names, as the docker build does (docker by default; podman with + * -Porg.xtclang.docker.command=podman). -PplaybookCases=X303,X337 selects cases, all of them without it. The first + * run builds the image; cases that run the XDK's tools or a Gradle project's build, such as X303, X336 and X337, need + * containerPlaybookBuild first. + */ +val playbookScript = layout.projectDirectory.file("lang/intellij-plugin/container/playbook.sh").asFile.absolutePath +val containerCommand = xdkProperties.string("org.xtclang.docker.command", "docker") +val playbookCases = providers.gradleProperty("playbookCases").orElse("all") + +fun containerPlaybookTask(name: String, description: String, vararg mode: String) = + tasks.register(name) { + group = "playbook" + this.description = description + commandLine(playbookScript, *mode) + environment("XTC_CONTAINER_COMMAND", containerCommand.get()) + } + +containerPlaybookTask("containerPlaybookImage", "Build the playbook container's image, localhost/xvm-playbook.", "image") +val containerPlaybookBuild = containerPlaybookTask( + "containerPlaybookBuild", + "Run xdk:installDist and publishLocal in the playbook container, for the cases that need them.", + "build" +) +val containerPlaybookIntellij = containerPlaybookTask( + "containerPlaybookIntellij", + "Run IntelliJ's features playbook in the container, for -PplaybookCases or all cases.", + "playbook", playbookCases.get() +) +val containerPlaybookVsCode = containerPlaybookTask( + "containerPlaybookVsCode", + "Run VS Code's playbook in the container, for -PplaybookCases or all cases.", + "vscode", playbookCases.get() +) +containerPlaybookIntellij.configure { mustRunAfter(containerPlaybookBuild) } +containerPlaybookVsCode.configure { mustRunAfter(containerPlaybookBuild, containerPlaybookIntellij) } +tasks.register("containerPlaybook") { + group = "playbook" + description = "Run both editors' playbooks in the container, IntelliJ's first, for -PplaybookCases or all cases." + dependsOn(containerPlaybookIntellij, containerPlaybookVsCode) +} diff --git a/doc/build.md b/doc/build.md index 5c5a8a3e0f9..43ba6b6b4f3 100644 --- a/doc/build.md +++ b/doc/build.md @@ -356,7 +356,7 @@ The current semi-manual process looks like this: 2) To publish the plugin to Gradle Plugin Portal: ./gradlew :plugin:publishPlugins (publish the plugin to gradlePortal) 3) To publish the XDK distro to Maven Central: (... TODO ... ) -You can already refer to the XDK and the XTC Plugin as external artifacts for your favourite +You can already refer to the XDK and the XTC Plugin as external artifacts for your favorite XTC project, either by manually setting up a link to the XTC Org GitHub Maven Repository like this: ``` diff --git a/doc/DOCUMENTATION.md b/doc/documentation.md similarity index 100% rename from doc/DOCUMENTATION.md rename to doc/documentation.md diff --git a/doc/xtc-cli.md b/doc/xtc-cli.md index e47688fb4f0..6feb529b3a2 100644 --- a/doc/xtc-cli.md +++ b/doc/xtc-cli.md @@ -251,6 +251,7 @@ xtc run [options] [args...] | `-I ` | Injection values | | `-J, --jit` | Enable JIT compiler | | `--no-recompile` | Disable automatic recompilation | +| `--debug ` | Run under a debugger outside the program, such as an IDE's (see below) | ### Examples @@ -259,8 +260,26 @@ xtc run myapp.xtc xtc run -L lib/ myapp.xtc xtc run -M main myapp.xtc xtc run -I config=prod myapp.xtc arg1 arg2 +xtc run --debug listen:127.0.0.1:5005,suspend myapp.xtc ``` +### Debugging from an editor + +`--debug listen|connect::[,suspend][,token=][,agent=]` runs the program, +with `xtc run`, `xec` or `xtc test`, under a debugger outside it, such as VS Code's or IntelliJ's: + +| Part | Meaning | +|------|---------| +| `listen` | Wait for a debugger to attach, as an editor's attach configuration does, and for the next once one has left | +| `connect` | Connect to a debugger that waits for the program, as the editors' Gradle-mode and command-line Debug do | +| `:` | This machine's; `listen` on port 0 takes a free port and prints it | +| `suspend` | Hold the program at its first line until a debugger has set it up | +| `token` | Refuse a debugger that does not send this token | +| `agent` | Load another debugger agent than the XDK's | + +The XDK's javatools has the debugger's agent, so `--debug` needs no `agent=`. A program run with +`--jit` cannot be debugged. + ## xtc test - Testing Runs tests in an Ecstasy module using the xunit framework. @@ -279,7 +298,9 @@ xtc test [options] | `-g, --test-group ` | Run tests with specific @Test group | | `-p, --test-package ` | Run tests in specific package | | `-t, --test-method ` | Run specific test method | -| `--xunit-out ` | Output directory for test results | +| `-x, --xunit-out ` | Output directory for test results | +| `-I xvm.xunit.report=json` | Also print xunit's `json` report, an event per line, for a tool that reads the run's output, such as an IDE | +| `--debug ` | Run the tests under a debugger outside the program, as `xtc run` does | ### Examples @@ -287,8 +308,17 @@ xtc test [options] xtc test myapp.xtc xtc test -c MyTests myapp.xtc xtc test --test-group integration myapp.xtc +xtc test -t tests.MathTest.adds -I xvm.xunit.report=json myapp.xtc ``` +With `--deduce`, `xtc test` also finds a test module by its name in the project's `src/test/x`, where +`xtc init` puts it, such as `xtc test --deduce myappTest` in the project's directory. Such a module +compiles with the project's modules in `src/main/x`, which it imports, into the project's `build` +directory, and again when one of them has changed; `xtc build --deduce myappTest` and `xtc run` +compile it the same way. + +See the [xunit README](../lib_xunit/README.md#running-tests) for selecting tests and the report's events. + ## xtc disass - Disassembly Disassembles a compiled `.xtc` module to inspect its contents. @@ -372,10 +402,10 @@ java -jar javatools.jar run -L xdk.xtc -L myapp.xtc myapp.xtc ## IntelliJ IDEA Integration -The XTC IntelliJ plugin provides a **New Project** wizard that offers the same project types and options as `xtc init`: +The Ecstasy IntelliJ plugin provides a **New Project** wizard that offers the same project types and options as `xtc init`: 1. **File → New → Project** -2. Select **XTC** from the generators list +2. Select **Ecstasy** from the generators list 3. Configure: - **Project name** and **Location** - **Project type**: Application, Library, or Service diff --git a/gradle.properties b/gradle.properties index 17ed32a6236..b0de4bad595 100644 --- a/gradle.properties +++ b/gradle.properties @@ -38,7 +38,7 @@ org.gradle.vfs.verbose=false # JVM and Gradle daemon flags. The default memory usage is 700M for a Daemon, which can be slow in extreme environments. # Note: --enable-native-access=ALL-UNNAMED suppresses warnings from Gradle's native platform library -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8 +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 # Logging and warning levels @@ -101,10 +101,11 @@ includeBuildAttachManualTests=false # 2. includeBuildAttachLang: Whether lang lifecycle tasks are wired to root lifecycle tasks, # so that ./gradlew build also builds lang. Requires includeBuildLang=true. # -# Use ./gradlew lang:lsp-server:build to build LSP server separately. -# CI overrides includeBuildLang to false for push/PR builds (see commit.yml). -# -# TODO: We want to have lang part of the default build, ideally, provided that it can be shown to bring no significant overhead with warm caches. +# Both are off: lang resolves a large amount (the IntelliJ Platform, the VS Code tooling and their +# dependencies), which an ordinary build of the XDK shouldn't pay for. Turn them on for a run that +# needs lang, with -PincludeBuildLang=true -PincludeBuildAttachLang=true, or with the environment +# variables ORG_GRADLE_PROJECT_includeBuildLang=true and ORG_GRADLE_PROJECT_includeBuildAttachLang=true. +# CI passes them to the steps that build lang (see commit.yml). # includeBuildLang=false includeBuildAttachLang=false @@ -114,9 +115,7 @@ includeBuildAttachLang=false # like being able to find Ecstasy plugin features in IntelliJ with Shift+Shift. lsp.buildSearchableOptions=false -# Semantic tokens are tested and should be enabled by default in the LSP -# Treesitter is tested and should be enabled by default in the LSP. The regex based mock adapter is trivially constrained, and the xdk adapter -# is still under development, and will need the SemanticModel to be finished and integrate with the real Ecstasy Lexer/Parser. Most of this work -# is done but incomplete. +# Compiler-backed language support is the default. Explicit alternatives remain available with +# -Plsp.adapter=treesitter (syntax only) or -Plsp.adapter=mock (test fallback). lsp.semanticTokens=true -lsp.adapter=treesitter +lsp.adapter=compiler diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5bbee0fa4b5..fb7105361ad 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -81,7 +81,7 @@ lang-node-gradle-plugin = "7.1.0" # ============================================================================= # Lang tooling - IntelliJ plugin # ============================================================================= -lang-intellij-ide = "2026.1" +lang-intellij-ide = "2026.2.3" lang-intellij-platform-gradle-plugin = "2.20.1" # JUnit 4.x is required at RUNTIME by the IntelliJ Platform test harness # (com.intellij.testFramework). IntelliJ's test infrastructure internally depends on JUnit 4 @@ -89,7 +89,17 @@ lang-intellij-platform-gradle-plugin = "2.20.1" # on the testRuntimeOnly classpath, IntelliJ platform tests fail with ClassNotFoundException # for org.junit.Test or org.junit.runners.BlockJUnit4ClassRunner. lang-intellij-junit4-compat = "4.13.2" -lang-intellij-lsp4ij = "0.19.4" +lang-intellij-lsp4ij = "0.21.0" +# Test launcher libraries aligned with the Starter framework for the selected IDE. +lang-intellij-kodein = "7.26.1" +lang-intellij-coroutines = "1.10.2-intellij-2" +lang-intellij-service-messages = "2024.07" +# The newer IDE the playbook can also drive, with -PintellijPlaybookIde=2026.3: the plugin still compiles against +# lang-intellij-ide, while IDE Starter and its driver follow the IDE they drive, and kodein and coroutines follow Starter. +lang-intellij-next-ide = "2026.3" +lang-intellij-next-starter = "263.7307.200" +lang-intellij-next-kodein = "7.33.0" +lang-intellij-next-coroutines = "1.11.0-intellij-1" # ============================================================================= # Lang tooling - Tree-sitter native library @@ -228,6 +238,11 @@ lang-slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "lang-slf # ============================================================================= # JUnit 4 runtime compat for IntelliJ Platform test harness (see version comment above) lang-intellij-junit4-compat = { module = "junit:junit", version.ref = "lang-intellij-junit4-compat" } +lang-intellij-kodein = { module = "org.kodein.di:kodein-di-jvm", version.ref = "lang-intellij-kodein" } +lang-intellij-coroutines = { module = "org.jetbrains.intellij.deps.kotlinx:kotlinx-coroutines-core-jvm", version.ref = "lang-intellij-coroutines" } +lang-intellij-next-kodein = { module = "org.kodein.di:kodein-di-jvm", version.ref = "lang-intellij-next-kodein" } +lang-intellij-next-coroutines = { module = "org.jetbrains.intellij.deps.kotlinx:kotlinx-coroutines-core-jvm", version.ref = "lang-intellij-next-coroutines" } +lang-intellij-service-messages = { module = "org.jetbrains.teamcity:serviceMessages", version.ref = "lang-intellij-service-messages" } # ============================================================================= # Lang tooling - Tree-sitter native library @@ -244,4 +259,29 @@ tukaani-xz = { module = "org.tukaani:xz", version.ref = "tukaani-xz" } lang-jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "lang-jackson" } [bundles] +# One library set for the distribution and embedded compiler hosts. +xdk-libraries = [ + "xdk-aggregate", + "xdk-cli", + "xdk-collections", + "xdk-convert", + "xdk-crypto", + "xdk-ecstasy", + "xdk-json", + "xdk-jsondb", + "xdk-metrics", + "xdk-net", + "xdk-oodb", + "xdk-runner", + "xdk-sec", + "xdk-web", + "xdk-webauth", + "xdk-webcli", + "xdk-xenia", + "xdk-xml", + "xdk-xunit", + "xdk-xunit-db", + "xdk-xunit-engine", + "javatools-bridge", +] unicode = ["jakarta-activation-api", "jakarta-xml-bind-api", "jaxb-runtime"] diff --git a/javatools/README.md b/javatools/README.md index 1b0f6ee22dc..b4c8143c2f6 100644 --- a/javatools/README.md +++ b/javatools/README.md @@ -7,8 +7,10 @@ compiler, IR, runtime, etc. This project uses the library produced by the "javatools_utils" project, and the `implicit.x` resource file from the "ecstasy" project. -Long term, this project will be used to help support IDE -plug-ins for common Java-based IDEs, such as IntelliJ IDEA. +The compiler also supports the compiler-backed language server. Start with +[Compiler support for the language server](lsp-compiler.md) for a complete walkthrough of the +compiler changes, their rationale, the LSP consumers and the regression tests. The +[architecture overview](#compiler-architecture-for-language-tooling) below provides a shorter introduction. (Note: The test portion of this project may have dependencies on test frameworks.) @@ -31,6 +33,111 @@ Status: Suitable for use * AST nodes are multi-pass compiled (with optional re-queuing) via `org.xvm.compiler.ast.StageManager` +## Compiler architecture for language tooling + +The language server needs the compiler's answers while a user is editing: which declaration a +name binds to, which overload an argument fits, what type an expression has, and which diagnostics +belong to the current document revision. Running the command-line compiler and parsing its output +cannot supply those answers. Successful compilation also discards or rewrites some of the source +information that navigation needs; an unfinished buffer may never reach emission at all. + +The [compiler guide](lsp-compiler.md) explains the design rationale and changes to `javatools` +that make this possible. The compiler remains independent of LSP, IntelliJ and VS Code. +It exposes compilation results and source facts; the adapter in `lang/lsp-server` converts those +facts into immutable editor models, applies document-version checks and implements the protocol. + +| Compiler mechanism | Why an editor needs it | Code and contract | +| --- | --- | --- | +| Owned diagnostics and explicit stopping | Associate every report with an attempt, retain speculative failures appropriately, and cancel obsolete work without manufacturing source errors. | `ErrorListener`, `ErrorList`, `Reporting`; [diagnostic design](lsp-compiler.md#diagnostics-cancellation-and-reporting-scopes). | +| Snapshot inputs and compilation outcomes | Compile unsaved module/member files together and distinguish successful output from ordinary partial progress after a failure. | `EmbeddingSupport` and the `ModuleInfo` source hooks; [host snapshots and outcomes](lsp-compiler.md#source-snapshots-and-compilation-outcomes). | +| Selected semantic facts | Navigate exact overloads and argument slots without reproducing compiler resolution in the editor. Preserve references lost to constant folding. | `InvocationBinding`, `InitializerBinding` and `EmbeddingSupport.Compilation`. | +| Declaration and incomplete-source analysis | Answer bounded scope, type and argument queries while the buffer is unfinished, without presenting invalid source as executable code. | `EmbeddingSupport.analyzeDeclarations`, `analyzeIncomplete`, `CursorBinding` and `compiler.ast.partial`. | +| Source-tree copying | Probe proposed syntax on an independently parented tree without changing the source tree being inspected. | `AstNode.copyTree` and its covariant specializations. | +| Tolerant repositories and explicit dependency validation | Preserve best-effort repository lookup while letting the adapter validate the specific dependency bytes it accepts. | `DirRepository`, `FileRepository`, `LinkedRepository` and the adapter's `XdkDependency`; no new repository reporting API. | +| Scoped compiler state and cache reporting | Release operation-owned listeners and temporary state, and replay cached diagnostics to the current attempt. | `Reporting`, `TransientThreadLocal` and `TypeInfo` diagnostic replay. | +| Running a built module for an IDE | Run a named entry method with exact arguments in a reusable runtime, without the runner's HTTP host, so editors can offer Run without Gradle. | `EmbeddingSupport.runMethod`, `InterpreterControl` and `lib_runner` tasks; the language server's [embedded build and run](../lang/lsp-server/README.md#embedded-build-and-run). | + +### Preserve selected semantic facts + +`InvocationBinding` records selected methods, function signatures, constructors and argument-to- +parameter mappings at the point where the compiler establishes them. The maps use AST identity; +they do not guess a target by spelling or make a dynamic function call look like a statically +selected method. `InitializerBinding` preserves validated source facts before a constant +initializer's temporary method is discarded. Without that step, a name can compile correctly but +disappear from references, rename and highlighting after constant folding. + +These facts belong to a compilation attempt. An immutable map does not make the contained AST, +type constants or constant pool safe to share across workers. The language adapter copies the +facts it needs into detached models before releasing the attempt. Keeping this boundary explicit +is essential for a long-running editor that repeatedly replaces analyses of unsaved text. + +The five source-binding classes live in `org.xvm.compiler.ast.bindings`, alongside `ast.partial`. +Compiler-owned collectors write the facts; a one-operation `ast.BindingAst` bridge associates +lambda parameter registers with written declarations without widening existing AST members. +See [binding ownership and package boundaries](lsp-compiler.md#keep-binding-facts-and-their-writers-together). + +### Analyze declarations and unfinished source explicitly + +`analyzeDeclarations` resolves declaration structure without promising a fully validated or +emitted module. `analyzeIncomplete` retains supported incomplete syntax and reports cursor facts +from its real lexical context. Its cursor overloads support a broader set of editing positions +than the deliberately narrow trailing-expression overload; their Javadoc defines each boundary. +Unsupported damage remains a diagnostic rather than a guessed semantic result. + +Incomplete nodes and their analysis helpers live in `org.xvm.compiler.ast.partial`: +`PartialQueries` coordinates inspection, `PartialCallResolver` and `PartialConstructionResolver` +fit candidates, `PartialArgument` builds disposable argument proposals, and `CursorScope` captures +visible names and resolves unfinished type headers. These helpers are package private. + +Java subpackages do not share package access. One internal bridge, `ast.PartialAst`, lets these +helpers reuse ordinary parenting, validation, argument fitting, construction preparation and +scope lookup. Existing AST members retain their protected or package-private visibility. The +bridge delegates compiler operations; the partial package decides which incomplete syntax and +candidate facts to retain. It neither introduces another type checker nor exposes compiler +contexts as a host API. Embedding hosts continue to consume `EmbeddingSupport` results. + +Candidate fitting uses ordinary compiler rules on trial syntax. A possible completion is not a +selected invocation, and partial analysis never supplies an executable module. This distinction +lets signature help and completion use authentic compiler information without weakening the +normal validation or emission contract. + +### Copy source trees and preserve ordinary compiler behavior + +The old pervasive `clone()` calls obscure an important distinction: `Object.clone()` starts with +a shallow field copy, while AST overrides selectively copy children and repair ownership. The +name alone does not tell a caller which bindings, transient state or parents remain shared, and +adding a field can silently change what a clone retains. A clone of a validated node is therefore +not equivalent to freshly parsed syntax. This is especially confusing for speculative editor +queries, which need to know exactly what a trial may mutate without affecting the original. + +`copyTree()` names the source-tree copy contract and gives expression/statement callers covariant +return types. Its contract covers child ownership, source metadata and the existing stage-aware +copy behavior; it is not a general deep copy of an entire compiler or constant pool. Specialized +incomplete nodes copy and adopt their children explicitly. The existing `clone()` entry point +continues to use the same copying machinery so the two APIs cannot drift independently. + +The compiler's callable typing and fitting corrections serve both ordinary compilation and +cursor probes. Boolean operand corrections preserve evaluation and side effects even when the +result is constant. These are compiler correctness rules, not alternate semantics enabled by an +LSP flag. The separate removal of the abandoned binary AST does not remove the source AST used +by tooling. The serialized format contains no BAST metadata slot; older `.xtc` files must be +recompiled with the current compiler. + +Repository lookup remains tolerant: an unreadable candidate can be skipped while another file +or repository supplies the module. An unrelated failed candidate does not establish why an import +is missing. The LSP validates its explicitly configured dependency bytes before accepting immutable +snapshots; it does not infer dependency errors from repository scans. The +[diagnostic guide](lsp-compiler.md#diagnostics-cancellation-and-reporting-scopes) explains the +intentional Java API migrations, retained CLI behavior and regression evidence; +it does not claim universal source or binary compatibility for old Java embedding code. + +## Compiler diagnostics and error listeners + +The detailed design and migration guide now lives in +[Compiler support for the language server](lsp-compiler.md#diagnostics-cancellation-and-reporting-scopes). +It explains [why probes and final propagated listeners replace implicit suppression](lsp-compiler.md#why-explicit-probes-and-a-final-propagated-listener), +diagnostic ownership, scoped reporting, cancellation, compatibility and regression evidence. + ## Assembler Status: Suitable for use @@ -50,6 +157,11 @@ LLVM-based adaptive compiler). * Command line is `org.xvm.tool.Runner` * Implementation in `org.xvm.runtime` package +* Debugging by a host such as an IDE: `org.xvm.runtime.SessionDebugger`, a debugger driven by + calls, and `org.xvm.runtime.debug`, its protocol (`DebugProtocol`, `DebugCommands`) and the + agent (`DebugAgent`), which javatools registers as its `org.xvm.api.DebuggerProvider`, so + `xec`, `xtc run` and `xtc test` take `--debug` without `agent=` + (see [the CLI reference](../doc/xtc-cli.md#debugging-from-an-editor)) **Warning:** This runtime is not speedy, by any stretch; this is expected, because it is only intended as a proof-of-concept. @@ -57,4 +169,4 @@ The runtime is currently implemented as an interpreter, and the interpreter (which would be naturally slow to begin with) has not been optimized. Its purpose is to be malleable and easy to test, so that we could prove out the design of the compiler -and the Ecstasy IR. \ No newline at end of file +and the Ecstasy IR. diff --git a/javatools/build.gradle.kts b/javatools/build.gradle.kts index 36368859a9f..49eff88bd05 100644 --- a/javatools/build.gradle.kts +++ b/javatools/build.gradle.kts @@ -57,6 +57,8 @@ dependencies { implementation(libs.slf4j.nop) // to avoid startup warnings while preserving no-op logging testCompileOnly(libs.jetbrains.annotations) testImplementation(libs.javatools.utils) + // the waits that go through the watchdog of hangs (org.xvm.testing.Await) + testImplementation(testFixtures(libs.javatools.utils)) // the runner and the runtime's system modules; the runner brings the libraries it needs testXtcModules(libs.xdk.ecstasy) testXtcModules(libs.xdk.runner) @@ -165,9 +167,11 @@ val generateBuildInfo = tasks.register("generateBuildInfo") { (project.extensions.extraProperties["versionDetails"] as groovy.lang.Closure).call() } - gitCommit.set(versionDetailsProvider.map { it.gitHashFull ?: "unknown" }) + // A copy of the tree without .git, such as the playbook container's, names its commit and branch in XTC_COMMIT + // and XTC_BRANCH: asking git there finds no repository, and its version plugin prints a stack trace for each. + gitCommit.set(providers.environmentVariable("XTC_COMMIT").orElse(versionDetailsProvider.map { it.gitHashFull ?: "unknown" })) - gitStatus.set(versionDetailsProvider.map { it.branchName ?: "detached-head" }) + gitStatus.set(providers.environmentVariable("XTC_BRANCH").orElse(versionDetailsProvider.map { it.branchName ?: "detached-head" })) // Put it under build/, so Gradle owns it outputFile.set(layout.buildDirectory.file("generated/resources/build-info/build-info.properties")) @@ -259,3 +263,19 @@ val versionOutputTest = tasks.register("versionOutputTest") { logger.info("[javatools] Verifying version output contains expected git and API information") } } + +// The error listeners' tests, which javatools/lsp-compiler.md runs on their own; they need no installed XDK modules +tasks.register("errorListenerTest") { + description = "Test the error listeners: branches, sites, silence, aborts, cancellation, the host boundary and the launcher" + group = VERIFICATION_GROUP + + val testSourceSet = sourceSets.test.get() + testClassesDirs = testSourceSet.output.classesDirs + classpath = testSourceSet.runtimeClasspath + useJUnitPlatform() + filter { + includeTestsMatching("org.xvm.asm.ErrorListener*Test") + includeTestsMatching("org.xvm.api.ErrorListenerBoundaryTest") + includeTestsMatching("org.xvm.tool.LauncherErrorHandlingTest") + } +} diff --git a/javatools/lsp-compiler.md b/javatools/lsp-compiler.md new file mode 100644 index 00000000000..2e6e4df0489 --- /dev/null +++ b/javatools/lsp-compiler.md @@ -0,0 +1,1248 @@ +# Compiler support for the language server + +The compiler-backed language server uses the Ecstasy compiler to answer questions about source +code: which declaration a name refers to, which overload a call selects, what types are visible +at a cursor, and whether an edit preserves those relationships. This document explains the +changes to `javatools` that make those answers available, why each change is needed, and where +the language server consumes it. + +The central change is to make compilation an operation that a host can own and inspect. A host +supplies a consistent source snapshot, dependencies, a diagnostic destination and a cancellation +policy. The compiler returns the structures and facts that the operation actually established. +An unsuccessful compilation can expose useful syntax and partial progress without claiming that +it produced an executable module. + +These mechanisms are general Java embedding APIs. `javatools` does not depend on LSP4J, LSP4IJ, +IntelliJ, VS Code, TextMate or tree-sitter. Protocol messages, document versions, scheduling, +editor presentation and refactoring edits remain in `lang/lsp-server`. + +This guide describes the combined implementation, including supporting changes that can be +reviewed or merged independently. The [diagnostic design and migration guide](#diagnostics-cancellation-and-reporting-scopes) +below includes the error-listener rationale and API examples. Java source links are relative to +this directory and can be followed from a rendered Markdown document. + +## Contents + +- [Why the previous compiler interface was insufficient](#why-the-previous-compiler-interface-was-insufficient) +- [One editor request through the compiler](#one-editor-request-through-the-compiler) +- [Source snapshots and compilation outcomes](#source-snapshots-and-compilation-outcomes) +- [Diagnostics, cancellation and reporting scopes](#diagnostics-cancellation-and-reporting-scopes) +- [Why explicit probes and a final propagated listener](#why-explicit-probes-and-a-final-propagated-listener) +- [Preserving semantic facts from successful validation](#preserving-semantic-facts-from-successful-validation) +- [Declaration analysis and unfinished source](#declaration-analysis-and-unfinished-source) +- [AST traversal, copying and the partial-analysis package](#ast-traversal-copying-and-the-partial-analysis-package) +- [Constant pools, caches and dependency inputs](#constant-pools-caches-and-dependency-inputs) +- [Why ordinary AST and runtime files also changed](#why-ordinary-ast-and-runtime-files-also-changed) +- [Compatibility and limits](#compatibility-and-limits) +- [Tests and reproduction](#tests-and-reproduction) +- [Appendix: file-by-file compiler changes](#appendix-file-by-file-compiler-changes) + +## Why the previous compiler interface was insufficient + +A batch compiler can report errors, abandon its intermediate state and return failure. An editor +must continue helping while that failure is the normal state of the buffer. It must also avoid +publishing an answer from an older version after the user has typed something else. + +| Previous limitation | Concrete editor failure | Compiler mechanism added or corrected | +| --- | --- | --- | +| A failed embedding compilation returned no module and lost access to its intermediate state. | An erroneous method body could leave the host with no way to inspect the declarations or source tree that had already been built. | `Compilation` retains available syntax, file structure, pool and validated binding facts independently of executable success. | +| Disk discovery and source reading were coupled to the CLI's module loading. | A saved module root and an unsaved member file could be analyzed as different revisions; virtual or newly created members were difficult to represent. | `ModuleInfo` exposes source-text and membership hooks while preserving normal package/resource assembly. | +| Listener substitution, null-to-discard conventions and implicit stop behavior obscured who owned diagnostics. | Reports could disappear during a probe or cached lookup; a host could be stopped by a budget it had not chosen. | Explicit reporting ownership, branches, suppression reasons, stateful collectors and abort queries. | +| Useful bindings were temporary validation decisions, or were rewritten away. | Name matching could select the wrong overload; constant folding could make a reference disappear from rename. | Attempt-owned call collectors, initializer provenance and source-binding accessors. | +| A valid parse and completed validation were prerequisites for many semantic queries. | Completion after a member-access dot or inside an unfinished argument could not use the real lexical scope. | Explicit cursor analysis, bounded recovery nodes and compiler-fitted candidate facts. | +| Compiler structures and working context were mutable and sometimes retained operation state. | Repeated editor requests could retain old listeners or accidentally inspect constants under the wrong working pool. | Scoped reporting/validation state, explicit pool scopes and detached adapter snapshots. | + +Parsing command-line output cannot recover selected overloads, local-register identity or facts +discarded by a rewrite. A separate editor type checker would have to reproduce Ecstasy's access, +generic inference, composition and overload rules. Instead, the compiler exports decisions where +it already makes them, and partial queries reuse its existing validation and fitting operations. + +## One editor request through the compiler + +```mermaid +flowchart TD + A[Editor document and workspace revisions] --> B[LSP adapter captures source and dependency inputs] + B --> C[Serialized compiler worker] + C --> D[EmbeddingSupport and the ordinary compiler pipeline] + D --> E[Attempt-owned diagnostics, syntax, structures and bindings] + E --> F[SemanticModelBuilder copies facts into detached models] + F --> G[Adapter checks that input revisions are still current] + G --> H[LSP response or diagnostic publication] +``` + +The ownership transition happens when **SemanticModelBuilder copies facts into detached models**. +An immutable collection of AST references is still a collection of compiler-owned objects. The host +must finish inspecting and copying them while it exclusively owns the attempt; it cannot hand the +AST or constant pool to arbitrary request threads. + +The production path is visible in these files: + +| Responsibility | Implementation | +| --- | --- | +| Capture module text, membership, URI aliases and resources; create fresh compilation inputs. | [XdkSources.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/XdkSources.kt) | +| Schedule and serialize work, cancel obsolete attempts, and gate publication against current inputs. | [XdkAdapter.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/XdkAdapter.kt) | +| Call and time the embedding APIs for ordinary, dependency and refactoring compilations. | [CompilerCalls.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/CompilerCalls.kt) | +| Drive parsing, linking, resolution, validation and emission, or stop at the requested analysis goal. | [EmbeddingSupport.java](src/main/java/org/xvm/api/EmbeddingSupport.java), [tool/Compiler.java](src/main/java/org/xvm/tool/Compiler.java), [compiler/Compiler.java](src/main/java/org/xvm/compiler/Compiler.java) | +| Convert compiler objects and source positions to independently owned semantic data. | [SemanticModelBuilder.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/SemanticModelBuilder.kt), [SemanticModel.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/SemanticModel.kt), [PartialSemanticModel.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/PartialSemanticModel.kt) | + +The compiler does not know a document's LSP version or decide whether a response is stale. The +adapter supplies cancellation and rejects superseded results. Compiler cancellation checkpoints +make that policy effective during compilation rather than only after it finishes. + +## Source snapshots and compilation outcomes + +### Configure compiler libraries without starting an interpreter + +`EmbeddingSupport.configure` accepts the core repository used by compilation. The `XDK_HOME` +fallback includes both `lib/` and `javatools/`: the compiler needs the bootstrap turtle module +(`mack.xtclang.org`) and native bridge (`_native.xtclang.org`), not just the ordinary libraries. +Missing a bootstrap module prevents compilation before the user's source can be analyzed. + +The language server instead supplies its bundled compiler libraries through +[XdkLibraries.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/XdkLibraries.kt). Library +setup, compiler attempts and runtime/interpreter initialization are separate responsibilities; +loading the compiler's required modules does not mean executing the user's program. + +### Reuse normal module assembly with host-provided inputs + +[ModuleInfo.java](src/main/java/org/xvm/tool/ModuleInfo.java) still implements module discovery, +member/package assembly and resource association. Its host hooks let the adapter replace the +inputs to that machinery: + +- `readSource(File)` supplies the captured characters for a source file. +- `sourceEntries(File)` supplies a directory's captured source membership, including added or + removed members, instead of rediscovering membership during the attempt. +- The protected `(File sourceFile, String moduleName)` constructor establishes a source identity + without requiring the source root to exist on disk. Subclasses provide the corresponding source + tree and resource behavior. +- `getSourceTree(errors, parser)` allows a selected member to use cursor-aware parsing while the + other members use ordinary parsing and the same assembly path. +- `getParsedSources()` exposes the available per-source trees, including syntax retained when an + error prevents construction of a usable assembled root. + +The LSP implementation overrides these hooks in `XdkSources`. Open buffers take precedence over +disk text. Source URIs, source membership and captured resource inputs belong to the same analysis +request. Refactoring proofs can replay captured inputs with proposed text or path changes without +first writing those changes to the user's files. + +**Use a fresh `ModuleInfo` for each attempt.** It caches assembled compiler trees. The adapter +therefore retains immutable input values and creates a fresh `XdkSources` when replaying them, +rather than treating a previously parsed `ModuleInfo` as a reusable immutable snapshot. + +### Return progress without calling failure success + +The `compileModule(Source, ...)` and `compileModule(ModuleInfo, ...)` overloads return +`EmbeddingSupport.Compilation`. Its fields have deliberately different meanings: + +| Result member | What the host can rely on | +| --- | --- | +| `module()` / `succeeded()` | An executable module was produced successfully. A failed attempt has no successful module. | +| `file()` / `pool()` | Registration got far enough to construct a file structure and its constant pool. Their existence alone does not establish successful validation. | +| `ast()` / `parsed()` | The assembled source root is available after loading/parsing succeeded. Semantic validation may still have failed. | +| `sourceTrees()` | Available per-source syntax, including recovered trees when parsing failed. Positions belong to each tree's own `Source`; recovered trees may not yet have parent links. | +| Binding maps | Facts established by the relevant successful validations and retained by the collector. Their presence does not turn the whole compilation into a successful one. | + +`EmbeddingSupport` creates a fresh embedded compiler for the attempt and drives the normal tool +compiler pipeline. It retains state as each phase reaches it, instead of converting every failure +into a single null. Expected compiler aborts remain distinct from unexpected exceptions and +assertions, which are reported as internal failures even if an earlier source diagnostic exists. + +The existing convenience APIs in [ModuleCompiler.java](src/main/java/org/xvm/api/ModuleCompiler.java) +and `EmbeddingSupport` keep their module/null or boolean result shapes. Hosts that need intermediate +state use `compileModule` explicitly. + +For example, with `embedding` already configured and `dependencies` supplied by the host: + +```java +var reports = new ErrorList(); +var obsolete = new AtomicBoolean(); +var errors = ErrorListener.cancellable(reports, obsolete::get); +var result = embedding.compileModule(new Source(text, documentName), dependencies, errors); + +if (result.succeeded()) { + ModuleStructure module = result.module(); + // The executable result is available. +} +if (result.pool() != null) { + try (var scope = ConstantPool.withPool(result.pool())) { + // Inspect available compiler facts and copy the values needed by the host. + // This is still the compiler worker, not an arbitrary LSP request thread. + } +} +// The host still checks its input revision before publishing reports or copied facts. +``` + +The host can set `obsolete` from its request lifecycle. Cancellation need not produce a source +error, so the absence of diagnostics is not a substitute for checking whether an attempt is current. + +## Diagnostics, cancellation and reporting scopes + +### Why the old reporting model had to change + +The old error-listener API did not provide a reliable contract for embedding the compiler. +Accepting a listener at the entry point was not enough: deeper calls could substitute `BLACKHOLE`, +accept null as an implicit request to discard reports, or find a different listener through a +shared structure. The caller could receive only part of an attempt's diagnostics without knowing +anything had been lost. Suppression, ownership and stopping policy were spread across unrelated +call sites, so adding a callback at the outer API could not repair what happened underneath it. + +This caused observable failures. The default branch for a host listener imposed a one-error +budget the host had never requested: the lexer stopped at the first recoverable error and never +reported the next one. A previously created branch could also ignore its parent's later abort +request. Two regression tests reproduce those failures on the old compiler with exactly the same +test source that passes on the repaired implementation. + +Reporting also had inconsistent control-flow effects. `log` returned an abort decision, but the +fallback `RuntimeErrorListener` instead threw an exception for ERROR and FATAL. Selecting a +diagnostic destination could therefore change how the operation failed. An editor must be able to +collect an ordinary source error as data, distinguish it from an internal compiler failure, and +let the compiler stop at an appropriate checkpoint. The destination receiving a report should not +introduce a different failure path merely because that report has a particular severity. + +### Changes and enhancements + +* **Separate reporting from stopping.** `log(ErrorInfo)` delivers a diagnostic; + `isAbortDesired()` answers whether work should stop. The lexer and parser retain explicit stop + checkpoints, and forwarding listeners preserve the owner's policy. +* **Make host callbacks usable by the compiler.** `collecting` forwards structured diagnostics + while remembering severity and error codes. A bare callback receives reports but does not + implement those state queries automatically. +* **Make branching consistent.** A host's default branch buffers without an arbitrary one-error + limit. `ErrorList` branches retain their configured budget, and both observe their parent's + current abort request. +* **Make suppression explicit.** Named reasons distinguish probes, cascade suppression and + intentional discards. A derived silence discards messages while retaining the owner's stop + policy. +* **Require a reporting destination at active compilation boundaries.** An omitted listener is + rejected before compilation starts; intentional silence is supplied as an explicit listener. +* **Keep ownership stable.** Constructor-owned listeners remain attached to their operation. + Temporary suppression derives a local listener instead of overwriting the original destination. +* **Preserve diagnostic structure.** `Site` describes source spans, structures or unpositioned + reports. Severity helpers take message arguments as trailing varargs; `tee` forwards reports to + two destinations while combining their state. + +### Removed behavior and its replacement + +* **The boolean return from `log`** is replaced by a reporting call followed by an explicit policy + query where the caller needs to stop. There is one continuation query for wrappers to preserve. +* **`BLACKHOLE` and `BlackholeErrorListener`** are replaced by named suppression factories. + Removing the old symbols makes unmigrated call sites fail to compile, requiring each caller to + identify its reason for silence and whether a parent's abort policy must survive. +* **Null-to-discard fallback at compilation boundaries** is replaced by a required listener. + Passing `silent(DISCARD)` still supports a caller that deliberately wants no reports; accidentally + omitting the destination no longer looks like a successful choice of silence. +* **The silent default in `ResolutionCollector.getErrorListener()`** is removed. Implementations + must provide their destination, and `SimpleCollector` retains the listener it was given. +* **Reassigning a TypeInfo build's listener to `BLACKHOLE`** is replaced by local cascade + suppression. The original listener continues to identify the operation and its stop policy. +* **Severity-triggered exceptions in the fallback runtime listener** are replaced by recorded + abort state. The compiler or runtime caller decides how to end its operation. This does not + remove the command-line launcher's deliberate fatal-error and exit handling. +* **The listener parameter on `Expression.testFitAsType`** is removed because that operation is + a probe. Its staging and fit checks use explicit probe silence; selected validation can still + retain and merge its diagnostics. + +The legacy positional reporting overloads remain available after recompilation, but are +deprecated in favor of `Site` and trailing arguments. Their source re-anchoring behavior is tested. + +### Why explicit probes and a final propagated listener + +A probe asks a question whose answer can legitimately be “no.” For example, overload resolution +can test whether a written argument fits an `Int` parameter, reject that candidate and then +select a `String` overload. Reporting the rejected fit as a source error would diagnose a valid +program. Completion performs similar trials on proposed arguments. Silent probing is therefore +necessary; the redesign makes its policy explicit instead of removing it. + +The old representations combined several different decisions: + +| Previous pattern | What a reviewer or embedding host could not determine reliably | Explicit replacement | +| --- | --- | --- | +| Pass `null` and let a deeper helper choose a sink. | Was silence deliberate, was the caller missing a listener, and which helpers would discard, substitute or dereference it? | Require a listener at the active boundary; pass an explicit silence when that is the intended policy. | +| Pass `BLACKHOLE` for every kind of suppression. | Was this an alternative fit, a cascade after an existing error, or a deliberately unused destination? Should the current attempt's abort request still apply? | Name the suppression reason and choose standalone silence versus a wrapper derived from the current owner. | +| Reassign the operation's listener to a discard sink. | Which later validations or callbacks still reach the original owner, and can they still see cancellation or its error budget? | Keep the owner stable and derive a local listener for only the affected calls. | +| Store a listener on a shared structure and find it implicitly. | Which request owns a later cached lookup, particularly after the structure outlives the request? | Pass the current attempt's listener through compiler operations; retain diagnostics as replayable data rather than retaining their previous destination. | + +The operation-owned references in `Compiler`, `StageMgr`, `Launcher` and resolution collectors +are final where their lifetime has one owner. This prevents accidental reassignment to a sink or +another request's destination. The compiler passes that owner, or a deliberately derived child, +through the calls that perform the work. It is one **logical reporting owner**, not a requirement +that every nested operation receive the exact same listener object. + +A branch, tee, cancellation wrapper or derived silence can legitimately be a different object. +What matters is preserving the required relationship to the owner. Diagnostic retention and +error counters can change while the reference remains final. Final references do not make the +listener, parser or compiler thread-safe, nor do they make an AST immutable. + +In particular, suppressing a message and suppressing a stop request are different decisions: + +| Operation | Appropriate listener | Reports and stopping | +| --- | --- | --- | +| Validate the selected source construct. | The operation's listener. | Real diagnostics reach their owner; its state and stop policy remain available. | +| Ask an alternative-fit question within an active attempt. | `errors.silence(PROBE)`. | Trial messages are discarded, but the current parent abort/cancellation request remains visible. | +| Perform a deliberately independent silent query with no parent policy. | `silent(PROBE)`. | No reports, retained errors or abort request; this is the standalone behavior formerly supplied by a blackhole. | +| Try work whose diagnostics may be needed if it is accepted. | `errors.branch(node)`. | Reports are buffered; merge accepted reports once. The branch also observes parent abort and its own configured policy. | +| Suppress follow-on errors from already incomplete information. | `errors.silence(CASCADE)`. | Avoids flooding the owner with consequences while retaining its stop request. | + +This distinction is why replacing every old `BLACKHOLE` with the real reporting listener would +also be wrong: rejected alternatives would become user errors. Conversely, replacing everything +with standalone `silent(PROBE)` would not preserve host cancellation. Existing standalone probe +sites remain standalone where that is their deliberate contract; they do not acquire cancellation +propagation merely because the factory has a clearer name. Work that must observe the active +attempt's policy needs a derived listener and a checkpoint that queries it. + +For instance, related trials within one operation can share a derived probe: + +```java +final ErrorListener probe = errors.silence(ErrorListener.Silence.PROBE); +var firstFit = first.testFit(context, candidateType, false, probe); +var secondFit = second.testFit(context, candidateType, false, probe); +// The fit results answer the trial questions; selected validation still receives errors. +``` + +The wrapper reads the parent's current abort state; it does not snapshot that state when created. +Creating it before two branches therefore does not lose a later cancellation request. It must +still belong to that operation, not be cached globally or reused for a different compilation. +The standalone `silent(PROBE)` factory already returns a shared stateless sink, so repeated calls +do not allocate a fresh listener. A local variable is useful when it makes a group of trials +clearer, not as a required allocation optimization. Buffers and branches are different: they +retain diagnostics and must not be shared between unrelated trials just to shorten the code. + +`PROBE`, `CASCADE` and `DISCARD` are explanatory reasons, not levels or compiler modes. Changing +the enum value does not change how messages are discarded. The substantive improvement is explicit +ownership, appropriate retention and stop propagation. Keeping a deprecated `BLACKHOLE` alias was +technically possible; removing it forces old call sites to be reviewed instead of silently keeping +an unexplained policy. A caller that deliberately wants the old independent discard behavior can +still choose it explicitly. + +Non-null active listeners also eliminate two meanings for the same value: an intentionally silent +operation and an accidentally omitted destination. Boundary checks reject the latter early; +annotations describe that contract to Java/Kotlin callers and IDEs. This does not forbid meaningful +absence elsewhere, such as an inactive resolver having no current reporting scope. + +Some temporary destination changes remain necessary for parser speculation and parameterless +resolution callbacks. They are enclosed in `Reporting.Scope`, with restoration on nested, normal +and exceptional exits, rather than arbitrary reassignment of the operation's owner. The scoped +mechanism below is the deliberate exception to permanent ownership, not a return to ambient +listener swapping. + +### Why this is a prerequisite for the compiler-backed LSP + +**Reliable diagnostic ownership and abort propagation are required for the compiler-backed LSP.** +The adapter must associate reports with the requested compilation, reject unusable semantic +results, and stop obsolete work at compiler checkpoints. It cannot reconstruct diagnostics that +the compiler discarded or obtain errors from work that an unintended one-error budget prevented. +Leaving those failures in place blocks a reliable compiler-backed adapter, regardless of how much +code is added around the public entry point. These changes establish the reporting foundation; +the result, source-snapshot and cached-diagnostic APIs described in this guide complete the host-facing path. +The exact helper names and the decision to remove a legacy alias are API design choices. The +delivery, state and stopping guarantees they implement are the essential requirements. + +### Compatibility with ordinary compilation + +Normal command-line compilation continues to use the existing compiler pipeline, console and +exit policy. Users do not need new flags or changes to their Ecstasy source. The CLI supplies an +explicit discard listener only for its external delegate; its console still reports diagnostics. +`ErrorList` retains its configured serious-error budget, fatal reports still request an abort, +and the lexer/parser still stop when the applicable policy requires it. The changes repair +diagnostic delivery and listener ownership without introducing an IDE dependency into the compiler. + +There is strong regression evidence for that continuity: the full Java suite for this listener +migration executed 418 tests successfully, and all 24 rebuilt XDK modules matched a freshly built +master baseline after normalizing only module creation timestamps. The [focused checks](#historical-listener-regression-evidence) add +direct coverage of reporting, aborts, boundary validation and launcher behavior. These historical results +support confidence in ordinary compilation; they are evidence for the tested paths, not an +exhaustive guarantee for every source program or external integration. + +**The Java embedding API deliberately changes at the planned 0.5.0 boundary.** Existing Java +callers and listener implementations must migrate and recompile. Changed return types, removed +symbols, rejected nulls and the runtime listener's new exception behavior prevent a claim of +100% source or binary compatibility, even for a non-LSP Java consumer. The migration steps below +identify those changes explicitly so ordinary consumers can preserve their intended behavior. + +### Code tour and practical usage + +#### 1. Give each compilation its own diagnostic owner + +Start at [ModuleCompiler](src/main/java/org/xvm/api/ModuleCompiler.java) or +[EmbeddingSupport](src/main/java/org/xvm/api/EmbeddingSupport.java). Supply the attempt's listener +at the entry point and keep it available until the result and diagnostics have been consumed. +For example, with a configured core repository and source text: + +```java +final var errors = new ErrorList(100); +var compiler = new ModuleCompiler(coreRepository); +var module = compiler.compile(sourceText, null, errors); +var diagnostics = List.copyOf(errors.getErrors()); + +if (module == null || errors.hasSeriousErrors()) { + // Publish the diagnostics and do not treat this as a successful compilation. + return; +} +// The caller can now use the successfully compiled module. +``` + +The null argument here is the optional dependency repository, not the listener. The required +listener also reaches launcher construction and dispatch, the compiler, lexer/parser, +[StageMgr](src/main/java/org/xvm/compiler/ast/StageMgr.java), anonymous-class construction and +[Component.SimpleCollector](src/main/java/org/xvm/asm/Component.java). +[ResolutionCollector](src/main/java/org/xvm/asm/ComponentResolver.java) implementations must +return their selected destination. AST reporting and validation helpers no longer silently +accept a missing listener. + +Use a fresh collector for an independent attempt. An `ErrorList` is mutable; even `clear()` keeps +its previously seen diagnostic UIDs. A new operation should not inherit that deduplication history. +Hosts must serialize compiler access and listener callbacks. Final references and non-null +parameters do not make compiler objects or their accumulated state thread-safe. + +#### 2. Report data, then make the control-flow decision + +[ErrorListener](src/main/java/org/xvm/asm/ErrorListener.java) remains the central interface. +Its nested `ErrorInfo` carries severity, code, message arguments and location. `Site.In`, `Site.At` +and `Site.None` describe the location independently of the message: + +```java +errors.error(code, ErrorListener.in(source, start, end), arguments); +if (errors.isAbortDesired()) { + return; +} +``` + +Use `in(source, start, end)` for source text, `at(structure)` for an assembled structure, and +`NOWHERE` for a report without either location. Source positions are the encoded values returned +by `Source.getPosition()`, not raw byte offsets or LSP positions. The `error`, `warn`, `info` and +`fatal` helpers select severity and delegate to `log`; they do not transfer control themselves. + +`isAbortDesired()` and `hasSeriousErrors()` answer different questions. An operation can have +errors while remaining within its budget, and a host may request a stop without reporting an +error. Callers use the first query at stop checkpoints and the second when deciding whether a +result is usable. `hasError(code)` checks whether a particular diagnostic was recorded. + +`RuntimeErrorListener` now prints diagnostics and records an abort request for ERROR/FATAL +instead of deliberately throwing from `log`. Its abort flag remains set for that instance's +lifetime; the shared `RUNTIME` sink is not a fresh per-attempt collector. A reporting callback +can still throw its own exception, which forwarding helpers propagate. The redesign removes the +fallback sink's severity-driven exception policy, not Java exception handling in general. + +#### 3. Choose retention, callback delivery or both + +[ErrorList](src/main/java/org/xvm/asm/ErrorList.java) retains the first report for each diagnostic +UID in insertion order. ERROR/FATAL reports spend its count budget; warnings do not. Its +no-argument constructor uses the shared `ErrorListener.DEFAULT_MAX_ERRORS` (512), `FIRST_ERROR` +selects one, and `UNLIMITED` disables the count limit. A retained FATAL requests an abort with +any budget. Pass an explicit limit to `ErrorList(int)` when an operation needs a different policy. + +When a host wants every report delivered to a callback, use: + +```java +ErrorListener errors = ErrorListener.collecting(hostDiagnostics::add); +``` + +`collecting` forwards every report, including duplicates, and tracks severity and codes before +calling the consumer. It has no count budget and requests an abort on FATAL. A bare lambda +implements only `log` and inherits state queries that return false, so it is not a substitute +when the compiler relies on those queries. + +`tee(first, second)` delivers to both destinations in order and observes either destination's +abort, serious-error and code state. This supports reporting to a host while recording diagnostics +for later replay. `ErrorList.logTo(destination)` replays retained reports without clearing them; +deduplication belongs to the destination. If the first destination throws, the second is not called. + +#### 4. Buffer speculative diagnostics until a result is accepted + +`branch(node)` creates an `ErrorList.BranchedErrorListener`. Pass it to speculative work and merge +it once if that work's diagnostics should be retained: + +```java +var trial = errors.branch(node); +// Run the candidate validation with trial as its listener. +if (keepDiagnostics) { + trial.merge(); +} +``` + +Unmerged diagnostics remain local to the branch. A default host branch has an unlimited count +budget, while an `ErrorList` branch copies its parent's configured limit. Both observe the +parent's current abort request, and a local FATAL still stops the branch. A node-associated +branch re-anchors structure reports to that node's source span; explicit source spans remain intact. + +Merging an unbranched listener returns that listener. Merging a branch forwards its reports and +returns the parent without clearing the buffer. Do not repeatedly merge an accepted branch: +an `ErrorList` parent can deduplicate replay, but an arbitrary callback need not. + +#### 5. Suppress reports without losing the operation's owner + +`Silence` and `SilentErrorListener`, nested in `ErrorListener`, make suppression explicit. +`PROBE`, `CASCADE` and `DISCARD` are **reasons, not severity levels**. They have identical discard +behavior; compiler algorithms must not depend on which name is selected. + +| Reason | Use it when | +|---|---| +| `PROBE` | Testing an alternative, such as whether an expression fits a candidate type. Failure is the answer to that query, not necessarily a problem in the selected program. | +| `CASCADE` | Examining already incomplete information whose follow-on errors would obscure the original failure. | +| `DISCARD` | Deliberately ignoring this destination, such as the CLI's external delegate when its console already reports diagnostics. | + +The important behavioral distinction is between a standalone sink and a derived silence: + +```java +// No parent listener or stop policy: an independent, deliberately silent probe. +ErrorListener independent = ErrorListener.silent(ErrorListener.Silence.PROBE); + +// Suppress follow-on reports but continue to honor this operation's abort request. +ErrorListener quiet = errors.silence(ErrorListener.Silence.CASCADE); +``` + +`independent` never requests an abort. `quiet.isAbortDesired()` observes `errors`, even when +no diagnostic caused that stop request. Neither retains discarded messages; use a branch when +diagnostics might later be needed. `suppressed()` exposes a derived silence's parent, not a buffer +of hidden diagnostics. Re-silencing an existing silence returns the same instance and reason. + +For a separately constructed standalone sink, `new SilentErrorListener(reason)` needs no +explicit null parent. The two-argument constructor retains an optional parent for callers that +need its stop policy. Required factory arguments and results use `@NotNull`; intentional absence +uses `@Nullable`, including `suppressed()` on a standalone sink. Public construction boundaries +still reject invalid null arguments immediately; annotations also expose the contract to IDEs +and Kotlin callers. + +The names make intent searchable and reviewable. `BLACKHOLE` did not distinguish those purposes, +and substituting it for an active listener also removed access to that listener's stop policy. +Removing the old constant and class makes missed migrations visible to the Java compiler. A +deprecated alias was possible, but would allow unexplained suppression to survive the migration. +Callers that want exactly the old standalone discard behavior can use `silent(DISCARD)`. + +Stable ownership is equally important. In +[TypeConstant](src/main/java/org/xvm/asm/constants/TypeConstant.java), `cascade(incomplete, errors)` +derives suppression at affected calls instead of assigning a sink to the `errors` parameter. +The original destination remains available throughout the build. Constructor-owned references, +such as those in `StageMgr`, `SimpleCollector` and `Launcher`, are final so they cannot drift to +another destination during the operation. Their listeners' counters can still change. + +Non-null means that an active reporting boundary has an explicit owner or an explicit discard +policy. It does not mean that every listener field everywhere must be permanently non-null or +final: nested parser/resolver work needs temporary destinations and scoped restoration. That +ownership work is described separately below. + +#### 6. Migrate Java callers and implementations + +The planned release boundary is **0.5.0**. Recompile Java clients and listener implementations +after applying these changes; Java cannot retain compatibility overloads that differ only by +the return type of `log`. + +| Previous use | Replacement and consequence | +|---|---| +| `if (errors.log(...))` | Report first, then query `errors.isAbortDesired()` and perform the same return/throw at that checkpoint. Update custom implementations to `void log(ErrorInfo)`. | +| `BLACKHOLE` or `BlackholeErrorListener` | Choose a named standalone silence or derive one from the current owner when its abort policy must survive. | +| Null at a compilation/launcher boundary | Pass the operation's listener or explicit `silent(DISCARD)`. Missing listeners now fail at the boundary. | +| A `ResolutionCollector` relying on the default sink | Implement `getErrorListener()` and retain the intended destination. | +| `testFitAsType(..., errors)` | Remove the listener argument; the operation owns its probe policy. | +| Depending on `RUNTIME.log` to throw on severity | Check `isAbortDesired()` and explicitly choose the operation's failure handling. | +| Legacy positional reporting overloads | They remain callable after recompilation but are deprecated; migrate to `Site` and trailing message arguments. | + +### Give temporary reporting state a bounded lifetime + +`Parser.attempt()` replaces `Parser.SafeLookAhead`: use `Attempt.keep()` in place of +`keepResults()`. A kept attempt merges its buffered diagnostics, including warnings; a discarded +attempt restores its tokens and recovery state. Nested attempts observe their parent's abort +request. This removes a public nested parser class and requires source migration/recompilation at +the same breaking release boundary described above. + +Parser and resolver destinations use `Reporting` scopes, closed in reverse order with +try-with-resources. `NameResolver.getErrorListener()` is null outside `resolve`; callbacks borrow +the caller's non-null listener only during that call. These scopes restore destinations on every +exit. They do not make a parser, resolver, AST or listener safe for concurrent use. Lexical +reports also follow the active attempt, preserving lexical rollback from the parser correction. +Module-name-only scans can supply an explicit discard listener when reports are unwanted. + +Loop and try/finally label variables retain their context/listener as one immutable +`ValidationScope` value. Each statement restores its previous value on normal, early and +exceptional exits, and `Statement.validate` likewise restores its common validation context. +The temporary references stay on the AST because label-variable and jump callbacks need the +currently validating statement. No shared mutable holder is added to cloned AST nodes. +`EvalCompiler` and `ModuleInfo.Node` own final diagnostic buffers from construction onward. + +The unit regressions cover nested reporting, failed host/resolver callbacks, token rollback, +abort propagation and exceptional statement exits without requiring installed modules. Compile +and run the existing `manualTests/src/main/x/loop.x` and `exceptions.x` exercises with the built +XDK to check lazy label variables and try/finally behavior through code generation and execution. +The separate Gradle compiler-consumer test supplies its compiled modules through dependencies. + +### Replay diagnostics from cached type information + +Structures no longer discover the current listener through a mutable listener attached to a file +or constant pool. Explicit inspection supplies it, for example through +`TypeConstant.ensureTypeInfo(errors)` and `XvmStructure.log(errors, ...)`. + +[TypeConstant.java](src/main/java/org/xvm/asm/constants/TypeConstant.java) retains diagnostics from +a successful TypeInfo build and replays them on later explicit lookups. This fixes the case where +a silent probe populated a cache before the selected source use: the real consumer must still +receive the warning. An `ErrorList` deduplicates repeated reports; a raw callback can observe each +replay. Invalidation replaces the cached facts and their retained reports. + +The no-argument TypeInfo lookup remains an explicitly silent metadata query. Diagnostic-sensitive +code must choose the listener-taking operation. Some inspections build TypeInfo and can report new +diagnostics; they are compiler work, not harmless getters safe to invoke on any thread. + +### Cancel obsolete work without manufacturing errors + +`ErrorListener.cancellable` decorates the operation listener, preserving reporting, state, +branching and merging while polling the host predicate at abort checkpoints. A valid token stream +also polls; cancellation cannot depend on finding a source error. Module-member loading and phase +boundaries retain the same policy. This is cooperative cancellation, not arbitrary interruption: +the host must keep its predicate true once an attempt becomes obsolete. Other attempts have +separate policies, and the adapter still checks input revisions before publishing their results. + +## Preserving semantic facts from successful validation + +### Keep binding facts and their writers together + +`org.xvm.compiler.ast.bindings` groups `InvocationBinding`, `InitializerBinding`, `CursorBinding`, +`LambdaBindings` and `AnonymousClassBindings` alongside the `ast.partial` analysis package. These +classes describe established source associations or inspected cursor candidates; the ordinary AST +still owns validation and register allocation. The LSP consumes the resulting facts through the +embedding results and source-node accessors. + +Lambda and anonymous-class bindings expose read-only accessors that return immutable collection +snapshots. Their nested `Collector` classes provide the compiler-facing recording operations and +retain the method/construction identity used to reject facts requested through a copied node. +Each writer stays with its existing validation context or construction. It does not introduce a +global cache, a second capture pass or a new lifecycle for the AST. + +The [BindingAst](src/main/java/org/xvm/compiler/ast/BindingAst.java) bridge has one operation: +associate an existing lambda parameter register with its written declaration and return the name +token. That operation needs the AST's protected parameter lists and package-private binding setter. +Keeping it in `ast` preserves their visibility. The other binding classes use public source +accessors and require no additional AST bridge. The collector and bridge APIs are public for Java +cross-package access, but are internal compiler operations; embedding hosts read published facts. + +### Names, declarations and source identity + +The source AST carries locations that assembled components alone cannot provide. Tooling accessors +expose written declaration-name tokens, documentation, local registers, resolved name targets, +import aliases and qualified-name components. Relevant changes are in: + +- [NameExpression](src/main/java/org/xvm/compiler/ast/NameExpression.java) and + [NamedTypeExpression](src/main/java/org/xvm/compiler/ast/NamedTypeExpression.java): resolved + targets and per-token type-name bindings. +- [ImportStatement](src/main/java/org/xvm/compiler/ast/ImportStatement.java): written alias and + qualified-name tokens, plus the imported identity. +- [Parameter](src/main/java/org/xvm/compiler/ast/Parameter.java), + [VariableDeclarationStatement](src/main/java/org/xvm/compiler/ast/VariableDeclarationStatement.java), + [MethodDeclarationStatement](src/main/java/org/xvm/compiler/ast/MethodDeclarationStatement.java), + [PropertyDeclarationStatement](src/main/java/org/xvm/compiler/ast/PropertyDeclarationStatement.java) + and [TypeCompositionStatement](src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java): + written names and the compiler identities associated with declarations. + +The adapter can consequently distinguish two same-spelled names in different scopes, a reference +to an import alias from a reference to its target, and a written declaration from a generated one. +These identities support navigation, references, rename and theme-based semantic highlighting. +They do not define colors in the compiler or introduce a public graphics/color library. + +### Capture selected calls before rewrites obscure them + +[InvocationBinding.java](src/main/java/org/xvm/compiler/ast/bindings/InvocationBinding.java) records a selected +method identity, its instantiated signature and the association between written argument spans +and visible parameter indices. Named-argument labels retain their written ranges. Synthetic +default arguments are not presented as arguments the user wrote. If a rewrite prevents a reliable +association, the collector omits that tooling fact rather than guessing or changing compilation. + +Method calls, function-valued calls and constructor calls remain distinct. A function-valued +invocation can have a validated signature without having a statically selected runtime method; +`FunctionCall` records the former without inventing the latter. Constructors are keyed by their +`NewExpression` nodes and carry the same kind of argument provenance. + +The collector belongs to the compilation attempt. It is passed through +[Compiler](src/main/java/org/xvm/compiler/Compiler.java), +[StageMgr](src/main/java/org/xvm/compiler/ast/StageMgr.java), method/root contexts, deferred bodies +and nested compilation. [InvocationExpression](src/main/java/org/xvm/compiler/ast/InvocationExpression.java) +and [NewExpression](src/main/java/org/xvm/compiler/ast/NewExpression.java) record facts when real +validation succeeds. Retried validation invalidates earlier entries. Publication retains only +eligible nodes surviving in the final tree, excluding discarded speculative clones, and clears +the collector's scratch state. + +Maps use AST identity, not structural equality. Their collection wrappers are immutable, but the +AST nodes and constants remain owned by the compiler worker. Ordinary compiler constructors use +`Collector.NONE`; normal compilation does not need to retain optional tooling call maps. + +This gives signature help and parameter hints the compiler's actual argument mapping. It also +lets rename and call hierarchy distinguish overloads, instead of treating all calls with the same +spelling as references to one declaration. + +### Keep references that constant folding would discard + +Successful constant-initializer compilation can discard the temporary method used to validate an +initializer. [InitializerBinding.java](src/main/java/org/xvm/compiler/ast/bindings/InitializerBinding.java) +captures source references, expression types and selected calls before that method disappears, +and anchors them to the surviving source property. + +For example, if an initializer refers to another constant, replacing the initializer with its +computed value must not erase the original occurrence from references or rename. The retained +record stores source spans and compiler constants, not the temporary clone nodes, contexts, +registers or mutable tokens. The adapter then copies the needed identities into its detached model. + +### Relate generated implementation details to written source + +[LambdaBindings.java](src/main/java/org/xvm/compiler/ast/bindings/LambdaBindings.java) distinguishes written +lambda parameters from synthetic capture parameters and maps capture registers back to their +source registers. [AnonymousClassBindings.java](src/main/java/org/xvm/compiler/ast/bindings/AnonymousClassBindings.java) +provides the corresponding origin information for anonymous-class captures. A captured local +must still rename as the same source variable even when generated methods or properties access it. + +For implementation and super navigation, +[MethodInfo.getSuperMethod](src/main/java/org/xvm/asm/constants/MethodInfo.java) exposes a written +super-dispatch target when one exists, without generating forwarding code. +[MethodBody.getIntoMethodInfo](src/main/java/org/xvm/asm/constants/MethodBody.java) exposes an +existing mixin constraint chain. Field, native and synthetic implementation details are not +misrepresented as source methods. The adapter's +[CompilerMethodRelations.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/CompilerMethodRelations.kt) +and [CompilerPropertyRelations.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/CompilerPropertyRelations.kt) +turn that metadata into navigation facts. + +## Declaration analysis and unfinished source + +These are separate operations with narrower guarantees than successful compilation: + +| Operation | Work performed | Result guarantee | +| --- | --- | --- | +| `compileModule` | Ordinary parse, registration, linking/resolution, validation and emission. | Successful module only when compilation succeeds; available partial progress otherwise. | +| `analyzeDeclarations` | Ordinary parsing and declaration resolution, stopping before body validation/emission. | `Optional` containing a resolved declaration graph; no executable artifact. Syntax, linkage, declaration failure or cancellation prevents a successful result. | +| `analyzeIncomplete` | Explicitly selected partial parsing plus ordinary compiler inspection of supported intact syntax. | `PartialAnalysis` with available syntax and proved cursor facts; never a promise of an executable module or a selected incomplete call. | + +### Recover syntax without validating damaged code as if it were complete + +[Parser.java](src/main/java/org/xvm/compiler/Parser.java) can retain useful syntax after errors by +recovering at statement/declaration boundaries. The host still sees those errors. An ordinary +`compileModule` attempt does not proceed to semantic compilation merely because a recovered tree +exists. This preserves structural information for diagnostics and outlines without making an +invalid program valid. + +Cursor-aware parsing is enabled explicitly with `Parser.forPartialAnalysis`, through +`EmbeddingSupport.analyzeIncomplete`. It retains the original source text and positions, including +declarations after the cursor. It does not truncate the document and rebase later locations, nor +does it replace every syntax error with invented valid code. + +The overload without an explicit cursor is deliberately narrow: a trailing incomplete expression +statement at EOF. The explicit-cursor overloads support a broader set of editing contexts, including +one selected member file in a module snapshot. They must receive a position produced by +`Source.getPosition()` for the exact source being analyzed. + +### Retain a bounded incomplete site and inspect its real context + +The explicit-cursor path supports the following groups; the +[API Javadoc](src/main/java/org/xvm/api/EmbeddingSupport.java) defines the detailed boundaries: + +| Editing context | Retained information | +| --- | --- | +| A member-access suffix, bare-name prefix or empty statement boundary. | Written prefix, receiver where available, visible variables/types and enclosing-instance access. | +| An unfinished call, including named and nested arguments. | Written argument positions and labels, candidate signatures, argument mappings and candidate value insertions. Assignment, initializer, return and compound-expression contexts remain attached to their real enclosing syntax. | +| Construction, including inner/virtual/annotated types. | Ordinary type preparation and constructor fitting, including available generic inference. Anonymous construction prepares the declaration and fits constructors without emitting forwarding constructors or running capture analysis. | +| A single-dimensional array size slot. | Fitting against the fixed-size constructor's size parameter. A following supplier is parsed for recovery but is not proved by this prefix analysis; multidimensional construction is unsupported. | +| Missing delimiters or a terminator around a supported site. | Bounded missing parentheses, brackets, angle/group closers and, at an EOF cursor, block braces. No invented operand, declaration name/type or literal contents. | +| Unfinished member/return/parameter types and declaration headers. | Visible types, qualified-name components and written formals with their shadowing constraints. Unfinished methods/properties do not register completed declarations or invented parameters. A root module header registers only its written namespace/core import. | + +The embedding implementation requires one selected innermost incomplete site and rejects semantic +analysis if other serious syntax errors remain outside its recognized boundary. Recognized boundary +diagnostics still reach the host. Repeated delivery of the same boundary report is deduplicated; +other errors and unexpected failures remain visible. + +The incomplete node inspects intact children in the actual compiler context and prevents the +incomplete operation from becoming an emitted method. A non-null pool only means semantic work +started. Child types are usable only when the child validated successfully with a fitting result; +placeholder types left by failed validation are not semantic proof. + +### Export scope and candidate facts while the context exists + +[CursorBinding.java](src/main/java/org/xvm/compiler/ast/bindings/CursorBinding.java) captures narrowed local +types and readability, the enclosing receiver, visible type names, method/function candidates, +written argument mappings and compiler-fitted insertion proposals. Written formals can hide outer +names without being assigned fictitious completed declaration identities. + +`callsInspected` distinguishes **no fitting candidates** from **the call was not inspected**. +That prevents an empty list from being interpreted as evidence that no overload is legal when +fitting never ran. A candidate whose written arguments fit is still a candidate: missing arguments +cannot establish that the complete operation will validate or select that method. + +Candidate fitting uses ordinary compiler operations on disposable syntax. Variable/property reads, +literals, enclosing-instance expressions and supported source templates are checked in context; +they are not accepted solely by comparing type names in the LSP server. The adapter consumes the +copied results through [XdkCursorQueries.kt](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/XdkCursorQueries.kt) +and its completion/signature-help code. + +## AST traversal, copying and the partial-analysis package + +### Give a source-tree copy a precise contract + +[AstNode.copyTree](src/main/java/org/xvm/compiler/ast/AstNode.java) names the operation that the +older `clone()` API left implicit. It preserves the concrete node type, source positions and +compilation stage, recursively copies registered syntax children, and parents copied children +to their copied owners. Copying does not reparent the original tree. The copy's root initially +retains its original parent until the caller adopts it elsewhere. + +This is **not a fresh parse or a deep copy of the compiler**. The default implementation still +starts with shallow cloning and reflectively copies registered child fields. Non-child bindings +and transient fields can remain shared unless an override handles them. A validated clone must +not be treated as an independent unvalidated program. + +`Expression`, `Statement`, `TypeExpression` and specialized nodes use covariant `copyTree` return +types, so callers do not need to cast a copied expression back from `AstNode`. Constructor-based +partial-node copies adopt their children explicitly and preserve root metadata with +`copyTreeMetadataTo`. Legacy `clone()` delegates to the same implementation, avoiding two competing +copy contracts. + +The related `childNodes()` API starts a fresh traversal for each iterator. `children()` retains +its existing one-shot mutable traversal semantics for compiler passes. Neither is a synchronized +snapshot; callers must own the tree and copy a list if they need a stable collection. + +### Keep partial-analysis policy out of ordinary node implementations + +The incomplete syntax and fitting helpers live under +[`org.xvm.compiler.ast.partial`](src/main/java/org/xvm/compiler/ast/partial): + +| Classes | Responsibility | +| --- | --- | +| `IncompleteStatement`, `IncompleteExpression` | Retain a selected unfinished operation and its intact children in statement/expression contexts. | +| `IncompleteLocalDeclaration`, `IncompleteDeclarationStatement`, `IncompleteTypeCompositionStatement` | Preserve written declaration/header syntax without inventing successfully registered declarations. | +| `PartialSyntax` | Inspect retained sites and deferred anonymous source bodies without changing child ownership. | +| `PartialQueries`, `CursorScope` | Coordinate cursor inspection and capture the visible lexical scope. | +| `PartialCallResolver`, `PartialConstructionResolver` | Fit call/constructor candidates with the ordinary compiler's rules. | +| `PartialArgument`, `ProposedLiteralToken` | Build disposable proposed arguments and retain appropriate source anchoring for trials. | + +Java subpackages do not share package-private access. The single documented +[PartialAst.java](src/main/java/org/xvm/compiler/ast/PartialAst.java) bridge exposes the ordinary +parenting, validation, type inspection, scope and fitting operations needed by these helpers. +Existing AST operations keep their protected/package-private visibility. The five coordinating +helpers (`PartialQueries`, `CursorScope`, `PartialCallResolver`, `PartialConstructionResolver` and +`PartialArgument`) remain package private. + +The bridge is public for Java package access, **not a supported embedding facade**. Its arguments +are live compiler nodes and contexts. Hosts use `EmbeddingSupport` results; the bridge neither +installs proposals in the source tree nor implements an alternative type checker. Parser hooks +and a few ordinary-node hooks remain necessary because only those components know the syntactic +or validation context in which the incomplete site occurs. + +## Constant pools, caches and dependency inputs + +### Make the working context explicit + +[ConstantPool.withPool](src/main/java/org/xvm/asm/ConstantPool.java) scopes the working pool and +restores its predecessor on close. A constant's owning pool is not automatically the working pool: +cross-pool resolution can deliberately register or resolve into another pool. Inspection after +compilation therefore establishes the appropriate scope explicitly. Scopes are opened and closed +on the same thread; asynchronous work must establish its own context. + +The adapter serializes access to compiler state and shared repositories. Its detached models do +not contain compiler objects; snapshot-local symbol/type identities are separate from live +registers and constants. Some worker-only projections deliberately build TypeInfo using the +attempt's listener, while ordinary snapshot extraction avoids resuming validation. Those two +operations have different cost and reporting contracts. + +`EmbeddingSupport.footprint(compilation)` inspects the supplied attempt's pool without starting an +interpreter. `ensureRuntimePool()` explicitly initializes runtime state when needed; the deprecated +`getConstantPool()` retains that runtime-initializing behavior. Neither should be confused with +the pool belonging to an unsuccessful compilation. The footprint's heap count is JVM-wide used +heap, not a measurement of memory owned exclusively by that compilation. + +The supporting [TransientThreadLocal](../javatools_utils/src/main/java/org/xvm/util/TransientThreadLocal.java) +correction treats null as absence instead of storing a null-valued entry that still retains its +key in the per-thread map. Setting or computing null removes the entry. This matters for repeated +compiler use, including deferred TypeInfo state, but is a utility lifecycle fix rather than an +editor feature or a new claim of compiler thread safety. + +### Preserve tolerant repositories and validate explicit dependency inputs + +[FileRepository](src/main/java/org/xvm/asm/FileRepository.java) and +[DirRepository](src/main/java/org/xvm/asm/DirRepository.java) retain their best-effort behavior: +an unreadable candidate is unavailable, and another candidate or repository may still satisfy +the request. A damaged unrelated file does not prove why a requested module is absent. + +**This does not add a repository error callback or progress API.** For explicit dependency inputs, +the LSP validates owned bytes through `FileStructure` before accepting them into +[XdkDependency](../lang/lsp-server/src/main/kotlin/org/xvm/lsp/adapter/xdk/XdkDependency.kt). It can then +retain immutable bytes and reopen compiler-owned structures for an attempt. This keeps useful +dependency diagnostics separate from tolerant repository scans and avoids sharing a mutable +`FileStructure` across requests. Compiler queue/progress presentation belongs to the adapter. + +## Why ordinary AST and runtime files also changed + +The number of touched AST files is larger than the number of new concepts. Several contracts must +be carried through existing compiler operations rather than implemented only at an entry point: + +| Change group | Why existing files participate | +| --- | --- | +| Listener migration across expressions, constants, structures and tools. | Every reporting/probing call must select an owner or an explicit silence and preserve the appropriate abort policy. Most such edits do not change an expression's language semantics. | +| Collector propagation through stages, root contexts, method bodies and nested construction. | The same attempt must see successful bindings established inside deferred or nested validation. An outer wrapper cannot reconstruct these decisions afterward. | +| Name/source accessors and capture provenance. | The relevant written token or resolved register is owned by the node that parsed or validated it. Exposing that fact avoids re-resolving the name in the adapter. | +| Copy and child-traversal overrides. | Each node must honor syntax-child ownership; specialized state cannot be made safe by changing only the base class. | +| Partial-syntax hooks in expressions and declarations. | The parser and enclosing node must retain the unfinished site in its real context while preventing normal emission of the unfinished operation. | +| Tool/runtime embedding call sites. | `Launcher`, tool drivers, `InterpreterControl`, `EvalCompiler`, `xRTCompiler` and `xRTType` must use the revised reporting/API contracts too. They do not gain an LSP dependency. | + +Compiler correctness fixes discovered while exercising the new consumers apply to ordinary +compilation as well: + +- Bound callable references must retain their callable type, including generic method references. + Function fitting must not return a usable type after an argument-count or argument-type failure + merely because the required return types match. The + [function-typing regressions](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/CompilerFunctionTypingTest.kt) + check the fitting contract, rejected calls, and successful execution after serialization. +- A constant Boolean result does not imply that evaluating its operands has no effects. + `probe() && False` must still evaluate `probe()` once, as must `probe() || True`. The + [conditional-effects regression](../manualTests/src/main/x/conditionalEffects.x) checks assignment, + conditional, negated and argument contexts as well as the corresponding nonconstant cases. + +Emitted programs and tooling probes must use one consistent set of rules. These are not alternate +semantics enabled by an editor flag; some supporting corrections are already present independently +of the remaining tooling changes. + +The source AST discussed here is distinct from the removed binary AST implementation. Tooling +does not restore that implementation or serialize its binding/cursor records into `.xtc` files. +The separate binary-format change requires older affected `.xtc` artifacts to be rebuilt; it +should not be mistaken for an LSP wire-format requirement. + +## Compatibility and limits + +- **Ordinary Ecstasy compilation uses the same pipeline and validation rules.** Optional binding + collectors default to `NONE` in ordinary compiler construction. Cursor analysis is explicitly + requested; recovered syntax does not authorize emission of an invalid program. +- **The Java embedding API is intentionally not universally source/binary compatible.** The + planned breaking release boundary covers changed listener signatures, removed `BLACKHOLE` and + `SafeLookAhead` APIs, required listeners and changed fallback runtime-listener behavior. Existing + Java consumers must migrate and recompile; see the [migration guide](#6-migrate-java-callers-and-implementations). +- **Partial facts have narrower guarantees than complete results.** An available syntax tree, + resolved declaration graph, fitted candidate and emitted module answer different questions. + Unsupported source damage yields diagnostics or absent facts, not invented semantic certainty. +- **Compiler objects remain mutable and confined to their worker.** Immutable binding-map wrappers, + final listeners and `copyTree()` do not make a compilation safe for concurrent access. +- **Cancellation is cooperative.** Polling and phase checks stop obsolete work where supported; + the adapter must still check revisions before publication. +- **This is not a fully incremental compiler API.** The adapter captures/reuses input and detached + result data, schedules work and avoids unnecessary requests. Fresh attempts still run compiler + phases for their module snapshot rather than incrementally mutating an AST shared with the IDE. +- **Editor policy stays outside the compiler.** LSP message handling, themes, text edits, URI/version + management, client workarounds and UI progress remain in `lang`. Syntax-only fallback behavior + and compiler-derived semantic answers remain distinct sources of information. + +### Java result records and nullable progress + +The five source-binding classes now live in `org.xvm.compiler.ast.bindings`. Java/Kotlin consumers +must update imports and recompile; moving a public class changes its binary name. This is part of +the planned embedding API migration, not a change to emitted Ecstasy code or the `.xtc` format. + +Retained convenience constructors do not preserve a record's old deconstruction shape. +`CursorBinding` has fourteen components; Java clients using record patterns must adapt to the +current shape even where an older constructor descriptor remains available. Prefer accessors +when the consumer does not need deconstruction. + +`PartialAnalysis` accepts and returns a nullable `ConstantPool`, like `Compilation`: null means +semantic analysis did not start. Its retained three- through six-argument constructors take that +pool directly, not an `Optional` parameter. Consumers of an earlier development API must recompile, +pass the pool or null and replace `pool().orElse(...)` with a null check. Optional return values +elsewhere remain appropriate where their API declares them. + +Collection snapshots retain compiler-owned values. Constructor compatibility and immutable +collection wrappers do not extend those values' lifetime or permit concurrent AST/pool access. + +## Tests and reproduction + +The useful evidence is at three levels: contracts tested directly in `javatools`, Java embedding +consumers exercised with compiled XDK modules, and packaged-server/editor acceptance tests. A +passing parser unit test alone does not demonstrate that the distributed LSP preserves a fact +through its transport and editor integration. + +| Contract | Representative tests | +| --- | --- | +| Callback delivery, budgets, branches, suppression and cancellation. | [ErrorListenerHostRegressionTest](src/test/java/org/xvm/asm/ErrorListenerHostRegressionTest.java), [ErrorListenerBranchTest](src/test/java/org/xvm/asm/ErrorListenerBranchTest.java), [ErrorListenerSilenceTest](src/test/java/org/xvm/asm/ErrorListenerSilenceTest.java), [ErrorListenerCancellationTest](src/test/java/org/xvm/asm/ErrorListenerCancellationTest.java), [CompilerCancellationTest](src/test/java/org/xvm/compiler/CompilerCancellationTest.java). | +| Reporting restoration and parser speculation. | [ReportingTest](src/test/java/org/xvm/asm/ReportingTest.java), [ParserAttemptTest](src/test/java/org/xvm/compiler/ParserAttemptTest.java), [NameResolverReportingTest](src/test/java/org/xvm/compiler/ast/NameResolverReportingTest.java), [ValidationScopeTest](src/test/java/org/xvm/compiler/ast/ValidationScopeTest.java). | +| Tree copying, traversal and source provenance. | [AstTreeCopyTest](src/test/java/org/xvm/compiler/ast/AstTreeCopyTest.java), [OrdinaryAstTreeCopyTest](src/test/java/org/xvm/compiler/ast/OrdinaryAstTreeCopyTest.java), [AstChildTraversalTest](src/test/java/org/xvm/compiler/ast/AstChildTraversalTest.java), [DeclarationProvenanceTest](src/test/java/org/xvm/compiler/ast/DeclarationProvenanceTest.java), [RegisteredChildFieldsTest](src/test/java/org/xvm/compiler/ast/partial/RegisteredChildFieldsTest.java). | +| Snapshot inputs, outcomes and cached diagnostics through the embedding boundary. | [EmbeddingSourceSnapshotTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/EmbeddingSourceSnapshotTest.kt), [EmbeddingDiagnosticsTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/EmbeddingDiagnosticsTest.kt), [TypeInfoDiagnosticsTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/TypeInfoDiagnosticsTest.kt), [TypeInfoFinalCompositionTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/TypeInfoFinalCompositionTest.kt). | +| Selected calls, functions, constructors and folded initializers. | [CompilerBindingFactsTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/CompilerBindingFactsTest.kt), [BindingIdentitySnapshotTest](src/test/java/org/xvm/api/BindingIdentitySnapshotTest.java). | +| Declaration analysis, cursor candidates and recovery. | [CompilerDeclarationAnalysisTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/CompilerDeclarationAnalysisTest.kt), [CompilerPartialFactsTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/CompilerPartialFactsTest.kt), [ParserRecoveryTest](src/test/java/org/xvm/compiler/ParserRecoveryTest.java), [XdkPartialAnalysisTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/XdkPartialAnalysisTest.kt), [XdkDeclarationHeaderTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/adapter/XdkDeclarationHeaderTest.kt). | +| Packaged compiler server and editor behavior. | [XdkStdioTest](../lang/lsp-server/src/test/kotlin/org/xvm/lsp/server/XdkStdioTest.kt) and the [manual/automated editor playbook](../lang/doc/manual-test-plan.md). | + +### Historical listener regression evidence + +The strongest evidence is the pair of +[host regression tests](src/test/java/org/xvm/asm/ErrorListenerHostRegressionTest.java). +Their source uses API shapes available on both sides of the reporting change. On the previous +implementation at `1d9352b6f`, both compile and fail for the expected behavioral reason. On the +repaired implementation, the same source passes both cases: + +* `hostBranchLetsLexerReportBothErrors` sends two bad string escapes through the real lexer. + Previously it threw after the first error because the default branch had a one-error budget. + It now completes the scan and publishes both reports when the branch is merged. +* `parentAbortReachesTheLexerThroughAnExistingBranch` aborts the parent after creating its branch. + Previously the lexer continued because the branch considered only its local state. It now + honors the parent at its reporting checkpoint. + +[ErrorListenerMigrationTest](src/test/java/org/xvm/asm/ErrorListenerMigrationTest.java) also proves +that derived silence preserves a host stop request without publishing an error. Its parser/lexer +budget tests and source-reanchoring test guard existing behavior. The branch, site, silence and +abort suites cover collection, forwarding, replay and suppression. +[ErrorListenerBoundaryTest](src/test/java/org/xvm/api/ErrorListenerBoundaryTest.java) checks required +listeners and destination retention; these assertions enforce the chosen API contract rather than +claiming that every API change reproduces an old bug. + +The 2026-10-09 focused run on this implementation passed **60 tests with zero failures, errors +or skips**. Earlier full-suite verification of the same production listener migration passed +418 executed Java tests, with 42 skips recorded separately. A forced XDK rebuild produced +24 modules matching the master baseline after normalizing creation timestamps. The separate +integration checkout combining the reporting changes with the embedding foundations also passed +the real compiler-consumer test with no skips. These checks cover different scopes; the 60 focused +tests are not presented as a substitute for compilation and integration validation. + +Run the focused tests, `errorListenerTest`, from the repository root: + +```bash +./gradlew :javatools:errorListenerTest --rerun-tasks --no-build-cache +``` + +These focused cases do not require installed XDK modules. The numerical receipts above +describe the listener migration at that time, not a fresh run of the complete implementation. +The broader commands below cover compiled-module consumers and packaged-server integration; +neither historical counts nor a green task substitute for checking current failures and skips. + +### Run the compiler and embedding consumers + +Run commands from the repository root. Build the XDK outputs first when exercising tests that +require installed modules: + +```bash +./gradlew :xdk:installDist +./gradlew :javatools:test --rerun-tasks --no-build-cache +``` + +A focused embedding-consumer run, `embeddingConsumerTest`, exercises the new API boundaries without +launching an IDE: + +```bash +./gradlew :lang:lsp-server:embeddingConsumerTest \ + -PincludeBuildLang=true -PincludeBuildAttachLang=true -Plsp.adapter=compiler \ + --rerun-tasks --no-build-cache +``` + +Exercise the packaged artifact separately: + +```bash +./gradlew :lang:lsp-server:compilerStdioTest \ + -PincludeBuildLang=true -PincludeBuildAttachLang=true -Plsp.adapter=compiler \ + --rerun-tasks --no-build-cache +``` + +Check the JUnit XML under `javatools/build/test-results/test` and +`lang/lsp-server/build/test-results/{test,compilerStdioTest}` for failures **and skips**. Some Java +tests conditionally skip when compiled modules are absent; a green task alone does not establish +that they ran. The test links and commands here describe coverage and reproduction, not a claim +that every suite has just been rerun. Current CI results and their artifacts provide that receipt. + +## Appendix: file-by-file compiler changes + +This inventory records the `javatools` portion of the full implementation review when this guide +was added and updated for binding-package organization: **148 files (97 modified and 51 added)**, +including this document. It describes what changed in each file and why, rather than restating +everything that file does. +Supporting changes already in the review's base and changes outside `javatools` are discussed +in the main guide but are not counted again here. + +The files are grouped by responsibility. **Modified** means an existing file changed; **Added** +means a new file in this review. A small listener migration is not a new language feature, and +a test file is not an additional production mechanism. Every file link opens the corresponding +source or document in this checkout. + +### Documentation and embedding entry points + +| File | Change | What changed and why | +| --- | --- | --- | +| [README.md](README.md) | Modified | Keeps a short architecture overview and links to this guide for the consolidated diagnostic design, listener migration and compiler contracts. | +| [lsp-compiler.md](lsp-compiler.md) | Added | Explains the compiler-to-LSP data flow, API guarantees, partial-analysis limits, compatibility implications and test coverage; this appendix maps those mechanisms to their files. | +| [src/main/java/org/xvm/api/EmbeddingSupport.java](src/main/java/org/xvm/api/EmbeddingSupport.java) | Modified | Returns compilation outcomes, retained source trees and identity-based semantic facts; adds declaration/cursor analysis over source snapshots, explicit bootstrap configuration and footprint inspection without starting the runtime. | +| [src/main/java/org/xvm/api/ModuleCompiler.java](src/main/java/org/xvm/api/ModuleCompiler.java) | Modified | Requires an explicit listener and reports compilation, input and output failures through it. Retains the simpler module/boolean result APIs and documents repository sharing constraints. | +| [src/main/java/org/xvm/api/InterpreterControl.java](src/main/java/org/xvm/api/InterpreterControl.java) | Modified | Routes an unhandled interpreted exception through the guaranteed listener with the module as its diagnostic site, removing the nullable-listener discard path. | + +### Diagnostics, structures and repositories + +| File | Change | What changed and why | +| --- | --- | --- | +| [Annotation.java](src/main/java/org/xvm/asm/Annotation.java) | Modified | Separates diagnostic emission from the abort query when rejecting a non-annotation type; logging no longer returns the stop decision. | +| [ClassStructure.java](src/main/java/org/xvm/asm/ClassStructure.java) | Modified | Uses an explicitly silent probe when looking up a Service member, replacing the global blackhole sentinel. | +| [Component.java](src/main/java/org/xvm/asm/Component.java) | Modified | Requires a destination for resolution collectors and reports cyclical contributions through the typed fatal/site API. | +| [ComponentResolver.java](src/main/java/org/xvm/asm/ComponentResolver.java) | Modified | Removes the default discard listener from the resolution-collector interface. Implementations must choose where resolution diagnostics go. | +| [ConstantPool.java](src/main/java/org/xvm/asm/ConstantPool.java) | Modified | Exposes implicit-import names without loading or resolving their components, allowing cursor scope queries to enumerate names using the language's own import table. | +| [ErrorList.java](src/main/java/org/xvm/asm/ErrorList.java) | Modified | Makes retained diagnostics, deduplication, shared default budgets, parent-aware branches and replay explicit. Branch delivery and abort policy remain distinct operations. | +| [ErrorListener.java](src/main/java/org/xvm/asm/ErrorListener.java) | Modified | Defines diagnostic sites, severity helpers, collection/forwarding/cancellation wrappers, branches and named silence reasons. Separates reporting from abort queries and replaces the blackhole/null conventions. | +| [FileStructure.java](src/main/java/org/xvm/asm/FileStructure.java) | Modified | Removes storage and inheritance of an ambient error listener, so a reused module structure cannot carry one host's diagnostic destination into another operation. | +| [MethodStructure.java](src/main/java/org/xvm/asm/MethodStructure.java) | Modified | Uses a shared probe listener for speculative common-type inference and the explicit diagnostic-site API for unsupported dynamic type parameters. | +| [Reporting.java](src/main/java/org/xvm/asm/Reporting.java) | Added | Adds a scoped diagnostic destination for callbacks that cannot take a listener parameter. Closing a scope restores the previous destination, including after nested calls or exceptions. | +| [XvmStructure.java](src/main/java/org/xvm/asm/XvmStructure.java) | Modified | Makes structural reporting use the listener supplied by the caller; removes ambient lookup and the boolean return from the logging helper. | + +### Type information and inherited method metadata + +| File | Change | What changed and why | +| --- | --- | --- | +| [ImmutableTypeConstant.java](src/main/java/org/xvm/asm/constants/ImmutableTypeConstant.java) | Modified | Reports redundant immutability and queries abort separately, preserving the validation decision under the revised listener contract. | +| [MethodBody.java](src/main/java/org/xvm/asm/constants/MethodBody.java) | Modified | Exposes the existing FromInto mixin constraint metadata for inspection without generating a forwarding implementation. | +| [MethodInfo.java](src/main/java/org/xvm/asm/constants/MethodInfo.java) | Modified | Adds a source-method query for super dispatch using the compiler's existing chain selection; synthetic, native, field and other non-source targets remain absent. | +| [PropertyInfo.java](src/main/java/org/xvm/asm/constants/PropertyInfo.java) | Modified | Anchors ignored duplicate/super annotation warnings to the contributing property's identity, so the reported structure matches the annotation being ignored. | +| [TypeCollector.java](src/main/java/org/xvm/asm/constants/TypeCollector.java) | Modified | Replaces blackhole use in speculative common-type inference with explicit probes, sharing one within a local sequence where appropriate. | +| [TypeConstant.java](src/main/java/org/xvm/asm/constants/TypeConstant.java) | Modified | Uses explicit reporting for TypeInfo construction, retains and replays cached diagnostics to later callers, and distinguishes speculative/cascade suppression from real validation. Cache invalidation also clears retained diagnostic state. | +| [TypeInfoReal.java](src/main/java/org/xvm/asm/constants/TypeInfoReal.java) | Modified | Marks the parent-newability check as a silent probe while retaining the caller's listener for the surrounding TypeInfo computation. | +| [UnresolvedTypeConstant.java](src/main/java/org/xvm/asm/constants/UnresolvedTypeConstant.java) | Modified | Reports an unresolved type through the error/site API, retaining the constant as the diagnostic location. | + +### Compiler phases, parsing and exported facts + +| File | Change | What changed and why | +| --- | --- | --- | +| [Compiler.java](src/main/java/org/xvm/compiler/Compiler.java) | Modified | Owns a final listener and optional invocation/cursor collectors and passes them through compiler stages. Registration recognizes host cancellation without inventing an error or retaining a listener on the file structure. | +| [EvalCompiler.java](src/main/java/org/xvm/compiler/EvalCompiler.java) | Modified | Creates its diagnostic buffer at construction so hosts can inspect errors before lambda compilation, with an explicit first-error policy. | +| [Lexer.java](src/main/java/org/xvm/compiler/Lexer.java) | Modified | Uses explicit sites/listeners and checks cancellation even while reading valid tokens. Unterminated string handling stops at EOF instead of repeatedly reporting when no error budget stops it. | +| [Parser.java](src/main/java/org/xvm/compiler/Parser.java) | Modified | Replaces implicit lookahead listener swapping with scoped attempts; adds bounded syntax recovery and opt-in cursor/header parsing while preserving written positions and later declarations. Unrelated parse errors still block semantic compilation. | +| [Token.java](src/main/java/org/xvm/compiler/Token.java) | Modified | Migrates token diagnostics to explicit source-span or no-location sites with trailing message arguments; the helper no longer returns an abort decision. | + +### Binding facts and collectors + +| File | Change | What changed and why | +| --- | --- | --- | +| [AnonymousClassBindings.java](src/main/java/org/xvm/compiler/ast/bindings/AnonymousClassBindings.java) | Added | Associates anonymous-class captures and generated properties with their enclosing source registers, so navigation and rename can follow the original declaration. | +| [CursorBinding.java](src/main/java/org/xvm/compiler/ast/bindings/CursorBinding.java) | Added | Defines immutable collections of scope, type, candidate-call and argument-proposal facts for an explicit cursor query. Its collector publishes inspected source sites without presenting candidates as selected executable calls. | +| [InitializerBinding.java](src/main/java/org/xvm/compiler/ast/bindings/InitializerBinding.java) | Added | Captures source references, types and call facts before successful constant folding discards an initializer's temporary method; excludes validation contexts and disposable AST identities from the published provenance. | +| [InvocationBinding.java](src/main/java/org/xvm/compiler/ast/bindings/InvocationBinding.java) | Added | Records selected methods, function types, constructors and source-to-parameter argument associations. Attempt-owned collectors discard stale/trial facts and publish maps keyed by source-node identity. | +| [LambdaBindings.java](src/main/java/org/xvm/compiler/ast/bindings/LambdaBindings.java) | Added | Maps written lambda parameters and generated capture parameters back to their source registers, distinguishing source declarations from synthetic compiler parameters. | + +### AST ownership, contexts and semantic provenance + +| File | Change | What changed and why | +| --- | --- | --- | +| [AnonInnerClass.java](src/main/java/org/xvm/compiler/ast/AnonInnerClass.java) | Modified | Enforces the non-null listener contract at construction even when assertions are disabled; this particular edit does not add a new anonymous-class algorithm. | +| [AstNode.java](src/main/java/org/xvm/compiler/ast/AstNode.java) | Modified | Adds repeatable child traversal and the documented copyTree contract, preserves the legacy clone entry point, supports inherited registered child fields and carries fact collectors into child staging. Reporting requires an explicit destination. | +| [BindingAst.java](src/main/java/org/xvm/compiler/ast/BindingAst.java) | Added | Supplies the single cross-package operation that associates a lambda parameter with its existing register, keeping AST field/setter visibility unchanged while binding collection lives in ast.bindings. | +| [CompositionNode.java](src/main/java/org/xvm/compiler/ast/CompositionNode.java) | Modified | Makes source lookup safe for unparented parsed compositions and implicit compositions with no type child, supporting inspection before normal tree adoption. | +| [Context.java](src/main/java/org/xvm/compiler/ast/Context.java) | Modified | Propagates the attempt's call/cursor collectors through lexical contexts and makes diagnostic/probe ownership explicit during name and this resolution. | +| [Expression.java](src/main/java/org/xvm/compiler/ast/Expression.java) | Modified | Adds the covariant copyTree result and uses it for exhaustive fit trials. Makes type-fitting probes explicitly silent instead of depending on whether a caller happened to pass null or a reporting listener. | +| [ForEachStatement.java](src/main/java/org/xvm/compiler/ast/ForEachStatement.java) | Modified | Holds the context/listener pair for lazy loop variables in ValidationScope and restores it on every validation exit; speculative fitting uses explicit probes. | +| [ForStatement.java](src/main/java/org/xvm/compiler/ast/ForStatement.java) | Modified | Replaces separate nullable callback fields with ValidationScope and restores the previous scope after normal, early and exceptional validation exits. | +| [ImportStatement.java](src/main/java/org/xvm/compiler/ast/ImportStatement.java) | Modified | Exposes the written alias, qualified-name tokens and already-resolved import identity without restarting name resolution, giving tooling exact import provenance. | +| [InvocationExpression.java](src/main/java/org/xvm/compiler/ast/InvocationExpression.java) | Modified | Publishes validated method/function/super call facts and written argument mappings, exposes the invoked expression/selected method and uses explicit probes. Keeps failed argument fitting from being overwritten by successful return-type matching. | +| [LambdaExpression.java](src/main/java/org/xvm/compiler/ast/LambdaExpression.java) | Modified | Exposes the arrow token and retained source bindings, carries collectors into generated-method validation and supplies a typed tree copy; diagnostic probes and sites use the explicit listener API. | +| [MethodDeclarationStatement.java](src/main/java/org/xvm/compiler/ast/MethodDeclarationStatement.java) | Modified | Exposes declaration names and documentation, retains formal-parameter identities and carries collectors through annotation/default/body validation and generated method compilation. | +| [NameExpression.java](src/main/java/org/xvm/compiler/ast/NameExpression.java) | Modified | Exposes the resolved constant or register and supplies a typed tree copy. Also preserves the callable type and hidden generic bindings of bound method references; speculative resolution selects an explicit probe. | +| [NameResolver.java](src/main/java/org/xvm/compiler/ast/NameResolver.java) | Modified | Scopes the caller's listener across collector callbacks and releases it on all exits. Retains successfully resolved qualified-name segments without restarting lookup when tooling inspects them. | +| [NamedTypeExpression.java](src/main/java/org/xvm/compiler/ast/NamedTypeExpression.java) | Modified | Exposes tokens and resolved identities for individual qualified-name segments. Migrates specialized copying, including the non-child dynamic expression, to copyTree and uses an explicit default-type probe. | +| [NewExpression.java](src/main/java/org/xvm/compiler/ast/NewExpression.java) | Modified | Shares construction preparation with partial queries, records selected constructors and capture origins, exposes written arguments/deferred bodies and propagates collectors. Cursor-containing bodies avoid publishing disposable capture-analysis clones or emitting an unfinished construction. | +| [Parameter.java](src/main/java/org/xvm/compiler/ast/Parameter.java) | Modified | Retains a parameter's resolved register, constructor-property identity or formal type constant for source consumers, with a typed copy that keeps the binding slot independent when a copy is rebound. | +| [PartialAst.java](src/main/java/org/xvm/compiler/ast/PartialAst.java) | Added | Provides one documented internal bridge from ast.partial to ordinary parenting, validation, scope, call-fitting and construction operations. Keeps those operations out of the embedding API and avoids widening existing AST members individually. | +| [PropertyDeclarationStatement.java](src/main/java/org/xvm/compiler/ast/PropertyDeclarationStatement.java) | Modified | Exposes the property name token, records initializer provenance before folding and carries collectors into real initializer methods. A cursor initializer is inspected through source-owned nodes rather than disposable folding clones. | +| [StageMgr.java](src/main/java/org/xvm/compiler/ast/StageMgr.java) | Modified | Retains a final non-null listener and the attempt's call/cursor collectors across stage processing. Existing constructors select disabled collectors for ordinary compilation. | +| [Statement.java](src/main/java/org/xvm/compiler/ast/Statement.java) | Modified | Adds the covariant copyTree result and restores the validation context in finally, including when partial validation supplies an explicit value-context callback. | +| [StatementBlock.java](src/main/java/org/xvm/compiler/ast/StatementBlock.java) | Modified | Threads collectors through method/root contexts, records parameter and capture associations, and adds a typed tree copy. TypeInfo queries during contextual lookup use the current listener. | +| [TryStatement.java](src/main/java/org/xvm/compiler/ast/TryStatement.java) | Modified | Keeps the finally-validation context and listener together and restores them after nested validation or failure, avoiding stale callback state on a retained tree. | +| [TypeCompositionStatement.java](src/main/java/org/xvm/compiler/ast/TypeCompositionStatement.java) | Modified | Exposes declaration/formal/constructor-parameter provenance and supports syntax-only incomplete subclasses. Rejects conflicting unconditional type declarations before an invalid composite cast, respects deferred children and propagates collectors into constructors. | +| [TypeExpression.java](src/main/java/org/xvm/compiler/ast/TypeExpression.java) | Modified | Adds a covariant copyTree result and replaces null-listener type queries with explicit probes while keeping reporting queries tied to their supplied listener. | +| [TypedefStatement.java](src/main/java/org/xvm/compiler/ast/TypedefStatement.java) | Modified | Exposes the written alias token, allowing navigation and rename to use its exact declaration range. | +| [ValidationScope.java](src/main/java/org/xvm/compiler/ast/ValidationScope.java) | Added | Represents a live context/listener pair for lazy validation callbacks. Null at the owning statement means no active scope; an active scope cannot contain a missing context or listener. | +| [VariableDeclarationStatement.java](src/main/java/org/xvm/compiler/ast/VariableDeclarationStatement.java) | Modified | Exposes the existing declaration register and documents matching narrowed uses through original-register identity, rather than matching local names by spelling. | +| [WhileStatement.java](src/main/java/org/xvm/compiler/ast/WhileStatement.java) | Modified | Uses ValidationScope for lazy loop variables and restores the previous scope after validation, including exceptions, rather than leaving context/listener fields attached to the statement. | + +### Expression call-site migration + +| File | Change | What changed and why | +| --- | --- | --- | +| [AnnotationExpression.java](src/main/java/org/xvm/compiler/ast/AnnotationExpression.java) | Modified | Uses a named probe when resolving an annotation argument as a method reference; real annotation validation keeps its reporting listener. | +| [ArrayAccessExpression.java](src/main/java/org/xvm/compiler/ast/ArrayAccessExpression.java) | Modified | Replaces null/blackhole listeners in index, slice, tuple, matrix and operator-fit queries with explicit probes, sharing a local probe across related candidate checks. | +| [AsExpression.java](src/main/java/org/xvm/compiler/ast/AsExpression.java) | Modified | Uses explicit probes for implicit cast-type lookup and operand-fit checks. | +| [AssignmentStatement.java](src/main/java/org/xvm/compiler/ast/AssignmentStatement.java) | Modified | Uses one local probe for speculative right-hand-side, tuple-unpacking and future-value fitting before the chosen assignment is validated. | +| [CaseManager.java](src/main/java/org/xvm/compiler/ast/CaseManager.java) | Modified | Uses a shared probe for case-value versus range fitting, leaving selected case validation responsible for diagnostics. | +| [CmpChainExpression.java](src/main/java/org/xvm/compiler/ast/CmpChainExpression.java) | Modified | Supplies an explicit probe when testing operands against a common comparison type. | +| [CmpExpression.java](src/main/java/org/xvm/compiler/ast/CmpExpression.java) | Modified | Uses an explicit probe when selecting a common operand type. | +| [ElseExpression.java](src/main/java/org/xvm/compiler/ast/ElseExpression.java) | Modified | Marks the alternative expression's multi-value fitting as a probe instead of passing a null listener. | +| [ElvisExpression.java](src/main/java/org/xvm/compiler/ast/ElvisExpression.java) | Modified | Uses explicit probes for alternative/conditional fitting and always branches the supplied listener where diagnostics must be retained for a chosen interpretation. | +| [ListExpression.java](src/main/java/org/xvm/compiler/ast/ListExpression.java) | Modified | Uses a probe for implicit lookup of the list's explicitly written type. | +| [MapExpression.java](src/main/java/org/xvm/compiler/ast/MapExpression.java) | Modified | Uses a probe for implicit lookup of the map's explicitly written type. | +| [NonBindingExpression.java](src/main/java/org/xvm/compiler/ast/NonBindingExpression.java) | Modified | Uses a probe to inspect the type of an unbound argument placeholder. | +| [NotNullExpression.java](src/main/java/org/xvm/compiler/ast/NotNullExpression.java) | Modified | Uses probes for type/value interpretations and conditional multi-value fitting, keeping speculative failures out of ordinary diagnostics. | +| [ParenthesizedExpression.java](src/main/java/org/xvm/compiler/ast/ParenthesizedExpression.java) | Modified | Shares an explicit probe between tuple and single-value interpretations before selecting the validation path. | +| [RelOpExpression.java](src/main/java/org/xvm/compiler/ast/RelOpExpression.java) | Modified | Uses explicit probes for operator and operand fitting and calls the revised type-fitting helper, whose silence policy is internal to that helper. | +| [ReturnStatement.java](src/main/java/org/xvm/compiler/ast/ReturnStatement.java) | Modified | Uses one probe for competing multi-value, conditional, future and tuple interpretations before validating the chosen return form. | +| [SequentialAssignExpression.java](src/main/java/org/xvm/compiler/ast/SequentialAssignExpression.java) | Modified | Adds the probe static imports alongside the expression migration. There is no executable behavior change in this file's diff. | +| [StatementExpression.java](src/main/java/org/xvm/compiler/ast/StatementExpression.java) | Modified | Uses probes when inspecting temporary statement bodies and an explicit source site when reporting a missing return. | +| [TemplateExpression.java](src/main/java/org/xvm/compiler/ast/TemplateExpression.java) | Modified | Shares a probe while choosing String versus Object handling for interpolated values. | +| [TernaryExpression.java](src/main/java/org/xvm/compiler/ast/TernaryExpression.java) | Modified | Uses local probes for the competing arm types and conditional False forms, replacing nullable listeners without changing which chosen arms are subsequently validated. | +| [TraceExpression.java](src/main/java/org/xvm/compiler/ast/TraceExpression.java) | Modified | Explicitly suppresses reports when adopting the already-validated wrapped expression's types and constants. | +| [TupleExpression.java](src/main/java/org/xvm/compiler/ast/TupleExpression.java) | Modified | Uses an explicit probe to obtain a written tuple type during implicit-type inspection. | +| [UnaryComplementExpression.java](src/main/java/org/xvm/compiler/ast/UnaryComplementExpression.java) | Modified | Marks required-type fitting as a probe before selecting the complement operator. | +| [UnaryMinusExpression.java](src/main/java/org/xvm/compiler/ast/UnaryMinusExpression.java) | Modified | Marks required-type fitting as a probe before selecting the negation operator. | + +### Partial syntax and compiler-backed cursor queries + +| File | Change | What changed and why | +| --- | --- | --- | +| [CursorScope.java](src/main/java/org/xvm/compiler/ast/partial/CursorScope.java) | Added | Captures visible variables, enclosing-instance information and type-name candidates through ordinary contextual lookup, retaining real accessibility and narrowing rules. | +| [IncompleteDeclarationStatement.java](src/main/java/org/xvm/compiler/ast/partial/IncompleteDeclarationStatement.java) | Added | Retains an unfinished declaration header and selected type prefix as syntax without registering an invented method/property component, parameter list or body context. | +| [IncompleteExpression.java](src/main/java/org/xvm/compiler/ast/partial/IncompleteExpression.java) | Added | Keeps an incomplete operation inside its real assignment, return or other value context. It inspects the intact prefix but never supplies a valid value or permits emission. | +| [IncompleteLocalDeclaration.java](src/main/java/org/xvm/compiler/ast/partial/IncompleteLocalDeclaration.java) | Added | Retains the written type/initializer when a local name is missing, without inventing a local register or inferred declaration identity. | +| [IncompleteStatement.java](src/main/java/org/xvm/compiler/ast/partial/IncompleteStatement.java) | Added | Owns the selected bare name, member or call prefix, including arguments and cursor metadata. Validation publishes prefix facts and then fails deliberately so the unfinished operation cannot emit code. | +| [IncompleteTypeCompositionStatement.java](src/main/java/org/xvm/compiler/ast/partial/IncompleteTypeCompositionStatement.java) | Added | Retains unfinished type headers and bodies without contributing inheritance/member facts. A root module can register its written namespace/core import solely to establish the query scope. | +| [PartialArgument.java](src/main/java/org/xvm/compiler/ast/partial/PartialArgument.java) | Added | Builds disposable argument trials with the correct lexical parentage, preserving labels and other written arguments without replacing retained source syntax. | +| [PartialCallResolver.java](src/main/java/org/xvm/compiler/ast/partial/PartialCallResolver.java) | Added | Uses normal lookup and argument fitting to inspect possible method/function calls at the cursor. Candidate signatures remain possibilities rather than a selected executable call. | +| [PartialConstructionResolver.java](src/main/java/org/xvm/compiler/ast/partial/PartialConstructionResolver.java) | Added | Reuses ordinary construction preparation and fitting for incomplete constructors, stopping before synthesis and emission instead of maintaining a second constructor type checker. | +| [PartialQueries.java](src/main/java/org/xvm/compiler/ast/partial/PartialQueries.java) | Added | Coordinates semantic inspection for incomplete nodes through PartialAst, collecting facts while their real compiler context is live and keeping compiler internals out of the host API. | +| [PartialSyntax.java](src/main/java/org/xvm/compiler/ast/partial/PartialSyntax.java) | Added | Provides read-only containment, cursor and argument-slot queries, including deferred anonymous bodies. Respects boundaries where a nested call or lambda owns a different argument context. | +| [ProposedLiteralToken.java](src/main/java/org/xvm/compiler/ast/partial/ProposedLiteralToken.java) | Added | Carries the spelling of a disposable zero-width literal proposal so normal literal conversion can inspect it; it is never installed in the source tree or published as a source binding. | + +### Tool drivers, runtime consumers and diagnostic text + +| File | Change | What changed and why | +| --- | --- | --- | +| [java/org/xvm/runtime/template/_native/lang/src/xRTCompiler.java](src/main/java/org/xvm/runtime/template/_native/lang/src/xRTCompiler.java) | Modified | Adapts the runtime compiler's diagnostics to explicit no-location sites and trailing varargs, preserving its existing result completion path. | +| [java/org/xvm/runtime/template/_native/reflect/xRTType.java](src/main/java/org/xvm/runtime/template/_native/reflect/xRTType.java) | Modified | Uses named probes for reflective newability queries instead of the removed blackhole sentinel; no editor dependency is introduced. | +| [java/org/xvm/tool/Bundler.java](src/main/java/org/xvm/tool/Bundler.java) | Modified | Aligns its constructor naming/documentation with the required listener passed to Launcher; bundling behavior is unchanged by this edit. | +| [java/org/xvm/tool/Compiler.java](src/main/java/org/xvm/tool/Compiler.java) | Modified | Extracts the ordinary declaration phases so embedding can stop before body validation/emission, while full compilation still runs them. Uses explicit diagnostics and abort queries without sentinel comparisons. | +| [java/org/xvm/tool/Disassembler.java](src/main/java/org/xvm/tool/Disassembler.java) | Modified | Aligns its constructor with Launcher's explicit listener contract; this is integration and documentation rather than a disassembly redesign. | +| [java/org/xvm/tool/Initializer.java](src/main/java/org/xvm/tool/Initializer.java) | Modified | Documents and forwards the explicit listener required by Launcher, retaining the initializer's existing work. | +| [java/org/xvm/tool/Launcher.java](src/main/java/org/xvm/tool/Launcher.java) | Modified | Requires a final diagnostic destination, chooses explicit discard for CLI-only dispatch and forwards a fatal diagnostic to the host before console handling can abort. | +| [java/org/xvm/tool/ModuleInfo.java](src/main/java/org/xvm/tool/ModuleInfo.java) | Modified | Adds snapshot-backed source identity, text and directory-entry hooks; accepts a parser strategy and exposes retained per-file syntax after loading failure. Node buffers are final and parsing observes host cancellation. | +| [java/org/xvm/tool/Runner.java](src/main/java/org/xvm/tool/Runner.java) | Modified | Aligns constructor naming/documentation with the explicit Launcher listener, retaining the execution path. | +| [java/org/xvm/tool/TestRunner.java](src/main/java/org/xvm/tool/TestRunner.java) | Modified | Forwards the listener through the revised Runner constructor and aligns its parameter naming; it adds no LSP-specific test-running behavior. | +| [resources/errors.properties](src/main/resources/errors.properties) | Modified | Adds the incomplete-expression diagnostic used by retained partial syntax, keeping the diagnostic code/message available through normal compiler reporting. | + +### Embedding and diagnostic contract tests + +| File | Change | What changed and why | +| --- | --- | --- | +| [api/BindingIdentitySnapshotTest.java](src/test/java/org/xvm/api/BindingIdentitySnapshotTest.java) | Added | Checks identity-based method/function/constructor maps at publication boundaries, detached immutable cursor maps and rejection of null entries. | +| [api/EmbeddingFootprintTest.java](src/test/java/org/xvm/api/EmbeddingFootprintTest.java) | Added | Checks that footprint inspection measures the supplied attempt and does not bootstrap the runtime. | +| [api/ErrorListenerBoundaryTest.java](src/test/java/org/xvm/api/ErrorListenerBoundaryTest.java) | Added | Checks rejection of missing listeners before compilation/file access and verifies that collectors, stage managers and AST reporting retain an explicitly chosen destination. | +| [asm/ErrorListenerAbortTest.java](src/test/java/org/xvm/asm/ErrorListenerAbortTest.java) | Added | Checks the shared default budget, unlimited mode, fatal abort policy and the fallback listener's reporting behavior. | +| [asm/ErrorListenerBranchTest.java](src/test/java/org/xvm/asm/ErrorListenerBranchTest.java) | Added | Checks host and ErrorList branches, full merge delivery, harmless merge on an unbranched listener and the distinction between a bare callback and a stateful collector. | +| [asm/ErrorListenerCancellationTest.java](src/test/java/org/xvm/asm/ErrorListenerCancellationTest.java) | Added | Checks cancellation through pre-existing branches and merges without replacing the wrapped budget or affecting a separate attempt. | +| [asm/ErrorListenerHostRegressionTest.java](src/test/java/org/xvm/asm/ErrorListenerHostRegressionTest.java) | Added | Exercises real lexer failures through a host branch, proving that two errors survive and that a later parent abort reaches an already-created branch. | +| [asm/ErrorListenerMigrationTest.java](src/test/java/org/xvm/asm/ErrorListenerMigrationTest.java) | Added | Checks duplicate delivery versus buffering, parent budget/abort propagation, retained branch source sites and parser/lexer stopping after the configured budget is spent. | +| [asm/ErrorListenerSilenceTest.java](src/test/java/org/xvm/asm/ErrorListenerSilenceTest.java) | Added | Checks silence reasons, standalone versus parent-derived silence, idempotent suppression and observation of later parent abort requests. | +| [asm/ErrorListenerSiteTest.java](src/test/java/org/xvm/asm/ErrorListenerSiteTest.java) | Added | Checks trailing varargs, severity helpers, exhaustive diagnostic sites, tee delivery/abort behavior and replay of retained diagnostics. | +| [asm/FileStructureErrorListenerTest.java](src/test/java/org/xvm/asm/FileStructureErrorListenerTest.java) | Modified | Replaces ambient-listener expectations with explicit delivery and verifies that reusing a structure does not reuse the previous request's listener. | +| [asm/ReportingTest.java](src/test/java/org/xvm/asm/ReportingTest.java) | Added | Checks nested destination restoration after exceptions, restoration of an inactive state and rejection of a null active destination. | + +### Parser, AST and partial-analysis tests + +| File | Change | What changed and why | +| --- | --- | --- | +| [compiler/CompilerCancellationTest.java](src/test/java/org/xvm/compiler/CompilerCancellationTest.java) | Added | Checks that canceled structure registration stops without throwing or fabricating a source error. | +| [compiler/ast/bindings/CursorBindingTest.java](src/test/java/org/xvm/compiler/ast/bindings/CursorBindingTest.java) | Added | Checks grouped cursor-fact construction, immutable snapshots and independent updates that preserve unrelated call, argument and enclosing-instance facts. | +| [compiler/EvalCompilerTest.java](src/test/java/org/xvm/compiler/EvalCompilerTest.java) | Added | Checks that diagnostics can be queried before compilation starts. | +| [compiler/ParserAttemptTest.java](src/test/java/org/xvm/compiler/ParserAttemptTest.java) | Added | Checks kept/discarded and nested attempts, token rewind, recovery restoration, lexer reports, callback exceptions, caller abort and module-name scanning. | +| [compiler/ParserDeclarationNameTest.java](src/test/java/org/xvm/compiler/ParserDeclarationNameTest.java) | Added | Checks unfinished declaration-name slots, retained written types/initializers, independent children and following declarations; rejects slots outside supported contexts and honors stopping policies. | +| [compiler/ParserRecoveryTest.java](src/test/java/org/xvm/compiler/ParserRecoveryTest.java) | Added | Exercises ordinary recovery and explicit cursor parsing across expressions, calls, delimiters, declaration/type headers and EOF. Checks positions, following declarations, ownership, speculation and cancellation/budgets. | +| [compiler/ParserTest.java](src/test/java/org/xvm/compiler/ParserTest.java) | Modified | Migrates the existing discarded-lookahead regression to Parser.Attempt and updates the explanation of speculative diagnostic suppression. | +| [compiler/ast/AstChildTraversalTest.java](src/test/java/org/xvm/compiler/ast/AstChildTraversalTest.java) | Added | Checks repeated/interleaved traversal, live views versus snapshots, exhaustion and preservation of the legacy mutable replacement/removal cursor. | +| [compiler/ast/AstTreeCopyTest.java](src/test/java/org/xvm/compiler/ast/AstTreeCopyTest.java) | Added | Checks recovery-node metadata, independent owned children, retained cursor tokens and immutable header lists when partial syntax is copied inside ordinary trees. | +| [compiler/ast/CompositionSourceTest.java](src/test/java/org/xvm/compiler/ast/CompositionSourceTest.java) | Added | Checks source lookup for a default composition before adoption, when no parent or explicit type supplies a source. | +| [compiler/ast/DeclarationProvenanceTest.java](src/test/java/org/xvm/compiler/ast/DeclarationProvenanceTest.java) | Added | Checks qualified-name prefixes across deferred/failed resolution and retries, resolver ownership after copying and independent rebinding of copied parameters. | +| [compiler/ast/NameResolverReportingTest.java](src/test/java/org/xvm/compiler/ast/NameResolverReportingTest.java) | Added | Checks that direct resolution and callbacks use the same caller, release the listener afterward and restore outer reporting through nested, deferred and exceptional calls. | +| [compiler/ast/OrdinaryAstTreeCopyTest.java](src/test/java/org/xvm/compiler/ast/OrdinaryAstTreeCopyTest.java) | Added | Checks typed and legacy copies of ordinary nodes for metadata preservation and independent child ownership. | +| [compiler/ast/ValidationScopeTest.java](src/test/java/org/xvm/compiler/ast/ValidationScopeTest.java) | Added | Exercises callback behavior after nested failures, finally-scope restoration and early loop exits; also checks that active scopes cannot contain missing state. | +| [compiler/ast/partial/PartialSyntaxTest.java](src/test/java/org/xvm/compiler/ast/partial/PartialSyntaxTest.java) | Added | Checks deferred anonymous syntax, exact cursor containment, argument ownership across labels/groups/nested calls, copying and disposable substitutions without changing written syntax. | +| [compiler/ast/partial/RegisteredChildFieldsTest.java](src/test/java/org/xvm/compiler/ast/partial/RegisteredChildFieldsTest.java) | Added | Checks inherited and declared child fields in traversal, dumping, copying, replacement and removal, including independent mutable child lists. | + +### Existing tool test integration + +| File | Change | What changed and why | +| --- | --- | --- | +| [BundlerTest.java](src/test/java/org/xvm/tool/BundlerTest.java) | Modified | Passes explicit discard listeners through existing launcher/bundler tests, preserving their CLI and artifact assertions under the new contract. | +| [LauncherErrorHandlingTest.java](src/test/java/org/xvm/tool/LauncherErrorHandlingTest.java) | Modified | Migrates callbacks and explicit listener choices and adds a regression proving that the host receives a fatal diagnostic before console handling throws. | +| [LauncherVersionTest.java](src/test/java/org/xvm/tool/LauncherVersionTest.java) | Modified | Uses explicit discard listeners for version commands, preserving the existing output-format checks. | +| [ModuleInfoTest.java](src/test/java/org/xvm/tool/ModuleInfoTest.java) | Modified | Adds a snapshot-source case that parses before its directory exists and checks one owned diagnostic buffer from node construction; migrates existing diagnostic assertions to the new API. | diff --git a/javatools/src/main/java/org/xvm/api/Connector.java b/javatools/src/main/java/org/xvm/api/Connector.java index 80193775baf..a5d2736a5ab 100644 --- a/javatools/src/main/java/org/xvm/api/Connector.java +++ b/javatools/src/main/java/org/xvm/api/Connector.java @@ -102,6 +102,14 @@ public void invoke0(MethodStructure method, List args) { */ public abstract int join() throws InterruptedException; + /** + * Release this connector's runtime once nothing is to run in it any more, so that a host that + * makes another connector does not keep this one's runtime: what holds it, such as pending + * timers and worker threads, lets it go. By default there is nothing to release. + */ + public void shutdown() { + } + // ----- data fields --------------------------------------------------------------------------- /** diff --git a/javatools/src/main/java/org/xvm/api/DebuggerProvider.java b/javatools/src/main/java/org/xvm/api/DebuggerProvider.java new file mode 100644 index 00000000000..dc5f05e43eb --- /dev/null +++ b/javatools/src/main/java/org/xvm/api/DebuggerProvider.java @@ -0,0 +1,61 @@ +package org.xvm.api; + +import java.io.IOException; + +import org.xvm.asm.ModuleRepository; +import org.xvm.asm.ModuleStructure; + +import org.xvm.runtime.Debugger; + +/** + * A debugger for a program that the launchers run with {@code --debug}, such as an IDE's, which + * {@link org.xvm.tool.Runner} and {@link org.xvm.tool.TestRunner} find with + * {@link java.util.ServiceLoader}: on their class path, or in the jar that the option's + * {@code agent=} names, which goes ahead of those on the class path. javatools' own, + * {@link org.xvm.runtime.debug.DebugAgent}, lets a language server's debug session attach; without + * {@code --debug}, the console debugger stays the debugger of {@code assert:debug}. + */ +public interface DebuggerProvider { + /** + * Start debugging a program: make the debugger that the launcher installs in the program's + * runtime before its module starts. With the option {@code suspend}, return once the debugger + * outside the program has sent what the program starts with, such as its breakpoints. + * + * @param options what {@code --debug} gave, without {@code agent=}, such as + * {@code listen:127.0.0.1:5005,suspend} + * @param repository the repository of the program's modules + * @param module the program's module + * + * @return the debugging of this run, which the launcher ends when the program does + * + * @throws IllegalArgumentException if the options cannot be understood + * @throws IOException if the debugger outside cannot be reached or waited for + */ + Session start(String options, ModuleRepository repository, ModuleStructure module) + throws IOException; + + /** + * The debugging of one run. + */ + interface Session extends AutoCloseable { + /** + * @return the debugger to install in the program's runtime + */ + Debugger debugger(); + + /** + * The program ended with the specified exit code; the debugger outside is told. + * + * @param code the exit code + */ + void ended(int code); + + /** + * End the debugging, also if the program did not end normally. Once this returns, the + * provider's own threads have ended: the launcher then closes the jar that {@code + * agent=} named, so no class of it can be loaded any more. + */ + @Override + void close(); + } +} diff --git a/javatools/src/main/java/org/xvm/api/EmbeddingSupport.java b/javatools/src/main/java/org/xvm/api/EmbeddingSupport.java index 11a36c56a07..54af91d9730 100644 --- a/javatools/src/main/java/org/xvm/api/EmbeddingSupport.java +++ b/javatools/src/main/java/org/xvm/api/EmbeddingSupport.java @@ -1,5 +1,10 @@ package org.xvm.api; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static java.util.Objects.requireNonNull; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -7,13 +12,23 @@ import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; + +import java.util.stream.Stream; + import org.xvm.asm.ConstantPool; import org.xvm.asm.DirRepository; +import org.xvm.asm.ErrorList; import org.xvm.asm.ErrorListener; import org.xvm.asm.FileStructure; import org.xvm.asm.LinkedRepository; @@ -24,11 +39,41 @@ import org.xvm.asm.constants.TypeConstant; +import org.xvm.compiler.BuildRepository; +import org.xvm.compiler.Compiler; +import org.xvm.compiler.CompilerException; import org.xvm.compiler.InstantRepository; +import org.xvm.compiler.Parser; +import org.xvm.compiler.Source; + +import org.xvm.compiler.Token.Id; +import org.xvm.compiler.ast.AstNode; +import org.xvm.compiler.ast.InvocationExpression; +import org.xvm.compiler.ast.NewExpression; +import org.xvm.compiler.ast.PropertyDeclarationStatement; +import org.xvm.compiler.ast.StatementBlock; +import org.xvm.compiler.ast.TypeCompositionStatement; +import org.xvm.compiler.ast.bindings.CursorBinding; +import org.xvm.compiler.ast.bindings.InitializerBinding; +import org.xvm.compiler.ast.bindings.InvocationBinding; +import org.xvm.compiler.ast.partial.IncompleteStatement; +import org.xvm.compiler.ast.partial.PartialSyntax; + +import org.xvm.runtime.Debugger; +import org.xvm.runtime.SessionDebugger; import org.xvm.runtime.template._native.io.xExternalConsole; +import org.xvm.tool.Console; +import org.xvm.tool.Launcher.LauncherException; +import org.xvm.tool.LauncherOptions.CompilerOptions; +import org.xvm.tool.ModuleInfo; +import org.xvm.tool.ModuleInfo.Node; + +import static org.xvm.asm.ErrorListener.NOWHERE; +import static org.xvm.asm.ErrorListener.at; import static org.xvm.util.Handy.quoted; +import static org.xvm.util.Handy.resolveFile; import static org.xvm.util.Severity.ERROR; /** @@ -39,7 +84,8 @@ * Ecstasy classes. Without configuration, EmbeddingSupport will attempt to locate the core Ecstasy * classes using the "XDK_HOME" OS property. * - *

The methods on EmbeddingSupport itself can be assumed to be thread-safe and concurrent. + *

Hosts must serialize compilations that share this instance's configured repository. Configuration + * must complete before compilation or execution begins. * *

EmbeddingSupport is a singleton because running a module requires the interpreter, and the * interpreter keeps JVM-wide state. When the Connector creates its NativeContainer, the container @@ -77,11 +123,88 @@ private static class Singleton { private static final Object LOCK = new Object(); + private static final Console SILENT_CONSOLE = new Console() { + @Override + public String out(Object value) { + return String.valueOf(value); + } + + @Override + public String err(Object value) { + return String.valueOf(value); + } + }; + private volatile boolean configured; private ModuleRepository cfgRepo; private String cfgInjector; private Connector connector; + /** + * Build a read-only repository over whichever of the given directories exist. + * + * @param dirs the directories, in search order + * + * @return the repository, or null if none of the directories exist + */ + private static ModuleRepository repoOver(File... dirs) { + var list = Arrays.stream(dirs) + .filter(File::isDirectory) + .map(dir -> new DirRepository(dir, true)) + .toList(); + return switch (list.size()) { + case 0 -> null; + case 1 -> list.getFirst(); + default -> new LinkedRepository(list.toArray(ModuleRepository.NO_REPOS)); + }; + } + + /** + * A snapshot of what the compiler is holding on to. + * + *

An embedding host that keeps a compiler alive across many compilations - a language server + * is the obvious one - needs some way to see whether it is accumulating. These are the cheap + * numbers: taking them costs a field read and a repository listing, so a host can log one per + * compilation without measuring itself instead of the compiler. + * + * @param modules how many modules the configured repository offers + * @param constants how many constants are interned in the current pool + * @param invalidations how many times cached type information has been invalidated + * @param heapBytes used heap, which is the JVM's figure and not the compiler's alone + */ + public record Footprint(int modules, int constants, int invalidations, long heapBytes) { + @Override + public String toString() { + return "modules=" + modules + ", constants=" + constants + + ", invalidations=" + invalidations + + ", heap=" + (heapBytes / (1024 * 1024)) + "MB"; + } + } + + /** + * Take a {@link Footprint} without selecting a compilation's pool. This does not initialize + * the runtime; pool counts are zero. Use {@link #footprint(Compilation)} for compiler counts. + * + * @return the snapshot; the counts are zero where nothing has been configured or built yet + */ + public Footprint footprint() { + return footprint(null); + } + + /** + * Take a passive snapshot of a compilation's pool and the configured repository. + * + * @param compilation the compilation to measure, or null for repository and heap counts only + * + * @return the snapshot; pool counts are zero if no file structure was built + */ + public Footprint footprint(Compilation compilation) { + var pool = compilation == null ? null : compilation.pool(); + var runtime = Runtime.getRuntime(); + return new Footprint(cfgRepo == null ? 0 : cfgRepo.getModuleNames().size(), pool == null ? 0 : pool.size(), + pool == null ? 0 : pool.getInvalidationCount(), runtime.totalMemory() - runtime.freeMemory()); + } + /** * @return true if configured * @throws IllegalStateException if not configured @@ -91,9 +214,15 @@ private boolean verifyConfigured() { // attempt to auto-configure String home = System.getenv("XDK_HOME"); if (home != null) { - File dir = new File(new File(home), "lib"); - if (dir.isDirectory()) { - configure(new DirRepository(dir, true), null); + // an XDK keeps its libraries in lib/ and the two modules the compiler bootstraps + // against - the turtle, mack.xtclang.org, and the native bridge - in javatools/. + // Configuring only lib/ produced a repository that could never compile anything: + // every compile failed in prelinkSystemLibraries with "Unable to load module: + // mack.xtclang.org", reported as an internal error with no location + var dirHome = new File(home); + var repo = repoOver(new File(dirHome, "lib"), new File(dirHome, "javatools")); + if (repo != null) { + configure(repo, null); } } @@ -126,6 +255,20 @@ public Connector ensureConnector() { } } + /** + * Shut down this instance's interpreter, if it has one, so that its runtime can be collected; + * a later run starts another. The singleton's interpreter lasts as long as the JVM; this is for + * the tests, which make instances of their own. + */ + void shutdownInterpreter() { + synchronized (LOCK) { + if (connector != null) { + connector.shutdown(); + connector = null; + } + } + } + /** * Diagnostics for a host that keeps the runtime across runs: the external consoles that runs * registered and that are still registered. A run releases its console when it ends, so this is @@ -139,6 +282,20 @@ public int registeredConsoles() { : 0; } + /** + * Diagnostics for a host that keeps the runtime across runs: the threads of the runtime's own + * pool, which runs share. The pool is fixed in size, so this is at most one thread per core. + * A thread that a pool thread starts, such as the JDK's for a CompletableFuture's timeout, + * joins the pool's thread group but is not one of these; JIT runs have no pool. + * + * @return the number of threads in the runtime's pool + */ + public int poolThreads() { + return ensureConnector() instanceof InterpreterConnector interpreter + ? interpreter.getNativeContainer().f_runtime.f_executorXVM.getPoolSize() + : 0; + } + // ----- API ----------------------------------------------------------------------------------- /** @@ -149,13 +306,16 @@ public static EmbeddingSupport instance() { } /** - * Provide configuration necessary for the underlying Ecstasy tools and libraries. Must be - * called exactly one time before any other method. + * Provide configuration for the underlying Ecstasy tools and libraries before compiling or + * running a module. Repeating the same configuration is allowed; a conflicting configuration + * is rejected. Passive configuration and footprint queries do not require initialization. * * @param coreRepo the ModuleRepository to load the core libraries from * @param customInjector (optional) "module:class" name of a custom injector implementation to * use to provide injectable resources in lieu of the default injector * for this implementation + * + * @return this configured host support instance */ public EmbeddingSupport configure(ModuleRepository coreRepo, String customInjector) { synchronized (LOCK) { @@ -195,14 +355,33 @@ public String getConfiguredInjector() { } /** - * @return the ConstantPool of the core Ecstasy libraries used by the runtime Connector instance - * that is instantiated by EmbeddingSupport + * Obtain the constant pool of the runtime, starting one if it is not running yet. + * + *

This is not the pool a compilation used - that belongs to the {@link Compilation} it + * produced. Asking for this one boots an interpreter: a connector builds a NativeContainer, + * which loads a native template for every core module, so it needs the whole library and not + * just the part the compiler bootstraps against. The old name said "get" and read like an + * accessor. + * + * @return the runtime's constant pool */ - public ConstantPool getConstantPool() { + public ConstantPool ensureRuntimePool() { verifyConfigured(); return ensureConnector().getConstantPool(); } + /** + * Obtain the runtime's constant pool, starting the runtime if necessary. + * + * @return the runtime's constant pool + * @deprecated use {@link #ensureRuntimePool()} to make runtime initialization explicit; + * use {@link Compilation#pool()} for a compilation's pool + */ + @Deprecated + public ConstantPool getConstantPool() { + return ensureRuntimePool(); + } + // ----- compiler support ---------------------------------------------------------------------- /** @@ -210,33 +389,770 @@ public ConstantPool getConstantPool() { * * @param source the source code for an entire module to compile * @param input (optional) the module repository to read any required modules from - * @param errs (optional) the ErrorListener to log any compiler messages to + * @param errs the ErrorListener to log any compiler messages to * * @return the resulting ModuleStructure, or null if a compiler error occurred * * @see ModuleCompiler#compile(String, ModuleRepository, ErrorListener) */ - public ModuleStructure compile(String source, ModuleRepository input, ErrorListener errs) { + public ModuleStructure compile(String source, ModuleRepository input, @NotNull ErrorListener errs) { + return compile(new Source(source), input, errs); + } + + /** + * Compile a module held in memory, as a named document. + * + *

A diagnostic's identity includes the name of the source it came from, so a host holding + * several documents that are not on disk - an editor's unsaved buffers - has to be able to + * tell them apart. Without a name, two documents with a problem at the same offset produce + * the same identity and a listener that deduplicates discards the second. The name belongs to + * the document rather than to the act of compiling it, which is why this takes a + * {@link Source}: {@code new Source(text, uri)} says it once, where a second String parameter + * would sit next to the first and be silently swappable with it. + * + * @param source the source to compile, carrying whatever name it was created with + * @param input (optional) the module repository to read any required modules from + * @param errs the ErrorListener to log any compiler messages to + * + * @return the resulting ModuleStructure, or null if a compiler error occurred + */ + public ModuleStructure compile(Source source, ModuleRepository input, @NotNull ErrorListener errs) { + return compileModule(source, input, errs).module(); + } + + /** + * The outcome of compiling a source or module source tree. + * + *

A failed compilation used to answer with nothing but null, which threw away everything the + * attempt had built. That is most of what a host wants when it fails: the structures a + * verification error was raised against still exist, and the pool they were interned in is + * the only way to reach them - to ask a type what building its TypeInfo had to say, for + * instance. + * + *

Recovered source trees have not entered compiler passes. Traverse their children using + * each root's Source; child parent pointers may not yet be installed. Binding maps are + * immutable identity snapshots: keys and values compare by reference, not structural equality. + * + * @param module the compiled module, or null if the compilation did not get that far + * @param file the file structure that was built, or null if it did not get that far + * @param ast the assembled source tree, or null if loading/parsing failed; linked + * member trees retain their original Sources and positions + * @param sourceTrees available per-source syntax, including recovered trees on parse failure; + * these are structural facts, not a promise of semantic validity + * @param callBindings immutable call facts keyed by surviving invocation identity; compiler + * objects remain worker-owned and must be copied before concurrent use + * @param functionBindings validated function signatures with no selected runtime method; + * ownership is the same as for callBindings + * @param constructorBindings validated constructor and argument provenance; the same ownership + * rules apply, with no additional state on the construction AST + * @param initializerBindings successful constant-initializer facts anchored to surviving source + * properties; no speculative clone nodes or contexts are retained + */ + public record Compilation(ModuleStructure module, FileStructure file, StatementBlock ast, + List sourceTrees, + Map callBindings, + Map functionBindings, + Map constructorBindings, + Map initializerBindings) { + public Compilation { + sourceTrees = List.copyOf(sourceTrees); + callBindings = identitySnapshot(callBindings); + functionBindings = identitySnapshot(functionBindings); + constructorBindings = identitySnapshot(constructorBindings); + initializerBindings = identitySnapshot(initializerBindings); + } + + /** + * Retain hosts that supply constructor bindings without folded initializer facts. + * + * @param module the compiled module, or null when compilation did not succeed + * @param file the file structure produced by the attempt, or null if none was built + * @param ast the assembled source root, or null if loading or parsing failed + * @param sourceTrees the available per-source syntax trees, including recovered trees + * @param callBindings validated method-call facts keyed by invocation identity + * @param functionBindings validated function-call facts keyed by invocation identity + * @param constructorBindings validated constructor-call facts keyed by construction identity + */ + public Compilation(ModuleStructure module, FileStructure file, StatementBlock ast, + List sourceTrees, Map callBindings, + Map functionBindings, + Map constructorBindings) { + this(module, file, ast, sourceTrees, callBindings, functionBindings, constructorBindings, Map.of()); + } + + /** + * Retain hosts that supply method and function bindings. + * + * @param module the compiled module, or null when compilation did not succeed + * @param file the file structure produced by the attempt, or null if none was built + * @param ast the assembled source root, or null if loading or parsing failed + * @param sourceTrees the available per-source syntax trees, including recovered trees + * @param callBindings validated method-call facts keyed by invocation identity + * @param functionBindings validated function-call facts keyed by invocation identity + */ + public Compilation(ModuleStructure module, FileStructure file, StatementBlock ast, + List sourceTrees, Map callBindings, + Map functionBindings) { + this(module, file, ast, sourceTrees, callBindings, functionBindings, Map.of()); + } + + /** + * Retain hosts that supply only statically selected method calls. + * + * @param module the compiled module, or null when compilation did not succeed + * @param file the file structure produced by the attempt, or null if none was built + * @param ast the assembled source root, or null if loading or parsing failed + * @param sourceTrees the available per-source syntax trees, including recovered trees + * @param callBindings validated method-call facts keyed by invocation identity + */ + public Compilation(ModuleStructure module, FileStructure file, StatementBlock ast, + List sourceTrees, Map callBindings) { + this(module, file, ast, sourceTrees, callBindings, Map.of()); + } + + /** + * Retain the construction API for hosts supplying structural source trees. + * + * @param module the compiled module, or null when compilation did not succeed + * @param file the file structure produced by the attempt, or null if none was built + * @param ast the assembled source root, or null if loading or parsing failed + * @param sourceTrees the available per-source syntax trees, including recovered trees + */ + public Compilation(ModuleStructure module, FileStructure file, StatementBlock ast, List sourceTrees) { + this(module, file, ast, sourceTrees, Map.of()); + } + + /** + * Retain the original construction API for hosts supplying one assembled tree. + * + * @param module the compiled module, or null when compilation did not succeed + * @param file the file structure produced by the attempt, or null if none was built + * @param ast the assembled source root, or null if loading or parsing failed + */ + public Compilation(ModuleStructure module, FileStructure file, StatementBlock ast) { + this(module, file, ast, ast == null ? List.of() : List.of(ast)); + } + + /** + * Represent file structures without a successful module or retained source tree. + * + * @param file the file structure produced so far + * + * @return a partial compilation + */ + public static Compilation forFile(FileStructure file) { + return new Compilation(null, requireNonNull(file, "file"), null); + } + + /** + * @return true iff a module came out of it + */ + public boolean succeeded() { + return module != null; + } + + /** + * @return the pool this compilation interned into, or null if there was no compilation + */ + public ConstantPool pool() { + return file == null ? null : file.getConstantPool(); + } + + /** + * Walk the assembled source after successful loading/parsing. + * + *

A host that wants to say where something is has to come through here: a + * {@link org.xvm.asm.Component} knows its name, its kind and its children, and nothing + * about the text it was written in. Only the AST carries positions. + * + * @return the assembled source root, or null after loading/parsing errors; use + * {@link #sourceTrees()} for available structural syntax in that case + */ + public StatementBlock parsed() { + return ast; + } + } + + /** + * Compile a module held in memory, and answer with everything the attempt produced. + * + * @param source the source to compile, carrying whatever name it was created with + * @param input (optional) the module repository to read any required modules from + * @param errs the ErrorListener to log any compiler messages to + * + * @return the outcome; never null, though its parts may be + */ + public Compilation compileModule(Source source, ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(source, "source"); + return compileModule(listener -> ParsedSources.parse(source, listener), input, errs); + } + + /** + * Compile a module's source tree using the same discovery, resource association and parse-tree + * assembly as the CLI. The returned AST includes the member files with their original Sources. + * Hosts may subclass {@link ModuleInfo} and override its protected {@code readSource(File)} + * and {@code sourceEntries(File)} hooks to supply a consistent snapshot of unsaved text and + * source membership. The default provider discovers and reads files from disk. + * + * @param sources a fresh ModuleInfo for this attempt; do not reuse a previously parsed tree + * @param input the optional repository of compiled dependencies + * @param errs the listener for this compilation + * + * @return the outcome, including the linked AST when source loading succeeded + */ + public Compilation compileModule(ModuleInfo sources, ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(sources, "sources"); + return compileModule(listener -> ParsedSources.parse(sources, listener), input, errs); + } + + /** + * A successfully resolved declaration graph, before expression validation and code generation. + * It contains no executable artifact. The calling worker exclusively owns these structures and + * may inspect TypeInfo with its listener; ordinary compilation still validates every body. + * + * @param file the file structure containing the resolved declarations + * @param ast the assembled source tree + * @param sourceTrees the available per-source trees, retained in source order + */ + public record DeclarationAnalysis(FileStructure file, StatementBlock ast, List sourceTrees) { + public DeclarationAnalysis { + requireNonNull(file, "file"); + requireNonNull(ast, "ast"); + sourceTrees = List.copyOf(sourceTrees); + } + + /** + * Obtain the pool that owns these resolved declarations. + * + * @return the constant pool owned by this declaration analysis + */ + public ConstantPool pool() { + return file.getConstantPool(); + } + } + + /** + * Resolve one source's declarations in a fresh attempt. Body errors are outside this operation's + * scope; syntax, linkage, declaration errors and cancellation prevent a result. + * + * @param source the source to parse + * @param input optional compiled dependencies + * @param errs the host listener + * @return resolved declarations, never a compiled module + */ + public Optional analyzeDeclarations(Source source, ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(source, "source"); + return analyzeDeclarations(listener -> ParsedSources.parse(source, listener), input, errs); + } + + /** + * Resolve a module tree's declarations using the same source/resource overlay contract as + * {@link #compileModule(ModuleInfo, ModuleRepository, ErrorListener)}. Use a fresh ModuleInfo. + * + * @param sources the module's source tree + * @param input optional compiled dependencies + * @param errs the host listener + * @return resolved declarations, empty after any declaration failure or cancellation + */ + public Optional analyzeDeclarations(ModuleInfo sources, ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(sources, "sources"); + return analyzeDeclarations(listener -> ParsedSources.parse(sources, listener), input, errs); + } + + private Optional analyzeDeclarations(Function parse, + ModuleRepository input, ErrorListener errs) { + var attempt = runCompiler(parse, input, errs, CursorBinding.Collector.NONE, CompilationGoal.DECLARATIONS); + var compiler = attempt.compiler(); + return attempt.succeeded() + ? Optional.of(new DeclarationAnalysis(compiler.file, compiler.ast, compiler.sourceTrees)) + : Optional.empty(); + } + + /** + * Facts retained by an explicit incomplete-source analysis, never a compiled module. + * + *

The available source syntax and sites may be unvalidated. A child expression supplies a + * semantic fact only if validation succeeded (isValidated and a fitting TypeFit) and its + * resolved target/type is available. Failed validation can leave placeholder types. An absent + * pool means semantic analysis did not start. Cursor bindings copy visible scope and candidate + * signatures/mappings fitted to written arguments; they never select the incomplete operation + * or invent a missing argument/result. Consumers must copy facts while exclusively + * owning the attempt, as with Compilation; ASTs and pools are not concurrent query objects. + * Binding maps preserve reference identity for both keys and values and reject null entries. + * + * @param sourceTrees available source syntax, including unvalidated recovered trees + * @param sites incomplete source sites retained by this attempt + * @param pool the attempt's constant pool, or null if semantic analysis did not start + * @param callBindings validated method calls keyed by source invocation identity + * @param cursorBindings visible scope and candidate facts keyed by incomplete site identity + * @param functionBindings validated function signatures keyed by source invocation identity + */ + public record PartialAnalysis(List sourceTrees, List sites, + @Nullable ConstantPool pool, + Map callBindings, + Map cursorBindings, + Map functionBindings) { + public PartialAnalysis { + sourceTrees = List.copyOf(sourceTrees); + sites = List.copyOf(sites); + cursorBindings = identitySnapshot(cursorBindings); + callBindings = identitySnapshot(callBindings); + functionBindings = identitySnapshot(functionBindings); + } + + /** + * Create partial facts with cursor and selected-method bindings. + * + * @param sourceTrees the available per-source syntax trees, including recovered trees + * @param sites the retained incomplete source sites + * @param pool the attempt's pool, or null if semantic analysis did not start + * @param callBindings validated method-call facts keyed by invocation identity + * @param cursorBindings visible scope and candidate facts keyed by incomplete-site identity + */ + public PartialAnalysis(List sourceTrees, List sites, + @Nullable ConstantPool pool, + Map callBindings, + Map cursorBindings) { + this(sourceTrees, sites, pool, callBindings, cursorBindings, Map.of()); + } + + /** + * Create partial facts with call and receiver information only. + * + * @param sourceTrees the available per-source syntax trees, including recovered trees + * @param sites the retained incomplete source sites + * @param pool the attempt's pool, or null if semantic analysis did not start + * @param callBindings validated method-call facts keyed by invocation identity + */ + public PartialAnalysis(List sourceTrees, List sites, + @Nullable ConstantPool pool, Map callBindings) { + this(sourceTrees, sites, pool, callBindings, Map.of()); + } + + /** + * Create partial facts with syntax and receiver information only. + * + * @param sourceTrees the available per-source syntax trees, including recovered trees + * @param sites the retained incomplete source sites + * @param pool the attempt's pool, or null if semantic analysis did not start + */ + public PartialAnalysis(List sourceTrees, List sites, @Nullable ConstantPool pool) { + this(sourceTrees, sites, pool, Map.of()); + } + } + + /** + * Analyze the intact prefix of one trailing incomplete expression statement in a single + * source module. Supports a member-access dot or unfinished call at EOF, with complete + * preceding declarations/statements and arguments. Other syntax errors prevent semantic + * analysis; returns, assignments, incomplete nested arguments and module member files are + * outside this bounded contract. Complete input has no incomplete site and is not compiled. + * + *

Parsing reports immediately to the host and respects its budget/cancellation. Only the + * recognized EOF boundary may enter compiler passes. The incomplete statement validates its + * intact children in the real method context, then fails validation before method emission. + * Its repeated EOF diagnostic stops the compiler internally without being delivered twice + * to the host. Other diagnostics and unexpected failures remain visible. + * + * @param source a fresh named source for this attempt + * @param input the optional repository of compiled dependencies + * @param errs the host's diagnostic listener + * + * @return partial facts with no success/module artifact contract + */ + public PartialAnalysis analyzeIncomplete(Source source, ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(source, "source"); + return analyzeIncomplete(listener -> { + var tree = Parser.forPartialAnalysis(source, listener).parseSource(); + return new ParsedSources(tree, List.of(tree)); + }, Parser.UNEXPECTED_EOF, input, errs); + } + + /** + * Analyze an incomplete member access or call at an explicit source position, including before + * existing closing braces or a semicolon. Source text is unchanged and later declarations retain + * their positions. The cursor must come from {@link Source#getPosition()} for this exact text. + * + *

Expression queries support standalone statements, assignment and initializer values, single + * return values, and nested call arguments, including named arguments and existing closing + * parentheses. Member tokens retain their completion prefix. A cursor before a call's closing + * parenthesis inspects its receiver and written arguments even when the selected syntax is + * complete. The selected member or call itself is not validated. Bare-name prefixes and empty + * statement boundaries expose visible scope; a named argument awaiting its value retains its + * label. Candidate signatures infer types from written arguments. Compound and conditional + * values, and arguments following the cursor, retain their real contexts. + * + *

Recovery retains missing call/group/tuple parentheses and index/list/map brackets at + * statement or outer-delimiter boundaries. Declaration values can retain missing terminators, + * parameter defaults can retain a missing closing parenthesis before a body, and a cursor at + * EOF can retain missing block braces. Recovery does not invent operands, declaration names or + * types, or literal contents, and does not repair unrelated errors. + * + *

Construction queries reuse normal receiver/type preparation, including virtual, inner and + * annotated types, required-type and argument-derived class inference, and parenthesized array + * initializers. Anonymous construction prepares the retained declaration and fits its own or + * superclass constructors without emitting forwarding constructors or running capture analysis. + * Single-dimensional array brackets fit the fixed-size constructor's size parameter, including + * an empty/final-name slot and a missing closing bracket. A following supplier is parsed for + * recovery but is outside this prefix proof. Multidimensional construction remains unsupported. + * + *

Declaration queries resolve member, return and method-parameter type prefixes through the + * enclosing compiler scope. Flat qualified names resolve their qualifier and visible nested + * types, replacing only the final written identifier. Unqualified empty parameter slots are + * supported. An unfinished header retains its written name and range but registers no method, + * property component or parameters. Written leaf names inside parameterized and compound types + * participate, including bounded missing angle/group closers; these are visible-type suggestions, + * not proof of generic constraints. Empty type/qualified slots, selected qualifier tokens, + * parameterized qualifiers, generic base names and function/sequence types are supported. + * + *

Generic-method, multiple-return and type-composition headers query their enclosing scope. + * Unregistered formals hide outer names without acquiring invented identities. A root module + * header registers only its written namespace/core import; its incomplete body and compositions + * cannot emit or contribute facts. Other syntax errors prevent semantic analysis, and cursors + * outside supported boundaries yield no site. Partial facts never promise executable output. + * + * @param source a fresh named source whose text remains unchanged during this attempt + * @param cursor the encoded source position at which to inspect incomplete syntax + * @param input compiled dependencies, or null to use only the configured core repository + * @param errs the host's diagnostic listener and abort policy + * + * @return retained source trees and available cursor/binding facts, without an executable module + */ + public PartialAnalysis analyzeIncomplete(Source source, long cursor, ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(source, "source"); + return analyzeIncomplete(listener -> { + var tree = Parser.forPartialAnalysis(source, cursor, listener).parseSource(); + return new ParsedSources(tree, List.of(tree)); + }, Parser.INCOMPLETE_EXPRESSION, input, errs); + } + + /** + * Analyze an explicit cursor in one file of a module snapshot. Uses the same source loading, + * overlay and resource contract as {@link #compileModule(ModuleInfo, ModuleRepository, ErrorListener)}. + * The cursor refers to the selected file's snapshot, which may contain unsaved text. Neither + * this ModuleInfo nor its parsed trees may be reused for another attempt or normal compilation. + * Only the recognized cursor diagnostic is deferred during assembly; other source errors abort it. + * + * @param sources a fresh module source snapshot for this attempt + * @param sourceFile the file in that snapshot containing the cursor + * @param cursor the selected position in the owning Source snapshot + * @param input the optional repository of compiled dependencies + * @param errs the diagnostic listener and abort policy for this operation + * + * @return retained syntax and any facts proved at the cursor; never a compiled module + */ + public PartialAnalysis analyzeIncomplete(ModuleInfo sources, File sourceFile, long cursor, + ModuleRepository input, @NotNull ErrorListener errs) { + requireNonNull(sources, "sources"); + var name = resolveFile(requireNonNull(sourceFile, "sourceFile")).getPath(); + return analyzeIncomplete(listener -> { + var assembly = ErrorListener.cancellable(ErrorListener.collecting(listener::log), listener::isAbortDesired); + var root = sources.getSourceTree(assembly, (source, nodeErrors) -> { + if (!name.equals(source.getFileName())) { + return new Parser(source, nodeErrors).parseSource(); + } + ErrorListener partial = ErrorListener.cancellable(ErrorListener.collecting(error -> { + if (error.getCode().equals(Parser.INCOMPLETE_EXPRESSION)) { + listener.log(error); + } else { + nodeErrors.log(error); + } + }), listener::isAbortDesired); + return Parser.forPartialAnalysis(source, cursor, partial).parseSource(); + }); + return new ParsedSources(root == null ? null : (StatementBlock) root.ast(), sources.getParsedSources()); + }, Parser.INCOMPLETE_EXPRESSION, input, errs); + } + + private PartialAnalysis analyzeIncomplete(Function parse, String boundaryCode, + ModuleRepository input, ErrorListener errs) { + verifyConfigured(); + requireNonNull(errs, "errs"); + Set delivered = new HashSet<>(); + var host = ErrorListener.cancellable(ErrorListener.collecting(error -> { + if (delivered.add(error.genUID())) { + errs.log(error); + } + }), errs::isAbortDesired); + var syntaxErrors = new ErrorList(ErrorList.UNLIMITED); + ParsedSources parsed; + try { + if (host.isAbortDesired()) { + return new PartialAnalysis(List.of(), List.of(), null); + } + parsed = parse.apply(ErrorListener.tee(syntaxErrors, host)); + } catch (CompilerException e) { + return new PartialAnalysis(List.of(), List.of(), null); + } catch (RuntimeException | AssertionError e) { + host.error(ERR_INTERNAL, NOWHERE, e, "Incomplete-source parsing failed"); + return new PartialAnalysis(List.of(), List.of(), null); + } + + if (parsed.root() == null) { + return new PartialAnalysis(parsed.sources(), List.of(), null); + } + var sites = incompleteSites(parsed.root()).distinct().toList(); + if (sites.size() != 1 || syntaxErrors.getErrors().stream().anyMatch(error -> + error.getSeverity().isAtLeast(ERROR) && !error.getCode().equals(boundaryCode))) { + return new PartialAnalysis(parsed.sources(), List.of(), null); + } + + var cursors = new CursorBinding.Collector(); + var attempt = runCompiler(listener -> parsed, input, host, cursors, CompilationGoal.PARTIAL).compiler().result(); + // Anonymous construction can replace its deferred body with an owned class/validation + // clone. Publish only surviving syntax, never the pre-validation cursor identity. + // Primary-constructor defaults can expose the same surviving node through both the + // written parameter and generated property getter. Publish each identity only once. + var surviving = incompleteSites(parsed.root()).distinct().toList(); + return new PartialAnalysis(parsed.sources(), surviving, attempt.pool(), + attempt.callBindings(), cursors.finish(parsed.sources()), attempt.functionBindings()); + } + + /** Parsed children exist before parent links; expose the innermost unfinished operations. */ + private static Stream incompleteSites(AstNode node) { + var nested = PartialSyntax.children(node).flatMap(EmbeddingSupport::incompleteSites); + if (node instanceof IncompleteStatement site) { + var descendants = nested.toList(); + return descendants.isEmpty() ? Stream.of(site) : descendants.stream(); + } + return nested; + } + + /** Preserve node identity, reject null entries and detach from the collector. */ + private static Map identitySnapshot(Map source) { + var copy = new IdentityHashMap(); + source.forEach((key, value) -> copy.put(requireNonNull(key), requireNonNull(value))); + return Collections.unmodifiableMap(copy); + } + + /** Available source trees, with an assembled root only when parsing and loading succeeded. */ + private record ParsedSources(StatementBlock root, List sources) { + /** Keep recovered syntax for diagnostics even when errors prevent compiling the root. */ + static ParsedSources parse(Source source, ErrorListener errs) { + var tree = new Parser(source, errs).parseSource(); + return new ParsedSources(errs.hasSeriousErrors() ? null : tree, List.of(tree)); + } + + /** Assemble the module before reading the per-source trees populated during parsing. */ + static ParsedSources parse(ModuleInfo sources, ErrorListener errs) { + var root = sources.getSourceTree(errs); + return new ParsedSources(root == null ? null : (StatementBlock) root.ast(), sources.getParsedSources()); + } + } + + private Compilation compileModule(Function parse, ModuleRepository input, ErrorListener errs) { + return runCompiler(parse, input, errs, CursorBinding.Collector.NONE, CompilationGoal.MODULE).compiler().result(); + } + + /** + * Selects declaration resolution alone, the full validation and emission pipeline, or that + * pipeline up to its first error, as a partial analysis needs: the diagnostic of its incomplete + * site is what stops the method that holds it from emitting. + */ + private enum CompilationGoal { DECLARATIONS, MODULE, PARTIAL } + + /** Keeps partial compiler state alongside whether the requested goal completed successfully. */ + private record CompilerAttempt(EmbeddingCompiler compiler, boolean succeeded) {} + + private CompilerAttempt runCompiler(Function parse, + ModuleRepository input, ErrorListener errs, + CursorBinding.Collector cursors, CompilationGoal goal) { verifyConfigured(); - return new ModuleCompiler(cfgRepo).compile(source, input, errs); + requireNonNull(errs, "errs"); + var compiler = new EmbeddingCompiler(parse, input, cfgRepo, errs, cursors, goal); + try { + return new CompilerAttempt(compiler, !errs.isAbortDesired() && compiler.process() == 0 + && !errs.hasSeriousErrors() && !errs.isAbortDesired()); + } catch (LauncherException e) { + // Expected aborts already have diagnostics or a cancellation request. + if (!errs.hasSeriousErrors() && !errs.isAbortDesired()) { + errs.error(ERR_INTERNAL, NOWHERE, e, "Compilation failed"); + } + } catch (RuntimeException | AssertionError e) { + // An earlier source error must not hide an unexpected compiler failure. + errs.error(ERR_INTERNAL, NOWHERE, e, "Compilation failed"); + } + return new CompilerAttempt(compiler, false); } /** - * Compile a module that is in a file, against the configured core repository. + * Compile a module that is in a file or directory, against the configured core repository. * - * @param file the module source file + * @param file the location of the module source code on disk, either the module source file + * or the directory containing a single .x file and nested contents thereof * @param input (optional) the module repository to read any required modules from * @param output (optional) the module repository to write any compiled modules to - * @param errs (optional) the ErrorListener to log any compiler messages to + * @param errs the ErrorListener to log any compiler messages to * * @return true if the compilation succeeded and the result was placed into the output * * @see ModuleCompiler#compile(File, ModuleRepository, ModuleRepository, ErrorListener) */ - public boolean compile(File file, ModuleRepository input, ModuleRepository output, - ErrorListener errs) { + public boolean compile(File file, ModuleRepository input, ModuleRepository output, @NotNull ErrorListener errs) { verifyConfigured(); - return new ModuleCompiler(cfgRepo).compile(file, input, output, errs); + requireNonNull(errs, "errs"); + ModuleStructure module; + try { + module = compileModule(new ModuleInfo(file, false), input, errs).module(); + } catch (IllegalArgumentException e) { + errs.error(ERR_INTERNAL, NOWHERE, e, "Unable to locate module " + file); + return false; + } + return ModuleCompiler.storeCompiled(module, output, errs); + } + + /** + * Adapter that supplies the source and repositories to the standard compiler pipeline and + * captures its single compiled module instead of writing it to disk. + */ + private static class EmbeddingCompiler + extends org.xvm.tool.Compiler { + private final Function parse; + private final InvocationBinding.Collector bindings = new InvocationBinding.Collector(); + private final CursorBinding.Collector cursors; + private final CompilationGoal goal; + + private final ModuleRepository inRepo; + private final ModuleRepository coreRepo; + private ModuleStructure module; + private FileStructure file; + private StatementBlock ast; + private List sourceTrees = List.of(); + + /** + * Everything this attempt produced. + * + *

Each part is present when the attempt got that far and null when it did not, which the + * fields already say: the module is only ever assigned on success, and the file structure + * and the AST only once they exist. So there is nothing for the caller to decide, and no + * success flag to pass in and get wrong. + * + * @return the outcome; never null, though its parts may be + */ + Compilation result() { + var facts = bindings.finishFacts(ast == null ? List.of() : List.of(ast)); + return new Compilation(module, file, ast, sourceTrees, facts.methods(), facts.functions(), + facts.constructors(), facts.initializers()); + } + + /** + * @return the file structure this compilation built, which exists whether or not the + * compilation went on to succeed + */ + FileStructure getFileStructure() { + return file; + } + + @Override + public boolean isAbortDesired() { + // Cancellation must also stop between phases; stopping only an individual compiler + // loop would let the following phase run against an unfinished predecessor. + return super.isAbortDesired() || m_errors.isAbortDesired(); + } + + protected EmbeddingCompiler(Function parse, + ModuleRepository input, ModuleRepository core, ErrorListener errs, + CursorBinding.Collector cursors, CompilationGoal goal) { + super(CompilerOptions.builder().build(), SILENT_CONSOLE, errs); + + this.parse = parse; + this.cursors = cursors; + this.goal = goal; + this.inRepo = input; + this.coreRepo = core; + } + + @Override + protected int process() { + var repoLib = ensureLibraryRepo(); + checkErrors("repository setup"); + + prelinkSystemLibraries(repoLib); + checkErrors("system library linking"); + + // The launcher stops the next compilation stage on any error. Each stage still goes on + // past its own errors, subject to the host's budget and cancellation, and so reports + // all of them: a stage logging to the launcher directly would stop at its first error, + // and a changed method would be an error in one of its callers instead of in each. A + // partial analysis is the exception, which stops at its incomplete site's error. + var errs = ErrorListener.cancellable(ErrorListener.collecting(this::log), m_errors::isAbortDesired); + StatementBlock block; + try { + var parsed = parse.apply(errs); + block = this.ast = parsed.root(); + this.sourceTrees = parsed.sources(); + } catch (CompilerException e) { + return 1; + } + if (checkErrors("source parsing") != 0) { + return 1; + } + + if (block == null) { + error(ERR_MODULE_SOURCE, NOWHERE); + return 1; + } + + if (block.getStatements().isEmpty()) { + block.log(this, ERROR, ERR_MODULE_SOURCE); + return checkErrors("source parsing"); + } + var stmt = block.getStatements().getLast(); + if (!(stmt instanceof TypeCompositionStatement stmtModule) || stmtModule.getCategory().getId() != Id.MODULE) { + stmt.log(this, ERROR, ERR_MODULE_SOURCE); + return checkErrors("source parsing"); + } + + var compiler = new Compiler(stmtModule, goal == CompilationGoal.PARTIAL ? this : errs, bindings, cursors); + var struct = compiler.generateInitialFileStructure(); + this.file = struct; + if (struct == null || checkErrors("module creation") != 0) { + return 1; + } + + try { + repoLib.storeModule(struct.getModule()); + } catch (IOException e) { + log(ERROR, e, "I/O exception storing module: {}", struct.getModule().getName()); + return 1; + } + + if (goal == CompilationGoal.DECLARATIONS) { + resolveDeclarations(List.of(compiler), repoLib); + return hasSeriousErrors() || isAbortDesired() ? 1 : 0; + } + + int result = super.compile(List.of(compiler), repoLib); + if (result == 0 && !isAbortDesired()) { + this.module = struct.getModule(); + } + return result; + } + + @Override + protected int compile(List compilers, ModuleRepository repoLib) { + throw new IllegalStateException("This method must not be called"); + } + + @Override + protected ModuleRepository configureLibraryRepo(List ignore) { + var build = new BuildRepository(); + return inRepo == null || inRepo == coreRepo + ? new LinkedRepository(true, build, coreRepo) + : new LinkedRepository(true, build, inRepo, coreRepo); + } + + @Override + protected int emitModules(List allNodes, ModuleRepository ignore) { + throw new IllegalStateException("This method must not be called"); + } + + /** + * @return the result of the compilation + */ + protected ModuleStructure getModule() { + return module; + } } /** @@ -268,7 +1184,7 @@ public interface ConsoleInput { * @return input that reads lines from the specified reader and ignores "suppressEcho" */ static ConsoleInput of(BufferedReader reader) { - Objects.requireNonNull(reader, "reader"); + requireNonNull(reader, "reader"); return _ -> reader.readLine(); } } @@ -286,7 +1202,7 @@ public record EntryPoint(String name, List args) { public static final EntryPoint RUN = new EntryPoint("run"); public EntryPoint { - Objects.requireNonNull(name, "name"); + requireNonNull(name, "name"); args = List.copyOf(args); } @@ -347,7 +1263,7 @@ public interface Control /** * Create a runtime container and execute the provided module. * - * A limited set of injections are made available to the module, including the console, clock, + *

A limited set of injections are made available to the module, including the console, clock, * and other "safe" injectable types. The FileSystem is provided as detailed by the "rootDir" * parameter. * @@ -358,7 +1274,7 @@ public interface Control * task-specific directory under "./.runner" that is deleted when the * returned Control is closed * @param injections (optional) additional "String" and "String[]" injections - * @param errs (optional) a means for the container to report uncaught exceptions and + * @param errs a means for the container to report uncaught exceptions and * other errors * * @return a Control object for the running module @@ -376,7 +1292,7 @@ public Control run( /** * Create a runtime container and execute the specified module. * - * The "customerInjector" option allows the caller to indicate an Ecstasy Injector class that + *

The "customerInjector" option allows the caller to indicate an Ecstasy Injector class that * will be loaded into its own container, and provided with the full set of injectable resources * that Ecstasy supports, also including any provided String injections; in turn, that * implementation provides the injections that will be available to the specified module within @@ -394,7 +1310,7 @@ public Control run( * @param customInjector (optional) "module:class" name of a custom injector implementation to * use to provide injectable resources; when used, the "rootDir" value is * ignored - * @param errs (optional) a means for the container to report uncaught exceptions and + * @param errs a means for the container to report uncaught exceptions and * other errors * * @return a Control object for the running module, or null if it could not be started, in @@ -408,7 +1324,7 @@ public Control run( File rootDir, Map> injections, String customInjector, - ErrorListener errs) { + @NotNull ErrorListener errs) { if (injections != null && !injections.isEmpty() || customInjector != null) { throw new UnsupportedOperationException("Custom injectors are not implemented yet"); } @@ -434,34 +1350,64 @@ public Control run( * as an empty line. Used only with a console * @param rootDir (optional) the root directory for the application's file system, as * described for {@link #run(ModuleStructure, PrintWriter, File, Map, ErrorListener)} - * @param errs (optional) a means for the container to report uncaught exceptions and - * other errors + * @param errs a means for the container to report uncaught exceptions and other errors * * @return a Control object for the running module, or null if it could not be started, in * which case the reason is reported to "errs" */ public Control runMethod( - ModuleRepository input, - String moduleName, - Version version, - EntryPoint entry, - PrintWriter console, - ConsoleInput consoleIn, - File rootDir, - ErrorListener errs) { - Objects.requireNonNull(entry, "entry"); - Objects.requireNonNull(consoleIn, "consoleIn"); + ModuleRepository input, + String moduleName, + Version version, + EntryPoint entry, + PrintWriter console, + ConsoleInput consoleIn, + File rootDir, + @NotNull ErrorListener errs) { + return runMethod(input, moduleName, version, entry, console, consoleIn, rootDir, null, errs); + } + + /** + * Create a runtime container and invoke the specified entry method of a module, as + * {@link #runMethod(ModuleRepository, String, Version, EntryPoint, PrintWriter, ConsoleInput, File, ErrorListener)} + * does, under a debugger. + * + *

The debugger, such as a {@link SessionDebugger}, is set on the runtime before the module + * starts, and replaced by {@link Debugger#NONE} when the run ends or is closed, which releases + * a program it holds stopped. Every run of this EmbeddingSupport shares that runtime, so a run + * with a debugger is meant to be its only run. Debugging needs the interpreter. + * + * @param debugger (optional) the debugger for this run, or null to run without one + * + * @return a Control object for the running module, or null if it could not be started, in + * which case the reason is reported to "errs" + * + * @throws UnsupportedOperationException if a debugger is given and the JIT is used + */ + public Control runMethod( + ModuleRepository input, + String moduleName, + Version version, + EntryPoint entry, + PrintWriter console, + ConsoleInput consoleIn, + File rootDir, + Debugger debugger, + @NotNull ErrorListener errs) { + if (debugger != null && useJit()) { + throw new UnsupportedOperationException("Debugging requires the interpreter"); + } + requireNonNull(entry, "entry"); + requireNonNull(consoleIn, "consoleIn"); verifyConfigured(); + requireNonNull(errs, "errs"); - ModuleRepository repository = new LinkedRepository(input, cfgRepo); - ModuleStructure module = version == null + var repository = new LinkedRepository(input, cfgRepo); + var module = version == null ? repository.loadModule(moduleName) : repository.loadModule(moduleName, version, true); if (module == null) { - if (errs != null) { - errs.log(ERROR, version == null ? ERR_NO_APP_MODULE : ERR_NO_APP_MODULE_VER, - new Object[] {moduleName, version}, null); - } + errs.error(version == null ? ERR_NO_APP_MODULE : ERR_NO_APP_MODULE_VER, NOWHERE, moduleName, version); return null; } @@ -470,20 +1416,17 @@ public Control runMethod( } try { - Connector connector = ensureConnector(); + var connector = ensureConnector(); return useJit() ? JitControl.create(connector, module, repository, console, rootDir, errs) : InterpreterControl.create(connector, module, repository, entry, console, - consoleIn, rootDir, errs); + consoleIn, rootDir, debugger, errs); } catch (RuntimeException | AssertionError e) { // an AssertionError is an Error, so the RuntimeException guard alone let a tripped // assertion in the structure code past this report and out to the host. Errors are // not caught wholesale: a VirtualMachineError says the JVM is in trouble, not that // this module failed to start, and handling one is not something to rely on - if (errs != null) { - errs.log(ERROR, ERR_CREATE_APP_CONTAINER, - new Object[] {e, "Unable to start " + moduleName}, module); - } + errs.error(ERR_CREATE_APP_CONTAINER, at(module), e, "Unable to start " + moduleName); return null; } } @@ -590,4 +1533,8 @@ static boolean passesArguments(MethodStructure method, EntryPoint entry) { * "%1" - why the module's entry method cannot be run, as the command-line runner reports it */ public static final String ERR_ENTRY_METHOD = "EMB-7"; + /** + * The source does not contain a module declaration. + */ + public static final String ERR_MODULE_SOURCE = "EMB-8"; } diff --git a/javatools/src/main/java/org/xvm/api/InterpreterConnector.java b/javatools/src/main/java/org/xvm/api/InterpreterConnector.java index a10281da91c..c6b98fcc1bc 100644 --- a/javatools/src/main/java/org/xvm/api/InterpreterConnector.java +++ b/javatools/src/main/java/org/xvm/api/InterpreterConnector.java @@ -18,8 +18,11 @@ import org.xvm.runtime.NativeContainer; import org.xvm.runtime.ObjectHandle; import org.xvm.runtime.Runtime; +import org.xvm.runtime.ServiceContext; import org.xvm.runtime.Utils; +import org.xvm.runtime.template._native.temporal.xNanosTimer; + import org.xvm.runtime.template.text.xString; /** @@ -71,6 +74,20 @@ public void setDebugger(Debugger debugger) { f_runtime.setDebugger(debugger); } + /** + * Replace the debugger if it is the expected one, as a run does with its own when it ends. + * + * @param expected the debugger to replace + * @param replacement the debugger to use instead + */ + public void replaceDebugger(Debugger expected, Debugger replacement) { + synchronized (f_runtime) { + if (f_runtime.getDebugger() == expected) { + f_runtime.setDebugger(replacement); + } + } + } + /** * @return true if no service of this connector's runtime is running or about to */ @@ -164,6 +181,18 @@ public int join() return nResult; } + @Override + public void shutdown() { + // the native container's timers keep their pending alarms on the JVM-wide Java timer, and + // with them this runtime, until they are due + for (ServiceContext service : f_containerNative.getServices()) { + if (service.getService() instanceof xNanosTimer.TimerHandle timer) { + timer.cancelAlarms(); + } + } + f_runtime.shutdownXVM(); + } + // ----- data fields --------------------------------------------------------------------------- /** diff --git a/javatools/src/main/java/org/xvm/api/InterpreterControl.java b/javatools/src/main/java/org/xvm/api/InterpreterControl.java index bb0f412475f..3bd32d5b7a6 100644 --- a/javatools/src/main/java/org/xvm/api/InterpreterControl.java +++ b/javatools/src/main/java/org/xvm/api/InterpreterControl.java @@ -38,13 +38,12 @@ import org.xvm.runtime.template._native.mgmt.xCoreRepository; import org.xvm.runtime.template._native.reflect.xRTModuleTemplate; +import static org.xvm.asm.ErrorListener.at; + import static org.xvm.api.EmbeddingSupport.ERR_ENTRY_METHOD; import static org.xvm.api.EmbeddingSupport.ERR_UNHANDLED_EXCEPTION; import static org.xvm.api.EmbeddingSupport.WARN_ARGUMENTS_IGNORED; -import static org.xvm.util.Severity.ERROR; -import static org.xvm.util.Severity.WARNING; - /** * Interpreter-backed management and monitoring for one runner task. */ @@ -67,12 +66,14 @@ static Connector createConnector(ModuleRepository repository) { * @param entry the entry method to invoke and its arguments * @param consoleIn the console's input, or {@link EmbeddingSupport.ConsoleInput#NONE}; * used only with a console + * @param debugger the run's debugger, or null; it is set before the task starts and + * replaced by {@link Debugger#NONE} when the run ends or is closed */ static EmbeddingSupport.Control create(Connector connector, ModuleStructure module, ModuleRepository repository, EmbeddingSupport.EntryPoint entry, PrintWriter console, EmbeddingSupport.ConsoleInput consoleIn, File rootDir, - ErrorListener errs) { + Debugger debugger, ErrorListener errs) { if (!(connector instanceof InterpreterConnector interpreter)) { throw new IllegalArgumentException("An InterpreterConnector is required"); } @@ -89,14 +90,12 @@ static EmbeddingSupport.Control create(Connector connector, ModuleStructure modu } catch (IllegalArgumentException e) { // the method is why the run does not start, as the command-line runner reports it; // its container was never the problem - if (errs != null) { - errs.log(ERROR, ERR_ENTRY_METHOD, new Object[] {e.getMessage()}, module); - } + errs.error(ERR_ENTRY_METHOD, at(module), e.getMessage()); return null; } - if (errs != null && !entry.args().isEmpty() && method.getParamCount() == 0) { + if (!entry.args().isEmpty() && method.getParamCount() == 0) { // the arguments are dropped, with the command-line runner's warning - errs.log(WARNING, WARN_ARGUMENTS_IGNORED, new Object[] {entry.name()}, module); + errs.warn(WARN_ARGUMENTS_IGNORED, at(module), entry.name()); } if (console != null) { @@ -124,7 +123,13 @@ static EmbeddingSupport.Control create(Connector connector, ModuleStructure modu CompletableFuture completion = new CompletableFuture<>(); InterpreterControl control = new InterpreterControl(interpreter, module, rootDir, - errs, taskId, consoleId, started, completion); + errs, taskId, consoleId, debugger, started, completion); + if (debugger != null) { + // the runner's own code is not the program; breakpoints are in place before the + // program's first op + debugger.setHost(main); + interpreter.setDebugger(debugger); + } postRequest(interpreter, "startTask", hTaskId).whenComplete((r, e) -> { if (e == null) { @@ -176,31 +181,42 @@ private static CompletableFuture postRequest( private InterpreterControl(InterpreterConnector connector, ModuleStructure module, File rootDir, ErrorListener errs, long taskId, Long consoleId, - Instant started, CompletableFuture completion) { + Debugger debugger, Instant started, + CompletableFuture completion) { this.connector = connector; this.module = module; this.rootDir = rootDir; this.errs = errs; this.taskId = taskId; this.consoleId = consoleId; + this.debugger = debugger; this.started = started; this.completion = completion; } + /** + * Replace this run's debugger, if it still is the runtime's, by {@link Debugger#NONE}, which + * releases a program it holds stopped. + */ + private void releaseDebugger() { + if (debugger != null) { + connector.replaceDebugger(debugger, Debugger.NONE); + } + } + private void finish(long result, String failure) { try { if (failure.isEmpty()) { this.result = result; } else { this.result = null; - if (errs != null) { - errs.log(ERROR, ERR_UNHANDLED_EXCEPTION, new Object[] {failure}, module); - } + errs.error(ERR_UNHANDLED_EXCEPTION, at(module), failure); } } finally { this.stopped = Instant.now(); this.running = false; unregisterConsole(connector, consoleId); + releaseDebugger(); } } @@ -245,6 +261,11 @@ public void close() { } try { + if (debugger != null) { + // a stopped program ends where it is, without running on until the runner kills it + debugger.terminate(); + } + releaseDebugger(); if (running) { try { postRequest(connector, "killTask", xInt64.makeHandle(taskId)).join(); @@ -283,6 +304,7 @@ private String taskStatus() { private final ErrorListener errs; private final long taskId; private final Long consoleId; + private final Debugger debugger; private final Instant started; private final CompletableFuture completion; private final AtomicBoolean closed = new AtomicBoolean(); diff --git a/javatools/src/main/java/org/xvm/api/ModuleCompiler.java b/javatools/src/main/java/org/xvm/api/ModuleCompiler.java index a70c520b812..1ecd5624321 100644 --- a/javatools/src/main/java/org/xvm/api/ModuleCompiler.java +++ b/javatools/src/main/java/org/xvm/api/ModuleCompiler.java @@ -6,6 +6,8 @@ import java.util.List; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import org.xvm.asm.ErrorList; import org.xvm.asm.ErrorListener; import org.xvm.asm.FileStructure; @@ -31,6 +33,9 @@ import static org.xvm.api.EmbeddingSupport.ERR_INTERNAL; +import static org.xvm.asm.ErrorListener.NOWHERE; +import static org.xvm.asm.ErrorListener.at; + import static org.xvm.util.Handy.readFileChars; import static org.xvm.util.Severity.ERROR; @@ -40,7 +45,9 @@ *

Unlike {@link EmbeddingSupport}, a ModuleCompiler is an ordinary object that needs no * configuration: compiling does not start the runtime, so it does not depend on the runtime's * JVM-wide state (see {@link EmbeddingSupport}), and any number of ModuleCompiler instances, each - * with its own core repository, can exist in one JVM. Each call to compile is independent. + * with its own core repository, can exist in one JVM. Each call to compile has its own attempt. + * Hosts must serialize calls that share repositories; separate instances do not isolate mutable + * compiler state held by a repository. * *

The compile methods report the same errors as the command-line compiler: every error found in * a compiler stage, with the compilation stopping at the end of the first stage that found any. @@ -74,11 +81,12 @@ public ModuleCompiler(ModuleRepository coreRepo) { * * @param source the source code for an entire module to compile * @param input (optional) the module repository to read any required modules from - * @param errs (optional) the ErrorListener to log any compiler messages to + * @param errs the ErrorListener to log any compiler messages to * * @return the resulting ModuleStructure, or null if a compiler error occurred */ - public ModuleStructure compile(String source, ModuleRepository input, ErrorListener errs) { + public ModuleStructure compile(String source, ModuleRepository input, @NotNull ErrorListener errs) { + Objects.requireNonNull(errs, "errs"); try { EmbeddingCompiler compiler = new EmbeddingCompiler(source, input, coreRepo, errs); return compiler.process() == 0 @@ -88,18 +96,15 @@ public ModuleStructure compile(String source, ModuleRepository input, ErrorListe // the compiler stops at the end of a stage that logged errors; those errors are already // in "errs", so this is an ordinary failed compile, as it is for the command-line // compiler. A tool-level failure is logged only to the silent console, so report it - if (errs != null && !errs.hasSeriousErrors()) { - errs.log(ERROR, ERR_INTERNAL, new Object[] {e, "Compilation failed"}, null); + if (!errs.hasSeriousErrors()) { + errs.error(ERR_INTERNAL, NOWHERE, e, "Compilation failed"); } return null; } catch (RuntimeException | AssertionError e) { // as in EmbeddingSupport.run(): the compiler runs over caller-supplied source, so a // failure in it is reported here rather than thrown at the caller, who was promised a // null instead - if (errs != null) { - errs.log(ERROR, ERR_INTERNAL, - new Object[] {e, "Compilation failed"}, null); - } + errs.error(ERR_INTERNAL, NOWHERE, e, "Compilation failed"); return null; } } @@ -110,25 +115,35 @@ public ModuleStructure compile(String source, ModuleRepository input, ErrorListe * @param file the module source file * @param input (optional) the module repository to read any required modules from * @param output (optional) the module repository to write any compiled modules to - * @param errs (optional) the ErrorListener to log any compiler messages to + * @param errs the ErrorListener to log any compiler messages to * * @return true if the compilation succeeded and the result was placed into the output */ public boolean compile(File file, ModuleRepository input, ModuleRepository output, - ErrorListener errs) { + @NotNull ErrorListener errs) { + Objects.requireNonNull(errs, "errs"); ModuleStructure module; try { module = compile(new String(readFileChars(file)), input, errs); } catch (IOException e) { - if (errs != null) { - errs.log(ERROR, ERR_INTERNAL, - new Object[] {e, "Unable to read module " + file}, null); - } + errs.error(ERR_INTERNAL, NOWHERE, e, "Unable to read module " + file); return false; } + return storeCompiled(module, output, errs); + } + /** + * Finish a file compilation by placing its module into the optional output repository. + * + * @param module the compiled module, or null if the compilation failed or was abandoned + * @param output (optional) the module repository to write the compiled module to + * @param errs the ErrorListener that received the compilation's messages + * + * @return true if there is a module and it was stored, or no output was requested + */ + static boolean storeCompiled(ModuleStructure module, ModuleRepository output, ErrorListener errs) { if (module == null) { - assert errs == null || errs.hasSeriousErrors(); + assert errs.hasSeriousErrors() || errs.isAbortDesired(); return false; } @@ -136,10 +151,7 @@ public boolean compile(File file, ModuleRepository input, ModuleRepository outpu try { output.storeModule(module); } catch (IOException e) { - if (errs != null) { - errs.log(ERROR, ERR_INTERNAL, - new Object[] {e, "Unable to store module " + module.getName()}, module); - } + errs.error(ERR_INTERNAL, at(module), e, "Unable to store module " + module.getName()); return false; } } diff --git a/javatools/src/main/java/org/xvm/asm/Annotation.java b/javatools/src/main/java/org/xvm/asm/Annotation.java index f32d829e8ea..b14f8dc0b96 100644 --- a/javatools/src/main/java/org/xvm/asm/Annotation.java +++ b/javatools/src/main/java/org/xvm/asm/Annotation.java @@ -321,8 +321,8 @@ public boolean validate(ErrorListener errs) { // it must be an annotation type if (getAnnotationType().getExplicitClassFormat() != Component.Format.ANNOTATION) { - fHalt |= log(errs, Severity.ERROR, VE_CLASS_NOT_ANNOTATION, - getAnnotationClass().getValueString()); + log(errs, Severity.ERROR, VE_CLASS_NOT_ANNOTATION, getAnnotationClass().getValueString()); + fHalt |= errs.isAbortDesired(); } return fHalt; diff --git a/javatools/src/main/java/org/xvm/asm/ClassStructure.java b/javatools/src/main/java/org/xvm/asm/ClassStructure.java index 82eb7fa18f0..d8114bcb444 100644 --- a/javatools/src/main/java/org/xvm/asm/ClassStructure.java +++ b/javatools/src/main/java/org/xvm/asm/ClassStructure.java @@ -37,6 +37,9 @@ import org.xvm.util.ListMap; import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + import static org.xvm.util.Handy.readIndex; import static org.xvm.util.Handy.readMagnitude; import static org.xvm.util.Handy.writeMagnitude; @@ -987,7 +990,7 @@ public ResolutionResult resolveName(String sName, Access access, ResolutionColle if (result == ResolutionResult.UNKNOWN && getFormat() == Format.SERVICE) { // look into the Service interface itself ClassStructure clzSvc = (ClassStructure) getConstantPool().clzService().getComponent(); - SimpleCollector collectorSvc = new SimpleCollector(ErrorListener.BLACKHOLE); + SimpleCollector collectorSvc = new SimpleCollector(silent(PROBE)); ResolutionResult resultSvc = clzSvc.resolveName(sName, Access.PROTECTED, collectorSvc); if (resultSvc == ResolutionResult.RESOLVED) { // only allow child classes; properties and methods are resolved by the TypeInfo diff --git a/javatools/src/main/java/org/xvm/asm/Component.java b/javatools/src/main/java/org/xvm/asm/Component.java index 53f60fd41bc..92cec3ad290 100644 --- a/javatools/src/main/java/org/xvm/asm/Component.java +++ b/javatools/src/main/java/org/xvm/asm/Component.java @@ -24,6 +24,8 @@ import java.util.stream.Stream; +import org.jetbrains.annotations.NotNull; + import org.xvm.asm.constants.ClassConstant; import org.xvm.asm.constants.ConditionalConstant; import org.xvm.asm.constants.IdentityConstant; @@ -46,7 +48,10 @@ import org.xvm.util.Handy; import org.xvm.util.Hash; import org.xvm.util.ListMap; -import org.xvm.util.Severity; + +import static java.util.Objects.requireNonNull; + +import static org.xvm.asm.ErrorListener.at; import static org.xvm.util.Handy.readIndex; import static org.xvm.util.Handy.readMagnitude; @@ -2012,8 +2017,8 @@ protected ResolutionResult resolveContributedName( } if (m_FVisited != null && m_FVisited.booleanValue() == fAllowInto) { // recursive contribution - collector.getErrorListener().log(Severity.FATAL, Constants.VE_CYCLICAL_CONTRIBUTION, - new Object[] {getName(), contrib.getComposition().toString().toLowerCase()}, this); + collector.getErrorListener().fatal(Constants.VE_CYCLICAL_CONTRIBUTION, at(this), + getName(), contrib.getComposition().toString().toLowerCase()); return ResolutionResult.ERROR; } @@ -3528,8 +3533,8 @@ public String toString() { */ public static class SimpleCollector implements ResolutionCollector { - public SimpleCollector(ErrorListener errs) { - m_errs = errs; + public SimpleCollector(@NotNull ErrorListener errs) { + m_errs = requireNonNull(errs, "errs"); } @Override diff --git a/javatools/src/main/java/org/xvm/asm/ComponentResolver.java b/javatools/src/main/java/org/xvm/asm/ComponentResolver.java index 930c81bd884..3f69ba5d8a1 100644 --- a/javatools/src/main/java/org/xvm/asm/ComponentResolver.java +++ b/javatools/src/main/java/org/xvm/asm/ComponentResolver.java @@ -63,9 +63,12 @@ default AstNode getNode() { /** * Provide an ErrorListener to report resolution issues to. + * + * Deliberately not defaulted. A default of "discard" would mean a collector that had not + * thought about diagnostics and one that had decided against them looked identical at the + * point of resolution, which is the distinction the listener exists to make. A collector + * that genuinely wants none says so by naming a silent listener. */ - default ErrorListener getErrorListener() { - return ErrorListener.BLACKHOLE; - } + ErrorListener getErrorListener(); } } diff --git a/javatools/src/main/java/org/xvm/asm/ConstantPool.java b/javatools/src/main/java/org/xvm/asm/ConstantPool.java index bc125aba077..8f18fa229a6 100755 --- a/javatools/src/main/java/org/xvm/asm/ConstantPool.java +++ b/javatools/src/main/java/org/xvm/asm/ConstantPool.java @@ -1277,6 +1277,15 @@ public static String getImplicitImportName(String sPath) { return IMPLICITS.byPath().get(sPath); } + /** + * The language's implicit import names, without resolving or loading their components. + * + * @return the names available through implicit imports + */ + public static Set getImplicitImportNames() { + return IMPLICITS.byName().keySet(); + } + /** * Given the specified typedef name and the context (module, package, class, method) within * which it exists, obtain a TypedefConstant that represents it. diff --git a/javatools/src/main/java/org/xvm/asm/ErrorList.java b/javatools/src/main/java/org/xvm/asm/ErrorList.java index 7c6133e7a71..1c69f7e8444 100644 --- a/javatools/src/main/java/org/xvm/asm/ErrorList.java +++ b/javatools/src/main/java/org/xvm/asm/ErrorList.java @@ -9,26 +9,79 @@ import org.xvm.util.Severity; /** - * Represents a list of errors collected from a process such as compilation, assembly, or the - * verifier, with an option to abort the process should a maximum number of errors be exceeded. + * An operation's diagnostic buffer with deduplication, severity state, and an optional count budget. + * + *

The first report for each {@link ErrorInfo#genUID()} is retained in insertion order. Each + * retained ERROR or FATAL counts toward the budget; warnings and informational reports do not. + * {@link #isAbortDesired()} becomes true when a positive budget is reached, or after any FATAL. + * Logging still records subsequent reports until the caller honors that stop request. + * + *

Use this for compilation results that must retain diagnostics, and + * {@link ErrorListener#collecting(java.util.function.Consumer)} when a host callback should receive + * every report without deduplication. Instances are mutable and are not thread-safe. */ public class ErrorList implements ErrorListener { // ----- constructors -------------------------------------------------------------------------- + /** + * Construct a buffer that requests an abort on its {@link ErrorListener#DEFAULT_MAX_ERRORS}-th serious report. + * + *

Use {@link #ErrorList(int)} with {@link #FIRST_ERROR}, {@link #UNLIMITED}, or a caller-specific + * positive budget when that policy is required. + */ + public ErrorList() { + this(ErrorListener.DEFAULT_MAX_ERRORS); + } + + /** + * Construct a buffer with the requested serious-error budget. + * + * @param cMaxErrors request an abort when this many retained ERROR/FATAL reports have arrived; + * non-positive values disable the count limit, conventionally {@link #UNLIMITED} + */ public ErrorList(int cMaxErrors) { f_cMaxErrors = cMaxErrors; } + /** + * Disable the count budget. A retained FATAL still requests an abort. + */ + public static final int UNLIMITED = 0; + + /** + * Request an abort after the first retained ERROR or FATAL. + */ + public static final int FIRST_ERROR = 1; + // ----- ErrorListener methods ----------------------------------------------------------------- + /** + * Create a diagnostic buffer with this list's count budget and parent abort policy. + * + *

The child counts its own retained reports; it does not subtract the parent's count from its + * budget. It requests an abort when either its own policy or the parent's policy says to stop. + * Reports reach this list only when the child is merged. + * + * @param node the optional source context for structure-site diagnostics + * + * @return a new child buffer + */ @Override public ErrorListener branch(AstNode node) { return new BranchedErrorListener(this, f_cMaxErrors, node); } + /** + * Record the first report for this diagnostic's UID and update severity and budget state. + * + *

Duplicate UIDs do not add entries or spend the count budget. This method does not stop the + * caller or refuse reports after the budget is reached; query {@link #isAbortDesired()} explicitly. + * + * @param err the diagnostic to record + */ @Override - public boolean log(ErrorInfo err) { + public void log(ErrorInfo err) { String uid = err.genUID(); if (f_setUID.add(uid)) { // remember the highest severity encountered @@ -46,10 +99,13 @@ public boolean log(ErrorInfo err) { ++m_cErrors; } } - - return isAbortDesired(); } + /** + * Test whether a retained FATAL or the configured serious-error count requests an abort. + * + * @return true for FATAL, or when a positive budget has been reached + */ @Override public boolean isAbortDesired() { return m_severity == Severity.FATAL || f_cMaxErrors > 0 && @@ -117,7 +173,10 @@ public List getErrors() { } /** - * Clear the list of errors, resetting the error collection state. + * Clear the retained reports, serious-error count, and worst severity. + * + *

Previously seen diagnostic UIDs remain recorded, so logging the same diagnostic after + * clearing still suppresses it. Use a new ErrorList for a new independent operation. */ public void clear() { f_list.clear(); @@ -126,9 +185,12 @@ public void clear() { } /** - * Log the errors from this ErrorList into another ErrorListener. + * Replay all retained diagnostics to another listener in insertion order. * - * @param errs the ErrorListener to log all the errors from this ErrorList to + *

This neither clears this list nor queries the destination's abort policy. Repeated calls + * replay the same reports; deduplication, if desired, is the destination's responsibility. + * + * @param errs the destination for the retained reports */ public void logTo(ErrorListener errs) { for (ErrorInfo err : getErrors()) { @@ -150,10 +212,21 @@ public String toString() { // ----- inner class: BranchedErrorListener ---------------------------------------------------- /** - * The ErrorListener that can be used to capture errors that may or may not be reported. + * A temporary diagnostic buffer whose accepted reports can be merged into a parent listener. + * + *

It retains its own deduplication and count state while observing the parent's abort request. + * Discarding the branch leaves the parent's diagnostics untouched. Merging forwards reports and + * returns the parent without clearing the branch, so callers should merge an accepted branch once. */ public static class BranchedErrorListener extends ErrorList { + /** + * Construct a speculative buffer associated with a parent and optional source node. + * + * @param listener the parent destination and additional abort policy + * @param cMaxErrors this branch's serious-error budget, or {@link ErrorList#UNLIMITED} + * @param node the optional source context for structure-site reports + */ public BranchedErrorListener(ErrorListener listener, int cMaxErrors, AstNode node) { super(cMaxErrors); @@ -168,11 +241,14 @@ public ErrorListener branch(AstNode node) { } @Override - public boolean log(Severity severity, String sCode, Object[] aoParam, XvmStructure xs) { - return f_node == null - ? super.log(severity, sCode, aoParam, xs) - : log(severity, sCode, aoParam, - f_node.getSource(), f_node.getStartPosition(), f_node.getEndPosition()); + public void log(Severity severity, String sCode, Site site, Object... aoParam) { + // a branch taken for a particular node re-anchors at that node: a diagnostic raised + // against a structure carries no source location of its own, and the node is the + // location the brancher knew about + if (f_node != null && site instanceof Site.At) { + site = ErrorListener.in(f_node.getSource(), f_node.getStartPosition(), f_node.getEndPosition()); + } + super.log(severity, sCode, site, aoParam); } @Override diff --git a/javatools/src/main/java/org/xvm/asm/ErrorListener.java b/javatools/src/main/java/org/xvm/asm/ErrorListener.java index a13fd78e50e..00e3944d75a 100644 --- a/javatools/src/main/java/org/xvm/asm/ErrorListener.java +++ b/javatools/src/main/java/org/xvm/asm/ErrorListener.java @@ -6,6 +6,15 @@ import java.util.Arrays; import java.util.ResourceBundle; +import java.util.Set; + +import java.util.concurrent.ConcurrentHashMap; + +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.xvm.compiler.Source; @@ -13,25 +22,67 @@ import org.xvm.util.Severity; +import static java.util.Objects.requireNonNull; + import static org.xvm.util.Handy.quotedString; /** - * A listener for errors being reported about source code, compilation, assembly, or verification of - * XVM structures. + * Receives diagnostics from compilation, assembly, and verification, and exposes the state that + * callers use to decide whether work may continue. + * + *

Reporting and continuation are separate operations. {@link #log(ErrorInfo)} delivers a + * diagnostic; {@link #isAbortDesired()} answers the current stop policy. A compiler caller that + * previously consumed the boolean result of {@code log} must query the policy explicitly: + *

{@code
+ * errors.error(code, ErrorListener.in(source, start, end), arguments);
+ * if (errors.isAbortDesired()) {
+ *     return;
+ * }
+ * }
+ *

This gives forwarding listeners one place to preserve a parent's error budget or abort + * request. It also lets a host query a stop policy independently of emitting another diagnostic. + * Changing {@code log} to {@code void} is an intentional source and binary API change; existing + * Java implementations and callers must migrate and recompile. + * + *

Use {@link #collecting(Consumer)} to deliver diagnostics to a host callback while maintaining + * severity and code state, {@link ErrorList} to retain and deduplicate reports with an error + * budget, and {@link #tee(ErrorListener, ErrorListener)} to forward to two destinations. A bare + * lambda implements only reporting and inherits state queries that always return {@code false}. + * + *

Use {@link #branch(AstNode)} when speculative diagnostics may need to be kept, or + * {@link #silence(Silence)} when they must be discarded while preserving this listener's abort + * policy. {@link #silent(Silence)} is a standalone discard sink with no parent policy. The named + * reasons replace the ambiguous intent of an unnamed discard sink such as {@code BLACKHOLE}; + * they do not select different compiler modes. + * + *

Keep an operation's original listener reference stable. Pass a derived silence or branch to + * temporary work instead of replacing the owner with a discard sink. This preserves the destination + * and its stop policy for later work; it does not make the listener's accumulated state immutable. + * + *

Listeners belong to an operation. Hosts must serialize callbacks and compiler access; + * these factories do not make compilation or listener state thread-safe. Reporting callbacks + * may throw, and the forwarding helpers do not catch or translate those exceptions. + * + *

Factory and constructor parameters annotated {@link NotNull} are required collaborators, + * not optional reporting destinations. Null arguments are rejected when the listener is created. + * {@link Nullable} marks intentional absence, such as a standalone silence's parent. */ @FunctionalInterface public interface ErrorListener { // ----- API ----------------------------------------------------------------------------------- /** - * Handles the logging of an error that originates in Ecstasy source code. + * Deliver one diagnostic to this listener. * - * @param err the error info + *

Implementations decide whether to retain, forward, deduplicate, or discard the report. + * Reporting supplies no continuation result. Call {@link #isAbortDesired()} at the caller's + * stop checkpoints, and use {@link #hasSeriousErrors()} or {@link #hasError(String)} when deciding + * whether a compiler result is usable. A custom reporting callback must maintain those queries + * itself or be wrapped with {@link #collecting(Consumer)}. * - * @return true to attempt to abort the process that reported the error, or - * false to attempt to continue the process + * @param err the diagnostic to report */ - boolean log(ErrorInfo err); + void log(ErrorInfo err); /** * Handles the logging of an error that originates in Ecstasy source code. @@ -45,12 +96,11 @@ public interface ErrorListener { * @param lPosStart the position in the source where the error was detected * @param lPosEnd the position in the source at which the error concluded * - * @return true to attempt to abort the process that reported the error, or - * false to attempt to continue the process + * @deprecated use {@link #log(Severity, String, Site, Object...)} with {@link #in} */ - default boolean log(Severity severity, String sCode, Object[] aoParam, - Source source, long lPosStart, long lPosEnd) { - return log(new ErrorInfo(severity, sCode, aoParam, source, lPosStart, lPosEnd)); + @Deprecated + default void log(Severity severity, String sCode, Object[] aoParam, Source source, long lPosStart, long lPosEnd) { + log(new ErrorInfo(severity, sCode, aoParam, source, lPosStart, lPosEnd)); } /** @@ -64,76 +114,580 @@ default boolean log(Severity severity, String sCode, Object[] aoParam, * @param xs the XvmStructure that the error is related to; may * be null * - * @return true to attempt to abort the process that reported the error, or - * false to attempt continue the process + * @deprecated use {@link #log(Severity, String, Site, Object...)} with {@link #at} + */ + @Deprecated + default void log(Severity severity, String sCode, Object[] aoParam, XvmStructure xs) { + log(severity, sCode, at(xs), aoParam); + } + + // ----- reporting ----------------------------------------------------------------------------- + + /** + * Construct and report a diagnostic with an explicit location and trailing message arguments. + * + *

Use {@link #in(Source, long, long)} for source text, {@link #at(XvmStructure)} for an + * assembled structure, or {@link #NOWHERE} when no location is available. Packaging the location + * as a {@link Site} leaves the final parameter available for varargs, so callers need not build + * an {@code Object[]} between the message code and location arguments. + * + *

The default implementation creates an {@link ErrorInfo} and delegates to {@link #log(ErrorInfo)}. + * It does not test the listener's abort policy or stop the caller. + * + * @param severity the diagnostic severity + * @param sCode the message code from the compiler's error resources + * @param site the diagnostic location; must not be null + * @param aoParam the message-format arguments, or an empty array when none are needed + * + * @throws NullPointerException if {@code site} is null + */ + default void log(Severity severity, String sCode, Site site, Object... aoParam) { + switch (site) { + case Site.In in -> log(new ErrorInfo(severity, sCode, aoParam, in.source(), in.lPosStart(), in.lPosEnd())); + case Site.At at -> log(new ErrorInfo(severity, sCode, aoParam, at.xs())); + case Site.None _ -> log(new ErrorInfo(severity, sCode, aoParam, null, 0, 0)); + } + } + + /** + * Report an {@link Severity#ERROR} diagnostic for an operation that failed validation. + * + *

The listener's error budget determines whether this also requests an abort. Reporting an + * error does not itself stop the caller. + * + * @param sCode the message code + * @param site the diagnostic location; must not be null + * @param aoParam the message-format arguments + * + * @see #log(Severity, String, Site, Object...) + */ + default void error(String sCode, Site site, Object... aoParam) { + log(Severity.ERROR, sCode, site, aoParam); + } + + /** + * Report a {@link Severity#WARNING} diagnostic without classifying it as a serious error. + * + * @param sCode the message code + * @param site the diagnostic location; must not be null + * @param aoParam the message-format arguments + * + * @see #log(Severity, String, Site, Object...) + */ + default void warn(String sCode, Site site, Object... aoParam) { + log(Severity.WARNING, sCode, site, aoParam); + } + + /** + * Report an {@link Severity#INFO} diagnostic, for example an informational compiler message. + * + * @param sCode the message code + * @param site the diagnostic location; must not be null + * @param aoParam the message-format arguments + * + * @see #log(Severity, String, Site, Object...) + */ + default void info(String sCode, Site site, Object... aoParam) { + log(Severity.INFO, sCode, site, aoParam); + } + + /** + * Report a {@link Severity#FATAL} diagnostic for work that cannot continue successfully. + * + *

{@link ErrorList} and {@link #collecting(Consumer)} request an abort after a fatal report, + * regardless of their count policy. This method does not itself throw or transfer control; + * the caller must return, throw, or honor its next stop checkpoint. Code that cannot safely + * continue must stop explicitly even when the chosen listener discards the report. + * + * @param sCode the message code + * @param site the diagnostic location; must not be null + * @param aoParam the message-format arguments + * + * @see #log(Severity, String, Site, Object...) + */ + default void fatal(String sCode, Site site, Object... aoParam) { + log(Severity.FATAL, sCode, site, aoParam); + } + + /** + * Create a stateful listener that forwards every report to a host callback. + * + *

Use this when an embedding host or editor collects diagnostics through a consumer, but the + * compiler still needs accurate state queries. Before invoking the consumer, the listener records + * the highest severity and the reported code. {@link #hasSeriousErrors()} becomes true for ERROR + * or FATAL, {@link #hasError(String)} recognizes codes from any severity, and + * {@link #isAbortDesired()} becomes true only for FATAL. There is no count budget. + * + *

Every report is forwarded, including duplicates; use {@link ErrorList} for deduplication or + * a count budget. If the consumer throws, its exception propagates and the recorded state remains. + * Create a new listener for each independent operation; there is no reset operation. + * + * @param consumer the destination for each diagnostic, invoked synchronously + * + * @return a new listener that retains severity and code state while forwarding reports + */ + @NotNull + static ErrorListener collecting(@NotNull Consumer consumer) { + requireNonNull(consumer, "consumer"); + return new ErrorListener() { + @Override + public void log(ErrorInfo err) { + Severity severity = err.getSeverity(); + if (severity.isWorseThan(m_severity)) { + m_severity = severity; + } + f_setCodes.add(err.getCode()); + consumer.accept(err); + } + + @Override + public boolean isAbortDesired() { + return m_severity == Severity.FATAL; + } + + @Override + public boolean hasSeriousErrors() { + return m_severity.isAtLeast(Severity.ERROR); + } + + @Override + public boolean hasError(String sCode) { + return f_setCodes.contains(sCode); + } + + @Override + public String toString() { + return "Collecting(worst=" + m_severity + ")"; + } + + private final Set f_setCodes = ConcurrentHashMap.newKeySet(); + private volatile Severity m_severity = Severity.NONE; + }; + } + + /** + * Wrap an operation listener with an independent, cooperatively polled cancellation policy. + * + *

For example, an editor can mark a completion request stale after a new edit without + * emitting a fake compiler error. Every abort query first polls {@code cancelled}; if it + * is false, the wrapped listener's abort policy is queried. Cancellation is not latched: + * a host that requires a permanent stop must keep its predicate true. + * + *

Reports and the other state queries delegate to the wrapped listener. Branching and + * merging preserve the cancellation predicate around the resulting child or parent. The + * wrapper does not interrupt a thread, discard reports, or stop work until a caller checks + * the abort policy. Predicate and listener exceptions propagate. + * + * @param errs the operation listener + * @param cancelled a predicate, cheap and safe to poll on the compiling thread + * + * @return a listener requesting a stop for cancellation or the wrapped listener's policy + */ + @NotNull + static ErrorListener cancellable(@NotNull ErrorListener errs, @NotNull BooleanSupplier cancelled) { + requireNonNull(errs, "errs"); + requireNonNull(cancelled, "cancelled"); + return new ErrorListener() { + @Override + public void log(ErrorInfo err) { + errs.log(err); + } + + @Override + public boolean isAbortDesired() { + return cancelled.getAsBoolean() || errs.isAbortDesired(); + } + + @Override + public boolean hasSeriousErrors() { + return errs.hasSeriousErrors(); + } + + @Override + public boolean hasError(String sCode) { + return errs.hasError(sCode); + } + + @Override + public boolean isSilent() { + return errs.isSilent(); + } + + @Override + public ErrorListener branch(AstNode node) { + // a branch of a cancellable listener is still cancellable: work inside it is just + // as pointless once the answer is not wanted + return cancellable(errs.branch(node), cancelled); + } + + @Override + public ErrorListener merge() { + // and it has to be mergeable, or branching it would quietly throw the branch away: + // the default merge() answers "this", which for a wrapper means the wrapped + // branch is never merged into what it branched from + return cancellable(errs.merge(), cancelled); + } + + @Override + @Nullable + public Silence silenceReason() { + return errs.silenceReason(); + } + + @Override + public String toString() { + return "Cancellable(" + errs + ")"; + } + }; + } + + /** + * Create a listener that forwards reports to two destinations in order. + * + *

For example, combine a host's budgeted listener with a recorder whose reports may later be + * replayed. Each report goes to {@code first}, then {@code second}; an abort request does not + * skip delivery to the second listener. If the first listener throws, the exception propagates + * and the second is not called. + * + *

The wrapper keeps no state of its own. Its abort, serious-error, and code queries are true + * when either destination answers true; it is silent only when both are silent. It uses this + * interface's default buffering {@link #branch(AstNode)} behavior rather than independently + * branching each destination. + * + * @param first the first destination + * @param second the second destination + * + * @return a listener combining delivery and the two destinations' current state + */ + @NotNull + static ErrorListener tee(@NotNull ErrorListener first, @NotNull ErrorListener second) { + requireNonNull(first, "first"); + requireNonNull(second, "second"); + return new ErrorListener() { + @Override + public void log(ErrorInfo err) { + first.log(err); + second.log(err); + } + + @Override + public boolean isAbortDesired() { + return first.isAbortDesired() || second.isAbortDesired(); + } + + @Override + public boolean hasSeriousErrors() { + return first.hasSeriousErrors() || second.hasSeriousErrors(); + } + + @Override + public boolean hasError(String sCode) { + return first.hasError(sCode) || second.hasError(sCode); + } + + @Override + public boolean isSilent() { + return first.isSilent() && second.isSilent(); + } + + @Override + public String toString() { + return "Tee(" + first + ", " + second + ")"; + } + }; + } + + /** + * Describe a diagnostic associated with an assembled structure. + * + *

This does not calculate a source span. A node-associated diagnostic branch can re-anchor + * the structure report to its node; otherwise an editor host must interpret the structure site. + * Prefer {@link #NOWHERE} when the caller has no location. + * + * @param xs the related structure; null is accepted for legacy unpositioned reports + * + * @return a structure site referring to {@code xs} + */ + static Site at(XvmStructure xs) { + return new Site.At(xs); + } + + /** + * Describe a diagnostic over a source span. + * + *

Positions use the encoding returned by {@link Source#getPosition()}, not byte offsets or + * LSP positions. This factory stores the supplied values without validating or copying the source. + * + * @param source the source being diagnosed; prefer {@link #NOWHERE} if absent + * @param lPosStart the encoded start position, inclusive + * @param lPosEnd the encoded end position, exclusive + * + * @return a site referring to the supplied source span + */ + static Site in(Source source, long lPosStart, long lPosEnd) { + return new Site.In(source, lPosStart, lPosEnd); + } + + /** + * The location of a diagnostic, independent of its severity and message arguments. + * + *

Hosts can switch exhaustively over source spans, structure references, and unpositioned + * reports. These values describe locations; they do not resolve a structure to source text or + * detach the referenced compiler objects for concurrent access. */ - default boolean log(Severity severity, String sCode, Object[] aoParam, XvmStructure xs) { - return log(new ErrorInfo(severity, sCode, aoParam, xs)); + sealed interface Site { + /** + * A source span using the encoded positions returned by {@link Source#getPosition()}. + * + * @param source the referenced source; not copied + * @param lPosStart the inclusive encoded start position + * @param lPosEnd the exclusive encoded end position + */ + record In(Source source, long lPosStart, long lPosEnd) implements Site {} + + /** + * A diagnostic associated with a structure rather than an explicit source span. + * + * @param xs the related structure; null represents a legacy unpositioned report + */ + record At(XvmStructure xs) implements Site {} + + /** + * A diagnostic with no source span or structure, such as an operation-wide setup failure. + */ + record None() implements Site {} } /** - * Branch this ErrorListener by creating a new one that will collect subsequent errors - * in the same manner as this one until it is {@link #merge() merged} or discarded in the - * (optional) context of the specified node. + * The shared location for a diagnostic with no source span or structure. + */ + Site NOWHERE = new Site.None(); + + /** + * Create a temporary buffer for diagnostics from work whose reports may later be retained. + * + *

For speculative validation, pass the branch to the trial operation and call {@link #merge()} + * once if its reports should reach this listener. Otherwise discard the branch. Reports are + * buffered and deduplicated in an {@link ErrorList}; they are not forwarded during the trial. + * Use {@link #silence(Silence)} instead when the reports must never be retained. * - * @param node (optional) the context ast node + *

The default branch has an unlimited count budget, still aborts on its own FATAL, and observes + * this parent's abort request. {@link ErrorList#branch(AstNode)} instead copies its configured + * count budget. A supplied node re-anchors structure-site reports to that node's source span; + * explicit source-site reports retain their own location. * - * @return the branched-out ErrorListener + * @param node the optional source context for structure-site diagnostics + * + * @return a new buffering child listener */ default ErrorListener branch(AstNode node) { - return new ErrorList.BranchedErrorListener(this, 0, node); + return new ErrorList.BranchedErrorListener(this, ErrorList.UNLIMITED, node); } /** - * Merge all errors collected by this ErrorListener into the one it was branched out of. + * Publish a branch's retained diagnostics to its parent and return the parent. + * + *

The default implementation is a no-op returning this listener, for a sink that has no parent. + * A {@link ErrorList.BranchedErrorListener} forwards its retained reports without clearing them; + * merging it again replays them again. A parent {@link ErrorList} can deduplicate that replay, + * but an arbitrary callback need not. Merge each accepted branch once. * - * @return the ErrorListener this one was {@link #branch branched out} of + * @return the parent for a branch, or this listener when there is nothing to merge */ default ErrorListener merge() { - throw new UnsupportedOperationException("nothing to merge"); + return this; + } + + /** + * The caller's reason for deliberately discarding diagnostics. + * + *

The legacy {@code BLACKHOLE} name did not distinguish a type-fit probe, follow-on errors + * from incomplete analysis, and a caller that deliberately wanted no reports. Naming the reason + * makes that decision reviewable and searchable. These reasons have the same discard behavior; + * compiler semantics must not depend on which reason was selected. + * + *

Suppression does not buffer diagnostics for later publication. Use {@link #branch(AstNode)} + * when deciding whether reports should eventually be kept. Use a derived {@link #silence(Silence)} + * when the parent listener's stop policy must remain active. + */ + enum Silence { + /** + * Speculative work whose return value answers the question, such as whether an expression fits + * a candidate type. Its failure is not a diagnostic for the user's program. Use a branch instead + * if the trial's reports may be needed when all alternatives fail. + */ + PROBE, + + /** + * Follow-on diagnostics from information already known to be incomplete. Discard them to avoid + * burying the original failure in its consequences. Derive this silence from the operation's + * listener when its abort policy must remain active. Discarded reports cannot later be recovered. + */ + CASCADE, + + /** + * An explicit decision not to receive diagnostics at this destination, for example a command-line + * launcher's external delegate when its console already reports them. This is the general-purpose + * replacement for an intentional {@code BLACKHOLE}; choose PROBE or CASCADE when those reasons apply. + */ + DISCARD + } + + /** + * Obtain a shared discard sink without a parent listener or inherited stop policy. + * + *

It retains no diagnostics, reports no serious errors or codes, never requests an abort, and + * is silent. All reasons behave identically. If the caller already has an operation listener, + * prefer {@link #silence(Silence)} to preserve that listener's abort or cancellation request. + * + * @param why the reason for discarding reports + * + * @return the shared stateless sink for the requested reason + */ + @NotNull + static ErrorListener silent(@NotNull Silence why) { + return switch (why) { + case PROBE -> SILENT_PROBE; + case CASCADE -> SILENT_CASCADE; + case DISCARD -> SILENT_DISCARD; + }; + } + + /** + * Create a discard wrapper for part of this listener's operation while retaining its stop policy. + * + *

The wrapper does not forward, retain, or replay reports, and its serious-error and code + * queries return false. Its abort query delegates to this listener, so suppressing reports cannot + * cancel a parent's stop request. {@link SilentErrorListener#suppressed()} exposes that parent, + * not the discarded diagnostics. The parent itself is not modified. + * + *

Silencing an existing {@link SilentErrorListener} returns the same instance and keeps its + * original reason, even when a different reason is requested. + * + * @param why the reason for discarding reports + * + * @return a wrapper retaining this listener's abort policy, or the existing silent wrapper + */ + @NotNull + default ErrorListener silence(@NotNull Silence why) { + return new SilentErrorListener(this, why); } /** - * @return true if the ErrorListener has decided to abort the process that reported the error + * Describe why reports are discarded, when the listener exposes a named suppression policy. + * + *

This is explanatory metadata, not a compiler mode. Wrappers do not necessarily expose a + * reason even when {@link #isSilent()} is true; use that predicate to test silence. + * + * @return the suppression reason, or null when no named reason is exposed + */ + @Nullable + default Silence silenceReason() { + return null; + } + + /** + * Ask whether the current operation should stop at the caller's next safe checkpoint. + * + *

This query is independent of reporting and need not imply that an error was emitted. A + * listener may request a stop because of an error budget, a fatal diagnostic, or host policy. + * Conversely, serious errors may be collected without exhausting the budget. The caller owns + * the control flow; the default implementation never requests an abort. + * + * @return true if the caller should stop the operation */ default boolean isAbortDesired() { return false; } /** - * @return true iff an error has been logged with at least the Severity of Error + * Ask whether this listener has recorded an ERROR or FATAL diagnostic. + * + *

Use this to decide whether a compiler result is usable, not as a substitute for the abort + * policy. The default is false because a reporting-only implementation retains no state. + * + * @return true if a serious diagnostic has been recorded */ default boolean hasSeriousErrors() { return false; } /** - * @return true iff an error has been logged with the specified code + * Ask whether this listener has recorded the specified diagnostic code at any severity. + * + *

The default returns false. Stateful implementations such as {@link ErrorList} and + * {@link #collecting(Consumer)} record codes for this query. + * + * @param sCode the diagnostic code to look for + * + * @return true if the code has been recorded */ default boolean hasError(String sCode) { return false; } /** - * Used for debugging only. + * Ask whether reports sent to this listener are ultimately discarded. + * + *

Callers can skip work done solely to construct diagnostics when the answer is true. They + * must still perform semantic work and honor {@link #isAbortDesired()}; a silent wrapper may + * retain a parent's stop request. The default is false. A buffering branch can be silent when + * its eventual parent discards the reports, even though the branch records its local state. * - * @return true iff this listener sits on top of the BlackHoleListener + * @return true if the diagnostic destination is silent */ default boolean isSilent() { return false; } - // ----- inner class: BlackholeErrorListener --------------------------------------------------- + // ----- inner class: SilentErrorListener ------------------------------------------------------ /** - * A simple implementation of the ErrorListener that converts reported errors to ErrorInfo - * objects and routes them to a single sink method. + * A discard listener carrying a reason and, optionally, a parent stop policy. + * + *

Reports are neither forwarded nor retained. The inherited serious-error and code queries + * return false; the abort query delegates to the parent when present. {@link #suppressed()} + * returns that parent only, and cannot recover discarded messages. Prefer the + * {@link ErrorListener#silent(Silence)} and {@link ErrorListener#silence(Silence)} factories. */ - class BlackholeErrorListener + class SilentErrorListener implements ErrorListener { + /** + * Construct a standalone discard listener with no parent abort policy. + * + * @param why the reason for discarding reports + */ + public SilentErrorListener(@NotNull Silence why) { + this(null, why); + } + + /** + * Construct a discard listener with an optional parent abort policy. + * + * @param errs the parent whose abort query is retained, or null for no parent policy + * @param why the reason for discarding reports + */ + public SilentErrorListener(@Nullable ErrorListener errs, @NotNull Silence why) { + f_errs = errs; + f_why = requireNonNull(why, "why"); + } + + /** + * Return the listener whose reports this wrapper suppresses. + * + *

This is a reference to the original destination, not a buffer of discarded diagnostics. + * The wrapper does not mutate or silence that destination for other callers. + * + * @return the parent listener, or null for a standalone discard sink + */ + @Nullable + public ErrorListener suppressed() { + return f_errs; + } + @Override - public boolean log(ErrorInfo err) { - return false; + public void log(ErrorInfo err) { + // discarded on purpose; f_why says which purpose } @Override @@ -141,36 +695,90 @@ public ErrorListener merge() { return this; } + @Override + public boolean isAbortDesired() { + // Suppressing reports does not make cancelled work useful again. + return f_errs != null && f_errs.isAbortDesired(); + } + @Override public boolean isSilent() { return true; } + @Override + @NotNull + public Silence silenceReason() { + return f_why; + } + + @Override + @NotNull + public ErrorListener silence(@NotNull Silence why) { + requireNonNull(why, "why"); + // already silent for a stated reason; a second reason would only add a layer + return this; + } + @Override public String toString() { - return "(Blackhole)"; + return f_errs == null ? "(" + f_why + ")" : "(" + f_why + " of " + f_errs + ")"; } + + @Nullable + private final ErrorListener f_errs; + @NotNull + private final Silence f_why; } // ----- inner class: Runtime ErrorListener ---------------------------------------------------- /** - * A simple implementation of the ErrorListener that can be used at runtime. Errors will throw, - * and non-errors will go to standard out. + * A forwarding sink that remembers an abort request after an ERROR or FATAL report. + * + *

Reports go to the supplied listener, or to {@link #STDERR} for standalone callers. + * Reporting does not deliberately throw to stop the operation; callers must query {@link #isAbortDesired()} and + * choose their own failure handling. The abort flag remains set for the life of this instance. + * + *

This sink does not collect diagnostic codes or override the serious-error query. Use + * {@link ErrorList} or {@link ErrorListener#collecting(Consumer)} for a stateful compiler host. */ class RuntimeErrorListener implements ErrorListener { + /** Construct a standalone listener that reports to standard error. */ + public RuntimeErrorListener() { + this(STDERR); + } + + /** + * @param errs the destination for runtime diagnostics + */ + public RuntimeErrorListener(ErrorListener errs) { + f_errs = requireNonNull(errs, "errs"); + } + @Override - public boolean log(ErrorInfo err) { - String s = err.toString(); - if (err.getSeverity().ordinal() >= Severity.ERROR.ordinal()) { - throw new IllegalStateException(s); - } else { - System.out.println(err.getSeverity() + ": " + s); - return false; + public void log(ErrorInfo err) { + if (err.getSeverity().isAtLeast(Severity.ERROR)) { + m_fAbort = true; } + f_errs.log(err); } + /** + * This used to throw from inside log(), which made the decision to stop - and the type of + * the exception that stopped it - a property of who was listening rather than of what went + * wrong. A caller that asks gets the same answer; a caller that does not is no longer + * interrupted in the middle of reporting. + */ + @Override + public boolean isAbortDesired() { + return m_fAbort; + } + + private final ErrorListener f_errs; + private volatile boolean m_fAbort; + @Override public String toString() { return "(Runtime error listener)"; @@ -213,7 +821,7 @@ public ErrorInfo(Severity severity, String sCode, Object[] aoParam, * {@link Severity#ERROR}, or {@link Severity#FATAL} * @param sCode the error code that identifies the error message * @param aoParam the parameters for the error message; may be null - * @param xs + * @param xs the related structure, or null for an unpositioned diagnostic */ public ErrorInfo(Severity severity, String sCode, Object[] aoParam, XvmStructure xs) { m_severity = severity; @@ -223,6 +831,21 @@ public ErrorInfo(Severity severity, String sCode, Object[] aoParam, XvmStructure // TODO need to be able to ask the XVM structure for the source & location } + /** + * Describe this diagnostic's location as a source span, structure, or unpositioned report. + * + *

A source takes precedence when present. Otherwise a non-null structure yields a structure + * site; if neither is present, the result is {@link ErrorListener#NOWHERE}. No structure-to-source + * lookup or copying of compiler objects occurs here. + * + * @return the diagnostic location + */ + public Site site() { + return m_source != null ? new Site.In(m_source, m_lPosStart, m_lPosEnd) + : m_xs != null ? new Site.At(m_xs) + : NOWHERE; + } + /** * @return the Severity of the error */ @@ -314,8 +937,11 @@ public String genUID() { .append(m_sCode); if (m_aoParam != null) { + // the parameters are part of the identity, so they have to be compared, not + // digested: a 32-bit hash collides, and a collision here does not merge two + // reports of the same thing, it discards one of two different things sb.append('#') - .append(Arrays.hashCode(m_aoParam)); + .append(Arrays.toString(m_aoParam)); } if (!m_sCode.startsWith("VERIFY")) { @@ -423,15 +1049,34 @@ public String toString() { ResourceBundle RESOURCES = ResourceBundle.getBundle("errors"); /** - * Stateless ErrorListeners. + * Shared standalone sink for speculative probes; prefer {@link #silent(Silence)} at call sites. + */ + ErrorListener SILENT_PROBE = new SilentErrorListener(Silence.PROBE); + /** + * Shared standalone sink for cascade suppression. Use {@link #silence(Silence)} on an operation + * listener when its abort policy must be preserved. + */ + ErrorListener SILENT_CASCADE = new SilentErrorListener(Silence.CASCADE); + /** + * Shared standalone sink for an explicitly unwanted diagnostic destination. + */ + ErrorListener SILENT_DISCARD = new SilentErrorListener(Silence.DISCARD); + + /** + * Stateless standard-error destination for standalone callers. Hosts should supply their own + * listener to receive diagnostics without writing to a process stream. + */ + ErrorListener STDERR = err -> System.err.println(err.getSeverity() + ": " + err); + + /** + * Shared standard-error fallback. Its abort flag is sticky across uses; it is not a fresh + * per-operation collector. Embedding hosts should supply their own listener. */ - ErrorListener BLACKHOLE = new BlackholeErrorListener(); - ErrorListener RUNTIME = new RuntimeErrorListener(); + ErrorListener RUNTIME = new RuntimeErrorListener(); /** - * The default maximum number of serious errors an {@link ErrorList} collects before it asks - * the process reporting them to abort; for example, the number of errors the compiler collects - * for one module in one stage. + * Default serious-error budget for compilation, shared by tools and {@link ErrorList#ErrorList()}. + * Pass an explicit budget to {@link ErrorList#ErrorList(int)} when a different policy is needed. */ int DEFAULT_MAX_ERRORS = 512; diff --git a/javatools/src/main/java/org/xvm/asm/FileStructure.java b/javatools/src/main/java/org/xvm/asm/FileStructure.java index fd523673249..0f91f99da36 100644 --- a/javatools/src/main/java/org/xvm/asm/FileStructure.java +++ b/javatools/src/main/java/org/xvm/asm/FileStructure.java @@ -205,7 +205,6 @@ public FileStructure(FileStructure that) { m_kind = that.m_kind; m_fLinked = that.m_fLinked; - m_errs = that.m_errs; resetModified(); } @@ -1452,29 +1451,6 @@ public boolean validateModuleConstants() { return true; } - @Override - public ErrorListener getErrorListener() { - ErrorListener errs = m_errs; - if (errs == null) { - // getCurrentPool() is an AMBIENT thread-local: it is null on any thread that has not had - // a pool pushed onto it, which is every thread driving the compiler or runtime from - // ordinary Java code. Dereferencing it unconditionally turned this diagnostic accessor - // into an NPE source. Ownership belongs in a parameter, not a thread-local - see the PR - // discussion - but the null guard is the minimal, behaviour-preserving fix. - ConstantPool poolCurrent = ConstantPool.getCurrentPool(); - if (poolCurrent != null && poolCurrent != m_pool) { - errs = poolCurrent.getErrorListener(); - } - } - return errs == null ? ErrorListener.RUNTIME : errs; - } - - @Override - public void setErrorListener(ErrorListener errs) { - // this is not considered a "mutation" of the FileStructure - m_errs = errs; - } - // ----- Object methods ------------------------------------------------------------------------ @Override @@ -1649,10 +1625,4 @@ public boolean isBundle() { */ private transient AssemblerContext m_ctx; - /** - * Holds an ErrorListener explicitly provided to this FileStructure. An absence of an - * ErrorListener implies that either the ErrorListener from the ConstantPool associated with - * the current thread should be used, or failing that, the runtime ErrorListener should be used. - */ - private transient ErrorListener m_errs; } diff --git a/javatools/src/main/java/org/xvm/asm/MethodStructure.java b/javatools/src/main/java/org/xvm/asm/MethodStructure.java index 41019bf95a4..3336acea3c9 100644 --- a/javatools/src/main/java/org/xvm/asm/MethodStructure.java +++ b/javatools/src/main/java/org/xvm/asm/MethodStructure.java @@ -59,6 +59,10 @@ import org.xvm.util.ListMap; import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.at; +import static org.xvm.asm.ErrorListener.silent; + import static org.xvm.util.Handy.indentLines; import static org.xvm.util.Handy.parseDelimitedString; import static org.xvm.util.Handy.readIndex; @@ -757,8 +761,10 @@ public List collectUnresolvedTypeParameters(Set setResolved) { private static boolean checkConflict(TypeConstant typeResult, FormalConstant constFormal, boolean fParam, Map mapTypeParams) { if (typeResult != null) { + ErrorListener probe = silent(PROBE); + // downgrade enum value types to their base type (e.g. True -> Boolean) - TypeInfo info = typeResult.ensureTypeInfo(ErrorListener.BLACKHOLE); + TypeInfo info = typeResult.ensureTypeInfo(probe); if (info.getFormat() == Format.ENUMVALUE) { typeResult = info.getExtends(); } @@ -774,7 +780,7 @@ private static boolean checkConflict(TypeConstant typeResult, FormalConstant con // the new parameter type is wider or the old return type is narrower; use it instead } else { // the type are not compatible; use the common type (TODO: consider union?) - typeResult = Op.selectCommonType(typePrev, typeResult, ErrorListener.BLACKHOLE); + typeResult = Op.selectCommonType(typePrev, typeResult, probe); if (typeResult == null) { // different arguments cause the formal type to resolve into // incompatible types @@ -1681,8 +1687,7 @@ public ResolutionResult resolveName(String sName, Access access, ResolutionColle // REVIEW need a better error? AstNode node = collector.getNode(); if (node == null) { - collector.getErrorListener().log(Severity.ERROR, - Compiler.UNSUPPORTED_DYNAMIC_TYPE_PARAMS, null, this); + collector.getErrorListener().error(Compiler.UNSUPPORTED_DYNAMIC_TYPE_PARAMS, at(this)); } else { node.log(collector.getErrorListener(), Severity.ERROR, Compiler.UNSUPPORTED_DYNAMIC_TYPE_PARAMS); diff --git a/javatools/src/main/java/org/xvm/asm/ModuleRepository.java b/javatools/src/main/java/org/xvm/asm/ModuleRepository.java index 284fa3ef99b..cc51a8cb624 100644 --- a/javatools/src/main/java/org/xvm/asm/ModuleRepository.java +++ b/javatools/src/main/java/org/xvm/asm/ModuleRepository.java @@ -1,14 +1,33 @@ package org.xvm.asm; +import java.io.File; import java.io.IOException; import java.util.Set; import java.util.TreeSet; +import static java.util.Objects.requireNonNull; + /** * An interface representing the ability to find Modules by identity. */ public interface ModuleRepository { + /** + * Create a filesystem repository. An existing directory selects a directory repository; every + * other path selects a file repository. A file need not exist yet, allowing a writable + * repository to create it. + * + * @param file a module directory or file + * @param readOnly true to make the repository read-only + * + * @return the repository for the given path + */ + static ModuleRepository of(File file, boolean readOnly) { + return requireNonNull(file, "file").isDirectory() + ? new DirRepository(file, readOnly) + : new FileRepository(file, readOnly); + } + /** * Obtain a set of domain names that are known by this repository. * diff --git a/javatools/src/main/java/org/xvm/asm/OpCallable.java b/javatools/src/main/java/org/xvm/asm/OpCallable.java index db7b012db32..2ae31b1075b 100644 --- a/javatools/src/main/java/org/xvm/asm/OpCallable.java +++ b/javatools/src/main/java/org/xvm/asm/OpCallable.java @@ -103,6 +103,16 @@ public boolean usesSuper() { return m_nFunctionId == A_SUPER; } + /** + * @return the function or constructor that this op calls, as the specified frame has it, or + * null if the op calls a function that a register holds, or the super function + */ + public MethodConstant getCalledMethod(Frame frame) { + return m_nFunctionId <= CONSTANT_OFFSET + ? frame.getConstant(m_nFunctionId, MethodConstant.class) + : null; + } + /** * A "virtual constant" indicating whether or not this op has multiple return values. * diff --git a/javatools/src/main/java/org/xvm/asm/OpInvocable.java b/javatools/src/main/java/org/xvm/asm/OpInvocable.java index 01f13f0ae6a..972ed614e10 100644 --- a/javatools/src/main/java/org/xvm/asm/OpInvocable.java +++ b/javatools/src/main/java/org/xvm/asm/OpInvocable.java @@ -106,6 +106,13 @@ protected boolean isTupleReturn() { return false; } + /** + * @return the method that this op invokes, as the specified frame has it + */ + public MethodConstant getInvokedMethod(Frame frame) { + return frame.getConstant(m_nMethodId, MethodConstant.class); + } + @Override public void resetSimulation() { if (isMultiReturn()) { diff --git a/javatools/src/main/java/org/xvm/asm/Reporting.java b/javatools/src/main/java/org/xvm/asm/Reporting.java new file mode 100644 index 00000000000..e96ca65868c --- /dev/null +++ b/javatools/src/main/java/org/xvm/asm/Reporting.java @@ -0,0 +1,67 @@ +package org.xvm.asm; + +import static java.util.Objects.requireNonNull; + +/** + * A diagnostic destination that can be diverted for a lexical scope. + * + *

The parser and name resolver use this when callbacks cannot take a listener parameter. + * Scopes must be closed in reverse order, normally by try-with-resources. Each scope restores + * its predecessor even when the operation throws. An inactive resolver has a null destination; + * an active scope always has a listener. + * + *

This holder controls lifetime, not ownership or concurrency. Owners remain confined to one + * compiler operation; sharing a holder across threads is not supported. + */ +public final class Reporting { + private ErrorListener destination; + + /** + * @param errs where diagnostics go to begin with; null if the owner has no destination + * until one is given to it + */ + public Reporting(ErrorListener errs) { + destination = errs; + } + + /** + * @return where diagnostics go right now, or null if the owner has none + */ + public ErrorListener get() { + return destination; + } + + /** + * Send diagnostics to the given listener until the returned scope is closed. + * + * @param errs the listener to report to for the duration of the scope + * + * @return the scope, which restores the previous destination when closed + */ + public Scope to(ErrorListener errs) { + return new Scope(errs); + } + + /** + * The scope opened by {@link #to}. + */ + public final class Scope + implements AutoCloseable { + private final ErrorListener previous; + + private Scope(ErrorListener errs) { + previous = destination; + destination = requireNonNull(errs, "errs"); + } + + @Override + public void close() { + destination = previous; + } + } + + @Override + public String toString() { + return "Reporting(" + destination + ")"; + } +} diff --git a/javatools/src/main/java/org/xvm/asm/XvmStructure.java b/javatools/src/main/java/org/xvm/asm/XvmStructure.java index 14000a384ec..138f9663ff9 100755 --- a/javatools/src/main/java/org/xvm/asm/XvmStructure.java +++ b/javatools/src/main/java/org/xvm/asm/XvmStructure.java @@ -463,41 +463,13 @@ public boolean validate(ErrorListener errs) { /** * Log an error against this structure. * - * @param errs the error list to log to, or null to use the runtime ErrorListener + * @param errs the error listener to log to * @param sev the severity of the error * @param sCode the error code * @param aoParam the parameters of the error */ - public boolean log(ErrorListener errs, Severity sev, String sCode, Object ... aoParam) { - return ensureErrorListener(errs).log(sev, sCode, aoParam, this); - } - - /** - * Make sure that an error listener is returned to use. - * - * @param errs an error listener, or null - * - * @return the error listener passed in, if it was not null, otherwise the previously specified - * error listener, otherwise the runtime error listener - */ - public ErrorListener ensureErrorListener(ErrorListener errs) { - return errs == null ? getErrorListener() : errs; - } - - /** - * @return the error listener, if provided, otherwise the runtime error listener - */ - public ErrorListener getErrorListener() { - return m_xsParent.getErrorListener(); - } - - /** - * Specify an error listener. - * - * @param errs the error listener - */ - public void setErrorListener(ErrorListener errs) { - m_xsParent.setErrorListener(errs); + public void log(ErrorListener errs, Severity sev, String sCode, Object ... aoParam) { + errs.log(sev, sCode, ErrorListener.at(this), aoParam); } // ----- debugging support --------------------------------------------------------------------- diff --git a/javatools/src/main/java/org/xvm/asm/constants/ImmutableTypeConstant.java b/javatools/src/main/java/org/xvm/asm/constants/ImmutableTypeConstant.java index 30310fe6107..6eb5b11e8ac 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/ImmutableTypeConstant.java +++ b/javatools/src/main/java/org/xvm/asm/constants/ImmutableTypeConstant.java @@ -238,7 +238,8 @@ public boolean validate(ErrorListener errs) { // type constant TypeConstant type = m_constType; if (type instanceof ImmutableTypeConstant) { - fHalt |= log(errs, Severity.WARNING, VE_IMMUTABLE_REDUNDANT); + log(errs, Severity.WARNING, VE_IMMUTABLE_REDUNDANT); + fHalt |= errs.isAbortDesired(); } // a service type cannot be immutable diff --git a/javatools/src/main/java/org/xvm/asm/constants/MethodBody.java b/javatools/src/main/java/org/xvm/asm/constants/MethodBody.java index 3bfc486ed29..b4cf3b8a9e3 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/MethodBody.java +++ b/javatools/src/main/java/org/xvm/asm/constants/MethodBody.java @@ -292,9 +292,13 @@ public boolean isInto() { } /** + * Inspect the existing mixin constraint chain without generating a forwarding body. + * The returned compiler metadata belongs to the current compilation; hosts must copy facts + * before publishing them to another thread. Only call this for a FromInto implementation. + * * @return the (potentially incomplete) MethodInfo from which the "into" MethodBody was created */ - MethodInfo getIntoMethodInfo() { + public MethodInfo getIntoMethodInfo() { assert isInto(); return (MethodInfo) m_target; } diff --git a/javatools/src/main/java/org/xvm/asm/constants/MethodInfo.java b/javatools/src/main/java/org/xvm/asm/constants/MethodInfo.java index c6436f087b5..3213373d95f 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/MethodInfo.java +++ b/javatools/src/main/java/org/xvm/asm/constants/MethodInfo.java @@ -6,6 +6,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -1348,6 +1349,33 @@ public boolean hasSuper(TypeInfo infoType) { return getSuper(infoType) != null; } + /** + * The written method body selected by super dispatch, when there is one. Field, native, + * delegation and into-constraint bodies have no source method target. Inspection uses the + * same chain selection as getSuper(), without generating or optimizing forwarding code. + * + * @param infoType the enclosing type information used to locate the inherited method + * + * @return the written super-dispatch target, or empty when no source method represents it + */ + public Optional getSuperMethod(TypeInfo infoType) { + MethodStructure method = getTopmostMethodStructure(infoType); + if (method.getParent().getParent() instanceof PropertyStructure property && + (method == property.getGetter() || method == property.getSetter()) && + infoType.findProperty(property.getIdentityConstant()).hasField()) { + return Optional.empty(); + } + MethodBody[] chain = m_aBodyResolved; + MethodBody body = chain == null ? findSuper(infoType, getChain()) + : chain.length > 1 ? chain[1] : null; + return Optional.ofNullable(body) + .filter(candidate -> candidate.getImplementation() == Implementation.Explicit || + candidate.getImplementation() == Implementation.Default) + .map(MethodBody::getMethodStructure) + .filter(target -> !target.isSynthetic()) + .map(MethodStructure::getIdentityConstant); + } + /** * @return the "super" signature */ diff --git a/javatools/src/main/java/org/xvm/asm/constants/PropertyInfo.java b/javatools/src/main/java/org/xvm/asm/constants/PropertyInfo.java index 4c0f38fc191..a0f786c1d2a 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/PropertyInfo.java +++ b/javatools/src/main/java/org/xvm/asm/constants/PropertyInfo.java @@ -304,7 +304,7 @@ else if (!(bodyAdd.isRO() && typeResult.isA(typeAdd)) for (Annotation annotation : aAnnoBase) { TypeConstant typeAnnoBase = annotation.getAnnotationType(); if (typeAnnoAdd.equals(typeAnnoBase)) { - idProp.log(errs, Severity.WARNING, VE_DUP_ANNOTATION_IGNORED, + that.getIdentity().log(errs, Severity.WARNING, VE_DUP_ANNOTATION_IGNORED, that.getIdentity().getParentConstant().getValueString(), getName(), annoAdd.getAnnotationClass().getValueString()); @@ -312,7 +312,7 @@ else if (!(bodyAdd.isRO() && typeResult.isA(typeAdd)) } if (typeAnnoAdd.isA(typeAnnoBase)) { - idProp.log(errs, Severity.WARNING, VE_SUP_ANNOTATION_IGNORED, + that.getIdentity().log(errs, Severity.WARNING, VE_SUP_ANNOTATION_IGNORED, that.getIdentity().getParentConstant().getValueString(), getName(), typeAnnoAdd.getValueString(), diff --git a/javatools/src/main/java/org/xvm/asm/constants/TypeCollector.java b/javatools/src/main/java/org/xvm/asm/constants/TypeCollector.java index ba27cbed5ee..99cb8e54fbb 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/TypeCollector.java +++ b/javatools/src/main/java/org/xvm/asm/constants/TypeCollector.java @@ -10,6 +10,9 @@ import org.xvm.asm.ErrorListener; import org.xvm.asm.Op; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * A TypeCollector is used to collect a number of types, such as would occur from return statements * within a lambda, in order to infer a type from that collection of types. @@ -167,19 +170,21 @@ public TypeConstant inferSingle(TypeConstant typeRequired) { return null; } + ErrorListener probe = silent(PROBE); + TypeConstant typeCommon = inferFrom(listTypes.toArray(new TypeConstant[cTypes]), f_pool); if (typeRequired != null && typeRequired.containsFormalType(true)) { typeCommon = typeRequired.resolvePending(f_pool, typeCommon); } - typeCommon = Op.selectCommonType(typeCommon, typeRequired, ErrorListener.BLACKHOLE); + typeCommon = Op.selectCommonType(typeCommon, typeRequired, probe); if (typeRequired != null && (typeCommon == null || !typeCommon.isAssignableTo(typeRequired))) { // approach above didn't quite work; try to match with individual types one-by-one - TypeConstant typeAlt = Op.selectCommonType(typeRequired, listTypes.get(0), ErrorListener.BLACKHOLE); + TypeConstant typeAlt = Op.selectCommonType(typeRequired, listTypes.get(0), probe); for (int i = 1; i < cTypes; i++) { - typeAlt = Op.selectCommonType(typeAlt, listTypes.get(i), ErrorListener.BLACKHOLE); + typeAlt = Op.selectCommonType(typeAlt, listTypes.get(i), probe); } if (typeAlt != null) { @@ -342,8 +347,7 @@ public TypeConstant[] inferMulti(TypeConstant[] atypeRequired) { for (int iCol = 0; iCol < cWidth; ++iCol) { TypeConstant typeRequired = iCol < cReqTypes ? atypeRequired[iCol] : null; if (typeRequired != null) { - aResult[iCol] = Op.selectCommonType(aResult[iCol], typeRequired, - ErrorListener.BLACKHOLE); + aResult[iCol] = Op.selectCommonType(aResult[iCol], typeRequired, silent(PROBE)); } } } @@ -429,7 +433,7 @@ public static TypeConstant inferFrom(TypeConstant[] aTypes, ConstantPool pool) { } } - typeCommon = Op.selectCommonType(type, typeCommon, ErrorListener.BLACKHOLE); + typeCommon = Op.selectCommonType(type, typeCommon, silent(PROBE)); if (typeCommon == null) { // no obvious common type return null; diff --git a/javatools/src/main/java/org/xvm/asm/constants/TypeConstant.java b/javatools/src/main/java/org/xvm/asm/constants/TypeConstant.java index fa2a018b02f..b63074d2443 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/TypeConstant.java +++ b/javatools/src/main/java/org/xvm/asm/constants/TypeConstant.java @@ -12,10 +12,10 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -39,6 +39,7 @@ import org.xvm.asm.ConstantPool; import org.xvm.asm.ErrorList; import org.xvm.asm.ErrorListener; +import org.xvm.asm.ErrorListener.ErrorInfo; import org.xvm.asm.GenericTypeResolver; import org.xvm.asm.MethodStructure; import org.xvm.asm.ModuleStructure; @@ -91,6 +92,11 @@ import static java.lang.constant.ConstantDescs.CD_boolean; import static java.lang.constant.ConstantDescs.CD_int; +import static org.xvm.asm.ErrorListener.Silence.CASCADE; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; +import static org.xvm.asm.ErrorListener.tee; + import static org.xvm.javajit.Builder.CD_Class; import static org.xvm.javajit.Builder.CD_Ctx; import static org.xvm.javajit.Builder.CD_nType; @@ -1780,12 +1786,27 @@ public TypeConstant widenEnumValueTypes() { // ----- TypeInfo support ---------------------------------------------------------------------- /** - * Obtain the information about this type, resolved from its recursive composition. + * Obtain the information about this type, resolved from its recursive composition, without + * reporting anything about the attempt. + * + *

A TypeConstant is an interned value shared by everything, so when it is asked to build a + * TypeInfo and given no listener it has no caller to ask. It used to walk up to its file + * structure and report to whatever that file was last told - an ambient lookup which, for the + * two thirds of these call sites that run after compilation is over, ended at a listener that + * prints to stdout. Measured across a full XDK build and test run, that listener never + * received anything, and during a compilation the file was parked on a silence anyway. So the + * silence is said here instead of arranged elsewhere. + * + *

This convenience read does not report diagnostics to a host. It is used by speculative and + * metadata queries, including provisional compositions. Diagnostics recorded by a completed + * build remain available for replay; a caller responsible for reporting a selected source use + * must pass its listener to {@link #ensureTypeInfo(ErrorListener)}. Silence here is not evidence + * that every diagnostic encountered is spurious. * * @return the flattened TypeInfo that represents the resolved type of this TypeConstant */ public TypeInfo ensureTypeInfo() { - return ensureTypeInfo(getErrorListener()); + return ensureTypeInfo(silent(CASCADE)); } /** @@ -1814,10 +1835,38 @@ public TypeInfo ensureTypeInfo(ErrorListener errs) { TypeInfo info = getTypeInfo(); if (isComplete(info) && isUpToDate(info)) { + // the cached answer was built for whoever asked first. If that was a speculative + // probe, the probe produced the diagnostics and this caller - which may be the one + // that actually cared - would otherwise hear nothing. Replay them instead, so what a + // caller is told does not depend on the order callers happen to arrive in + replayDiagnostics(errs); return info; } - return ensureTypeInfo(info, errs); + // record what building it has to say, so a later caller can be told the same + ErrorList recorder = new ErrorList(ErrorList.UNLIMITED); + info = ensureTypeInfo(info, tee(errs, recorder)); + + // freeze it. An ErrorList is a listener, which is what the recording needed to be while + // it was being made and not what it should be once it is made: keeping one per + // TypeConstant for the life of the pool holds an error budget, a severity, a dedup set + // and a mutable list, to say something a List already says. Most builds report nothing - + // measured over an XDK build, all of them - and copyOf gives back the shared empty list + // for those, so saying "nothing" costs no allocation at all + m_diagnostics = List.copyOf(recorder.getErrors()); + return info; + } + + /** + * Report the diagnostics that were produced while this type's TypeInfo was built. + * + *

Deduplication in the receiving listener makes this idempotent: a caller that asks twice, or + * two callers sharing a listener, are told once. + * + * @param errs the listener to replay them to + */ + private void replayDiagnostics(ErrorListener errs) { + m_diagnostics.forEach(errs::log); } private synchronized TypeInfo ensureTypeInfo(TypeInfo info, ErrorListener errs) { @@ -2104,6 +2153,16 @@ protected void setTypeInfo(TypeInfo info) { } } + /** + * The diagnostics produced while this type's TypeInfo was built, replayed to later callers + * that get the memoized result; see {@link #replayDiagnostics}. + * + *

Empty until the TypeInfo has been built, and empty afterwards when building it had nothing + * to say - which is the ordinary case. Never null: "not built yet" and "built quietly" are + * the same answer to everyone who reads this, so there is nothing for a third state to say. + */ + private transient volatile List m_diagnostics = List.of(); + /** * @return the invalidation count that this TypeConstant has already processed */ @@ -2183,6 +2242,26 @@ public static boolean isComplete(TypeInfo info) { return rankTypeInfo(info) == 3; } + /** + * Choose the listener to report the rest of a TypeInfo build to. + * + * Once a contribution has turned out to be incomplete, what follows is reported against a + * type that is known to be missing pieces, so the diagnostics are consequences of what is + * absent rather than faults in the source. They are suppressed for the remainder of the + * build; the incompleteness itself is what the caller is told, by the return value. + * + * The choice is made at each use rather than by rebinding the listener, so that the errs + * parameter still means what its signature says all the way down the method. + * + * @param fIncomplete whether the build is already known to be incomplete + * @param errs the caller's listener + * + * @return the listener to report to + */ + private static ErrorListener cascade(boolean fIncomplete, ErrorListener errs) { + return fIncomplete ? errs.silence(CASCADE) : errs; + } + /** * Determine if the passed TypeInfo is up-to-date for this type. * @@ -2378,7 +2457,7 @@ private TypeInfo buildBaseTypeInfoImpl( // validate the type parameters against the properties checkTypeParameterProperties(mapTypeParams, mapVirtProps, - fComplete && !errs.hasSeriousErrors() ? errs : ErrorListener.BLACKHOLE); + fComplete && !errs.hasSeriousErrors() ? errs : errs.silence(CASCADE)); Annotation[] aAnnoMixin = fComplete ? collectMixinAnnotations(listProcess) @@ -2759,7 +2838,7 @@ private TypeInfo buildStructInfo(ErrorListener errs) { typeContrib = typeContrib.removeAccess(); typeContrib = pool.ensureAccessTypeConstant(typeContrib, Access.STRUCT); - TypeInfo infoContrib = typeContrib.ensureTypeInfoInternal(errs); + TypeInfo infoContrib = typeContrib.ensureTypeInfoInternal(cascade(fIncomplete, errs)); if (isComplete(infoContrib)) { for (Map.Entry entry : infoContrib.getProperties().entrySet()) { PropertyInfo prop = entry.getValue(); @@ -2771,14 +2850,13 @@ private TypeInfo buildStructInfo(ErrorListener errs) { } } else { fIncomplete = true; - errs = ErrorListener.BLACKHOLE; } break; }} } // add Object.toString() method - MethodInfo infoToString = pool.typeObject().ensureTypeInfo(errs). + MethodInfo infoToString = pool.typeObject().ensureTypeInfo(cascade(fIncomplete, errs)). getMethodBySignature(pool.sigToString()); mapMethods.putIfAbsent(infoToString.getIdentity(), infoToString); @@ -3567,18 +3645,14 @@ private boolean createCallChains( case Into: { // append to the call chain TypeConstant typeContrib = contrib.getTypeConstant(); // already resolved - TypeInfo infoContrib = typeContrib.adjustAccess(constId).ensureTypeInfoInternal(errs); + TypeInfo infoContrib = typeContrib.adjustAccess(constId).ensureTypeInfoInternal(cascade(fIncomplete, errs)); if (!isComplete(infoContrib)) { fIncomplete |= computeIncomplete(composition, typeContrib, infoContrib, setDepends); - if (fIncomplete) { - errs = ErrorListener.BLACKHOLE; - } } if (infoContrib != null) { - infoContrib.contributeChains(listmapClassChain, listmapDefaultChain, - listmapRootChain, composition); - layerOnTypeParams(mapTypeParams, typeContrib, infoContrib.getTypeParams(), errs); + infoContrib.contributeChains(listmapClassChain, listmapDefaultChain, listmapRootChain, composition); + layerOnTypeParams(mapTypeParams, typeContrib, infoContrib.getTypeParams(), cascade(fIncomplete, errs)); } break; } @@ -3737,18 +3811,16 @@ private boolean collectMemberInfo( int nBasePropRank = mapProps.size(); int nBaseMethRank = mapMethods.size(); - if (!collectSelfTypeParameters(struct, mapTypeParams, mapContribProps, nBasePropRank, errs)) { + if (!collectSelfTypeParameters(struct, mapTypeParams, mapContribProps, nBasePropRank, cascade(fIncomplete, errs))) { fIncomplete = true; - errs = ErrorListener.BLACKHOLE; } var listExplode = new ArrayList(); boolean fInterface = struct.getFormat() == Component.Format.INTERFACE; if (!collectChildInfo(constId, fInterface, struct, mapTypeParams, mapContribProps, mapContribMethods, mapContribChildren, listExplode, - mapVirtProps, nBasePropRank, nBaseMethRank, errs)) { + mapVirtProps, nBasePropRank, nBaseMethRank, cascade(fIncomplete, errs))) { fIncomplete = true; - errs = ErrorListener.BLACKHOLE; } // the order in which the properties are layered on and exploded is extremely @@ -3770,21 +3842,19 @@ private boolean collectMemberInfo( // layer on the property so its information is all correct before we have to // make any decisions about how to process the property prop = layerOnProp(constId, ContribSource.Self, null, mapProps, mapVirtProps, - typeContrib, idProp, prop, errs); + typeContrib, idProp, prop, cascade(fIncomplete, errs)); // now that the necessary data is in place, explode the property - if (!fNative && !explodeProperty(constId, struct, idProp, prop, - mapProps, mapVirtProps, mapMethods, mapVirtMethods, errs)) { + if (!fNative && !explodeProperty(constId, struct, idProp, prop, mapProps, + mapVirtProps, mapMethods, mapVirtMethods, cascade(fIncomplete, errs))) { fIncomplete = true; - errs = ErrorListener.BLACKHOLE; } } } else { - infoContrib = typeContrib.adjustAccess(constId).ensureTypeInfoInternal(errs); + infoContrib = typeContrib.adjustAccess(constId).ensureTypeInfoInternal(cascade(fIncomplete, errs)); if (!isComplete(infoContrib)) { if (computeIncomplete(composition, typeContrib, infoContrib, setDepends)) { fIncomplete = true; - errs = ErrorListener.BLACKHOLE; } if (infoContrib == null) { // even if the contribution has an incomplete info we can still proceed @@ -3838,7 +3908,7 @@ private boolean collectMemberInfo( // process properties layerOnProps(constId, contribSource, idDelegate, mapProps, mapVirtProps, - typeContrib, mapContribProps, errs); + typeContrib, mapContribProps, cascade(fIncomplete, errs)); // if there are any remaining declared-but-not-overridden properties originating from // an interface on a class once the "self" layer is applied, then those need to be @@ -3846,7 +3916,7 @@ private boolean collectMemberInfo( if (fSelf && !isInterface(constId, struct) && !struct.isExplicitlyAbstract()) { for (Entry entry : mapProps.entrySet()) { PropertyInfo infoOld = entry.getValue(); - PropertyInfo infoNew = infoOld.finishAdoption(fNative, errs); + PropertyInfo infoNew = infoOld.finishAdoption(fNative, cascade(fIncomplete, errs)); if (infoNew != infoOld) { entry.setValue(infoNew); if (infoNew.isVirtual()) { @@ -3863,7 +3933,7 @@ private boolean collectMemberInfo( if (!mapContribMethods.isEmpty()) { assert contrib.getComposition() != Composition.Annotation; layerOnMethods(constId, contribSource, idDelegate, mapMethods, mapVirtMethods, - typeContrib, mapContribMethods, errs); + typeContrib, mapContribMethods, cascade(fIncomplete, errs)); } // process children @@ -3875,7 +3945,7 @@ private boolean collectMemberInfo( if (infoPrev != null) { ChildInfo infoNew = infoPrev.layerOn(infoChild); if (infoNew == null) { - log(errs, Severity.ERROR, VE_CHILD_COLLISION, + log(cascade(fIncomplete, errs), Severity.ERROR, VE_CHILD_COLLISION, constId, sName, contrib.getTypeConstant(), @@ -3893,7 +3963,7 @@ private boolean collectMemberInfo( // to be processed by "finishAdoption" for (Entry entry : mapMethods.entrySet()) { MethodInfo infoOld = entry.getValue(); - MethodInfo infoNew = infoOld.finishAdoption(fNative, errs); + MethodInfo infoNew = infoOld.finishAdoption(fNative, cascade(fIncomplete, errs)); if (infoNew != infoOld) { entry.setValue(infoNew); if (infoNew.isVirtual()) { @@ -3975,7 +4045,6 @@ protected boolean explodeProperty( infoProp.getHead().markExploded(); } else { fComplete = false; - errs = ErrorListener.BLACKHOLE; } // layer on any annotations, if any @@ -3993,13 +4062,12 @@ protected boolean explodeProperty( } typeAnno = pool.ensureAccessTypeConstant(typeAnno, Access.PROTECTED); - TypeInfo infoAnno = typeAnno.ensureTypeInfoInternal(errs); + TypeInfo infoAnno = typeAnno.ensureTypeInfoInternal(cascade(!fComplete, errs)); if (infoAnno == null) { fComplete = false; - errs = ErrorListener.BLACKHOLE; } else { nestAndLayerOn(constId, idProp, mapProps, mapVirtProps, mapMethods, mapVirtMethods, - typeAnno, infoAnno, ContribSource.Annotation, errs); + typeAnno, infoAnno, ContribSource.Annotation, cascade(!fComplete, errs)); } } @@ -4054,11 +4122,11 @@ protected boolean explodeProperty( idGet.getValueString() + " at " + this.getValueString()); } infoGet = infoGet.layerOn(new MethodInfo(new MethodBody(idGet, - idGet.getSignature(), Implementation.Implicit), nRank), false, errs); + idGet.getSignature(), Implementation.Implicit), nRank), false, cascade(!fComplete, errs)); if (infoSet != null) { infoSet = infoSet.layerOn(new MethodInfo(new MethodBody(idSet, - idSet.getSignature(), Implementation.Implicit), nRank + 1), false, errs); + idSet.getSignature(), Implementation.Implicit), nRank + 1), false, cascade(!fComplete, errs)); } } @@ -7661,7 +7729,7 @@ public TypeConstant getJitCCType() { public TypeConstant getJitICType() { // TODO CP: plug in the new logic TypeConstant type = getJitCCType(); - assert type.isNativeRef() || type.ensureTypeInfo().isNewable(false, ErrorListener.BLACKHOLE); + assert type.isNativeRef() || type.ensureTypeInfo().isNewable(false, silent(PROBE)); return type; } diff --git a/javatools/src/main/java/org/xvm/asm/constants/TypeInfoReal.java b/javatools/src/main/java/org/xvm/asm/constants/TypeInfoReal.java index 7d84ae258be..c506f4069cf 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/TypeInfoReal.java +++ b/javatools/src/main/java/org/xvm/asm/constants/TypeInfoReal.java @@ -37,6 +37,9 @@ import org.xvm.util.ListMap; import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * The fully realized "flattened" information about a type. */ @@ -803,7 +806,7 @@ public boolean isNewable(boolean fSingleton, ErrorListener errs) { return false; } TypeConstant typeParent = f_type.getParentType(); - if (!typeParent.ensureTypeInfo(errs).isNewable(false, ErrorListener.BLACKHOLE)) { + if (!typeParent.ensureTypeInfo(errs).isNewable(false, silent(PROBE))) { // the parent is abstract, so the virtual child "new-ability" will be checked // by concrete parent's subclasses return true; diff --git a/javatools/src/main/java/org/xvm/asm/constants/UnresolvedTypeConstant.java b/javatools/src/main/java/org/xvm/asm/constants/UnresolvedTypeConstant.java index 1ff5142b810..5cdcf2c5ae0 100644 --- a/javatools/src/main/java/org/xvm/asm/constants/UnresolvedTypeConstant.java +++ b/javatools/src/main/java/org/xvm/asm/constants/UnresolvedTypeConstant.java @@ -18,7 +18,8 @@ import org.xvm.compiler.CompilerException; import org.xvm.util.Hash; -import org.xvm.util.Severity; + +import static org.xvm.asm.ErrorListener.at; /** * Represent a type constant that will eventually be replaced with a real type constant. @@ -517,7 +518,7 @@ public boolean validate(ErrorListener errs) { return getResolvedType().validate(errs); } - errs.log(Severity.ERROR, Compiler.NAME_UNRESOLVABLE, new Object[]{getValueString()}, this); + errs.error(Compiler.NAME_UNRESOLVABLE, at(this), getValueString()); return true; } diff --git a/javatools/src/main/java/org/xvm/asm/op/MoveThis.java b/javatools/src/main/java/org/xvm/asm/op/MoveThis.java index 7a4e15d9708..187b22943d7 100644 --- a/javatools/src/main/java/org/xvm/asm/op/MoveThis.java +++ b/javatools/src/main/java/org/xvm/asm/op/MoveThis.java @@ -139,14 +139,13 @@ public int process(Frame frame, int iPC) { } if (m_nAccess != 0) { - Access access = switch (m_nAccess) { + hOuter = hOuter.ensureAccess(switch (m_nAccess) { case A_PUBLIC -> Access.PUBLIC; case A_PROTECTED -> Access.PROTECTED; case A_PRIVATE -> Access.PRIVATE; case A_STRUCT -> Access.STRUCT; default -> throw new IllegalStateException(); - }; - hOuter = hOuter.ensureAccess(access); + }); } int nToValue = m_nToValue; diff --git a/javatools/src/main/java/org/xvm/compiler/Compiler.java b/javatools/src/main/java/org/xvm/compiler/Compiler.java index cbeb25b45c0..6aebf5b71e8 100644 --- a/javatools/src/main/java/org/xvm/compiler/Compiler.java +++ b/javatools/src/main/java/org/xvm/compiler/Compiler.java @@ -1,5 +1,7 @@ package org.xvm.compiler; +import static java.util.Objects.requireNonNull; + import org.xvm.asm.ConstantPool; import org.xvm.asm.ErrorListener; import org.xvm.asm.FileStructure; @@ -9,6 +11,8 @@ import org.xvm.compiler.ast.StageMgr; import org.xvm.compiler.ast.TypeCompositionStatement; +import org.xvm.compiler.ast.bindings.CursorBinding; +import org.xvm.compiler.ast.bindings.InvocationBinding; /** * A module compiler for Ecstasy code. @@ -27,18 +31,39 @@ public class Compiler { * @param errs the listener to log any errors to during the various phases of compilation */ public Compiler(TypeCompositionStatement stmtModule, ErrorListener errs) { + this(stmtModule, errs, InvocationBinding.Collector.NONE); + } + + /** + * Construct a compiler that records call facts in the supplied attempt-owned collector. + * + * @param stmtModule the assembled module declaration to compile + * @param errs the diagnostic listener and abort policy for this operation + * @param bindings the attempt-owned call-fact collector, or its NONE instance to disable collection + */ + public Compiler(TypeCompositionStatement stmtModule, ErrorListener errs, InvocationBinding.Collector bindings) { + this(stmtModule, errs, bindings, CursorBinding.Collector.NONE); + } + + /** + * Construct a module compiler using the supplied attempt-owned fact collectors. + * + * @param stmtModule the assembled module declaration to compile + * @param errs the diagnostic listener and abort policy for this operation + * @param bindings the attempt-owned call-fact collector, or its NONE instance to disable collection + * @param cursors the attempt-owned cursor-fact collector, or its NONE instance to disable collection + */ + public Compiler(TypeCompositionStatement stmtModule, ErrorListener errs, InvocationBinding.Collector bindings, CursorBinding.Collector cursors) { if (stmtModule == null) { throw new IllegalArgumentException("AST node for module required"); } if (stmtModule.getCategory().getId() != Token.Id.MODULE) { throw new IllegalArgumentException("AST node for module is not a module statement"); } - if (errs == null) { - throw new IllegalArgumentException("ErrorListener required"); - } - m_stmtModule = stmtModule; - m_errs = errs; + f_errs = requireNonNull(errs, "errs"); + f_bindings = requireNonNull(bindings, "bindings"); + f_cursors = requireNonNull(cursors, "cursors"); } // ----- accessors ----------------------------------------------------------------------------- @@ -56,7 +81,7 @@ public TypeCompositionStatement getModuleStatement() { */ public ErrorListener getErrorListener() { validateCompiler(); - return m_errs; + return f_errs; } /** @@ -91,7 +116,7 @@ public boolean alreadyReached(Stage stage) { * @return true if the compiler has decided to abort the process */ public boolean isAbortDesired() { - return m_errs.isAbortDesired(); + return f_errs.isAbortDesired(); } // ----- public API ---------------------------------------------------------------------------- @@ -114,15 +139,14 @@ public FileStructure generateInitialFileStructure() { if (getStage() == Stage.Initial) { setStage(Stage.Registering); - StageMgr mgr = new StageMgr(m_stmtModule, Stage.Registered, m_errs); + StageMgr mgr = new StageMgr(m_stmtModule, Stage.Registered, f_errs, f_bindings, f_cursors); if (!mgr.processComplete()) { - if (m_errs.hasSeriousErrors()) { + if (f_errs.hasSeriousErrors() || f_errs.isAbortDesired()) { return null; } throw new CompilerException("failed to create module"); } m_structFile = m_stmtModule.getComponent().getFileStructure(); - m_structFile.setErrorListener(ErrorListener.BLACKHOLE); setStage(Stage.Registered); } @@ -188,7 +212,7 @@ public boolean resolveNames(boolean fLastAttempt) { if (!alreadyReached(Stage.Resolving)) { // first time through: resolve starting from the module, and recurse down setStage(Stage.Resolving); - m_mgr = new StageMgr(m_stmtModule, Stage.Resolved, m_errs); + m_mgr = new StageMgr(m_stmtModule, Stage.Resolved, f_errs, f_bindings, f_cursors); } if (fLastAttempt) { @@ -232,7 +256,7 @@ public boolean validateExpressions(boolean fLastAttempt) { if (!alreadyReached(Stage.Validating)) { // first time through: resolve starting from the module, and recurse down setStage(Stage.Validating); - m_mgr = new StageMgr(m_stmtModule, Stage.Validated, m_errs); + m_mgr = new StageMgr(m_stmtModule, Stage.Validated, f_errs, f_bindings, f_cursors); } if (fLastAttempt) { @@ -276,7 +300,7 @@ public boolean generateCode(boolean fLastAttempt) { if (!alreadyReached(Stage.Emitting)) { // first time through: resolve starting from the module, and recurse down setStage(Stage.Emitting); - m_mgr = new StageMgr(m_stmtModule, Stage.Emitted, m_errs); + m_mgr = new StageMgr(m_stmtModule, Stage.Emitted, f_errs, f_bindings, f_cursors); } if (fLastAttempt) { @@ -286,11 +310,11 @@ public boolean generateCode(boolean fLastAttempt) { if (m_mgr.processComplete()) { setStage(Stage.Emitted); - if (!m_errs.hasSeriousErrors()) { - // "purge" the constant pool and do a final validation on the entire module structure + if (!f_errs.hasSeriousErrors()) { + // "purge" the constant pool and do a final validation on the entire module + // structure m_structFile.reregisterConstants(true); - m_structFile.validate(m_errs); - m_structFile.setErrorListener(null); + m_structFile.validate(f_errs); } } } @@ -303,8 +327,8 @@ public boolean generateCode(boolean fLastAttempt) { * method will report any unresolved names as fatal errors. */ public void logRemainingDeferredAsErrors() { - if (!m_errs.hasSeriousErrors()) { - m_mgr.logDeferredAsErrors(m_errs); + if (!f_errs.hasSeriousErrors()) { + m_mgr.logDeferredAsErrors(f_errs); } } @@ -378,7 +402,11 @@ private void setStage(Stage stage) { /** * The ErrorListener to report errors to. */ - private final ErrorListener m_errs; + private final ErrorListener f_errs; + + /** Optional tooling output, confined to this compiler attempt. */ + private final InvocationBinding.Collector f_bindings; + private final CursorBinding.Collector f_cursors; /** * The FileStructure that this compiler is putting together in a series of passes. diff --git a/javatools/src/main/java/org/xvm/compiler/EvalCompiler.java b/javatools/src/main/java/org/xvm/compiler/EvalCompiler.java index dc4afbdf977..c482747acdf 100644 --- a/javatools/src/main/java/org/xvm/compiler/EvalCompiler.java +++ b/javatools/src/main/java/org/xvm/compiler/EvalCompiler.java @@ -39,7 +39,6 @@ import org.xvm.runtime.Utils; import org.xvm.util.ListMap; -import org.xvm.util.Severity; /** * The compiler of the "eval" script used by the debugger. @@ -65,7 +64,7 @@ public EvalCompiler(Frame frame, String sMethod) { */ public MethodStructure createLambda(TypeConstant typeReturn) { ConstantPool pool = f_frame.poolContext(); - ErrorList errs = m_errs = new ErrorList(1); + ErrorList errs = f_errs; MethodStructure method = f_frame.f_function; ClassStructure clz = method.getContainingClass(); @@ -134,8 +133,8 @@ public MethodStructure createLambda(TypeConstant typeReturn) { return lambda; } catch (Exception e) { if (!errs.hasSeriousErrors()) { - errs.log(Severity.FATAL, Parser.FATAL_ERROR, null, - f_source, f_source.getPosition(), f_source.getPosition()); + errs.fatal(Parser.FATAL_ERROR, ErrorListener.in( + f_source, f_source.getPosition(), f_source.getPosition())); } return discard(lambda); } @@ -155,7 +154,7 @@ private static MethodStructure discard(MethodStructure lambda) { * @return a list of errors */ public List getErrors() { - return m_errs.getErrors(); + return f_errs.getErrors(); } /** @@ -319,9 +318,9 @@ protected Argument getLocalVar(String sName, Branch branch) { private final Source f_source; /** - * The errors. + * Diagnostics owned by this evaluation, available before lambda compilation starts. */ - private ErrorList m_errs; + private final ErrorList f_errs = new ErrorList(ErrorList.FIRST_ERROR); /** * A synthetic MethodDeclarationStatement that contains the eval body. diff --git a/javatools/src/main/java/org/xvm/compiler/Lexer.java b/javatools/src/main/java/org/xvm/compiler/Lexer.java index 89cbfb0ca36..fd3d89fb33b 100644 --- a/javatools/src/main/java/org/xvm/compiler/Lexer.java +++ b/javatools/src/main/java/org/xvm/compiler/Lexer.java @@ -19,6 +19,10 @@ import org.xvm.util.PackedInteger; import org.xvm.util.Severity; +import static java.util.Objects.requireNonNull; + +import static org.xvm.asm.ErrorListener.in; + import static org.xvm.util.Handy.hexitValue; import static org.xvm.util.Handy.isAsciiLetter; import static org.xvm.util.Handy.isDigit; @@ -39,16 +43,14 @@ public class Lexer * * @param source the source to parse */ - public Lexer(Source source, ErrorListener errorListener) { + public Lexer(Source source, ErrorListener errs) { if (source == null) { throw new IllegalArgumentException("Source required"); } - if (errorListener == null) { - throw new IllegalArgumentException("ErrorListener required"); - } + requireNonNull(errs, "errs"); m_source = source; - m_errorListener = errorListener; + m_errorListener = errs; eatWhitespace(); } @@ -131,6 +133,10 @@ public boolean hasNext() { @Override public Token next() { + // Valid input must also observe cancellation; no diagnostic may be emitted. + if (m_errorListener.isAbortDesired()) { + throw new CompilerException("Tokenization aborted"); + } if (hasNext()) { return atoken[iNext++]; } @@ -231,6 +237,10 @@ public static boolean isValidQualifiedModule(String sName) { @Override public Token next() { + // Valid input must also observe cancellation; no diagnostic may be emitted. + if (m_errorListener.isAbortDesired()) { + throw new CompilerException("Tokenization aborted"); + } boolean fWhitespaceBefore = m_fWhitespace; Token token = eatToken(); boolean fWhitespaceAfter = eatWhitespace(); @@ -407,7 +417,7 @@ protected Token eatBinaryLiteral(long lInitPos, boolean fMultiline) { } else if (ch == '_') { if (fFirst) { // it's an error to start with an underscore - log(Severity.ERROR, ILLEGAL_HEX, new Object[] {String.valueOf(ch)}, lInitPos); + log(Severity.ERROR, ILLEGAL_HEX, span(lInitPos), String.valueOf(ch)); } // ignore the _ (it's used for spacing within the literal) } else if (!fMultiline) { @@ -422,7 +432,7 @@ protected Token eatBinaryLiteral(long lInitPos, boolean fMultiline) { } } else { // error - log(Severity.ERROR, ILLEGAL_HEX, new Object[] {String.valueOf(ch)}, lInitPos); + log(Severity.ERROR, ILLEGAL_HEX, span(lInitPos), String.valueOf(ch)); source.rewind(); break; } @@ -851,7 +861,7 @@ protected Token eatToken() { default: if (!isIdentifierStart(chInit)) { - log(Severity.ERROR, ILLEGAL_CHAR, new Object[]{quotedChar(chInit)}, lInitPos); + log(Severity.ERROR, ILLEGAL_CHAR, span(lInitPos), quotedChar(chInit)); } // fall through case 'A':case 'B':case 'C':case 'D':case 'E':case 'F':case 'G': @@ -1045,9 +1055,8 @@ protected Token eatToken() { case LIT_FLOAT: if (!fFloat) { // we were expecting an integer - log(Severity.ERROR, ILLEGAL_NUMBER, new Object[] { - extractSource(lInitPos, tokNum.getEndPosition())}, - lInitPos); + log(Severity.ERROR, ILLEGAL_NUMBER, span(lInitPos), + extractSource(lInitPos, tokNum.getEndPosition())); return tokNum; } // fall through @@ -1836,8 +1845,8 @@ protected PackedInteger eatDigits(boolean fNeg, int radix, int[] digitCount) { protected Token eatTime(long lInitPos) { Token tokDate = eatDate(lInitPos, true); if (!(match('t') || expect('T'))) { - log(Severity.ERROR, BAD_TIME, new Object[] {tokDate.getValue()}, - tokDate.getStartPosition(), tokDate.getEndPosition()); + log(Severity.ERROR, BAD_TIME, + span(tokDate.getStartPosition(), tokDate.getEndPosition()), tokDate.getValue()); return tokDate; } @@ -1887,7 +1896,7 @@ protected Token eatTimeZone(long lInitPos) { if (nHour >= 0 && nMin >= 0) { if (nHour > 16 || nMin > 59) { - log(Severity.ERROR, BAD_TIMEZONE, new Object[] {sZone}, lStart, lEnd); + log(Severity.ERROR, BAD_TIMEZONE, span(lStart, lEnd), sZone); } } @@ -1907,7 +1916,7 @@ protected int eatDigits(int digitCount) { int n = 0; for (int i = 0; i < digitCount; ++i) { if (!isNextCharDigit(10)) { - log(Severity.ERROR, EXPECTED_DIGITS, new Object[] {digitCount, i}, lPosStart); + log(Severity.ERROR, EXPECTED_DIGITS, span(lPosStart), digitCount, i); return -n; } @@ -2136,7 +2145,7 @@ protected Token eatDate(long lInitPos, boolean fContinued) { String sDate = extractSource(lLitPos, lEndPos); if (nYear >= 0 && nMonth >= 0 && nDay >= 0) { if (nYear < 1582 || nMonth < 1 || nMonth > 12 || nDay < 1 || nDay > 31) { - log(Severity.ERROR, BAD_DATE, new Object[] {sDate}, lLitPos, lEndPos); + log(Severity.ERROR, BAD_DATE, span(lLitPos, lEndPos), sDate); } } @@ -2196,7 +2205,7 @@ protected Token eatTimeOfDay(long lInitPos, boolean fContinued) { String sTime = extractSource(lStart, lEnd); if (nHour >= 0 && nMin >= 0 && nSec >= 0) { if (nHour > 23 || nMin > 59 || nSec > 59 && !(nHour == 23 && nMin == 59 && nSec == 60)) { - log(Severity.ERROR, BAD_TIME_OF_DAY, new Object[] {sTime}, lStart, lEnd); + log(Severity.ERROR, BAD_TIME_OF_DAY, span(lStart, lEnd), sTime); } } @@ -2343,7 +2352,7 @@ protected Token eatDuration(long lInitPos) { String sDuration = extractSource(lStart, lEnd).toUpperCase(); if (fErr) { - log(Severity.ERROR, BAD_DURATION, new Object[] {sDuration}, lStart, lEnd); + log(Severity.ERROR, BAD_DURATION, span(lStart, lEnd), sDuration); } else { peekNotIdentifierOrNumber(); } @@ -2447,9 +2456,8 @@ protected boolean expect(char ch) { } if (chActual != ch) { - log(Severity.ERROR, EXPECTED_CHAR, - new Object[] {String.valueOf(ch), String.valueOf(chActual)}, - m_source.getPosition()); + log(Severity.ERROR, EXPECTED_CHAR, span(m_source.getPosition()), + String.valueOf(ch), String.valueOf(chActual)); return false; } @@ -2477,7 +2485,7 @@ protected void peekNotIdentifierOrNumber() { long lEnd = m_source.getPosition(); m_source.rewind(); if (ch >= '0' && ch <= '9' || isIdentifierPart(ch)) { - log(Severity.ERROR, UNEXPECTED_CHAR, new Object[] {ch}, m_source.getPosition(), lEnd); + log(Severity.ERROR, UNEXPECTED_CHAR, span(m_source.getPosition(), lEnd), ch); } } @@ -2507,7 +2515,7 @@ protected void log(Severity severity, String sCode, long lPosPrev, long lPosStar m_source.next(); } - log(severity, sCode, new Object[]{extractSource(lPosPrev, lPosEnd)}, lPosStart, lPosEnd); + log(severity, sCode, span(lPosStart, lPosEnd), extractSource(lPosPrev, lPosEnd)); } /** @@ -2521,11 +2529,42 @@ protected void log(Severity severity, String sCode, Object[] aoParam, long lPosS * Log an error. */ protected void log(Severity severity, String sCode, Object[] aoParam, long lPosStart, long lPosEnd) { - if (m_errorListener.log(severity, sCode, aoParam, m_source, lPosStart, lPosEnd)) { + log(severity, sCode, span(lPosStart, lPosEnd), aoParam); + } + + /** + * Log an error, with the message parameters as a trailing varargs. + * + * The location is a {@link ErrorListener.Site} rather than a pair of positions so that the + * parameters can be the tail; taking them as an array in the middle is what forces a call site + * to build one by hand. + */ + protected void log(Severity severity, String sCode, ErrorListener.Site site, Object... aoParam) { + m_errorListener.log(severity, sCode, site, aoParam); + if (m_errorListener.isAbortDesired()) { throw new CompilerException("error list is full: " + m_errorListener); } } + /** + * @param lPosStart where the diagnostic starts + * + * @return the span from there to the current position in the script + */ + protected ErrorListener.Site span(long lPosStart) { + return in(m_source, lPosStart, m_source.getPosition()); + } + + /** + * @param lPosStart where the diagnostic starts + * @param lPosEnd where it ends + * + * @return that span of the script + */ + protected ErrorListener.Site span(long lPosStart, long lPosEnd) { + return in(m_source, lPosStart, lPosEnd); + } + // ----- helper methods ------------------------------------------------------------------------ /** diff --git a/javatools/src/main/java/org/xvm/compiler/Parser.java b/javatools/src/main/java/org/xvm/compiler/Parser.java index 20fdd3b0e89..b9fe2e06d16 100644 --- a/javatools/src/main/java/org/xvm/compiler/Parser.java +++ b/javatools/src/main/java/org/xvm/compiler/Parser.java @@ -1,5 +1,7 @@ package org.xvm.compiler; +import static java.util.Objects.requireNonNull; + import java.io.File; import java.io.IOException; @@ -9,15 +11,27 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + import org.xvm.asm.ErrorList; import org.xvm.asm.ErrorListener; +import org.xvm.asm.Reporting; import org.xvm.asm.Version; import org.xvm.compiler.Token.Id; import org.xvm.compiler.ast.*; +import org.xvm.compiler.ast.partial.IncompleteDeclarationStatement; +import org.xvm.compiler.ast.partial.IncompleteExpression; +import org.xvm.compiler.ast.partial.IncompleteLocalDeclaration; +import org.xvm.compiler.ast.partial.IncompleteStatement; +import org.xvm.compiler.ast.partial.IncompleteTypeCompositionStatement; +import org.xvm.compiler.ast.partial.PartialSyntax; import org.xvm.tool.ResourceDir; @@ -25,6 +39,11 @@ import org.xvm.util.ListMap; import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.in; + +import static org.xvm.asm.ErrorListener.Silence.DISCARD; +import static org.xvm.asm.ErrorListener.silent; + /** * A recursive descent parser for Ecstasy source code. */ @@ -38,7 +57,42 @@ public class Parser { * @param listener the error listener */ public Parser(Source source, ErrorListener listener) { - this(source, listener, null); + this(source, listener, null, false, NO_CURSOR); + } + + /** + * Create a parser that can retain a trailing incomplete expression statement for explicit + * partial analysis. Errors are still reported; this does not make the source compilable. + * + * @param source the source snapshot to parse + * @param listener the diagnostic listener and abort policy for this parse + * + * @return a parser that retains only the explicitly supported incomplete syntax boundaries + */ + public static Parser forPartialAnalysis(Source source, ErrorListener listener) { + return new Parser(source, listener, null, true, NO_CURSOR); + } + + /** + * Retain a supported incomplete statement at a cursor, without truncating the source. + * The cursor uses a position token obtained from {@link Source#getPosition()} for this text. + * Supports missing member/call suffixes, typed member prefixes and calls before an existing + * closing parenthesis at statement and final call-argument boundaries. Cursor-selected syntax + * need not be malformed; it is inspected as a prefix, without validating the selected operation. + * Missing call/group/index closers around that operation retain their enclosing syntax at a + * statement or outer-delimiter boundary; a cursor at EOF also retains unfinished blocks. + * + * @param source the source snapshot to parse + * @param cursor the selected position in the owning Source snapshot + * @param listener the diagnostic listener and abort policy for this parse + * + * @return a parser that retains only the explicitly supported incomplete syntax boundaries + */ + public static Parser forPartialAnalysis(Source source, long cursor, ErrorListener listener) { + if (cursor == NO_CURSOR) { + throw new IllegalArgumentException("A source cursor position is required"); + } + return new Parser(source, listener, null, true, cursor); } /** @@ -48,30 +102,34 @@ public Parser(Source source, ErrorListener listener) { * @param atoken the tokens to parse */ protected Parser(Parser parent, Token[] atoken) { - this(parent.m_source, parent.m_errorListener, parent.m_lexer.createLexer(atoken)); + this(parent.m_source, parent.f_errs.get(), parent.m_lexer.createLexer(atoken), false, + NO_CURSOR); } - private Parser(Source source, ErrorListener errs, Lexer lexer) { + private Parser(Source source, ErrorListener errs, Lexer lexer, boolean fPartialAnalysis, + long cursor) { if (source == null) { throw new IllegalArgumentException("Source required"); } - if (errs == null) { - throw new IllegalArgumentException("ErrorListener required"); - } - - m_source = source; - m_errorListener = errs; + m_source = source; + f_errs = new Reporting(requireNonNull(errs, "errs")); + // Lexical reports and cancellation follow the active speculative attempt. + ErrorListener lexicalReports = new ErrorListener() { + @Override + public void log(ErrorInfo error) { + f_errs.get().log(error); + } - // we need to route lexical reports through the active lookahead attempt, if any - ErrorListener listenerWrapper = err -> { - if (m_lookAhead == null) { - return m_errorListener.log(err); + @Override + public boolean isAbortDesired() { + return f_errs.get().isAbortDesired(); } - m_lookAhead.log(err); - return false; }; - m_lexer = lexer == null ? new Lexer(source, listenerWrapper) : lexer; + m_lexer = lexer == null ? new Lexer(source, lexicalReports) : lexer; + + f_partialAnalysis = fPartialAnalysis; + f_cursor = cursor; // prime the token stream next(); @@ -87,7 +145,10 @@ private Parser(Source source, ErrorListener errs, Lexer lexer) { * AliasStatements-opt TypeDeclaration * } * - * @return the top level type declaration + *

Recover at statement/declaration boundaries when possible. Recovered syntax is incomplete; + * callers must check their listener for errors before entering semantic compilation. + * + * @return the top level syntax, possibly omitting malformed statements or declarations * * @throws CompilerException if a parsing error occurs while parsing the * source code that forces the parser to abandon its progress before @@ -100,9 +161,13 @@ public StatementBlock parseSource() { // during parsing m_fDone = true; + long lStart = peek().getStartPosition(); List stmts = parseTypeCompositionComponents(null, new ArrayList<>(), true); - m_root = new StatementBlock(stmts, m_source, stmts.getFirst().getStartPosition(), - stmts.getLast().getEndPosition()); + if (stmts.isEmpty() && !f_errs.get().hasSeriousErrors()) { + log(Severity.ERROR, NO_TYPE_FOUND, lStart, m_source.getPosition()); + } + m_root = new StatementBlock(stmts, m_source, lStart, + stmts.isEmpty() ? m_source.getPosition() : stmts.getLast().getEndPosition()); // there shouldn't be more in the file; (note that a zero-length token doesn't count, // since it is probably the synthetic closing '}') @@ -139,47 +204,58 @@ public Map parseImplicits() { } /** - * Quick-scan the file for the module name. + * Quick-scan the source for the name of the module it declares, ignoring everything else. * - * @return the module name + *

Parser diagnostics from the scan use a {@link ErrorListener.Silence#DISCARD} silence. + * The module name itself is parsed against a separate listener so a malformed name is not + * accepted. Lexical diagnostics + * still use the lexer's original listener; callers performing only this scan should supply + * a discard listener when constructing the parser. + * + * @return the module's dotted name, or null if the source declares no module - or declares + * one whose name does not parse */ public String parseModuleNameIgnoreEverythingElse() { - ErrorListener errsPrev = m_errorListener; - try { - m_errorListener = ErrorListener.BLACKHOLE; - - Loop: while (!eof()) { - if (match(Id.MODULE) != null) { - if (!eof()) { - m_errorListener = new ErrorList(1); - List tokens = parseQualifiedName(); - if (!m_errorListener.hasSeriousErrors()) { - StringBuilder sb = new StringBuilder(); - for (int i = 0, c = tokens.size(); i < c; ++i) { - if (i > 0) { - sb.append('.'); - } - sb.append(tokens.get(i).getValueText()); - } - return sb.toString(); - } + try (Reporting.Scope quiet = reportingTo(silent(DISCARD))) { + while (!eof()) { + if (match(Id.MODULE) == null) { + // not a module declaration; skip it, and give up at the first body we meet, + // because a module declaration cannot follow one + Id id = current().getId(); + if (id == Id.L_CURLY || id == Id.R_CURLY) { + return null; } - } else { - switch (current().getId()) { - case L_CURLY: - case R_CURLY: - break Loop; + } else if (!eof()) { + String sName = parseModuleName(); + if (sName != null) { + return sName; } } } - } catch (RuntimeException ignore) { - } finally { - m_errorListener = errsPrev; + } catch (RuntimeException _) { + // a quick scan answers or gives up without propagating parsing failures } return null; } + /** + * Parse the qualified name of a module that has just been announced by its keyword. + * + * @return the dotted name, or null if it did not parse cleanly + */ + private String parseModuleName() { + ErrorList errs = new ErrorList(ErrorList.FIRST_ERROR); + List tokens; + try (Reporting.Scope reporting = reportingTo(errs)) { + tokens = parseQualifiedName(); + } + + return errs.hasSeriousErrors() + ? null + : tokens.stream().map(Token::getValueText).collect(Collectors.joining(".")); + } + /** * As part of the runtime (NOT compile-time), parse the name of the class that is in the source. * @@ -351,14 +427,20 @@ TypeCompositionStatement parseTypeDeclarationStatementAfterModifiers(long lStart name = qualified.get(0); } - // optional type parameters - List typeParams = parseTypeParameterList(false); - - // optional constructor definition - List constructorParams = parseParameterList(false); - - // sequence of compositions - List compositions = parseCompositions(); + Mark header = mark(); + List typeParams = null; + List constructorParams; + List compositions; + try { + typeParams = parseTypeParameterList(false, true); + constructorParams = parseParameterList(false); + compositions = parseCompositions(true); + } catch (IncompleteHeader error) { + return retainTypeDeclaration(header, lStartPos, category, name, qualified, + List.of(error.site), error.withFormals(typeParams)); + } catch (CompilerException error) { + return retainTypeDeclaration(header, lStartPos, category, name, qualified, List.of(), error); + } // TypeCompositionBody StatementBlock body = null; @@ -381,7 +463,7 @@ TypeCompositionStatement parseTypeDeclarationStatementAfterModifiers(long lStart * * @return a list of compositions */ - List parseCompositions() { + List parseCompositions(boolean header) { List compositions = new ArrayList<>(); while (true) { // the keywords below require "match()" to extract them, because they are context @@ -389,18 +471,19 @@ List parseCompositions() { Token keyword; if ((keyword = match(Id.EXTENDS)) != null) { do { - TypeExpression type = parseExtendedTypeExpression(); + TypeExpression type = header ? parseDeclarationType(true) : parseExtendedTypeExpression(); List args = parseArgumentList(false, false, false); compositions.add(new CompositionNode.Extends(null, keyword, type, args, prev().getEndPosition())); } while (match(Id.COMMA) != null); } else if ((keyword = match(Id.IMPLEMENTS)) != null) { do { - compositions.add(new CompositionNode.Implements(null, keyword, parseExtendedTypeExpression())); + compositions.add(new CompositionNode.Implements(null, keyword, + header ? parseDeclarationType(true) : parseExtendedTypeExpression())); } while (match(Id.COMMA) != null); } else if ((keyword = match(Id.DELEGATES)) != null) { do { - TypeExpression type = parseExtendedTypeExpression(); + TypeExpression type = header ? parseDeclarationType(true) : parseExtendedTypeExpression(); expect(Id.L_PAREN); Expression expr = parseExpression(); Token tokEnd = expect(Id.R_PAREN); @@ -416,7 +499,7 @@ List parseCompositions() { TypeExpression type = null; List constraints = null; if (match(Id.CONDITIONAL) == null) { - type = parseTypeExpression(); + type = header ? parseDeclarationType() : parseTypeExpression(); } else { do { // parse the type parameter list e.g. "", @@ -450,7 +533,8 @@ List parseCompositions() { compositions.add(new CompositionNode.Incorporates(null, keyword, type, args, constraints)); } while (match(Id.COMMA) != null); } else if ((keyword = match(Id.INTO)) != null) { - compositions.add(new CompositionNode.Into(null, keyword, parseExtendedTypeExpression())); + compositions.add(new CompositionNode.Into(null, keyword, + header ? parseDeclarationType(true) : parseExtendedTypeExpression())); } else { // not context-sensitive keywords switch (peek().getId()) { case IMPORT: { @@ -582,58 +666,62 @@ StatementBlock parseTypeCompositionBody(Token category) { List parseTypeCompositionComponents(Expression exprCondition, List stmts, boolean fFileLevel) { boolean fFoundType = false; - while (match(Id.R_CURLY) == null) { - Statement stmt; - switch (peek().getId()) { - case IMPORT: - stmt = parseImportStatement(exprCondition); - break; + while (!atBlockEnd(fFileLevel)) { + Mark statementStart = mark(); + try { + Statement stmt; + switch (peek().getId()) { + case IMPORT: + stmt = parseImportStatement(exprCondition); + break; - case TYPEDEF: - stmt = parseTypeDefStatement(exprCondition, null); - break; + case TYPEDEF: + stmt = parseTypeDefStatement(exprCondition, null); + break; - default: { - Token start = peek(); - - stmt = parseTypeCompositionComponent(exprCondition, false); - fFoundType = true; - - if (fFileLevel) { - // module cannot have any statements before it in the source - if (stmt instanceof TypeCompositionStatement stmtType) { - if (stmtType.getCategory().getId() == Id.MODULE && !stmts.isEmpty()) { - if (stmts.stream().allMatch(ImportStatement.class::isInstance)) { - // transfer the imports from outside the module body to inside - // the module body - stmtType.ensureBody().getStatements().addAll(0, stmts); - stmts.clear(); - } else { - log(Severity.ERROR, MODULE_NOT_ROOT, start.getStartPosition(), - start.getEndPosition()); + default: { + Token start = peek(); + + stmt = parseTypeCompositionComponent(exprCondition, false); + fFoundType = true; + + if (fFileLevel) { + // module cannot have any statements before it in the source + if (stmt instanceof TypeCompositionStatement stmtType) { + if (stmtType.getCategory().getId() == Id.MODULE && !stmts.isEmpty()) { + if (stmts.stream().allMatch(ImportStatement.class::isInstance)) { + // transfer the imports from outside the module body to inside + // the module body + stmtType.ensureBody().getStatements().addAll(0, stmts); + stmts.clear(); + } else { + log(Severity.ERROR, MODULE_NOT_ROOT, start.getStartPosition(), + start.getEndPosition()); + } } + } else { + log(Severity.ERROR, NO_TYPE_FOUND, start.getStartPosition(), + start.getEndPosition()); } - } else { - log(Severity.ERROR, NO_TYPE_FOUND, start.getStartPosition(), - start.getEndPosition()); } + break; + } } - break; - } - } - stmts.add(stmt); - if (stmt instanceof MethodDeclarationStatement) { - MethodDeclarationStatement stmtFinally = - ((MethodDeclarationStatement) stmt).getConstructorFinally(); - if (stmtFinally != null) { - stmts.add(stmtFinally); + stmts.add(stmt); + if (stmt instanceof MethodDeclarationStatement method) { + MethodDeclarationStatement stmtFinally = method.getConstructorFinally(); + if (stmtFinally != null) { + stmts.add(stmtFinally); + } } - } - if (fFileLevel && fFoundType) { - // at the file level, there is nothing after the outermost type's conclusion - break; + if (fFileLevel && fFoundType) { + // at the file level, there is nothing after the outermost type's conclusion + break; + } + } catch (CompilerException e) { + recoverStatement(statementStart, e); } } @@ -719,7 +807,7 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet // evaluate annotations if (annotations != null) { for (AnnotationExpression annotation : annotations) { - annotation.log(m_errorListener, Severity.ERROR, Compiler.ANNOTATION_UNEXPECTED); + annotation.log(f_errs.get(), Severity.ERROR, Compiler.ANNOTATION_UNEXPECTED); } } @@ -737,7 +825,8 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet } // fall through default: - modifier.log(m_errorListener, m_source, Severity.ERROR, Compiler.KEYWORD_UNEXPECTED, modifier.getValueText()); + modifier.log(f_errs.get(), m_source, Severity.ERROR, Compiler.KEYWORD_UNEXPECTED, + modifier.getValueText()); break; } } @@ -749,12 +838,19 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet case CONDITIONAL: case VOID: { // it's definitely a method or a function - List typeVars = peek(Id.COMP_LT) ? parseTypeParameterList(true) : null; - Token conditional = match(Id.CONDITIONAL); - List returns = parseReturnList(); - Token name = expect(Id.IDENTIFIER); - return parseMethodDeclarationAfterName(lStartPos, exprCondition, doc, - modifiers, annotations, typeVars, conditional, returns, name); + Mark header = mark(); + List typeVars = null; + try { + typeVars = peek(Id.COMP_LT) ? parseTypeParameterList(true, true) : null; + Token conditional = match(Id.CONDITIONAL); + List returns = parseDeclarationReturns(); + Token name = expect(Id.IDENTIFIER); + return parseMethodDeclarationAfterName(lStartPos, exprCondition, doc, + modifiers, annotations, typeVars, conditional, returns, name); + } catch (IncompleteHeader error) { + return retainDeclaration(header, lStartPos, null, IncompleteDeclarationStatement.Kind.METHOD, + List.of(error.site), error.withFormals(typeVars)); + } } case CONSTRUCT: { @@ -779,6 +875,22 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet } case L_PAREN: { + if (!fInMethod && f_cursor != NO_CURSOR) { + // Outside a method this is declaration syntax, not a tuple value. Probe the + // type grammar without letting a discarded interpretation publish diagnostics. + Mark header = mark(); + var branch = f_errs.get().branch(null); + try (var ignored = reportingTo(branch)) { + parseDeclarationReturns(); + } catch (IncompleteHeader error) { + branch.merge(); + return retainDeclaration(header, lStartPos, null, IncompleteDeclarationStatement.Kind.METHOD, + List.of(error.site), error); + } catch (CompilerException ignored) { + // The ordinary declaration grammar below owns non-cursor failures. + } + restore(header); + } // it's a property or a method, but the property type is parenthesized (odd!) or // the method return types (with optional names) are parenthesized (multi-return), // which means that to know it's a method, we have to find one of: @@ -847,7 +959,7 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet if (expr.isLValueSyntax()) { listLVals.add(expr); } else { - expr.log(m_errorListener, Severity.ERROR, NOT_ASSIGNABLE); + expr.log(f_errs.get(), Severity.ERROR, NOT_ASSIGNABLE); } } else { listLVals.add(new VariableDeclarationStatement( @@ -911,6 +1023,14 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet throw new CompilerException("var or val keyword outside of method"); } + var missingName = declarationNameSlot(new VariableTypeExpression(tokType)); + if (missingName.isPresent() && peek(Id.ASN)) { + current(); + Expression initializer = parseExpression(); + expect(Id.SEMICOLON); + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + return new IncompleteLocalDeclaration(missingName.orElseThrow(), initializer, prev().getEndPosition()); + } Token tokName = matchNameOrAny(); if (tokName == null) { // var and val are not reserved keywords; they are context sensitive types @@ -918,8 +1038,10 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet } else { VariableTypeExpression typeDecl = new VariableTypeExpression(tokType); VariableDeclarationStatement stmtDecl = new VariableDeclarationStatement(typeDecl, tokName, false); - AssignmentStatement stmtAsn = new AssignmentStatement(stmtDecl, match(Id.ASN), parseExpression()); - expect(Id.SEMICOLON); + Token op = match(Id.ASN); + Expression value = parsePartialValue(); + AssignmentStatement stmtAsn = new AssignmentStatement(stmtDecl, op, value); + finishValueStatement(value); return stmtAsn; } } @@ -934,7 +1056,15 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet type = expr.toTypeExpression(); } else { - type = parseTypeExpression(); + Mark header = mark(); + try { + type = parseDeclarationType(); + } catch (IncompleteHeader e) { + Token name = matchNameOrAny(); + var kind = peek(Id.L_PAREN) ? IncompleteDeclarationStatement.Kind.METHOD + : IncompleteDeclarationStatement.Kind.PROPERTY; + return retainDeclaration(header, lStartPos, name, kind, List.of(e.site), e); + } } if (type instanceof BadTypeExpression) { @@ -942,6 +1072,14 @@ Statement parseTypeCompositionComponent(Expression exprCondition, boolean fInMet } // it's a constant, property, or method + if (!fInMethod) { + var missingName = declarationNameSlot(type); + if (missingName.isPresent()) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + return retainDeclaration(mark(), lStartPos, null, IncompleteDeclarationStatement.Kind.PROPERTY, + List.of(missingName.orElseThrow()), new CompilerException("Missing declaration name")); + } + } Token name = expectNameOrAny(); if (peek(Id.COMP_LT) || peek(Id.L_PAREN)) { // '<' indicates redundant return type list @@ -992,15 +1130,25 @@ Statement parsePossibleExpressionOrAssignmentStatement(Expression expr) { case COND_OR_ASN: case COND_NN_ASN: case COND_ELSE_ASN: { - AssignmentStatement stmt = new AssignmentStatement(expr, current(), parseExpression()); - expect(Id.SEMICOLON); + Token op = current(); + Expression value = op.getId() == Id.ASN ? parsePartialValue() : parseExpression(); + AssignmentStatement stmt = new AssignmentStatement(expr, op, value); + finishValueStatement(value); return stmt; } } if (isMissingStatementSemicolon()) { long lPos = expr.getEndPosition(); - log(Severity.ERROR, MISSING_SEMICOLON, lPos, lPos); + if (!containsCursorHole(expr)) { + log(Severity.ERROR, MISSING_SEMICOLON, lPos, lPos); + } + return new ExpressionStatement(expr); + } + + // what is being typed at the end of a line ends its statement there; the later line is the next one + if (containsCursorHole(expr) && !eof() + && Source.calculateLine(peek().getStartPosition()) > Source.calculateLine(expr.getEndPosition())) { return new ExpressionStatement(expr); } @@ -1054,9 +1202,9 @@ Statement parseVariableDeclarationAfterName(List annotatio Expression value = null; Token op = match(Id.ASN); if (op != null) { - value = parseExpression(); + value = parsePartialValue(); } - expect(Id.SEMICOLON); + finishValueStatement(value); // apply any annotations to the variable type; "@A @B @C T" is "@A of (@B of (@C of T))" if (annotations != null) { @@ -1082,15 +1230,27 @@ Statement parseVariableDeclarationAfterName(List annotatio * StatementBlock * } * - * @return a MethodDeclarationStatement + * @return a method declaration, or retained incomplete header syntax after a parse error */ - MethodDeclarationStatement parseMethodDeclarationAfterName(long lStartPos, Expression exprCondition, + Statement parseMethodDeclarationAfterName(long lStartPos, Expression exprCondition, Token doc, List modifiers, List annotations, List typeVars, Token conditional, List returns, Token name) { - List redundantReturns = parseTypeParameterTypeList(false, true); - List params = parseParameterList(true); - long lEndPos = prev().getEndPosition(); - StatementBlock body; + Mark header = mark(); + List redundantReturns; + List params; + try { + redundantReturns = parseTypeParameterTypeList(false, true); + params = parseParameterList(true); + } catch (IncompleteHeader e) { + // This incomplete method has no registered type parameters or own scope yet. + return retainDeclaration(header, lStartPos, name, IncompleteDeclarationStatement.Kind.METHOD, + List.of(e.site), e.withFormals(typeVars)); + } catch (CompilerException e) { + return retainDeclaration(header, lStartPos, name, IncompleteDeclarationStatement.Kind.METHOD, + List.of(), e); + } + long lEndPos = prev().getEndPosition(); + StatementBlock body; switch (peek().getId()) { case SEMICOLON: body = null; @@ -1099,10 +1259,10 @@ MethodDeclarationStatement parseMethodDeclarationAfterName(long lStartPos, Expre case ASN: Token eq = expect(Id.ASN); Expression expr = parseExpression(); - Token semi = expect(Id.SEMICOLON); + finishValueStatement(expr); ReturnStatement stmt = new ReturnStatement(eq, expr); body = new StatementBlock(Arrays.asList(stmt), stmt.getStartPosition(), - semi.getEndPosition()); + prev().getEndPosition()); break; default: body = parseStatementBlock(); @@ -1179,7 +1339,7 @@ PropertyDeclarationStatement parsePropertyDeclarationFinish(long lStartPos, // "=" Expression ";" value = parseExpression(); lEndPos = value.getEndPosition(); - expect(Id.SEMICOLON); + finishValueStatement(value); } else if (body == null || fNeedsSemi) { expect(Id.SEMICOLON); } @@ -1196,13 +1356,47 @@ PropertyDeclarationStatement parsePropertyDeclarationFinish(long lStartPos, StatementBlock parseStatementBlock() { Token tokStart = expect(Id.L_CURLY); List stmts = new ArrayList<>(); - while (match(Id.R_CURLY) == null) { - stmts.add(parseStatement()); + retainEmptyCursor(stmts); + while (!atBlockEnd(false)) { + Mark statementStart = mark(); + long lStart = peek().getStartPosition(); + try { + stmts.add(parseStatement()); + retainEmptyCursor(stmts); + } catch (IncompleteSyntax e) { + // Do not reinterpret a return, assignment, condition or nested call as a + // standalone expression. Only the intact statement prefix is supported. + if (e.statement.getStartPosition() == lStart) { + stmts.add(e.statement); + // The cursor boundary has already consumed the intact prefix. Keep the + // enclosing brace and following declarations instead of scanning the open + // call again as an unmatched parenthesis. + if (f_cursor != NO_CURSOR) { + match(Id.SEMICOLON); + continue; + } + } + recoverStatement(statementStart, e); + } catch (CompilerException e) { + recoverStatement(statementStart, e); + } } return new StatementBlock(stmts, tokStart.getStartPosition(), prev().getEndPosition()); } + /** A zero-width insertion anchor after an intact statement or opening brace. */ + private void retainEmptyCursor(List statements) { + if (f_cursor != NO_CURSOR && (prev().getId() == Id.L_CURLY + || prev().getId() == Id.R_CURLY || prev().getId() == Id.SEMICOLON) + && endsAt(f_cursor)) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + statements.add(new IncompleteStatement( + new Token(f_cursor, f_cursor, Id.IDENTIFIER, ""), f_cursor, INCOMPLETE_EXPRESSION)); + match(Id.SEMICOLON); + } + } + /** * Parse an Ecstasy statement. * @@ -1706,16 +1900,16 @@ AstNode parseCondition(boolean fNegated) { // test for a negated conditional assignment Token tokNot = null; AssignmentStatement stmtAsn = null; - try (SafeLookAhead attempt = new SafeLookAhead()) { + try (Attempt attempt = attempt()) { tokNot = expect(Id.NOT); if (match(Id.L_PAREN) != null) { AstNode stmtPeek = parseCondition(true); if (attempt.isClean() && stmtPeek instanceof AssignmentStatement) { stmtAsn = (AssignmentStatement) stmtPeek; - attempt.keepResults(); + attempt.keep(); } } - } catch (CompilerException ignore) {} + } catch (CompilerException _) {} if (stmtAsn != null) { stmtAsn.negate(tokNot, expect(Id.R_PAREN)); @@ -1839,8 +2033,18 @@ ReturnStatement parseReturnStatement() { // note: it is possible that the expression list is parenthesized, in which case it will be // parsed as a single expression (a tuple literal), and the compiler will have to // it out later - List exprs = parseExpressionList(); - expect(Id.SEMICOLON); + long start = peek().getStartPosition(); + List exprs; + try { + exprs = parseExpressionList(); + } catch (IncompleteSyntax e) { + exprs = List.of(retainPartialValue(start, e)); + } + if (exprs.size() == 1) { + finishValueStatement(exprs.getFirst()); + } else { + expect(Id.SEMICOLON); + } return new ReturnStatement(keyword, exprs); } @@ -2452,7 +2656,7 @@ AssignmentStatement parseVariableInitializer(boolean fAllowExpr) { */ Expression parseLinkerCondition() { Expression expr = parseExpression(); - expr.validateCondition(m_errorListener); + expr.validateCondition(f_errs.get()); return expr; } @@ -3039,6 +3243,18 @@ private Expression parsePrefixExpressionBody(boolean fExtended) { * @return an expression */ Expression parsePostfixExpression(boolean fExtended) { + try { + return parsePostfixValue(fExtended); + } catch (IncompleteSyntax e) { + if (f_cursor == NO_CURSOR) { + throw e; + } + return new IncompleteExpression(e.statement); + } + } + + /** Retain a cursor hole as syntax so enclosing operators establish their real contexts. */ + private Expression parsePostfixValue(boolean fExtended) { Expression expr = parsePrimaryExpression(fExtended); while (true) { switch (peek().getId()) { @@ -3060,7 +3276,10 @@ Expression parsePostfixExpression(boolean fExtended) { break; case DOT: { - expect(Id.DOT); + Token dot = expect(Id.DOT); + if (endsAt(f_cursor)) { + throw incomplete(expr, dot, List.of(), List.of()); + } switch (peek().getId()) { case NEW: { expr = parseNewExpression(expr); @@ -3090,21 +3309,33 @@ Expression parsePostfixExpression(boolean fExtended) { name = expect(Id.IDENTIFIER); } } + // a cursor at the name's start, right after the dot, completes the member too, with + // nothing of it typed + if (noDeRef == null && name.getId() == Id.IDENTIFIER + && name.getStartPosition() <= f_cursor && f_cursor <= name.getEndPosition() + && endsAt(name.getEndPosition(), true)) { + var hole = incomplete(expr, dot, name); + if (peek(Id.L_PAREN) || peek(Id.ASYNC_PAREN)) { + expr = new IncompleteExpression(hole.statement); + break; + } + throw hole; + } long lEndPos = name.getEndPosition(); List params = null; Token tokPeekLT = match(Id.COMP_LT); if (tokPeekLT != null) { putBack(tokPeekLT); - try (SafeLookAhead attempt = new SafeLookAhead()) { + try (Attempt attempt = attempt()) { params = parseTypeParameterTypeList(true, true); if (attempt.isClean()) { - attempt.keepResults(); + attempt.keep(); lEndPos = prev().getEndPosition(); } else { params = null; } - } catch (CompilerException ignore) {} + } catch (CompilerException _) {} } if (expr instanceof NamedTypeExpression) { @@ -3128,8 +3359,7 @@ Expression parsePostfixExpression(boolean fExtended) { case L_PAREN: case ASYNC_PAREN: // ArgumentList - expr = new InvocationExpression(expr, peek(Id.ASYNC_PAREN), - parseArgumentList(true, true, false), prev().getEndPosition()); + expr = parseInvocation(expr); break; case L_SQUARE: { @@ -3150,7 +3380,8 @@ Expression parsePostfixExpression(boolean fExtended) { } else if (match(Id.COND) == null) { // "someArray[3]" List indexes = parseExpressionList(); - expr = new ArrayAccessExpression(expr, indexes, expect(Id.R_SQUARE)); + expr = new ArrayAccessExpression(expr, indexes, + expectPartialClose(Id.R_SQUARE, indexes)); } else { // "SomeClass[?,?]" int cExplicitDims = 1; @@ -3191,7 +3422,10 @@ NewExpression parseNewExpression(Expression left) { // argument list for a "virtual new"; assume it's a virtual new, and we'll back // up if we were wrong Mark mark = mark(); - args = parseArgumentList(true, false, false); + args = f_partialAnalysis + ? parsePartialArguments(new NewExpression(left, keyword, null, List.of(), + -1, null, keyword.getEndPosition()), false) + : parseArgumentList(true, false, false); lEndPos = prev().getEndPosition(); Token.Id idNext = peek().getId(); @@ -3212,7 +3446,19 @@ NewExpression parseNewExpression(Expression left) { boolean fArray = type instanceof ArrayTypeExpression; if (fArray) { if (peek(Id.L_SQUARE)) { - args = parseArgumentList(true, false, true); + try { + args = f_partialAnalysis + ? parsePartialArguments(new NewExpression(left, keyword, type, List.of(), + ((ArrayTypeExpression) type).getDimensions(), null, lEndPos), false) + : parseArgumentList(true, false, true); + } catch (IncompleteSyntax failure) { + // Finish the construction's suffix so a written supplier does not hide + // the dimension cursor. This prefix query does not validate the supplier. + if (prev().getId() == Id.R_SQUARE) { + parseArgumentList(false, false, false); + } + throw failure; + } dims = args.size(); lEndPos = prev().getEndPosition(); } else { @@ -3220,7 +3466,10 @@ NewExpression parseNewExpression(Expression left) { } // parenthesized arguments after the dims - List argsTrailing = parseArgumentList(false, false, false); + List argsTrailing = f_partialAnalysis && peek(Id.L_PAREN) + ? parsePartialArguments(new NewExpression(left, keyword, type, args, + dims, null, lEndPos), false) + : parseArgumentList(false, false, false); if (argsTrailing != null) { if (!argsTrailing.isEmpty()) { if (args.isEmpty()) { @@ -3232,7 +3481,22 @@ NewExpression parseNewExpression(Expression left) { lEndPos = prev().getEndPosition(); } } else { - args = parseArgumentList(true, false, false); + try { + args = f_partialAnalysis + ? parsePartialArguments(new NewExpression(left, keyword, type, List.of(), + -1, null, type.getEndPosition()), false) + : parseArgumentList(true, false, false); + } catch (IncompleteSyntax failure) { + if (left == null && peek(Id.L_CURLY)) { + // Preserve the anonymous body as construction syntax. A cursor probe + // must not mistake it for ordinary construction followed by a block. + body = parseTypeCompositionBody(keyword); + var site = failure.statement; + site.replaceChild(site.getTarget(), new NewExpression(left, keyword, type, + List.of(), -1, body, prev().getEndPosition())); + } + throw failure; + } lEndPos = prev().getEndPosition(); } @@ -3285,6 +3549,11 @@ NewExpression parseNewExpression(Expression left) { * @return an expression */ Expression parsePrimaryExpression(boolean fExtended) { + if (f_cursor != NO_CURSOR && endsAt(f_cursor)) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + return new IncompleteExpression(new IncompleteStatement( + new Token(f_cursor, f_cursor, Id.IDENTIFIER, ""), f_cursor, INCOMPLETE_EXPRESSION)); + } switch (peek().getId()) { case ANY: { IgnoredNameExpression exprIgnore = new IgnoredNameExpression(current()); @@ -3406,6 +3675,10 @@ Expression parsePrimaryExpression(boolean fExtended) { boolean fQuit = false; Token dot; while (nameNDR == null && (dot = match(Id.DOT)) != null) { + if (fNormal && atLineEnd(f_cursor)) { + // the name on the next line starts the next statement, not this access + throw incomplete(new NameExpression(left, nameNDR, name, null, lEndPos), dot, List.of(), List.of()); + } Token nameNext = match(Id.IDENTIFIER); if (nameNext == null) { // it's not a ".name" construct, so the name expression ended BEFORE the dot @@ -3414,7 +3687,15 @@ Expression parsePrimaryExpression(boolean fExtended) { break; } - left = new NameExpression(left, nameNDR, name, null, lEndPos); + left = new NameExpression(left, nameNDR, name, null, lEndPos); + if (fNormal && nameNext.getStartPosition() <= f_cursor && f_cursor <= nameNext.getEndPosition() + && endsAt(nameNext.getEndPosition(), true)) { + var hole = incomplete(left, dot, nameNext); + if (peek(Id.L_PAREN) || peek(Id.ASYNC_PAREN)) { + return new IncompleteExpression(hole.statement); + } + throw hole; + } nameNDR = null; // only gets applied once name = nameNext; lEndPos = name.getEndPosition(); @@ -3433,6 +3714,12 @@ Expression parsePrimaryExpression(boolean fExtended) { return expr; } + if (left == null && name.getStartPosition() < f_cursor && f_cursor <= name.getEndPosition() + && endsAt(name.getEndPosition())) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + throw new IncompleteSyntax(new IncompleteStatement(name, f_cursor, INCOMPLETE_EXPRESSION)); + } + // test for a non-auto-narrowing modifier ("!") Token tokNoNarrow = !peek().hasLeadingWhitespace() ? match(Id.NOT) @@ -3444,17 +3731,17 @@ Expression parsePrimaryExpression(boolean fExtended) { Token tokPeekLT = match(Id.COMP_LT); if (tokPeekLT != null) { putBack(tokPeekLT); - try (SafeLookAhead attempt = new SafeLookAhead()) { + try (Attempt attempt = attempt()) { params = parseTypeParameterTypeList(true, true); if (attempt.isClean() // "index>1" is a comparison and a shift, not a type && (peek().getId() != Id.COMP_GT || peek().hasLeadingWhitespace())) { - attempt.keepResults(); + attempt.keep(); lEndPos = prev().getEndPosition(); } else { params = null; } - } catch (CompilerException ignore) {} + } catch (CompilerException _) {} } // test to see if this is a tuple literal of the form "Tuple:(", or some other @@ -3530,7 +3817,10 @@ Expression parsePrimaryExpression(boolean fExtended) { // parsed to see if it's followed by a lambda operator List exprs = new ArrayList<>(); exprs.add(expr); - while (match(Id.R_PAREN, match(Id.COMMA) == null) == null) { + Token tupleClose; + while ((tupleClose = match(Id.COMMA) == null + ? expectPartialClose(Id.R_PAREN, exprs) + : matchPartialClose(Id.R_PAREN, exprs)) == null) { exprs.add(parseExpression()); } @@ -3540,7 +3830,7 @@ Expression parsePrimaryExpression(boolean fExtended) { } // it's a Tuple literal - return new TupleExpression(null, exprs, tokLParen.getStartPosition(), prev().getEndPosition()); + return new TupleExpression(null, exprs, tokLParen.getStartPosition(), tupleClose.getEndPosition()); case R_PAREN: // this is either a parenthesized expression or a single parameter for a @@ -3572,9 +3862,8 @@ Expression parsePrimaryExpression(boolean fExtended) { } default: - expect(Id.R_PAREN); - skipToNextStatement(); - return expr; + return new ParenthesizedExpression(expr, tokLParen.getStartPosition(), + expectPartialClose(Id.R_PAREN, List.of(expr)).getEndPosition()); } } @@ -3678,7 +3967,7 @@ Expression parsePrimaryExpression(boolean fExtended) { byte[] abData = null; try { abData = m_source.includeBinary(sFile); - } catch (IOException ignore) {} + } catch (IOException _) {} if (abData == null) { abData = new byte[0]; fErr = true; @@ -3689,7 +3978,7 @@ Expression parsePrimaryExpression(boolean fExtended) { try { Source source = m_source.includeString(sFile); sData = source == null ? null : source.toRawString(); - } catch (IOException ignore) {} + } catch (IOException _) {} if (sData == null) { sData = ""; fErr = true; @@ -4058,12 +4347,13 @@ Expression parseComplexLiteral(TypeExpression type) { if (peek().getId() == Id.ASN) { List keys = new ArrayList<>(); List values = new ArrayList<>(); - while (match(Id.R_SQUARE) == null) { + Token close; + while ((close = match(Id.R_SQUARE)) == null) { keys.add(keys.isEmpty() ? expr : parseExpression()); expect(Id.ASN); values.add(parseExpression()); if (match(Id.COMMA) == null) { - expect(Id.R_SQUARE); + close = expectPartialClose(Id.R_SQUARE, values); break; } } @@ -4072,12 +4362,12 @@ Expression parseComplexLiteral(TypeExpression type) { Token tokName = new Token(ofMap, ofMap, Id.IDENTIFIER, "Map"); TypeExpression exprType = new NamedTypeExpression(null, Collections.singletonList(tokName), null, null, null, ofMap); - return new MapExpression(exprType, keys, values, prev().getEndPosition()); + return new MapExpression(exprType, keys, values, close.getEndPosition()); } } - expect(Id.R_SQUARE); - break; + return new ListExpression(type, exprs, lStartPos, + expectPartialClose(Id.R_SQUARE, exprs).getEndPosition()); } } return new ListExpression(type, exprs, lStartPos, prev().getEndPosition()); @@ -4106,8 +4396,8 @@ Expression parseComplexLiteral(TypeExpression type) { expect(Id.ASN); values.add(parseExpression()); if (match(Id.COMMA) == null) { - expect(Id.R_SQUARE); - break; + return new MapExpression(type, keys, values, + expectPartialClose(Id.R_SQUARE, values).getEndPosition()); } } return new MapExpression(type, keys, values, prev().getEndPosition()); @@ -4115,17 +4405,16 @@ Expression parseComplexLiteral(TypeExpression type) { case "Tuple": { expect(Id.L_PAREN); - List exprs = null; - while (match(Id.R_PAREN) == null) { - if (exprs == null) { - exprs = new ArrayList<>(); - } else { + List exprs = new ArrayList<>(); + Token close; + while ((close = matchPartialClose(Id.R_PAREN, exprs)) == null) { + if (!exprs.isEmpty()) { expect(Id.COMMA); } exprs.add(parseExpression()); } return new TupleExpression(type, exprs, type.getStartPosition(), - prev().getEndPosition()); + close.getEndPosition()); } case "Path": @@ -4210,11 +4499,16 @@ public TypeExpression parseExtendedTypeExpression() { * @return a type expression */ TypeExpression parseIntersectingTypeExpression(boolean fExtended) { - TypeExpression expr = parseUnionedTypeExpression(fExtended); + return parseIntersectingTypeExpression(fExtended, false); + } + + /** Pass header recovery explicitly through the type grammar; other type contexts stay strict. */ + private TypeExpression parseIntersectingTypeExpression(boolean fExtended, boolean fHeader) { + TypeExpression expr = parseUnionedTypeExpression(fExtended, fHeader); Token tokOp; do { if ((tokOp = match(Id.ADD)) != null || (tokOp = match(Id.SUB)) != null) { - expr = new BiTypeExpression(expr, tokOp, parseUnionedTypeExpression(fExtended)); + expr = new BiTypeExpression(expr, tokOp, parseUnionedTypeExpression(fExtended, fHeader)); } } while (tokOp != null); return expr; @@ -4237,10 +4531,10 @@ TypeExpression parseIntersectingTypeExpression(boolean fExtended) { * * @return a type expression */ - TypeExpression parseUnionedTypeExpression(boolean fExtended) { - TypeExpression expr = parseNonBiTypeExpression(fExtended); + TypeExpression parseUnionedTypeExpression(boolean fExtended, boolean fHeader) { + TypeExpression expr = parseNonBiTypeExpression(fExtended, fHeader); while (peek(Id.BIT_OR)) { - expr = new BiTypeExpression(expr, expect(Id.BIT_OR), parseNonBiTypeExpression(fExtended)); + expr = new BiTypeExpression(expr, expect(Id.BIT_OR), parseNonBiTypeExpression(fExtended, fHeader)); } return expr; } @@ -4300,9 +4594,9 @@ TypeExpression parseUnionedTypeExpression(boolean fExtended) { * * @return a type expression */ - TypeExpression parseNonBiTypeExpression(boolean fExtended) { + TypeExpression parseNonBiTypeExpression(boolean fExtended, boolean fHeader) { try (var ignore = nest()) { - return parseNonBiTypeExpressionBody(fExtended); + return parseNonBiTypeExpressionBody(fExtended, fHeader); } } @@ -4313,22 +4607,23 @@ TypeExpression parseNonBiTypeExpression(boolean fExtended) { * * @return a type expression */ - private TypeExpression parseNonBiTypeExpressionBody(boolean fExtended) { + private TypeExpression parseNonBiTypeExpressionBody(boolean fExtended, boolean fHeader) { + retainEmptyTypeArgument(fHeader); TypeExpression type; Token tokAccess = null; switch (peek().getId()) { case L_PAREN: expect(Id.L_PAREN); - type = parseExtendedTypeExpression(); - expect(Id.R_PAREN); + type = parseIntersectingTypeExpression(true, fHeader); + expectHeaderTypeClose(Id.R_PAREN, List.of(type), fHeader); break; case AT: - type = parseAnnotatedTypeExpression(fExtended); + type = parseAnnotatedTypeExpression(fExtended, fHeader); break; case FUNCTION: - type = parseFunctionTypeExpression(); + type = parseFunctionTypeExpression(fHeader); break; case IMMUTABLE: { @@ -4348,7 +4643,7 @@ private TypeExpression parseNonBiTypeExpressionBody(boolean fExtended) { } } - type = new DecoratedTypeExpression(tokImmut, parseNonBiTypeExpression(fExtended)); + type = new DecoratedTypeExpression(tokImmut, parseNonBiTypeExpression(fExtended, fHeader)); break; } @@ -4377,54 +4672,67 @@ private TypeExpression parseNonBiTypeExpressionBody(boolean fExtended) { } tokAccess = match(Id.STRUCT); } - type = parseNamedTypeExpression(tokAccess); + type = parseNamedTypeExpression(tokAccess, fHeader); break; } while (true) { switch (peek().getId()) { - case L_SQUARE: - // this could be either: - // -> NonBiTypeExpression ArrayDims - // -> NonBiTypeExpression ArrayIndexes - // in the case of the ArrayIndexes, we do NOT consume that portion of the - // expression; we use it to give us a dimension count, as if it were ArrayDims + case L_SQUARE: { + // Sizes belong to the surrounding construction, not to its type. Look ahead + // with a branch so a cursor hole is reported only when the sizes are reparsed + // with their owning NewExpression. No diagnostic escapes a discarded lookahead. Mark mark = mark(); - - expect(Id.L_SQUARE); - int cDims = 0; - int cIndexes = 0; - while (match(Id.R_SQUARE) == null) { - if (cDims + cIndexes > 0) { - expect(Id.COMMA); - } - - Token dim = peek(); // just for error reporting - if (match(Id.COND) == null) { - parseExpression(); - if (cIndexes == 0 && cDims > 0) { - // just log the first one that deviates - log(Severity.ERROR, ALL_OR_NO_DIMS, dim.getStartPosition(), dim.getEndPosition()); + var branch = f_errs.get().branch(null); + try (var ignored = reportingTo(branch)) { + expect(Id.L_SQUARE); + int cDims = 0; + int cIndexes = 0; + while (true) { + if (f_cursor != NO_CURSOR && canRetainIncomplete()) { + int dimensions = cDims + cIndexes + 1 + (peek(Id.COMMA) ? 1 : 0); + restore(mark); + return new ArrayTypeExpression(type, dimensions, f_cursor); } - ++cIndexes; - } else { // we ate the "?" - if (cDims == 0 && cIndexes > 0) { - // just log the first one that deviates - log(Severity.ERROR, ALL_OR_NO_DIMS, dim.getStartPosition(), dim.getEndPosition()); + if (match(Id.R_SQUARE) != null) { + break; + } + if (cDims + cIndexes > 0) { + expect(Id.COMMA); } - ++cDims; - } - } - long lEndPos = prev().getEndPosition(); - type = new ArrayTypeExpression(type, cDims + cIndexes, lEndPos); - // if there were only indexes, then we need to leave them in place because the - // type expression does not consume them - if (cDims == 0 && cIndexes > 0) { - restore(mark); - return type; + Token dim = peek(); + if (match(Id.COND) == null) { + Expression index = f_partialAnalysis ? parsePartialValue() : parseExpression(); + ++cIndexes; + if (f_cursor != NO_CURSOR && (containsCursorHole(index) || canRetainIncomplete())) { + int dimensions = cDims + cIndexes + (peek(Id.COMMA) ? 1 : 0); + long end = Math.max(f_cursor, prev().getEndPosition()); + restore(mark); + return new ArrayTypeExpression(type, dimensions, end); + } + if (cIndexes == 1 && cDims > 0) { + log(Severity.ERROR, ALL_OR_NO_DIMS, dim.getStartPosition(), dim.getEndPosition()); + } + } else { + if (cDims == 0 && cIndexes > 0) { + log(Severity.ERROR, ALL_OR_NO_DIMS, dim.getStartPosition(), dim.getEndPosition()); + } + ++cDims; + } + } + type = new ArrayTypeExpression(type, cDims + cIndexes, prev().getEndPosition()); + if (cDims == 0 && cIndexes > 0) { + restore(mark); + return type; + } + } catch (CompilerException error) { + branch.merge(); + throw error; } + branch.merge(); break; + } case COND: if (!peek().hasLeadingWhitespace()) { @@ -4455,9 +4763,9 @@ private TypeExpression parseNonBiTypeExpressionBody(boolean fExtended) { * * @return an AnnotatedTypeExpression */ - AnnotatedTypeExpression parseAnnotatedTypeExpression(boolean fExtended) { + AnnotatedTypeExpression parseAnnotatedTypeExpression(boolean fExtended, boolean fHeader) { AnnotationExpression annotation = parseAnnotation(true); - TypeExpression type = parseNonBiTypeExpression(fExtended); + TypeExpression type = parseNonBiTypeExpression(fExtended, fHeader); return new AnnotatedTypeExpression(annotation, type); } @@ -4477,19 +4785,23 @@ AnnotatedTypeExpression parseAnnotatedTypeExpression(boolean fExtended) { * @return a FunctionTypeExpression */ FunctionTypeExpression parseFunctionTypeExpression() { + return parseFunctionTypeExpression(false); + } + + private FunctionTypeExpression parseFunctionTypeExpression(boolean header) { Token function = expect(Id.FUNCTION); // return values Token conditional = match(Id.CONDITIONAL); - List listReturn = parseReturnList(); + List listReturn = parseReturnList(header); // see if the parameters precede the name - List listParam = parseParameterTypeList(false); + List listParam = parseParameterTypeList(false, header); if (listParam == null) { // name optionally comes before or after the parameters Token name = expect(Id.IDENTIFIER); - listParam = parseParameterTypeList(true); + listParam = parseParameterTypeList(true, header); // pretend the name is the next token (as if we didn't eat it already) putBack(name); @@ -4520,6 +4832,10 @@ FunctionTypeExpression parseFunctionTypeExpression() { * @return a NamedTypeExpression */ NamedTypeExpression parseNamedTypeExpression(Token tokAccess) { + return parseNamedTypeExpression(tokAccess, false); + } + + private NamedTypeExpression parseNamedTypeExpression(Token tokAccess, boolean fHeader) { NamedTypeExpression expr = null; do { if (expr != null) { @@ -4527,7 +4843,22 @@ NamedTypeExpression parseNamedTypeExpression(Token tokAccess) { } // QualifiedName - List names = parseQualifiedName(); + List names = new ArrayList<>(); + do { + boolean hole = fHeader && f_cursor != NO_CURSOR && canRetainTypeSlot(); + Token name = hole ? new Token(f_cursor, f_cursor, Id.IDENTIFIER, "") : expect(Id.IDENTIFIER); + names.add(name); + // A cursor in a qualifier owns this token only; the remaining dotted suffix + // stays in source. A trailing dot owns a zero-width insertion slot. + if (fHeader && (hole || peek(Id.DOT) && name.getStartPosition() < f_cursor + && f_cursor <= name.getEndPosition())) { + var selected = expr == null + ? new NamedTypeExpression(null, names, tokAccess, null, null, name.getEndPosition()) + : new NamedTypeExpression(expr, names, null, name.getEndPosition()); + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + throw new IncompleteHeader(IncompleteStatement.forDeclarationType(selected, f_cursor)); + } + } while (match(Id.DOT) != null); if (tokAccess != null && expr != null) { log(Severity.ERROR, NO_CHILD_ACCESS, tokAccess.getStartPosition(), @@ -4545,7 +4876,7 @@ NamedTypeExpression parseNamedTypeExpression(Token tokAccess) { } // TypeParameterTypeList - List params = parseTypeParameterTypeList(false, true); + List params = parseTypeParameterTypeList(false, true, fHeader); if (expr == null) { expr = new NamedTypeExpression(null, names, tokAccess, tokNarrow, params, @@ -4763,6 +5094,10 @@ AnnotationExpression parseAnnotation(boolean required) { * @return a list of zero or more type parameters, or null if there were no angle brackets */ List parseTypeParameterList(boolean required) { + return parseTypeParameterList(required, false); + } + + private List parseTypeParameterList(boolean required, boolean header) { List typeParams = null; if (match(Id.COMP_LT, required) != null) { typeParams = new ArrayList<>(); @@ -4777,7 +5112,12 @@ List parseTypeParameterList(boolean required) { Token param = expect(Id.IDENTIFIER); TypeExpression type = null; if (match(Id.EXTENDS) != null) { - type = parseExtendedTypeExpression(); + try { + type = header ? parseDeclarationType(true) : parseExtendedTypeExpression(); + } catch (IncompleteHeader error) { + throw error.withFormals(Stream.concat(typeParams.stream(), + Stream.of(new Parameter(new BadTypeExpression((Expression) error.writtenType.clone()), param))).toList()); + } } typeParams.add(new Parameter(type, param)); } @@ -4807,13 +5147,19 @@ List parseTypeParameterList(boolean required) { * @return a list of zero or more types, or null if there were no angle brackets */ List parseTypeParameterTypeList(boolean required, boolean fAllowTypeSequence) { + return parseTypeParameterTypeList(required, fAllowTypeSequence, false); + } + + private List parseTypeParameterTypeList(boolean required, boolean fAllowTypeSequence, + boolean fHeader) { List types = null; if (match(Id.COMP_LT, required) != null) { + retainEmptyTypeArgument(fHeader); if (match(Id.COMP_GT) != null) { types = Collections.emptyList(); } else { - types = parseTypeExpressionList(fAllowTypeSequence); - expect(Id.COMP_GT); + types = parseTypeExpressionList(fAllowTypeSequence, fHeader); + expectHeaderTypeClose(Id.COMP_GT, types, fHeader); } } return types; @@ -4831,23 +5177,48 @@ List parseTypeParameterTypeList(boolean required, boolean fAllow * @return a list of type expressions */ List parseTypeExpressionList(boolean fAllowTypeSequence) { + return parseTypeExpressionList(fAllowTypeSequence, false); + } + + private List parseTypeExpressionList(boolean fAllowTypeSequence, boolean fHeader) { List types = new ArrayList<>(); while (true) { if (!types.isEmpty() && match(Id.COMMA) == null) { return types; } + retainEmptyTypeArgument(fHeader); + if (fAllowTypeSequence && peek(Id.COMP_LT)) { Token tokStart = peek(); - List listSeq = parseTypeParameterTypeList(true, false); + List listSeq = parseTypeParameterTypeList(true, false, fHeader); Token tokEnd = prev(); types.add(new TupleTypeExpression(listSeq, tokStart.getStartPosition(), tokEnd.getEndPosition())); } else { - types.add(parseExtendedTypeExpression()); + types.add(parseIntersectingTypeExpression(true, fHeader)); } } } + /** An explicit empty generic slot has no invented type expression or generic owner. */ + private void retainEmptyTypeArgument(boolean header) { + if (header && f_cursor != NO_CURSOR && canRetainTypeSlot()) { + throw incompleteHeader(new Token(f_cursor, f_cursor, Id.IDENTIFIER, "")); + } + } + + /** Only explicit cursor probes can select an empty type slot at a grammar boundary. */ + private boolean canRetainTypeSlot() { + return m_cSpeculating == 0 && !m_fAvoidRecovery && !f_errs.get().isAbortDesired() + && prev().getEndPosition() <= f_cursor + && f_cursor <= (eof() ? m_source.getPosition() : peek().getStartPosition()) + && (switch (peek().getId()) { + case COMP_GT, COMP_GTEQ, SHR, SHR_ASN, USHR, USHR_ASN, BIT_OR, ADD, SUB -> true; + case IDENTIFIER -> f_cursor < peek().getStartPosition(); + default -> canRetainIncomplete(); + }); + } + /** * Parse a list of package injections, starting with the opening parenthesis. * @@ -4919,21 +5290,227 @@ List parseParameterList(boolean required) { List params = null; if (match(Id.L_PAREN, required) != null) { params = new ArrayList<>(); + if (f_cursor != NO_CURSOR && canRetainIncomplete()) { + throw incompleteHeader(new Token(f_cursor, f_cursor, Id.IDENTIFIER, "")); + } if (match(Id.R_PAREN) == null) { - do { - TypeExpression type = parseTypeExpression(); - Token name = expect(Id.IDENTIFIER); - Expression value = null; + while (true) { + TypeExpression type = parseDeclarationType(); + var missingName = declarationNameSlot(type); + if (missingName.isPresent()) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + throw new IncompleteHeader(missingName.orElseThrow()); + } + Token name = expect(Id.IDENTIFIER); + Expression value = null; if (match(Id.ASN) != null) { value = parseExpression(); } params.add(new Parameter(type, name, value)); - } while (match(Id.R_PAREN, (match(Id.COMMA) == null)) == null); + Token comma = match(Id.COMMA); + if (comma != null && f_cursor != NO_CURSOR && canRetainIncomplete()) { + throw incompleteHeader(new Token(f_cursor, f_cursor, Id.IDENTIFIER, "")); + } + if ((comma == null ? expectPartialClose(Id.R_PAREN, params) : match(Id.R_PAREN)) != null) { + break; + } + } } } return params; } + /** A header type prefix is queried in its enclosing declaration, never as a value. */ + private TypeExpression parseDeclarationType() { + return parseDeclarationType(false); + } + + /** + * Only a grammar position that requires a name may retain an empty name slot. In a method + * body, a bare expression followed by whitespace is not evidence of a variable declaration. + * Keep the complete written type as syntax; no parameter or property is registered. + */ + private Optional declarationNameSlot(TypeExpression type) { + return f_cursor != NO_CURSOR && m_cSpeculating == 0 && !m_fAvoidRecovery + && !f_errs.get().isAbortDesired() && !(type instanceof BadTypeExpression) + && prev().getEndPosition() < f_cursor + && f_cursor <= (eof() ? m_source.getPosition() : peek().getStartPosition()) + && m_source.toString(prev().getEndPosition(), f_cursor).isBlank() + && (eof() || peek(Id.ASN) || peek(Id.SEMICOLON) || peek(Id.R_CURLY) + || peek(Id.R_PAREN) || peek(Id.COMMA)) + ? Optional.of(IncompleteStatement.forDeclarationName(type, prev(), f_cursor)) + : Optional.empty(); + } + + private TypeExpression parseDeclarationType(boolean extended) { + if (f_cursor != NO_CURSOR && canRetainIncomplete()) { + throw incompleteHeader(new Token(f_cursor, f_cursor, Id.IDENTIFIER, "")); + } + boolean header = f_cursor != NO_CURSOR && m_cSpeculating == 0 && !m_fAvoidRecovery + && !f_errs.get().isAbortDesired(); + TypeExpression type = parseIntersectingTypeExpression(extended, header); + if (header) { + var prefix = declarationTypePrefix(type); + if (prefix.isPresent()) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + throw new IncompleteHeader(IncompleteStatement.forDeclarationType(prefix.orElseThrow(), f_cursor), type, List.of()); + } + } + return type; + } + + /** + * Select a written type name, including a generic base whose arguments follow the cursor. The + * caller transfers this original node into the cursor site; ordinary adoption and cloning + * then own it. This is syntax selection, not validation of the enclosing type's constraints. + */ + private Optional declarationTypePrefix(TypeExpression type) { + if (type instanceof NamedTypeExpression named && named.getModule() == null + && named.getNameToken() != null && named.getNameToken().getStartPosition() < f_cursor + && f_cursor <= named.getNameToken().getEndPosition()) { + return Optional.of(named); + } + var types = type instanceof FunctionTypeExpression function + ? Stream.concat(function.getReturnValues().stream().map(Parameter::getType), + function.getParamTypes().stream()) + : StreamSupport.stream(type.children().spliterator(), false) + .filter(TypeExpression.class::isInstance).map(TypeExpression.class::cast); + return types.flatMap(child -> declarationTypePrefix(child).stream()).findFirst(); + } + + /** Missing type closers are tolerated only around a selected header prefix, never in normal parsing. */ + private void expectHeaderTypeClose(Id close, List types, boolean header) { + if (match(close) != null) { + return; + } + if (header && m_cSpeculating == 0 && !m_fAvoidRecovery && !f_errs.get().isAbortDesired() + && (atMissingClose() || peek(Id.IDENTIFIER) || peek(Id.COMP_GT)) + && types.stream().anyMatch(type -> declarationTypePrefix(type).isPresent())) { + return; + } + expect(close); + } + + private IncompleteHeader incompleteHeader(Token prefix) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + return new IncompleteHeader(new IncompleteStatement(prefix, f_cursor, INCOMPLETE_EXPRESSION)); + } + + /** Unwinds header parsing while carrying the incomplete site, written type and formal parameters. */ + private static class IncompleteHeader extends CompilerException { + private IncompleteHeader(IncompleteStatement site) { + this(site, site.getTarget(), List.of()); + } + + private IncompleteHeader(IncompleteStatement site, Expression writtenType, List formals) { + super("Incomplete declaration header"); + this.site = site; + this.formals = formals; + this.writtenType = writtenType; + } + + private IncompleteHeader withFormals(List parameters) { + return parameters == null ? this : new IncompleteHeader(site, writtenType, + Stream.concat(formals.stream(), parameters.stream()).distinct().toList()); + } + + private final IncompleteStatement site; + private final List formals; + private final Expression writtenType; + } + + /** Keep a type's real body for structure, but never register an unfinished inheritance header. */ + private TypeCompositionStatement retainTypeDeclaration(Mark header, long start, Token category, + Token name, List qualified, List sites, CompilerException error) { + if (m_cSpeculating != 0 || m_fAvoidRecovery || f_errs.get().isAbortDesired()) { + throw error; + } + restore(header); + while (!eof() && !peek(Id.R_CURLY)) { + if (f_errs.get().isAbortDesired()) { + throw error; + } + if (peek(Id.L_CURLY)) { + StatementBlock body = parseTypeCompositionBody(category); + return new IncompleteTypeCompositionStatement(m_source, category, name, qualified, body, + start, body.getEndPosition(), sites, + error instanceof IncompleteHeader incomplete ? incomplete.formals : List.of()); + } + if (match(Id.SEMICOLON) != null) { + break; + } + current(); + } + return new IncompleteTypeCompositionStatement(m_source, category, name, qualified, null, + start, retainedDeclarationEnd(sites), sites, + error instanceof IncompleteHeader incomplete ? incomplete.formals : List.of()); + } + + /** + * Retain only the known declaration name and source extent. A brace or semicolon bounds + * this recovery; the body is skipped as one unit so its names cannot leak into the owner. + * Unfinished parameter syntax never registers a partial method signature. + */ + private IncompleteDeclarationStatement retainDeclaration(Mark header, long start, Token name, + IncompleteDeclarationStatement.Kind kind, List sites, CompilerException error) { + if (m_cSpeculating != 0 || m_fAvoidRecovery || f_errs.get().isAbortDesired()) { + throw error; + } + restore(header); + while (!eof() && !peek(Id.R_CURLY)) { + if (f_errs.get().isAbortDesired()) { + throw error; + } + if (peek(Id.L_CURLY)) { + current(); + // Consume the body by brace depth so missing parameter delimiters cannot eat + // its siblings. current() also preserves the actual closing token's range. + int depth = 1; + while (depth > 0 && !eof()) { + if (f_errs.get().isAbortDesired()) { + throw error; + } + switch (current().getId()) { + case L_CURLY -> ++depth; + case R_CURLY -> --depth; + default -> {} + } + } + break; + } + if (match(Id.SEMICOLON) != null) { + break; + } + current(); + } + return new IncompleteDeclarationStatement(kind, name, start, retainedDeclarationEnd(sites), sites, + error instanceof IncompleteHeader incomplete ? incomplete.formals : List.of()); + } + + /** An empty name slot at EOF includes the whitespace after its last written type token. */ + private long retainedDeclarationEnd(List sites) { + return Math.max(prev().getEndPosition(), sites.stream().mapToLong(IncompleteStatement::getEndPosition) + .max().orElse(prev().getEndPosition())); + } + + /** Declaration return slots select types without publishing an unfinished callable. */ + private List parseDeclarationReturns() { + if (f_cursor == NO_CURSOR) { + return parseReturnList(); + } + if (match(Id.VOID) != null) { + return List.of(); + } + if (match(Id.L_PAREN) == null) { + return List.of(new Parameter(parseDeclarationType())); + } + List returns = new ArrayList<>(); + do { + returns.add(new Parameter(parseDeclarationType(), matchNameOrAny())); + } while (match(Id.R_PAREN, match(Id.COMMA) == null) == null); + return returns; + } + /** * Parse a list of parameter types (without parameter names). * @@ -4946,12 +5523,16 @@ List parseParameterList(boolean required) { * @return */ List parseParameterTypeList(boolean required) { + return parseParameterTypeList(required, false); + } + + private List parseParameterTypeList(boolean required, boolean header) { List types = null; if (match(Id.L_PAREN, required) != null) { types = peek(Id.R_PAREN) ? Collections.emptyList() - : parseTypeExpressionList(false); - expect(Id.R_PAREN); + : parseTypeExpressionList(false, header); + expectHeaderTypeClose(Id.R_PAREN, types, header); } return types; } @@ -5022,62 +5603,324 @@ List parseArgumentList(boolean required, boolean allowCurrying, bool List args = new ArrayList<>(); if (match(idClose) == null) { do { - Token label = null; - if (!fArray) { - // special case where the parameter names are being specified with the arguments - if (peek(Id.IDENTIFIER)) { - Token name = expect(Id.IDENTIFIER); - if (match(Id.ASN) == null) { - // oops, it wasn't a "name=value" argument - putBack(name); - } else { - label = name; - } - } - } + args.add(parseArgument(allowCurrying, fArray)); + } while (match(idClose, (match(Id.COMMA) == null)) == null); + } - Expression expr; - if (allowCurrying && !fArray) { - switch (peek().getId()) { - case ANY: { - // one of two cases: an unbound argument, or a lambda whose one parameter is - // ignored - Token tokUnbound = expect(Id.ANY); - if (peek(Id.LAMBDA)) { - putBack(tokUnbound); - expr = parseExpression(); - } else { - expr = new NonBindingExpression(tokUnbound.getStartPosition(), - tokUnbound.getEndPosition(), null); - } - break; - } + return args; + } - case COMP_LT: { - Token tokOpen = expect(Id.COMP_LT); - TypeExpression type = parseTypeExpression(); - Token tokClose = expect(Id.COMP_GT); - Token tokUnbound = expect(Id.ANY); - expr = new NonBindingExpression(tokOpen.getStartPosition(), - tokUnbound.getEndPosition(), type); - break; - } + /** Parse one argument, shared by calls, constructors and annotation argument lists. */ + private Expression parseArgument(boolean allowCurrying, boolean fArray) { + return parseArgument(allowCurrying, fArray, false); + } - default: - expr = parseExpression(); - break; - } + private Expression parseArgument(boolean allowCurrying, boolean fArray, boolean partial) { + Token label = null; + if (!fArray) { + // special case where the parameter names are being specified with the arguments + if (peek(Id.IDENTIFIER)) { + Token name = expect(Id.IDENTIFIER); + if (match(Id.ASN) == null) { + // oops, it wasn't a "name=value" argument + putBack(name); } else { + label = name; + } + } + } + + if (partial && canRetainIncomplete() && peek(Id.COMMA)) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + Expression hole = new IncompleteExpression(new IncompleteStatement( + new Token(f_cursor, f_cursor, Id.IDENTIFIER, ""), f_cursor, INCOMPLETE_EXPRESSION)); + return label == null ? hole : new LabeledExpression(label, hole); + } + if (partial && label != null && endsAt(f_cursor)) { + throw new IncompleteArgument(label); + } + Expression expr; + if (allowCurrying && !fArray) { + switch (peek().getId()) { + case ANY: { + // one of two cases: an unbound argument, or a lambda whose one parameter is + // ignored + Token tokUnbound = expect(Id.ANY); + if (peek(Id.LAMBDA)) { + putBack(tokUnbound); expr = parseExpression(); + } else { + expr = new NonBindingExpression(tokUnbound.getStartPosition(), + tokUnbound.getEndPosition(), null); } + break; + } - args.add(label == null ? expr : new LabeledExpression(label, expr)); - } while (match(idClose, (match(Id.COMMA) == null)) == null); + case COMP_LT: { + Token tokOpen = expect(Id.COMP_LT); + TypeExpression type = parseTypeExpression(); + Token tokClose = expect(Id.COMP_GT); + Token tokUnbound = expect(Id.ANY); + expr = new NonBindingExpression(tokOpen.getStartPosition(), + tokUnbound.getEndPosition(), type); + break; + } + + default: + expr = partial ? parsePartialValue() : parseExpression(); + break; + } + } else { + expr = partial ? parsePartialValue() : parseExpression(); + } + + return label == null ? expr : new LabeledExpression(label, expr); + } + + /** Retain the opening token and separators only for the explicit partial-analysis parser. */ + private Expression parseInvocation(Expression callee) { + if (!f_partialAnalysis) { + return new InvocationExpression(callee, peek(Id.ASYNC_PAREN), + parseArgumentList(true, true, false), prev().getEndPosition()); } + boolean async = peek(Id.ASYNC_PAREN); + List args = parsePartialArguments(callee, true); + return new InvocationExpression(callee, async, args, prev().getEndPosition()); + } + + /** Shared cursor syntax for ordinary calls and constructor arguments. */ + private List parsePartialArguments(Expression callee, boolean allowBindings) { + boolean array = callee instanceof NewExpression && peek(Id.L_SQUARE); + Id close = array ? Id.R_SQUARE : Id.R_PAREN; + Token open = array || allowBindings && peek(Id.ASYNC_PAREN) ? current() : expect(Id.L_PAREN); + List args = new ArrayList<>(); + List separators = new ArrayList<>(); + boolean incomplete = containsCursorHole(callee); + if (endsAt(f_cursor) && !peek(Id.COMMA)) { + match(close); + throw incomplete(callee, open, args, separators); + } + if (match(close) == null) { + while (true) { + Expression argument; + try { + argument = parseArgument(allowBindings, array, f_cursor != NO_CURSOR); + } catch (IncompleteArgument e) { + match(close); + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + throw new IncompleteSyntax(IncompleteStatement.forNamedArgument( + callee, open, args, separators, f_cursor, e.name)); + } + Expression value = argument instanceof LabeledExpression labeled + ? labeled.getUnderlyingExpression() : argument; + // A direct final argument prefix belongs to this call's fitting context. Keep + // its token rather than validating it as a value or retaining a second site. + // Compound/member expressions and slots before later arguments keep their sites. + if (value instanceof IncompleteExpression partial && partial.getSite().isNameCompletion() + && endsAt(f_cursor) && !peek(Id.COMMA)) { + match(close); + throw new IncompleteSyntax(IncompleteStatement.forArgumentPrefix(callee, open, + args, separators, f_cursor, + argument instanceof LabeledExpression labeled ? labeled.getNameToken() : null, + partial.getSite().getMemberName().orElseThrow())); + } + args.add(argument); + if (containsCursorHole(value)) { + incomplete = true; + } + Token comma = match(Id.COMMA); + if (comma != null) { + separators.add(comma); + } + if (endsAt(f_cursor)) { + match(close); + throw incomplete(callee, open, args, separators); + } + if (comma == null && incomplete && !peek(close) && atMissingClose()) { + break; + } + if (match(close, comma == null) != null) { + break; + } + } + } + if (incomplete) { + throw incomplete(callee, open, args, separators); + } return args; } + private boolean canRetainIncomplete() { + return canRetainIncompleteAt(f_cursor); + } + + /** A mid-token value cursor uses the same recovery boundary after its complete written token. */ + private boolean canRetainIncompleteAt(long cursor) { + return canRetainIncompleteAt(cursor, false); + } + + /** A selected member name can precede a call whose arguments must still be parsed and owned. */ + private boolean canRetainIncompleteAt(long cursor, boolean allowCall) { + if (!f_partialAnalysis || m_cSpeculating != 0 || m_fAvoidRecovery || f_errs.get().isAbortDesired()) { + return false; + } + if (cursor == NO_CURSOR) { + return eof(); + } + return prev().getEndPosition() <= cursor + && cursor <= (eof() ? m_source.getPosition() : peek().getStartPosition()) + && (eof() || peek(Id.R_CURLY) || peek(Id.SEMICOLON) || peek(Id.R_PAREN) + || peek(Id.R_SQUARE) || peek(Id.COMMA) || peek(Id.COLON) || peek(Id.L_CURLY) + || prev().getId() == Id.IDENTIFIER && switch (peek().getId()) { + case ADD, SUB, MUL, DIV, MOD, DIVREM, SHL, SHR, USHR, + BIT_AND, BIT_OR, BIT_XOR, COND_AND, COND_OR, COND_XOR, + COMP_EQ, COMP_NEQ, COMP_LT, COMP_LTEQ, COMP_GT, COMP_GTEQ, + COMP_ORD, COND_ELSE -> true; + case COND -> peek().hasLeadingWhitespace(); + default -> false; + } + || allowCall && (peek(Id.L_PAREN) || peek(Id.ASYNC_PAREN))); + } + + /** + * Whether the operation being typed at the cursor ends there: the next token ends it, or the + * cursor ends a line that the next token cannot continue. + * + * @param cursor the position after the operation's last written token, or the cursor itself + * + * @return whether partial analysis retains the operation as the cursor's site + */ + private boolean endsAt(long cursor) { + return endsAt(cursor, false); + } + + /** A selected member name can also end before a call whose arguments must still be parsed. */ + private boolean endsAt(long cursor, boolean allowCall) { + return canRetainIncompleteAt(cursor, allowCall) || atLineEnd(cursor); + } + + /** + * What is being typed at the end of its line, before code on a later line that cannot continue + * it, ends there: the later line is the next statement, not the rest of a name, a member access, + * a value or a call's arguments. Without this, what is typed and that statement parse as one, + * which is an error, and the cursor has no site. A cursor after a comment at the line's end is + * in the comment, not at the end of the code. + */ + private boolean atLineEnd(long cursor) { + if (!f_partialAnalysis || m_cSpeculating != 0 || m_fAvoidRecovery || f_errs.get().isAbortDesired() + || cursor == NO_CURSOR || eof()) { + return false; + } + Token next = peek(); + return prev().getEndPosition() <= cursor && cursor <= next.getStartPosition() + && Source.calculateLine(next.getStartPosition()) > Source.calculateLine(cursor) + && m_source.toString(prev().getEndPosition(), cursor).isBlank() + && switch (next.getId()) { + case DOT, L_PAREN, ASYNC_PAREN, L_SQUARE, COND, + ADD, SUB, MUL, DIV, MOD, DIVREM, SHL, SHR, USHR, + BIT_AND, BIT_OR, BIT_XOR, COND_AND, COND_OR, COND_XOR, + COMP_EQ, COMP_NEQ, COMP_LT, COMP_LTEQ, COMP_GT, COMP_GTEQ, + COMP_ORD, COND_ELSE, ASN, AS, IS -> false; + default -> true; + }; + } + + /** + * Retain enclosing syntax around a cursor hole at a statement/outer-delimiter boundary. + * The existing incomplete-expression diagnostic covers the hole and its missing suffix; + * this zero-width end marker neither edits the source nor consumes an enclosing delimiter. + * Without that hole, ordinary parsing (including speculation and error budgets) still fails. + */ + private Token expectPartialClose(Id close, List expressions) { + Token token = matchPartialClose(close, expressions); + return token == null ? expect(close) : token; + } + + private Token matchPartialClose(Id close, List expressions) { + Token token = match(close); + if (token != null) { + return token; + } + if (atMissingClose() && expressions.stream().anyMatch(this::containsCursorHole)) { + long end = Math.max(f_cursor, prev().getEndPosition()); + return new Token(end, end, close); + } + return null; + } + + private boolean atMissingClose() { + return f_cursor != NO_CURSOR && m_cSpeculating == 0 && !m_fAvoidRecovery + && !f_errs.get().isAbortDesired() + && (eof() || peek(Id.SEMICOLON) || peek(Id.R_CURLY) + || peek(Id.R_PAREN) || peek(Id.R_SQUARE) || peek(Id.L_CURLY)); + } + + /** Query syntax ownership rather than caching cursor state on ordinary AST nodes. */ + private boolean containsCursorHole(AstNode node) { + return f_cursor != NO_CURSOR && PartialSyntax.containsAt(node, f_cursor); + } + + private IncompleteSyntax incomplete(Expression target, Token operator, + List arguments, List separators) { + long position = f_cursor == NO_CURSOR ? m_source.getPosition() : f_cursor; + String code = f_cursor == NO_CURSOR ? UNEXPECTED_EOF : INCOMPLETE_EXPRESSION; + log(Severity.ERROR, code, position, position); + return new IncompleteSyntax( + new IncompleteStatement(target, operator, arguments, separators, position, code)); + } + + private IncompleteSyntax incomplete(Expression receiver, Token dot, Token memberName) { + log(Severity.ERROR, INCOMPLETE_EXPRESSION, f_cursor, f_cursor); + return new IncompleteSyntax(new IncompleteStatement( + receiver, dot, memberName, f_cursor, INCOMPLETE_EXPRESSION)); + } + + /** Preserve the enclosing statement only when its whole value is the incomplete operation. */ + private Expression parsePartialValue() { + long start = peek().getStartPosition(); + try { + return parseExpression(); + } catch (IncompleteSyntax e) { + return retainPartialValue(start, e); + } + } + + private Expression retainPartialValue(long start, IncompleteSyntax error) { + if (f_cursor == NO_CURSOR || error.statement.getStartPosition() != start) { + throw error; + } + return new IncompleteExpression(error.statement); + } + + private void finishValueStatement(Expression value) { + if (value instanceof IncompleteExpression || atMissingClose() && containsCursorHole(value)) { + match(Id.SEMICOLON); + } else { + expect(Id.SEMICOLON); + } + } + + /** Unwind to the enclosing statement without pretending the unfinished expression has a value. */ + private static class IncompleteSyntax extends CompilerException { + private IncompleteSyntax(IncompleteStatement statement) { + super("Incomplete expression"); + this.statement = statement; + } + + private final IncompleteStatement statement; + } + + /** Unwind only to the owning call, retaining a written label without inventing a value. */ + private static class IncompleteArgument extends CompilerException { + private IncompleteArgument(Token name) { + super("Incomplete named argument"); + this.name = name; + } + + private final Token name; + } + /** * Parse a declared list of return types. * @@ -5099,15 +5942,19 @@ List parseArgumentList(boolean required, boolean allowCurrying, bool * } */ List parseReturnList() { + return parseReturnList(false); + } + + private List parseReturnList(boolean header) { List listReturn; if (match(Id.VOID) != null) { listReturn = Collections.emptyList(); } else if (match(Id.L_PAREN) == null) { - listReturn = Collections.singletonList(new Parameter(parseTypeExpression())); + listReturn = List.of(new Parameter(parseIntersectingTypeExpression(false, header))); } else { listReturn = new ArrayList<>(); do { - listReturn.add(new Parameter(parseTypeExpression(), matchNameOrAny())); + listReturn.add(new Parameter(parseIntersectingTypeExpression(false, header), matchNameOrAny())); } while (match(Id.R_PAREN, (match(Id.COMMA) == null)) == null); } return listReturn; @@ -5218,19 +6065,70 @@ LiteralExpression parseVersionLiteral(boolean fRequired) { return tokVer == null ? null : new LiteralExpression(tokVer); } + /** + * Finish a block without consuming the synthetic EOF token. Missing braces are errors, but + * completed declarations and statements remain useful to source tools. A speculative parse + * still fails immediately, and cancellation/error budgets always stop recovery. + */ + private boolean atBlockEnd(boolean fFileLevel) { + if (f_errs.get().isAbortDesired()) { + throw new CompilerException("Parsing aborted"); + } + if (eof()) { + // An explicit EOF cursor already reports its unfinished suffix, including containers. + // Other cursors and ordinary parsing must still diagnose missing block braces. + if (!fFileLevel && !f_errs.get().hasError(UNEXPECTED_EOF) + && !(f_cursor == m_source.getPosition() + && f_errs.get().hasError(INCOMPLETE_EXPRESSION))) { + log(Severity.ERROR, UNEXPECTED_EOF, m_source.getPosition(), m_source.getPosition()); + } + // Retain an end marker for recovered containers without advancing past EOF. + m_tokenPrev = new Token(m_source.getPosition(), m_source.getPosition(), Id.R_CURLY); + return true; + } + return match(Id.R_CURLY) != null; + } + + /** + * Drop one malformed statement and resume at a lexical statement boundary. Rewind to its + * beginning so delimiters consumed by the failed parse cannot change the enclosing scope. + * Always consume its first token before scanning, even if that token is a statement keyword. + */ + private void recoverStatement(Mark start, CompilerException error) { + if (m_cSpeculating != 0 || m_fAvoidRecovery || f_errs.get().isAbortDesired()) { + throw error; + } + if (eof()) { + return; + } + restore(start); + Token token = current(); + switch (token.getId()) { + case L_CURLY, L_PAREN, L_SQUARE: + skipEnclosed(token.getId()); + break; + default: + skipToNextStatement(); + break; + } + } + /** * Attempt to get out of whatever parsing mess we got ourselves into by * figuring out where the current statement ends, and starting anew from * there. */ void skipToNextStatement() { - while (true) { + while (!eof()) { switch (peek().getId()) { case SEMICOLON: next(); return; case L_CURLY: + skipEnclosed(current().getId()); + return; + case L_PAREN: case L_SQUARE: skipEnclosed(current().getId()); @@ -5291,7 +6189,7 @@ void skipToNextStatement() { * @param idOpen the opening parenthesis / curlies / brackets */ void skipEnclosed(Id idOpen) { - while (true) { + while (!eof()) { switch (peek().getId()) { case L_CURLY: case L_PAREN: @@ -5441,12 +6339,11 @@ protected void putBack(Token token) { } protected static class Mark { - long pos; - Token token; - Token putBack; - Token lastMatch; - Token doc; - boolean noRec; + long pos; + Token token; + Token putBack; + Token lastMatch; + Token doc; } protected Mark mark() { @@ -5456,17 +6353,17 @@ protected Mark mark() { mark.putBack = m_tokenPutBack == null ? null : m_tokenPutBack.clone(); mark.lastMatch = m_tokenPrev == null ? null : m_tokenPrev .clone(); mark.doc = m_doc; - mark.noRec = m_fAvoidRecovery; return mark; } protected void restore(Mark mark) { + // A nesting-limit or listener abort ends this parse, including after speculative rewind. + // Token backtracking must not re-enable recovery from an exhausted parser budget. m_lexer.restore(mark.pos); - m_token = mark.token; - m_tokenPutBack = mark.putBack; - m_tokenPrev = mark.lastMatch; - m_doc = mark.doc; - m_fAvoidRecovery = mark.noRec; + m_token = mark.token; + m_tokenPutBack = mark.putBack; + m_tokenPrev = mark.lastMatch; + m_doc = mark.doc; } /** @@ -5622,30 +6519,29 @@ protected Token takeDoc() { * @param aoParam */ protected void log(Severity severity, String sCode, long lPosStart, long lPosEnd, Object... aoParam) { - if (m_lookAhead != null) { - m_lookAhead.log(severity, sCode, aoParam, lPosStart, lPosEnd); - } else if (m_errorListener.log(severity, sCode, aoParam, m_source, lPosStart, lPosEnd)) { + f_errs.get().log(severity, sCode, in(m_source, lPosStart, lPosEnd), aoParam); + if (f_errs.get().isAbortDesired()) { m_fAvoidRecovery = true; - throw new CompilerException("error list is full: " + m_errorListener); + throw new CompilerException("error list is full: " + f_errs.get()); } } /** * Count one level of nested parsing, to be released when the returned handle is closed: * - *

+     * 

      * try (var ignore = nest()) {
      *     ...
      * }
-     * 
+ *
* - * The parser is a recursive descent, and a level of nesting in the source costs a dozen or so + *

The parser is a recursive descent, and a level of nesting in the source costs a dozen or so * Java frames. Without a limit, sufficiently nested source exhausts the thread's stack, and a * StackOverflowError is not something a caller can be asked to handle: it escapes the compiler * and reaches whatever embeds it. Counting the levels turns that into an ordinary error with a * source position. * - * The count is shared by every construct that nests, because they nest through each other - + *

The count is shared by every construct that nests, because they nest through each other - * a type inside an expression inside a statement - so one budget bounds the descent no matter * how the source alternates between them. It counts levels of parsing rather than levels of * source: one construct can pass more than one guard on its way down, so the limit is a bound @@ -5657,6 +6553,7 @@ protected void log(Severity severity, String sCode, long lPosStart, long lPosEnd */ private Nesting nest() { if (m_cNesting >= MAX_NESTING_DEPTH) { + m_fAvoidRecovery = true; Token token = peek(); log(Severity.ERROR, NESTING_TOO_DEEP, token.getStartPosition(), token.getEndPosition(), MAX_NESTING_DEPTH); @@ -5678,57 +6575,113 @@ public void close() { } } - public class SafeLookAhead - implements AutoCloseable { - public SafeLookAhead() { - m_oldLookAhead = m_lookAhead; - m_lookAhead = this; - m_mark = mark(); - } + /** + * Report to the given listener until the returned scope is closed. + * + * @param errs the listener to report to for the duration of the scope + */ + private Reporting.Scope reportingTo(ErrorListener errs) { + return f_errs.to(errs); + } + + /** + * Begin a speculative parse: an attempt whose tokens and whose diagnostics both count only if + * it is kept. + * + *

The parser used to hand-roll this, buffering nothing and throwing away every diagnostic + * below ERROR even when the attempt was kept - which lost those diagnostics for good, because + * a kept attempt is the real parse and nothing re-reads those tokens. An attempt is a + * {@link ErrorListener#branch} instead, so keeping it merges what it had to say and dropping + * it drops the lot, which is the behaviour the rest of the compiler already gets. + * + * @return a scope that restores tokens and discards diagnostics unless the attempt is kept + */ + public Attempt attempt() { + return new Attempt(); + } - public void log(Severity severity, String sCode, Object[] aoParam, long lPosStart, long lPosEnd) { - log(new ErrorList.ErrorInfo(severity, sCode, aoParam, m_source, lPosStart, lPosEnd)); + /** + * The scope opened by {@link #attempt()}. + */ + public class Attempt + implements AutoCloseable, ErrorListener { + /** Save the token position and install a diagnostic branch until this attempt is closed. */ + public Attempt() { + // read the destination before moving it: the branch buffers into whatever this parser + // was reporting to when the attempt began + f_branch = f_errs.get().branch(null); + f_scope = f_errs.to(this); + f_mark = mark(); + ++m_cSpeculating; } - private void log(ErrorListener.ErrorInfo err) { - if (err.getSeverity().ordinal() >= Severity.ERROR.ordinal()) { - m_err = err; - m_fKeepResults = false; - throw new CompilerException("err=" + m_err); + @Override + public void log(ErrorInfo err) { + f_branch.log(err); + + // an attempt that has gone wrong has answered; abandoning it here is what stops the + // parser from trying to recover inside a guess + if (err.getSeverity().isAtLeast(Severity.ERROR)) { + m_fKeep = false; + throw new CompilerException("err=" + err); } } + @Override + public boolean isAbortDesired() { + return f_branch.isAbortDesired(); + } + + @Override + public boolean hasSeriousErrors() { + return f_branch.hasSeriousErrors(); + } + + @Override + public boolean hasError(String sCode) { + return f_branch.hasError(sCode); + } + + /** + * @return true iff nothing serious has been reported against this attempt + */ public boolean isClean() { - return m_err == null; + return !f_branch.hasSeriousErrors(); } - public void keepResults() { - m_fKeepResults = true; + /** + * Keep what this attempt parsed, and with it whatever it had to say. + */ + public void keep() { + m_fKeep = true; } @Override public void close() { - assert m_lookAhead == this; - m_lookAhead = m_oldLookAhead; + assert f_errs.get() == this; + f_scope.close(); + --m_cSpeculating; - if (m_fKeepResults) { - assert m_err == null; + if (m_fKeep) { + assert isClean(); + f_branch.merge(); } else { - restore(m_mark); + // the attempt did not happen: its tokens are put back and its branch is dropped + restore(f_mark); } } - SafeLookAhead m_oldLookAhead; - Mark m_mark; - ErrorList.ErrorInfo m_err; - boolean m_fKeepResults; + private final ErrorListener f_branch; + private final Reporting.Scope f_scope; + private final Mark f_mark; + private boolean m_fKeep; } /** * @return true iff it's ok to try to recover from a parsing error */ protected boolean recoverable() { - return !eof() && !m_fAvoidRecovery && m_lookAhead == null; + return !eof() && !m_fAvoidRecovery && m_cSpeculating == 0; } // ----- Object methods ------------------------------------------------------------------------ @@ -5871,6 +6824,10 @@ public String toString() { * Semicolon is missing. */ public static final String MISSING_SEMICOLON = "PARSER-29"; + /** + * An explicit partial-analysis cursor marks an unfinished expression. + */ + public static final String INCOMPLETE_EXPRESSION = "PARSER-31"; /** * Nesting is too deep. @@ -5886,6 +6843,9 @@ public String toString() { // ----- data members -------------------------------------------------------------------------- + /** Reserved value that cannot be produced by Source.getPosition(). */ + private static final long NO_CURSOR = -1; + /** * The Source to parse. */ @@ -5904,13 +6864,23 @@ public String toString() { /** * The ErrorListener to report errors to. */ - private ErrorListener m_errorListener; + /** + * Where this parser's diagnostics go: the listener it was handed, or - for as long as a + * scope or an {@link Attempt} is open - somewhere else. + */ + private final Reporting f_errs; /** * The lexical analyzer. */ private final Lexer m_lexer; + /** Whether trailing incomplete expression statements may be retained for partial analysis. */ + private final boolean f_partialAnalysis; + + /** Explicit cursor, or NO_CURSOR for ordinary parsing and trailing-EOF partial analysis. */ + private final long f_cursor; + /** * The "put back" token. */ @@ -5950,5 +6920,9 @@ public String toString() { /** * Object supporting unpredictable amount of look-ahead. */ - private SafeLookAhead m_lookAhead; + /** + * How many speculative attempts are open. Recovery is for the real parse: inside an attempt a + * syntax error is an answer, so there is nothing to recover from. + */ + private int m_cSpeculating; } diff --git a/javatools/src/main/java/org/xvm/compiler/Token.java b/javatools/src/main/java/org/xvm/compiler/Token.java index 54faab17062..8d9c18e152c 100644 --- a/javatools/src/main/java/org/xvm/compiler/Token.java +++ b/javatools/src/main/java/org/xvm/compiler/Token.java @@ -7,6 +7,9 @@ import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.NOWHERE; +import static org.xvm.asm.ErrorListener.in; + import static org.xvm.util.Handy.appendChar; import static org.xvm.util.Handy.appendString; @@ -316,17 +319,15 @@ public String getString(Source source) { * {@link Severity#WARNING}, {@link Severity#ERROR}, or {@link Severity#FATAL} * @param sCode the error code that identifies the error message * @param aoParam the parameters for the error message; may be null - * - * @return true to attempt to abort the process that reported the error, or false to attempt to - * continue the process */ - public boolean log(ErrorListener errs, Source source, Severity severity, String sCode, Object... aoParam) { + public void log(ErrorListener errs, Source source, Severity severity, String sCode, Object... aoParam) { if (aoParam == null || aoParam.length == 0) { aoParam = new Object[] {source == null ? toString() : getString(source)}; } - return errs.log(severity, sCode, aoParam, source, - source == null ? 0L : getStartPosition(), source == null ? 0L : getEndPosition()); + errs.log(severity, sCode, source == null + ? NOWHERE + : in(source, getStartPosition(), getEndPosition()), aoParam); } // ----- Object methods ------------------------------------------------------------------------ diff --git a/javatools/src/main/java/org/xvm/compiler/ast/AnnotationExpression.java b/javatools/src/main/java/org/xvm/compiler/ast/AnnotationExpression.java index 43ef4064b15..a1af49acbf3 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/AnnotationExpression.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/AnnotationExpression.java @@ -40,6 +40,9 @@ import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * A type annotation is used for type annotations with an optional argument list. */ @@ -61,6 +64,12 @@ public AnnotationExpression(Annotation anno, AstNode node) { this.lEndPos = node.getEndPosition(); } + /** Return an independent syntax tree with the same node type. */ + @Override + public AnnotationExpression copyTree() { + return (AnnotationExpression) super.copyTree(); + } + // ----- accessors ----------------------------------------------------------------------------- @Override @@ -300,7 +309,7 @@ public void validateContent(StageMgr mgr, ErrorListener errs) { if (exprNew instanceof NameExpression exprName && exprName.getMeaning() == NameExpression.Meaning.Method) { MethodConstant idMethod = (MethodConstant) - exprName.resolveRawArgument(ctx, false, ErrorListener.BLACKHOLE); + exprName.resolveRawArgument(ctx, false, silent(PROBE)); method = (MethodStructure) idMethod.getComponent(); } else if (exprNew instanceof LambdaExpression exprLambda) { method = exprLambda.getLambda(); diff --git a/javatools/src/main/java/org/xvm/compiler/ast/AnonInnerClass.java b/javatools/src/main/java/org/xvm/compiler/ast/AnonInnerClass.java index 8fb3762c434..f8eaabcbf6e 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/AnonInnerClass.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/AnonInnerClass.java @@ -21,6 +21,8 @@ import org.xvm.util.Severity; +import static java.util.Objects.requireNonNull; + /** * The AnonInnerClass represents the suggested shape for an anonymous inner class. */ @@ -33,7 +35,7 @@ public class AnonInnerClass { */ public AnonInnerClass(TypeExpression expr, ErrorListener errs) { assert expr != null; - assert errs != null; + requireNonNull(errs, "errs"); m_exprType = expr; f_errs = errs; diff --git a/javatools/src/main/java/org/xvm/compiler/ast/ArrayAccessExpression.java b/javatools/src/main/java/org/xvm/compiler/ast/ArrayAccessExpression.java index 97a9d77a931..faf549c4dc3 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/ArrayAccessExpression.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/ArrayAccessExpression.java @@ -19,10 +19,10 @@ import org.xvm.asm.constants.ArrayConstant; import org.xvm.asm.constants.IntConstant; -import org.xvm.asm.constants.RangeConstant; import org.xvm.asm.constants.MapConstant; import org.xvm.asm.constants.MethodConstant; import org.xvm.asm.constants.MethodInfo; +import org.xvm.asm.constants.RangeConstant; import org.xvm.asm.constants.SignatureConstant; import org.xvm.asm.constants.TypeConstant; import org.xvm.asm.constants.TypeInfo; @@ -38,6 +38,9 @@ import org.xvm.util.PackedInteger; import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * An array access expression is an expression followed by an array index expression. * @@ -191,6 +194,8 @@ public TypeFit testFit(Context ctx, TypeConstant typeRequired, boolean fExhausti } } + ErrorListener probe = silent(PROBE); + // at this point, either the "array" expression has no idea what type it will result in, or // its best guess doesn't have an array operator that results in the required type; "probe" // for some likely types that the array access operators are known to exist on @@ -206,7 +211,7 @@ public TypeFit testFit(Context ctx, TypeConstant typeRequired, boolean fExhausti // (which must be determinable at compile time in order to know which tuple elements // are being accessed) if (typeTarget != null && typeTarget.isTuple() - || expr.testFit(ctx, pool.typeTuple(), fExhaustive, null) == TypeFit.Fit) { + || expr.testFit(ctx, pool.typeTuple(), fExhaustive, probe) == TypeFit.Fit) { TypeConstant typeTest = determineTupleTestType(typeRequired); return typeTest == null // the only thing that we can say for sure at this point is that the return @@ -227,7 +232,7 @@ public TypeFit testFit(Context ctx, TypeConstant typeRequired, boolean fExhausti // this:type) and index could be a Range // array[index] - if (!isSliceOp() && exprIndex.testFit(ctx, pool.typeInt64(), fExhaustive, null).isFit()) { + if (!isSliceOp() && exprIndex.testFit(ctx, pool.typeInt64(), fExhaustive, probe).isFit()) { return exprTarget.testFit(ctx, pool.ensureParameterizedTypeConstant( pool.typeList(), typeRequired), fExhaustive, errs); } @@ -235,7 +240,7 @@ public TypeFit testFit(Context ctx, TypeConstant typeRequired, boolean fExhausti // array[index..index] or array[index..index) // REVIEW what if it is a Range or Range ??? if (exprTarget.testFit(ctx, typeRequired, fExhaustive, errs) == TypeFit.Fit - && exprIndex.testFit(ctx, pool.ensureRangeType(pool.typeInt64()), fExhaustive, null).isFit()) { + && exprIndex.testFit(ctx, pool.ensureRangeType(pool.typeInt64()), fExhaustive, probe).isFit()) { return TypeFit.Fit; } } else { // not a List, but might still be UniformIndexed and/or Sliceable @@ -261,7 +266,7 @@ public TypeFit testFit(Context ctx, TypeConstant typeRequired, boolean fExhausti // Index -> Element - probably does not matter what indexes are, since // Sliceable ops return this:type if (testType(ctx, exprTarget, typeTarget, pool.typeSliceable()) - && exprIndex.testFit(ctx, pool.typeRange(), fExhaustive, null).isFit()) { + && exprIndex.testFit(ctx, pool.typeRange(), fExhaustive, probe).isFit()) { fit = exprTarget.testFit(ctx, typeRequired, fExhaustive, errs); if (fit.isFit()) { return fit; @@ -291,16 +296,17 @@ && testType(ctx, exprTarget, typeTarget, pool.typeMap())) { Expression exprRow = indexes.get(1); // matrix[index,index] - if (exprCol.testFit(ctx, pool.typeInt64(), fExhaustive, null).isFit() && - exprRow.testFit(ctx, pool.typeInt64(), fExhaustive, null).isFit()) { + if (exprCol.testFit(ctx, pool.typeInt64(), fExhaustive, probe).isFit() && + exprRow.testFit(ctx, pool.typeInt64(), fExhaustive, probe).isFit()) { return expr.testFit(ctx, pool.ensureParameterizedTypeConstant( pool.typeMatrix(), typeRequired), fExhaustive, errs); } // matrix[index..index,index..index] TypeConstant typeInterval = pool.ensureRangeType(pool.typeInt64()); - if (typeRequired.isA(pool.typeMatrix()) && exprCol.testFit(ctx, typeInterval, fExhaustive, null).isFit() - && exprRow.testFit(ctx, typeInterval, fExhaustive, null).isFit()) { + if (typeRequired.isA(pool.typeMatrix()) + && exprCol.testFit(ctx, typeInterval, fExhaustive, probe).isFit() + && exprRow.testFit(ctx, typeInterval, fExhaustive, probe).isFit()) { // REVIEW same issue as array above TypeConstant typeElement = typeRequired.resolveGenericType("Element"); return typeElement == null @@ -327,14 +333,16 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe int cIndexes = indexes.size(); Expression[] aexprIndexes = indexes.toArray(new Expression[cIndexes]); + ErrorListener probe = silent(PROBE); + // test if the access is against a tuple expression if (typeArray != null && typeArray.isTuple() - || typeArray == null && exprArray.testFit(ctx, pool.typeTuple(), false, null) == TypeFit.Fit) { + || typeArray == null && exprArray.testFit(ctx, pool.typeTuple(), false, probe) == TypeFit.Fit) { if (typeArray == null) { typeArrayReq = pool.typeTuple(); } else { TypeConstant typeTupleTest = determineTupleTestType(typeRequired); - if (exprArray.testFit(ctx, typeTupleTest, false, null).isFit()) { + if (exprArray.testFit(ctx, typeTupleTest, false, probe).isFit()) { typeArrayReq = typeTupleTest; } } @@ -344,11 +352,11 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe if (typeRequired != null) { // array[index] TypeConstant typeElement = null; - if (!isSliceOp() && aexprIndexes[0].testFit(ctx, pool.typeInt64(), false, null).isFit()) { + if (!isSliceOp() && aexprIndexes[0].testFit(ctx, pool.typeInt64(), false, probe).isFit()) { typeElement = typeRequired; // array[index..index] or array[index..index) } else if (typeRequired.isA(pool.typeList()) && aexprIndexes[0].testFit(ctx, - pool.ensureRangeType(pool.typeInt64()), false, null).isFit()) { + pool.ensureRangeType(pool.typeInt64()), false, probe).isFit()) { // REVIEW keep this in sync with testFit() typeElement = typeRequired.resolveGenericType("Element"); } @@ -356,7 +364,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe if (typeElement != null) { TypeConstant typeArrayTest = pool.ensureParameterizedTypeConstant(typeArrayReq, typeElement); - if (exprArray.testFit(ctx, typeArrayTest, false, null).isFit()) { + if (exprArray.testFit(ctx, typeArrayTest, false, probe).isFit()) { typeArrayReq = typeArrayTest; } } @@ -369,7 +377,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe TypeConstant typeIndex = typeArray == null ? null : typeArray.resolveGenericType("Index"); - if (typeIndex == null || !aexprIndexes[0].testFit(ctx, typeIndex, false, null).isFit()) { + if (typeIndex == null || !aexprIndexes[0].testFit(ctx, typeIndex, false, probe).isFit()) { typeIndex = aexprIndexes[0].getImplicitType(ctx); typeIndex = determineIndexType(ctx, exprArray, typeArray, aexprIndexes, typeIndex); } @@ -379,7 +387,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe if (typeRequired != null) { TypeConstant typeTest = pool.ensureParameterizedTypeConstant( pool.typeIndexed(), typeIndex, typeRequired); - if (exprArray.testFit(ctx, typeTest, false, null).isFit()) { + if (exprArray.testFit(ctx, typeTest, false, probe).isFit()) { // we figured out what to ask for, including the index and element types typeElement = typeRequired; typeArrayReq = typeTest; @@ -390,7 +398,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe // we can only figure out the index type TypeConstant typeArrayTest = pool.ensureParameterizedTypeConstant(typeArrayReq, typeIndex); - if (exprArray.testFit(ctx, typeArrayTest, false, null).isFit()) { + if (exprArray.testFit(ctx, typeArrayTest, false, probe).isFit()) { typeArrayReq = typeArrayTest; } } @@ -404,14 +412,14 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe // matrix[index,index] TypeConstant typeElement = null; - if (exprCol.testFit(ctx, pool.typeInt64(), false, null).isFit() && - exprRow.testFit(ctx, pool.typeInt64(), false, null).isFit()) { + if (exprCol.testFit(ctx, pool.typeInt64(), false, probe).isFit() && + exprRow.testFit(ctx, pool.typeInt64(), false, probe).isFit()) { typeElement = typeRequired; // array[index..index] } else if (typeRequired.isA(pool.typeInterval())) { TypeConstant typeIntInterval = pool.ensureRangeType(pool.typeInt64()); - if (exprCol.testFit(ctx, typeIntInterval, false, null).isFit() && - exprRow.testFit(ctx, typeIntInterval, false, null).isFit()) { + if (exprCol.testFit(ctx, typeIntInterval, false, probe).isFit() && + exprRow.testFit(ctx, typeIntInterval, false, probe).isFit()) { // REVIEW keep this in sync with testFit() typeElement = typeRequired.resolveGenericType("Element"); } @@ -420,7 +428,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe if (typeElement != null) { TypeConstant typeArrayTest = pool.ensureParameterizedTypeConstant(typeArrayReq, typeRequired); - if (exprArray.testFit(ctx, typeArrayTest, false, null).isFit()) { + if (exprArray.testFit(ctx, typeArrayTest, false, probe).isFit()) { typeArrayReq = typeArrayTest; } } @@ -493,8 +501,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe System.arraycopy(aexprIndexes, 0, aExprSet, 0, cIndexes); // note: cannot fill in the expression that represents the value being set because it // hasn't yet been validated (for example, if our parent is an AssignmentStatement) - m_idSet = findOpMethod(ctx, typeArray, "setElement", "[]=", aExprSet, null, - ErrorListener.BLACKHOLE); + m_idSet = findOpMethod(ctx, typeArray, "setElement", "[]=", aExprSet, null, probe); } // tuple is different from other container types (like array) in that every field in the @@ -669,6 +676,8 @@ private MethodConstant findArrayAccessor( Set setMatch = new HashSet<>(); Map mapMethods = new HashMap<>(); + ErrorListener probe = silent(PROBE); + Set setAll = findPotentialOps(infoTarget, cArgs); NextOp: for (MethodConstant idMethod : setAll) { SignatureConstant sig = idMethod.getSignature(); @@ -701,7 +710,7 @@ private MethodConstant findArrayAccessor( TypeConstant typeArg = atypeArgs[i]; if (typeArg == null || !isAssignable(ctx, typeArg, typeParam)) { Expression exprArg = aexprArgs[i]; - if (!exprArg.testFit(ctx, typeParam, false, null).isFit()) { + if (!exprArg.testFit(ctx, typeParam, false, probe).isFit()) { continue NextOp; } } @@ -722,7 +731,7 @@ private MethodConstant findArrayAccessor( return switch (setMatch.size()) { case 0 -> null; case 1 -> setMatch.iterator().next(); - default -> chooseBest(setMatch, typeTarget, mapMethods, ErrorListener.BLACKHOLE); + default -> chooseBest(setMatch, typeTarget, mapMethods, probe); }; } @@ -860,7 +869,7 @@ private boolean indexesFit(Context ctx, MethodConstant method) { int cTypes = atypes.length; if (cTypes == indexes.size()) { for (int i = 0; i < cTypes; ++i) { - if (!indexes.get(i).testFit(ctx, atypes[i], false, null).isFit()) { + if (!indexes.get(i).testFit(ctx, atypes[i], false, silent(PROBE)).isFit()) { return false; } } @@ -1111,7 +1120,7 @@ TypeConstant estimateIndexType(Context ctx, TypeConstant typeTarget, String sInd TypeConstant typeIndex = typeTarget == null ? null : typeTarget.resolveGenericType(sIndexParam); - if (typeIndex == null || !exprIndex.testFit(ctx, typeIndex, false, null).isFit()) { + if (typeIndex == null || !exprIndex.testFit(ctx, typeIndex, false, silent(PROBE)).isFit()) { typeIndex = exprIndex.getImplicitType(ctx); } return typeIndex; @@ -1131,11 +1140,12 @@ TypeConstant estimateIndexType(Context ctx, TypeConstant typeTarget, String sInd */ private TypeConstant determineIndexType(Context ctx, Expression exprArray, TypeConstant typeArray, Expression[] aexprIndexes, TypeConstant typeIndex) { + ErrorListener probe = silent(PROBE); + ConstantPool pool = pool(); if (typeIndex == null) { if (typeArray != null) { - MethodConstant id = findOpMethod(ctx, typeArray, "getElement", "[]", aexprIndexes, - null, ErrorListener.BLACKHOLE); + MethodConstant id = findOpMethod(ctx, typeArray, "getElement", "[]", aexprIndexes, null, probe); if (id != null && id.getRawReturns().length >= 1) { typeIndex = id.getRawReturns()[0]; if (!pool.typeObject().isA(typeIndex)) { @@ -1148,7 +1158,7 @@ private TypeConstant determineIndexType(Context ctx, Expression exprArray, } if (exprArray.testFit(ctx, - pool.ensureParameterizedTypeConstant(pool.typeIndexed(), typeIndex), false, null).isFit()) { + pool.ensureParameterizedTypeConstant(pool.typeIndexed(), typeIndex), false, probe).isFit()) { return typeIndex; } @@ -1156,7 +1166,7 @@ private TypeConstant determineIndexType(Context ctx, Expression exprArray, for (MethodInfo info : setInfos) { typeIndex = info.getSignature().getRawReturns()[0]; if (exprArray.testFit(ctx, - pool.ensureParameterizedTypeConstant(pool.typeIndexed(), typeIndex), false, null).isFit()) { + pool.ensureParameterizedTypeConstant(pool.typeIndexed(), typeIndex), false, probe).isFit()) { return typeIndex; } } @@ -1179,7 +1189,7 @@ private boolean testType(Context ctx, Expression expr, TypeConstant typeExpr, Ty return true; } - return expr.testFit(ctx, typeTest, false, null) == TypeFit.Fit; + return expr.testFit(ctx, typeTest, false, silent(PROBE)) == TypeFit.Fit; } /** diff --git a/javatools/src/main/java/org/xvm/compiler/ast/AsExpression.java b/javatools/src/main/java/org/xvm/compiler/ast/AsExpression.java index 1c0afec533a..50eb62a2c90 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/AsExpression.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/AsExpression.java @@ -13,6 +13,9 @@ import org.xvm.compiler.Token; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * Expression for "expression as type". */ @@ -35,7 +38,7 @@ public long getEndPosition() { @Override public TypeConstant getImplicitType(Context ctx) { - return ((TypeExpression) expr2).ensureTypeConstant(ctx, null); + return ((TypeExpression) expr2).ensureTypeConstant(ctx, silent(PROBE)); } @Override @@ -59,7 +62,7 @@ protected Expression validate(Context ctx, TypeConstant typeRequired, ErrorListe TypeConstant type = exprType.ensureTypeConstant(ctx, errs).resolveAutoNarrowingBase(); - if (!exprType.isDynamic() && expr1.testFit(ctx, type, false, null).isFit()) { + if (!exprType.isDynamic() && expr1.testFit(ctx, type, false, silent(PROBE)).isFit()) { typeRequest = type; m_fCastRequired = false; } diff --git a/javatools/src/main/java/org/xvm/compiler/ast/AssertStatement.java b/javatools/src/main/java/org/xvm/compiler/ast/AssertStatement.java index 0d46b4e1c4b..e3256469227 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/AssertStatement.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/AssertStatement.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.xvm.asm.Argument; import org.xvm.asm.Assignment; @@ -591,11 +592,7 @@ public String toString() { if (!conds.isEmpty()) { sb.append(' ') - .append(conds.get(0)); - for (int i = 1, c = conds.size(); i < c; ++i) { - sb.append(", ") - .append(conds.get(i)); - } + .append(conds.stream().map(Object::toString).collect(Collectors.joining(", "))); } sb.append(';'); diff --git a/javatools/src/main/java/org/xvm/compiler/ast/AssignmentStatement.java b/javatools/src/main/java/org/xvm/compiler/ast/AssignmentStatement.java index 8ebc52bc5b8..a87a50065b5 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/AssignmentStatement.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/AssignmentStatement.java @@ -34,6 +34,9 @@ import org.xvm.util.Severity; +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * An assignment statement specifies an l-value, an assignment operator, and an r-value. * @@ -386,6 +389,8 @@ protected Statement validateImpl(Context ctx, ErrorListener errs) { return null; } + ErrorListener probe = silent(PROBE); + // regardless of whether the LValue is a statement or expression, all L-Values must be able // to provide an expression as a representative form Expression exprLeft = nodeLeft.getLValueExpression(); @@ -407,12 +412,12 @@ protected Statement validateImpl(Context ctx, ErrorListener errs) { // contributions Context ctxInfer = ctxRValue.enterInferring(atypeLeft[0]); - TypeFit fit = rvalue.testFitMulti(ctxInfer, atypeTest, false, null); + TypeFit fit = rvalue.testFitMulti(ctxInfer, atypeTest, false, probe); if (!fit.isFit() && cLeft > 1) { Expression exprUnpack = new UnpackExpression(rvalue, null); - fit = exprUnpack.testFitMulti(ctxInfer, atypeTest, false, null); + fit = exprUnpack.testFitMulti(ctxInfer, atypeTest, false, probe); if (fit.isFit()) { rvalue = exprUnpack; } @@ -456,7 +461,7 @@ protected Statement validateImpl(Context ctx, ErrorListener errs) { if (exprLeft instanceof NameExpression exprName && exprName.isDynamicVar()) { // test for a future assignment first TypeConstant typeFuture = pool.ensureFuture(typeLeft); - if (rvalue.testFit(ctxRValue, typeFuture, false, null).isFit()) { + if (rvalue.testFit(ctxRValue, typeFuture, false, probe).isFit()) { typeLeft = typeFuture; } } diff --git a/javatools/src/main/java/org/xvm/compiler/ast/AstNode.java b/javatools/src/main/java/org/xvm/compiler/ast/AstNode.java index d6d613d4ea7..88756897bcf 100644 --- a/javatools/src/main/java/org/xvm/compiler/ast/AstNode.java +++ b/javatools/src/main/java/org/xvm/compiler/ast/AstNode.java @@ -18,8 +18,12 @@ import java.util.NoSuchElementException; import java.util.Set; +import java.util.function.BiConsumer; + import java.util.stream.Collectors; +import org.jetbrains.annotations.NotNull; + import org.xvm.asm.Argument; import org.xvm.asm.Component; import org.xvm.asm.ComponentResolver; @@ -51,12 +55,20 @@ import org.xvm.compiler.Token; import org.xvm.compiler.ast.Expression.TypeFit; import org.xvm.compiler.ast.NameExpression.Meaning; +import org.xvm.compiler.ast.bindings.CursorBinding; +import org.xvm.compiler.ast.bindings.InvocationBinding; import org.xvm.util.ListMap; import org.xvm.util.Severity; import static org.xvm.util.Handy.indentLines; +import static org.xvm.asm.ErrorListener.NOWHERE; +import static org.xvm.asm.ErrorListener.in; + +import static org.xvm.asm.ErrorListener.Silence.PROBE; +import static org.xvm.asm.ErrorListener.silent; + /** * Common base class for all statements and expressions. */ @@ -124,15 +136,29 @@ protected T adopt(T child) { } /** - * Return an Iterable/Iterator that represents all the child nodes of this node. + * Return a one-shot cursor over the child nodes of this node. The cursor supports replacement + * and removal during compiler traversal. Its Iterable view uses the same cursor, not a new + * traversal; use {@link #childNodes()} when an independently iterable view is needed. * - * @return an Iterable of child nodes (from whence an Iterator can be obtained) + * @return a mutable traversal cursor */ public ChildIterator children() { Field[] fields = getChildFields(); return fields.length == 0 ? ChildIterator.EMPTY : new ChildIteratorImpl(fields); } + /** + * Return a view of this node's children that starts an independent traversal for each iterator. + * This is a live view, not a snapshot: subsequent traversals see subsequent AST changes. + * Callers that need a stable list must copy it while they own the tree; iteration does not + * provide synchronization with compiler mutation. + * + * @return an iterable view of the child nodes + */ + public Iterable childNodes() { + return this::children; + } + /** * Replace the specified child of this AstNode with a new child. * @@ -201,8 +227,39 @@ protected Field[] getChildFields() { return NO_FIELDS; } + /** + * Compatibility entry point for {@link #copyTree()}. + * + * @return a tree copy of this node + */ @Override + @SuppressWarnings("MethodDoesntCallSuperMethod") // copyTree owns allocation and copying. public AstNode clone() { + return copyTree(); + } + + /** + * Copy this node and its registered syntax children, preserving the concrete node type, source + * positions and compilation stage. Copied children belong to their copied owners. The root + * retains its original parent until a caller adopts it into another tree; copying does not + * replace or reparent any original node. + * + *

This is not a fresh parse or an isolated copy of all compiler state. The default + * implementation shallow-copies non-child fields, including resolved bindings and transient + * fields. Overrides must handle owned state that cannot be shared, such as a lambda's + * generated method. Recovery nodes construct fresh shells and copy their written syntax + * without carrying inherited validation or emission state. + * + *

The default implementation still uses shallow cloning followed by reflective copying of + * registered children; copied child lists are mutable. Constructor-based overrides must copy + * and adopt their children and preserve root metadata with {@link #copyTreeMetadataTo(AstNode)}. + * Specialize this method rather than {@link #clone()} so both entry points behave alike. + * Declare a covariant return type when specializing, so callers can copy that node type without + * a cast. A caller-selected method type parameter would not enforce that relationship. + * + * @return a tree copy of this node + */ + public AstNode copyTree() { AstNode that; try { that = (AstNode) super.clone(); @@ -222,21 +279,19 @@ public AstNode clone() { if (oVal != null) { if (oVal instanceof AstNode node) { - AstNode nodeNew = node.clone(); + var nodeNew = node.copyTree(); that.adopt(nodeNew); oVal = nodeNew; - } else if (oVal instanceof List list) { - ArrayList listNew = new ArrayList<>(); - for (AstNode node : (List) list) { - listNew.add(node.clone()); - } - + } else if (oVal instanceof List list) { + var listNew = list.stream() + .map(AstNode.class::cast) + .map(AstNode::copyTree) + .collect(Collectors.toCollection(ArrayList::new)); that.adopt(listNew); oVal = listNew; } else { - throw new IllegalStateException( - "unsupported container type: " + oVal.getClass().getSimpleName()); + throw new IllegalStateException("unsupported container type: " + oVal.getClass().getSimpleName()); } try { @@ -250,6 +305,21 @@ public AstNode clone() { return that; } + /** + * Preserve the root metadata of a constructor-based tree copy. The caller is responsible for + * copying and adopting children; this method does not transfer validation or emission state. + * + * @param copy the newly constructed node + * @param the concrete node type + * + * @return the supplied copy with this node's parent and compilation stage + */ + protected final T copyTreeMetadataTo(T copy) { + copy.setParent(getParent()); + copy.setStage(getStage()); + return copy; + } + /** * @return the current compilation stage for this node */ @@ -371,12 +441,15 @@ protected AstNode getCodeContainer() { /** * @return a unique counter in the context of the code container + * + * @throws IllegalStateException if this node has no enclosing code container */ protected int getCodeContainerCounter() { AstNode parent = getParent(); - return parent == null - ? null - : parent.getCodeContainerCounter(); + if (parent == null) { + throw new IllegalStateException("No enclosing code container for " + getClass().getSimpleName()); + } + return parent.getCodeContainerCounter(); } /** @@ -638,16 +711,12 @@ public Access getDefaultAccess() { * @param sCode the error code that identifies the error message * @param aoParam the parameters for the error message; may be null * - * @return true to attempt to abort the process that reported the error, or - * false to attempt to continue the process */ - public boolean log(ErrorListener errs, Severity severity, String sCode, Object... aoParam) { + public void log(ErrorListener errs, Severity severity, String sCode, Object... aoParam) { Source source = getSource(); - return errs == null - ? severity.ordinal() >= Severity.ERROR.ordinal() - : errs.log(severity, sCode, aoParam, source, - source == null ? 0L : getStartPosition(), - source == null ? 0L : getEndPosition()); + errs.log(severity, sCode, source == null + ? NOWHERE + : in(source, getStartPosition(), getEndPosition()), aoParam); } // ----- compile phases ------------------------------------------------------------------------ @@ -660,9 +729,9 @@ public boolean log(ErrorListener errs, Severity severity, String sCode, Object.. * checking for errors that are obvious from "this point down" (no lateral evaluation of * structures, because we can't count on them even existing yet.) * - *

  • The general idea is that this method recurses through the structure, allowing each node - * to introduce itself as the parent of each node under it, and that the nodes which will be - * structures in the resulting FileStructure will register themselves.
  • + *
  • This method recurses through the tree, establishing each node as the parent of its + * children. Nodes that represent structures in the resulting FileStructure also register + * themselves.
  • * *
  • Type parameters for the types must also be registered, because they are also types, and * they will be required to already be present when the second pass begins.
  • @@ -683,9 +752,8 @@ protected void registerStructures(StageMgr mgr, ErrorListener errs) { * compilation is not only unpredictable, but the potential exists for dependencies in either * direction (first to last and/or vice versa). * - *

    As a result, some questions may come to an AstNode to resolve that it is not yet prepared to - * resolve, in which case the caller (another AstNode) has to add itself to the list of nodes - * that require another pass. + *

    A node may be asked to resolve a name before its dependencies are ready. In that case, + * the calling node must defer its own resolution to another pass. * *