diff --git a/.github/actions/scala-native-build/action.yml b/.github/actions/scala-native-build/action.yml new file mode 100644 index 000000000..0eb876367 --- /dev/null +++ b/.github/actions/scala-native-build/action.yml @@ -0,0 +1,43 @@ +name: Build Scala Native FPP +description: Link the Scala Native FPP binary with an optional PGO mode + +inputs: + jvm-heap: + description: Maximum sbt JVM heap, such as 12G; empty uses JVM ergonomics + required: false + default: '' + pgo-mode: + description: Empty for a normal build, or generate/apply for PGO + required: false + default: '' + profile-directory: + description: Directory for raw profiles during PGO generation + required: false + default: '' + profile-file: + description: Merged profile file to consume during PGO application + required: false + default: '' + +runs: + using: composite + steps: + - shell: bash + working-directory: compiler + env: + FPP_PGO: ${{ inputs.pgo-mode }} + FPP_PGO_DIR: ${{ inputs.profile-directory }} + FPP_PGO_PROFILE: ${{ inputs.profile-file }} + SBT_JVM_HEAP: ${{ inputs.jvm-heap }} + run: | + sbt_args=(--batch) + if [[ -n "$SBT_JVM_HEAP" ]] + then + sbt_args+=("-J-Xmx$SBT_JVM_HEAP") + fi + if [[ "$FPP_PGO" == generate ]] + then + test -n "$FPP_PGO_DIR" + mkdir -p "$FPP_PGO_DIR" + fi + sbt "${sbt_args[@]}" "nativeFpp/nativeLink" diff --git a/.github/actions/scala-native-profile/action.yml b/.github/actions/scala-native-profile/action.yml new file mode 100644 index 000000000..339c440f9 --- /dev/null +++ b/.github/actions/scala-native-profile/action.yml @@ -0,0 +1,75 @@ +name: Profile Scala Native FPP +description: Exercise instrumented FPP, build a matching F Prime revision, and merge profiles + +inputs: + fpp-version: + description: FPP version required by the F Prime revision + required: true + profile-directory: + description: Directory containing raw LLVM profiles + required: true + profile-file: + description: Output path for the merged LLVM profile + required: true + python: + description: Python interpreter used for the F Prime environment + required: true + fprime-directory: + description: Temporary F Prime checkout path + required: true + fprime-venv: + description: Temporary F Prime virtual environment path + required: true + +runs: + using: composite + steps: + - name: Exercise FPP acceptance suite + shell: bash + working-directory: compiler + env: + LLVM_PROFILE_FILE: ${{ inputs.profile-directory }}/fpp-%p.profraw + run: ./test + + - name: Build matching F Prime revision + shell: bash + env: + FPP_VERSION: ${{ inputs.fpp-version }} + FPRIME_DIR: ${{ inputs.fprime-directory }} + FPRIME_VENV: ${{ inputs.fprime-venv }} + LLVM_PROFILE_FILE: ${{ inputs.profile-directory }}/fpp-%p.profraw + run: | + git clone https://github.com/nasa/fprime.git "$FPRIME_DIR" + cd "$FPRIME_DIR" + pin="fprime-fpp==$FPP_VERSION" + if grep -Fqx "$pin" requirements.txt + then + commit=HEAD + else + boundary=$(git log -1 -G "^$pin$" --format=%H -- requirements.txt) + test -n "$boundary" + commit="$boundary^" + fi + git checkout "$commit" + grep -Fqx "$pin" requirements.txt + git submodule update --init --recursive + git show -s --format='Profiling against F Prime %H (%cs): %s' + + uv venv "$FPRIME_VENV" --python '${{ inputs.python }}' + uv pip install --python "$FPRIME_VENV/bin/python" -r requirements.txt + export PATH="$GITHUB_WORKSPACE/compiler/bin:$FPRIME_VENV/bin:$PATH" + test "$(command -v fpp)" = "$GITHUB_WORKSPACE/compiler/bin/fpp" + fpp --version + cd TestDeploymentsProject + fprime-util generate + fprime-util build -j4 + + - name: Merge profiles + shell: bash + env: + PROFILE_DIRECTORY: ${{ inputs.profile-directory }} + PROFILE_FILE: ${{ inputs.profile-file }} + run: | + mkdir -p "$(dirname "$PROFILE_FILE")" + llvm-profdata merge -o "$PROFILE_FILE" "$PROFILE_DIRECTORY"/*.profraw + llvm-profdata show "$PROFILE_FILE" diff --git a/.github/actions/scala-native-stage/action.yml b/.github/actions/scala-native-stage/action.yml new file mode 100644 index 000000000..0d3d9879e --- /dev/null +++ b/.github/actions/scala-native-stage/action.yml @@ -0,0 +1,20 @@ +name: Stage Scala Native FPP +description: Stage the Scala Native dispatcher and per-tool wrappers + +runs: + using: composite + steps: + - shell: bash + working-directory: compiler + run: | + native_binary=$(find tools/fpp -type f -name fpp-out -print -quit) + test -n "$native_binary" + rm -rf bin + mkdir -p bin + install -m 755 "$native_binary" bin/fpp + while IFS= read -r tool + do + printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \ + > "bin/fpp-$tool" + chmod +x "bin/fpp-$tool" + done < tools.txt diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml index 1c1de107b..a29b19f38 100644 --- a/.github/workflows/build-native.yml +++ b/.github/workflows/build-native.yml @@ -1,4 +1,4 @@ -name: Build Native Images +name: Build Graal CE Native Images on: push: diff --git a/.github/workflows/build-scala-native.yml b/.github/workflows/build-scala-native.yml index 68fc7ba3c..f2b49ea25 100644 --- a/.github/workflows/build-scala-native.yml +++ b/.github/workflows/build-scala-native.yml @@ -19,12 +19,12 @@ jobs: include: - runner: macos-15 artifact: darwin-arm64 - python: python3 + python: '3.10' wheel_tag: macosx_14_0_arm64 - runner: ubuntu-22.04 artifact: manylinux_2_28_x86_64 container: quay.io/pypa/manylinux_2_28_x86_64 - python: /opt/python/cp39-cp39/bin/python + python: /opt/python/cp310-cp310/bin/python wheel_tag: manylinux_2_28_x86_64 runs-on: ${{ matrix.runner }} container: @@ -53,7 +53,7 @@ jobs: - if: runner.os == 'Linux' name: Install Linux build tools run: | - dnf install --assumeyes clang + dnf install --assumeyes clang llvm - name: Set up uv uses: astral-sh/setup-uv@v6 - name: Show toolchain versions @@ -92,11 +92,39 @@ jobs: fi util=lib/src/main/scala/util echo "Updating version to $version" + echo "FPP_VERSION=${version#v}" >> "$GITHUB_ENV" sed -i.update.bak -e "s/val v = .*/val v = \"$version\"/" \ $util/Version.scala - - name: Build Scala Native tools - working-directory: compiler - run: sbt --batch "nativeFpp/nativeLink" + - if: runner.os == 'macOS' + name: Build Scala Native tools + uses: ./.github/actions/scala-native-build + - if: runner.os == 'Linux' + name: Build instrumented Scala Native tools + uses: ./.github/actions/scala-native-build + with: + jvm-heap: 12G + pgo-mode: generate + profile-directory: ${{ runner.temp }}/fpp-pgo + - if: runner.os == 'Linux' + name: Stage instrumented Scala Native tools + uses: ./.github/actions/scala-native-stage + - if: runner.os == 'Linux' + name: Profile instrumented Scala Native tools + uses: ./.github/actions/scala-native-profile + with: + fpp-version: ${{ env.FPP_VERSION }} + profile-directory: ${{ runner.temp }}/fpp-pgo + profile-file: ${{ github.workspace }}/compiler/pgo/fpp.profdata + python: ${{ matrix.python }} + fprime-directory: ${{ runner.temp }}/fprime + fprime-venv: ${{ runner.temp }}/fprime-venv + - if: runner.os == 'Linux' + name: Build profile-guided Scala Native tools + uses: ./.github/actions/scala-native-build + with: + jvm-heap: 12G + pgo-mode: apply + profile-file: ${{ github.workspace }}/compiler/pgo/fpp.profdata - name: Restore Version.scala if: always() working-directory: compiler @@ -108,19 +136,7 @@ jobs: sed -i.restore.bak -e "s/val v = .*/val v = \"[unknown version]\"/" \ $util/Version.scala - name: Stage Scala Native tools - working-directory: compiler - run: | - native_binary=$(find tools/fpp -type f -name fpp-out -print -quit) - test -n "$native_binary" - rm -rf bin - mkdir -p bin - install -m 755 "$native_binary" bin/fpp - while IFS= read -r tool - do - printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \ - > "bin/fpp-$tool" - chmod +x "bin/fpp-$tool" - done < tools.txt + uses: ./.github/actions/scala-native-stage - name: Test Scala Native tools working-directory: compiler run: ./test diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 79f254995..baa80301a 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -1,4 +1,4 @@ -name: 'fprime-build-native-scala' +name: 'Build Graal CE Native' on: workflow_call: inputs: @@ -109,7 +109,7 @@ jobs: "tag": "manylinux_2_28_aarch64", "container": "quay.io/pypa/manylinux_2_28_aarch64" } - + ] } tags = {"tag": ["jar"] + [run["tag"] for run in matrix["run"]]} diff --git a/.gitignore b/.gitignore index 436b5be07..f176eb65f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .* !.gitignore !.gitattributes +!.github/ __SHADOW__ __pycache__ dist/ diff --git a/compiler/.gitignore b/compiler/.gitignore index 4296a0b8d..601176618 100644 --- a/compiler/.gitignore +++ b/compiler/.gitignore @@ -1,9 +1,12 @@ bin num_failed.txt target/ +target-native/ test-output.txt *.o native-fpp-* *.class # Version is not checked in during typical development lib/src/main/scala/util/Version.scala +# PGO profile +pgo/fpp.profdata diff --git a/compiler/build.sbt b/compiler/build.sbt index 2d8f91f7d..6c3a052a4 100644 --- a/compiler/build.sbt +++ b/compiler/build.sbt @@ -17,24 +17,23 @@ lazy val settings = Seq( Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oNCXELOPQRM"), ) -lazy val jvmDependencies = Seq( - "com.github.scopt" %% "scopt" % "4.0.1", - "io.circe" %% "circe-core" % "0.14.3", - "io.circe" %% "circe-generic" % "0.14.3", - "io.circe" %% "circe-parser" % "0.14.3", - "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1", - "org.scala-lang.modules" %% "scala-xml" % "2.1.0", - "org.scalatest" %% "scalatest" % "3.2.12" % "test", +// Shared (org, artifact, version); JVM cross-builds with %%, Scala Native with %%%. +lazy val sharedDependencies = Seq( + ("com.github.scopt", "scopt", "4.0.1"), + ("io.circe", "circe-core", "0.14.3"), + ("io.circe", "circe-generic", "0.14.3"), + ("io.circe", "circe-parser", "0.14.3"), + ("org.scala-lang.modules", "scala-parser-combinators", "2.1.1"), + ("org.scala-lang.modules", "scala-xml", "2.1.0"), ) -lazy val nativeDependencies = Def.setting(Seq( - "com.github.scopt" %%% "scopt" % "4.0.1", - "io.circe" %%% "circe-core" % "0.14.3", - "io.circe" %%% "circe-generic" % "0.14.3", - "io.circe" %%% "circe-parser" % "0.14.3", - "org.scala-lang.modules" %%% "scala-parser-combinators" % "2.1.1", - "org.scala-lang.modules" %%% "scala-xml" % "2.1.0", -)) +lazy val jvmDependencies = + sharedDependencies.map { case (o, a, v) => o %% a % v } :+ + ("org.scalatest" %% "scalatest" % "3.2.12" % "test") + +lazy val nativeDependencies = Def.setting( + sharedDependencies.map { case (o, a, v) => o %%% a % v } +) lazy val jvmSettings = settings ++ Seq( libraryDependencies ++= jvmDependencies, @@ -77,10 +76,36 @@ lazy val nativeFpp = (project in file("tools/fpp")) .settings(nativeSettings) .settings( name := "fpp", - nativeConfig ~= { config => - config.withLTO(LTO.thin).withMode(Mode.releaseFast).withGC(GC.none) - .withLinkStubs(true) - .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions) + nativeConfig := { + // FPP_PGO selects an explicit build path: + // unset normal release/test build (no PGO; releaseFast with full LTO) + // generate matching build instrumented to write raw profiles + // apply matching build that consumes FPP_PGO_PROFILE, or the + // local pgo/fpp.profdata when FPP_PGO_PROFILE is unset. + val (compile, link) = sys.env.get("FPP_PGO").map(_.trim).filter(_.nonEmpty) match { + case Some("generate") => // instrumented + val dir = sys.env.getOrElse("FPP_PGO_DIR", "/tmp/fpp-pgo") + val rt = sys.env.get("FPP_PGO_RUNTIME").map(_.trim).filter(_.nonEmpty) + (Seq("-fprofile-generate=" + dir), + Seq("-fprofile-generate=" + dir) ++ + rt.toSeq.flatMap(a => Seq("-Wl,--whole-archive", a, "-Wl,--no-whole-archive"))) + case Some("apply") => + val localProf = (ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata" + val prof = sys.env.get("FPP_PGO_PROFILE").map(new java.io.File(_)).getOrElse(localProf) + if (!prof.exists) + sys.error("FPP_PGO=apply requires FPP_PGO_PROFILE or pgo/fpp.profdata") + (Seq("-fprofile-use=" + prof.getAbsolutePath, + "-Wno-profile-instr-out-of-date", "-Wno-profile-instr-unprofiled"), + Seq.empty[String]) + case None => + (Seq.empty[String], Seq.empty[String]) + case Some(value) => + sys.error(s"Unsupported FPP_PGO value '$value'; use generate or apply") + } + val config = nativeConfig.value + config.withMode(Mode.releaseFast).withLTO(LTO.full).withGC(GC.none).withLinkStubs(true) + .withCompileOptions(config.compileOptions ++ compile) + .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions ++ link) } ) .dependsOn(nativeLib) diff --git a/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala b/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala index 6a463802d..79245d9a5 100644 --- a/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala +++ b/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala @@ -162,9 +162,9 @@ object LocateDefsFppWriter extends AstVisitor with LineUtils { case Some(dir) => dir case None => "" } - val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath - val relativePath = baseDirPath.relativize(path) - val fileNode = AstNode.create(relativePath.normalize.toString) + val baseDirPath = java.nio.file.Paths.get(baseDir) + val relativePath = File.relativize(baseDirPath)(path) + val fileNode = AstNode.create(relativePath.toString) val specLocNode = AstNode.create(Ast.SpecLoc(kind, qualIdentNode, fileNode, isDictionaryDef)) val specLocAnnotatedNode = (Nil, specLocNode, Nil) FppWriter.specLocAnnotatedNode((), specLocAnnotatedNode) diff --git a/compiler/lib/src/main/scala/util/File.scala b/compiler/lib/src/main/scala/util/File.scala index cbe74fe82..83dac9883 100644 --- a/compiler/lib/src/main/scala/util/File.scala +++ b/compiler/lib/src/main/scala/util/File.scala @@ -64,6 +64,35 @@ object File { /** Construct a file from a string representing a file path */ def fromString(s: String): File = Path(getJavaPath(s)) + /** Relativize a path without relying on Path.relativize. + * + * Scala Native 0.4.17 drops parent segments when the target is outside the + * base directory. Its Path.normalize also cancels consecutive leading `..` + * segments, so construct and return the clean lexical path directly. + */ + def relativize(base: JavaPath)(target: JavaPath): JavaPath = { + val normalizedBase = base.toAbsolutePath.normalize + val normalizedTarget = target.toAbsolutePath.normalize + if normalizedBase.getRoot != normalizedTarget.getRoot + then throw new IllegalArgumentException( + s"cannot relativize paths with different roots: $normalizedBase and $normalizedTarget" + ) + + def getNames(path: JavaPath) = + (0 until path.getNameCount).map(path.getName(_).toString) + + val baseNames = getNames(normalizedBase) + val targetNames = getNames(normalizedTarget) + val commonCount = baseNames.zip(targetNames).takeWhile { + case (baseName, targetName) => baseName == targetName + }.length + val relativeNames = + Seq.fill(baseNames.length - commonCount)("..") ++ targetNames.drop(commonCount) + relativeNames.foldLeft(java.nio.file.Paths.get("")) { + case (path, name) => path.resolve(name) + } + } + /** Remove the longest prefix from a Java path */ def removeLongestPrefix(prefixes: List[String])(path: JavaPath): JavaPath = { def removePrefix(s: String) = { diff --git a/compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt b/compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt new file mode 100644 index 000000000..5e885bc5e --- /dev/null +++ b/compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt @@ -0,0 +1,55 @@ +locate component C at "../../defs-1.fpp" +locate component M.C at "../../defs-2.fpp" +locate constant C.a at "../../defs-1.fpp" +locate constant M.C.a at "../../defs-2.fpp" +locate constant M.SM.a at "../../defs-2.fpp" +locate constant M.a at "../../defs-2.fpp" +locate constant SM.a at "../../defs-1.fpp" +locate constant a at "../../defs-1.fpp" +locate dictionary constant a2 at "../../defs-1.fpp" +locate dictionary type Alias2 at "../../defs-1.fpp" +locate dictionary type C.A2 at "../../defs-1.fpp" +locate dictionary type M.C.E2 at "../../defs-2.fpp" +locate dictionary type M.C.S2 at "../../defs-2.fpp" +locate instance M.T at "../../defs-2.fpp" +locate instance M.c at "../../defs-2.fpp" +locate instance T at "../../defs-1.fpp" +locate instance c at "../../defs-1.fpp" +locate interface I at "../../defs-1.fpp" +locate port M.P at "../../defs-2.fpp" +locate port P at "../../defs-1.fpp" +locate state machine M.C.S at "../../defs-2.fpp" +locate state machine M.SM at "../../defs-2.fpp" +locate state machine SM at "../../defs-1.fpp" +locate type A at "../../defs-1.fpp" +locate type Alias at "../../defs-1.fpp" +locate type C.A at "../../defs-1.fpp" +locate type C.Alias at "../../defs-1.fpp" +locate type C.E at "../../defs-1.fpp" +locate type C.S at "../../defs-1.fpp" +locate type C.T at "../../defs-1.fpp" +locate type E at "../../defs-1.fpp" +locate type M.A at "../../defs-2.fpp" +locate type M.Alias at "../../defs-2.fpp" +locate type M.C.A at "../../defs-2.fpp" +locate type M.C.Alias at "../../defs-2.fpp" +locate type M.C.E at "../../defs-2.fpp" +locate type M.C.S at "../../defs-2.fpp" +locate type M.C.T at "../../defs-2.fpp" +locate type M.E at "../../defs-2.fpp" +locate type M.S at "../../defs-2.fpp" +locate type M.SM.A at "../../defs-2.fpp" +locate type M.SM.E at "../../defs-2.fpp" +locate type M.SM.S at "../../defs-2.fpp" +locate type M.SM.State at "../../defs-2.fpp" +locate type M.SM.T at "../../defs-2.fpp" +locate type M.SM.X at "../../defs-2.fpp" +locate type M.T at "../../defs-2.fpp" +locate type S at "../../defs-1.fpp" +locate type SM.A at "../../defs-1.fpp" +locate type SM.E at "../../defs-1.fpp" +locate type SM.S at "../../defs-1.fpp" +locate type SM.State at "../../defs-1.fpp" +locate type SM.T at "../../defs-1.fpp" +locate type SM.X at "../../defs-1.fpp" +locate type T at "../../defs-1.fpp" diff --git a/compiler/tools/fpp-locate-defs/test/run b/compiler/tools/fpp-locate-defs/test/run index 2a88ece40..a326b4e71 100755 --- a/compiler/tools/fpp-locate-defs/test/run +++ b/compiler/tools/fpp-locate-defs/test/run @@ -53,4 +53,9 @@ defs_dir() run_test '-d defs' 'defs/defs-1.fpp defs/defs-2' defs_dir } +defs_parent_dir() +{ + run_test '-d defs/build/subdir' 'defs/defs-1.fpp defs/defs-2' defs_parent_dir +} + run_suite $tests diff --git a/compiler/tools/fpp-locate-defs/test/tests.sh b/compiler/tools/fpp-locate-defs/test/tests.sh index a2423bb00..9d6fc598d 100644 --- a/compiler/tools/fpp-locate-defs/test/tests.sh +++ b/compiler/tools/fpp-locate-defs/test/tests.sh @@ -1,6 +1,7 @@ tests=" defs defs_dir +defs_parent_dir include stdin " diff --git a/compiler/tools/fpp-locate-defs/test/update-ref b/compiler/tools/fpp-locate-defs/test/update-ref index 6b296e32c..c5b359c20 100755 --- a/compiler/tools/fpp-locate-defs/test/update-ref +++ b/compiler/tools/fpp-locate-defs/test/update-ref @@ -45,6 +45,11 @@ defs_dir() update '-d defs' 'defs/defs-1.fpp defs/defs-2' defs_dir } +defs_parent_dir() +{ + update '-d defs/build/subdir' 'defs/defs-1.fpp defs/defs-2' defs_parent_dir +} + stdin() { $fpp_locate_defs < defs/defs-1.fpp > stdin.ref.txt diff --git a/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala b/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala index dffe190a0..50d4bd944 100644 --- a/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala +++ b/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala @@ -55,9 +55,9 @@ object FPPLocateUses { case Some(dir) => dir case None => "" } - val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath - val relativePath = baseDirPath.relativize(path) - val fileNode = AstNode.create(relativePath.normalize.toString) + val baseDirPath = java.nio.file.Paths.get(baseDir) + val relativePath = File.relativize(baseDirPath)(path) + val fileNode = AstNode.create(relativePath.toString) val kind = s match { case _: Symbol.AbsType => Ast.SpecLoc.Type case _: Symbol.AliasType => Ast.SpecLoc.Type