From 5ea77f7fea410ddc36e4d175b7a4f895adcdba8e Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 10:50:29 -0500 Subject: [PATCH 01/10] PGO and full bore optimization --- compiler/.gitignore | 3 +++ compiler/build.sbt | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler/.gitignore b/compiler/.gitignore index 4296a0b8d..601176618 100644 --- a/compiler/.gitignore +++ b/compiler/.gitignore @@ -1,9 +1,12 @@ bin num_failed.txt target/ +target-native/ test-output.txt *.o native-fpp-* *.class # Version is not checked in during typical development lib/src/main/scala/util/Version.scala +# PGO profile +pgo/fpp.profdata diff --git a/compiler/build.sbt b/compiler/build.sbt index 2d8f91f7d..9ee0670bd 100644 --- a/compiler/build.sbt +++ b/compiler/build.sbt @@ -77,9 +77,18 @@ lazy val nativeFpp = (project in file("tools/fpp")) .settings(nativeSettings) .settings( name := "fpp", - nativeConfig ~= { config => - config.withLTO(LTO.thin).withMode(Mode.releaseFast).withGC(GC.none) + nativeConfig := { + val config = nativeConfig.value + // Absolute path to the committed PGO profile (clang resolves it at compile time). + val profile = ((ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata") + val pgoOpts = + if (profile.exists) Seq("-fprofile-use=" + profile.getAbsolutePath, + "-Wno-profile-instr-out-of-date", + "-Wno-profile-instr-unprofiled") + else Seq.empty[String] + config.withLTO(LTO.full).withMode(Mode.releaseFull).withGC(GC.none) .withLinkStubs(true) + .withCompileOptions(config.compileOptions ++ Seq("-O3") ++ pgoOpts) .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions) } ) From 9266f0f0290e06c73cf2b6b4e082e9ec51bca101 Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 12:27:35 -0500 Subject: [PATCH 02/10] PGO gen and PGO consume --- compiler/build.sbt | 61 ++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/compiler/build.sbt b/compiler/build.sbt index 9ee0670bd..8d7c436a2 100644 --- a/compiler/build.sbt +++ b/compiler/build.sbt @@ -17,24 +17,23 @@ lazy val settings = Seq( Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oNCXELOPQRM"), ) -lazy val jvmDependencies = Seq( - "com.github.scopt" %% "scopt" % "4.0.1", - "io.circe" %% "circe-core" % "0.14.3", - "io.circe" %% "circe-generic" % "0.14.3", - "io.circe" %% "circe-parser" % "0.14.3", - "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1", - "org.scala-lang.modules" %% "scala-xml" % "2.1.0", - "org.scalatest" %% "scalatest" % "3.2.12" % "test", +// Shared (org, artifact, version); JVM cross-builds with %%, Scala Native with %%%. +lazy val sharedDependencies = Seq( + ("com.github.scopt", "scopt", "4.0.1"), + ("io.circe", "circe-core", "0.14.3"), + ("io.circe", "circe-generic", "0.14.3"), + ("io.circe", "circe-parser", "0.14.3"), + ("org.scala-lang.modules", "scala-parser-combinators", "2.1.1"), + ("org.scala-lang.modules", "scala-xml", "2.1.0"), ) -lazy val nativeDependencies = Def.setting(Seq( - "com.github.scopt" %%% "scopt" % "4.0.1", - "io.circe" %%% "circe-core" % "0.14.3", - "io.circe" %%% "circe-generic" % "0.14.3", - "io.circe" %%% "circe-parser" % "0.14.3", - "org.scala-lang.modules" %%% "scala-parser-combinators" % "2.1.1", - "org.scala-lang.modules" %%% "scala-xml" % "2.1.0", -)) +lazy val jvmDependencies = + sharedDependencies.map { case (o, a, v) => o %% a % v } :+ + ("org.scalatest" %% "scalatest" % "3.2.12" % "test") + +lazy val nativeDependencies = Def.setting( + sharedDependencies.map { case (o, a, v) => o %%% a % v } +) lazy val jvmSettings = settings ++ Seq( libraryDependencies ++= jvmDependencies, @@ -78,18 +77,26 @@ lazy val nativeFpp = (project in file("tools/fpp")) .settings( name := "fpp", nativeConfig := { + // PGO via env FPP_PGO; see compiler/pgo/README.md. + val localProf = (ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata" + val (mode, lto, compile, link) = sys.env.get("FPP_PGO").map(_.trim).filter(_.nonEmpty) match { + case Some("generate") => // instrumented + val dir = sys.env.getOrElse("FPP_PGO_DIR", "/tmp/fpp-pgo") + val rt = sys.env.get("FPP_PGO_RUNTIME").map(_.trim).filter(_.nonEmpty) + (Mode.releaseFast, LTO.thin, Seq("-fprofile-generate=" + dir), + Seq("-fprofile-generate=" + dir) ++ + rt.toSeq.flatMap(a => Seq("-Wl,--whole-archive", a, "-Wl,--no-whole-archive"))) + case other => // optimized, applying a profile when present + val prof = other.map(new java.io.File(_)).filter(_.exists).orElse(Some(localProf).filter(_.exists)) + (Mode.releaseFull, LTO.full, + prof.toSeq.flatMap(p => Seq("-fprofile-use=" + p.getAbsolutePath, + "-Wno-profile-instr-out-of-date", "-Wno-profile-instr-unprofiled")), + Seq.empty[String]) + } val config = nativeConfig.value - // Absolute path to the committed PGO profile (clang resolves it at compile time). - val profile = ((ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata") - val pgoOpts = - if (profile.exists) Seq("-fprofile-use=" + profile.getAbsolutePath, - "-Wno-profile-instr-out-of-date", - "-Wno-profile-instr-unprofiled") - else Seq.empty[String] - config.withLTO(LTO.full).withMode(Mode.releaseFull).withGC(GC.none) - .withLinkStubs(true) - .withCompileOptions(config.compileOptions ++ Seq("-O3") ++ pgoOpts) - .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions) + config.withMode(mode).withLTO(lto).withGC(GC.none).withLinkStubs(true) + .withCompileOptions(config.compileOptions ++ Seq("-O3") ++ compile) + .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions ++ link) } ) .dependsOn(nativeLib) From b0416f8083f7ae3427a8f2b63d96996a1819868d Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 15:22:55 -0500 Subject: [PATCH 03/10] Normalize before relativize --- .../lib/src/main/scala/codegen/LocateDefsFppWriter.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala b/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala index 6a463802d..9e271f056 100644 --- a/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala +++ b/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala @@ -162,8 +162,9 @@ object LocateDefsFppWriter extends AstVisitor with LineUtils { case Some(dir) => dir case None => "" } - val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath - val relativePath = baseDirPath.relativize(path) + // Normalize both operands before relativize (fixes Scala Native bug) + val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath.normalize + val relativePath = baseDirPath.relativize(path.toAbsolutePath.normalize) val fileNode = AstNode.create(relativePath.normalize.toString) val specLocNode = AstNode.create(Ast.SpecLoc(kind, qualIdentNode, fileNode, isDictionaryDef)) val specLocAnnotatedNode = (Nil, specLocNode, Nil) From 6bee641dfef59dfad79537811832c4bc5e23ee6d Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 15:32:17 -0500 Subject: [PATCH 04/10] One more normalize cluster --- compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala b/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala index dffe190a0..ff9f21fcb 100644 --- a/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala +++ b/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala @@ -55,8 +55,9 @@ object FPPLocateUses { case Some(dir) => dir case None => "" } - val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath - val relativePath = baseDirPath.relativize(path) + // Normalize both operands before relativize (fixes Scala Native bug) + val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath.normalize + val relativePath = baseDirPath.relativize(path.toAbsolutePath.normalize) val fileNode = AstNode.create(relativePath.normalize.toString) val kind = s match { case _: Symbol.AbsType => Ast.SpecLoc.Type From f539c354cc6389cedfeb44a5f2b129fc2c96b7aa Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 21:44:26 -0500 Subject: [PATCH 05/10] PGO gen, apply, or no-PGO flows (all releaseFast for now) --- .github/workflows/build-scala-native.yml | 52 +++++++++++++++++++++++- compiler/build.sbt | 31 +++++++++----- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-scala-native.yml b/.github/workflows/build-scala-native.yml index 68fc7ba3c..4d3f2d301 100644 --- a/.github/workflows/build-scala-native.yml +++ b/.github/workflows/build-scala-native.yml @@ -53,7 +53,7 @@ jobs: - if: runner.os == 'Linux' name: Install Linux build tools run: | - dnf install --assumeyes clang + dnf install --assumeyes clang llvm - name: Set up uv uses: astral-sh/setup-uv@v6 - name: Show toolchain versions @@ -94,9 +94,57 @@ jobs: echo "Updating version to $version" sed -i.update.bak -e "s/val v = .*/val v = \"$version\"/" \ $util/Version.scala - - name: Build Scala Native tools + - if: runner.os == 'macOS' + name: Build Scala Native tools working-directory: compiler run: sbt --batch "nativeFpp/nativeLink" + - if: runner.os == 'Linux' + name: Build instrumented Scala Native tools + working-directory: compiler + env: + FPP_PGO: generate + FPP_PGO_DIR: ${{ runner.temp }}/fpp-pgo + run: | + mkdir -p "$FPP_PGO_DIR" + sbt -J-Xmx12G --batch "nativeFpp/nativeLink" + - if: runner.os == 'Linux' + name: Stage instrumented Scala Native tools + working-directory: compiler + run: | + native_binary=$(find tools/fpp -type f -name fpp-out -print -quit) + test -n "$native_binary" + mkdir -p bin + install -m 755 "$native_binary" bin/fpp + while IFS= read -r tool + do + printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \ + > "bin/fpp-$tool" + chmod +x "bin/fpp-$tool" + done < tools.txt + - if: runner.os == 'Linux' + name: Exercise instrumented Scala Native tools + working-directory: compiler + env: + # Each test launches a new fpp process; keep every process's counters. + LLVM_PROFILE_FILE: ${{ runner.temp }}/fpp-pgo/fpp-%p.profraw + run: ./test + - if: runner.os == 'Linux' + name: Merge Scala Native profiles + working-directory: compiler + env: + FPP_PGO_DIR: ${{ runner.temp }}/fpp-pgo + run: | + mkdir -p pgo + llvm-profdata merge -o pgo/fpp.profdata "$FPP_PGO_DIR"/*.profraw + llvm-profdata show pgo/fpp.profdata + - if: runner.os == 'Linux' + name: Build profile-guided Scala Native tools + working-directory: compiler + env: + FPP_PGO: apply + # -J forwards the heap limit through sbt to its JVM. The runner has + # 16 GB, leaving 4 GB for Clang, the linker, and other processes. + run: sbt -J-Xmx12G --batch "nativeFpp/nativeLink" - name: Restore Version.scala if: always() working-directory: compiler diff --git a/compiler/build.sbt b/compiler/build.sbt index 8d7c436a2..6c3a052a4 100644 --- a/compiler/build.sbt +++ b/compiler/build.sbt @@ -77,25 +77,34 @@ lazy val nativeFpp = (project in file("tools/fpp")) .settings( name := "fpp", nativeConfig := { - // PGO via env FPP_PGO; see compiler/pgo/README.md. - val localProf = (ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata" - val (mode, lto, compile, link) = sys.env.get("FPP_PGO").map(_.trim).filter(_.nonEmpty) match { + // FPP_PGO selects an explicit build path: + // unset normal release/test build (no PGO; releaseFast with full LTO) + // generate matching build instrumented to write raw profiles + // apply matching build that consumes FPP_PGO_PROFILE, or the + // local pgo/fpp.profdata when FPP_PGO_PROFILE is unset. + val (compile, link) = sys.env.get("FPP_PGO").map(_.trim).filter(_.nonEmpty) match { case Some("generate") => // instrumented val dir = sys.env.getOrElse("FPP_PGO_DIR", "/tmp/fpp-pgo") val rt = sys.env.get("FPP_PGO_RUNTIME").map(_.trim).filter(_.nonEmpty) - (Mode.releaseFast, LTO.thin, Seq("-fprofile-generate=" + dir), + (Seq("-fprofile-generate=" + dir), Seq("-fprofile-generate=" + dir) ++ rt.toSeq.flatMap(a => Seq("-Wl,--whole-archive", a, "-Wl,--no-whole-archive"))) - case other => // optimized, applying a profile when present - val prof = other.map(new java.io.File(_)).filter(_.exists).orElse(Some(localProf).filter(_.exists)) - (Mode.releaseFull, LTO.full, - prof.toSeq.flatMap(p => Seq("-fprofile-use=" + p.getAbsolutePath, - "-Wno-profile-instr-out-of-date", "-Wno-profile-instr-unprofiled")), + case Some("apply") => + val localProf = (ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata" + val prof = sys.env.get("FPP_PGO_PROFILE").map(new java.io.File(_)).getOrElse(localProf) + if (!prof.exists) + sys.error("FPP_PGO=apply requires FPP_PGO_PROFILE or pgo/fpp.profdata") + (Seq("-fprofile-use=" + prof.getAbsolutePath, + "-Wno-profile-instr-out-of-date", "-Wno-profile-instr-unprofiled"), Seq.empty[String]) + case None => + (Seq.empty[String], Seq.empty[String]) + case Some(value) => + sys.error(s"Unsupported FPP_PGO value '$value'; use generate or apply") } val config = nativeConfig.value - config.withMode(mode).withLTO(lto).withGC(GC.none).withLinkStubs(true) - .withCompileOptions(config.compileOptions ++ Seq("-O3") ++ compile) + config.withMode(Mode.releaseFast).withLTO(LTO.full).withGC(GC.none).withLinkStubs(true) + .withCompileOptions(config.compileOptions ++ compile) .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions ++ link) } ) From 0d6e5a0a8761bdeb349318e1da8043833c5934b3 Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 21:52:24 -0500 Subject: [PATCH 06/10] Build, Profile, Rebuild (w/ multiple action files) --- .github/actions/scala-native-build/action.yml | 43 ++++++++++ .../actions/scala-native-profile/action.yml | 75 ++++++++++++++++++ .github/actions/scala-native-stage/action.yml | 20 +++++ .github/workflows/build-scala-native.yml | 78 ++++++------------- .gitignore | 1 + 5 files changed, 162 insertions(+), 55 deletions(-) create mode 100644 .github/actions/scala-native-build/action.yml create mode 100644 .github/actions/scala-native-profile/action.yml create mode 100644 .github/actions/scala-native-stage/action.yml diff --git a/.github/actions/scala-native-build/action.yml b/.github/actions/scala-native-build/action.yml new file mode 100644 index 000000000..ca2d069a7 --- /dev/null +++ b/.github/actions/scala-native-build/action.yml @@ -0,0 +1,43 @@ +name: Build Scala Native FPP +description: Link the Scala Native FPP binary with an optional PGO mode + +inputs: + jvm-heap: + description: Maximum sbt JVM heap, such as 12G; empty uses JVM ergonomics + required: false + default: '' + pgo-mode: + description: Empty for a normal build, or generate/apply for PGO + required: false + default: '' + profile-directory: + description: Directory for raw profiles during PGO generation + required: false + default: '' + profile-file: + description: Merged profile file to consume during PGO application + required: false + default: '' + +runs: + using: composite + steps: + - shell: bash + working-directory: compiler + env: + FPP_PGO: ${{ inputs.pgo-mode }} + FPP_PGO_DIR: ${{ inputs.profile-directory }} + FPP_PGO_PROFILE: ${{ inputs.profile-file }} + SBT_JVM_HEAP: ${{ inputs.jvm-heap }} + run: | + sbt_args=(--batch) + if [[ -n "$SBT_JVM_HEAP" ]] + then + sbt_args=("-J-Xmx$SBT_JVM_HEAP" "${sbt_args[@]}") + fi + if [[ "$FPP_PGO" == generate ]] + then + test -n "$FPP_PGO_DIR" + mkdir -p "$FPP_PGO_DIR" + fi + sbt "${sbt_args[@]}" "nativeFpp/nativeLink" diff --git a/.github/actions/scala-native-profile/action.yml b/.github/actions/scala-native-profile/action.yml new file mode 100644 index 000000000..339c440f9 --- /dev/null +++ b/.github/actions/scala-native-profile/action.yml @@ -0,0 +1,75 @@ +name: Profile Scala Native FPP +description: Exercise instrumented FPP, build a matching F Prime revision, and merge profiles + +inputs: + fpp-version: + description: FPP version required by the F Prime revision + required: true + profile-directory: + description: Directory containing raw LLVM profiles + required: true + profile-file: + description: Output path for the merged LLVM profile + required: true + python: + description: Python interpreter used for the F Prime environment + required: true + fprime-directory: + description: Temporary F Prime checkout path + required: true + fprime-venv: + description: Temporary F Prime virtual environment path + required: true + +runs: + using: composite + steps: + - name: Exercise FPP acceptance suite + shell: bash + working-directory: compiler + env: + LLVM_PROFILE_FILE: ${{ inputs.profile-directory }}/fpp-%p.profraw + run: ./test + + - name: Build matching F Prime revision + shell: bash + env: + FPP_VERSION: ${{ inputs.fpp-version }} + FPRIME_DIR: ${{ inputs.fprime-directory }} + FPRIME_VENV: ${{ inputs.fprime-venv }} + LLVM_PROFILE_FILE: ${{ inputs.profile-directory }}/fpp-%p.profraw + run: | + git clone https://github.com/nasa/fprime.git "$FPRIME_DIR" + cd "$FPRIME_DIR" + pin="fprime-fpp==$FPP_VERSION" + if grep -Fqx "$pin" requirements.txt + then + commit=HEAD + else + boundary=$(git log -1 -G "^$pin$" --format=%H -- requirements.txt) + test -n "$boundary" + commit="$boundary^" + fi + git checkout "$commit" + grep -Fqx "$pin" requirements.txt + git submodule update --init --recursive + git show -s --format='Profiling against F Prime %H (%cs): %s' + + uv venv "$FPRIME_VENV" --python '${{ inputs.python }}' + uv pip install --python "$FPRIME_VENV/bin/python" -r requirements.txt + export PATH="$GITHUB_WORKSPACE/compiler/bin:$FPRIME_VENV/bin:$PATH" + test "$(command -v fpp)" = "$GITHUB_WORKSPACE/compiler/bin/fpp" + fpp --version + cd TestDeploymentsProject + fprime-util generate + fprime-util build -j4 + + - name: Merge profiles + shell: bash + env: + PROFILE_DIRECTORY: ${{ inputs.profile-directory }} + PROFILE_FILE: ${{ inputs.profile-file }} + run: | + mkdir -p "$(dirname "$PROFILE_FILE")" + llvm-profdata merge -o "$PROFILE_FILE" "$PROFILE_DIRECTORY"/*.profraw + llvm-profdata show "$PROFILE_FILE" diff --git a/.github/actions/scala-native-stage/action.yml b/.github/actions/scala-native-stage/action.yml new file mode 100644 index 000000000..0d3d9879e --- /dev/null +++ b/.github/actions/scala-native-stage/action.yml @@ -0,0 +1,20 @@ +name: Stage Scala Native FPP +description: Stage the Scala Native dispatcher and per-tool wrappers + +runs: + using: composite + steps: + - shell: bash + working-directory: compiler + run: | + native_binary=$(find tools/fpp -type f -name fpp-out -print -quit) + test -n "$native_binary" + rm -rf bin + mkdir -p bin + install -m 755 "$native_binary" bin/fpp + while IFS= read -r tool + do + printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \ + > "bin/fpp-$tool" + chmod +x "bin/fpp-$tool" + done < tools.txt diff --git a/.github/workflows/build-scala-native.yml b/.github/workflows/build-scala-native.yml index 4d3f2d301..7cdadc9de 100644 --- a/.github/workflows/build-scala-native.yml +++ b/.github/workflows/build-scala-native.yml @@ -92,59 +92,39 @@ jobs: fi util=lib/src/main/scala/util echo "Updating version to $version" + echo "FPP_VERSION=${version#v}" >> "$GITHUB_ENV" sed -i.update.bak -e "s/val v = .*/val v = \"$version\"/" \ $util/Version.scala - if: runner.os == 'macOS' name: Build Scala Native tools - working-directory: compiler - run: sbt --batch "nativeFpp/nativeLink" + uses: ./.github/actions/scala-native-build - if: runner.os == 'Linux' name: Build instrumented Scala Native tools - working-directory: compiler - env: - FPP_PGO: generate - FPP_PGO_DIR: ${{ runner.temp }}/fpp-pgo - run: | - mkdir -p "$FPP_PGO_DIR" - sbt -J-Xmx12G --batch "nativeFpp/nativeLink" + uses: ./.github/actions/scala-native-build + with: + jvm-heap: 12G + pgo-mode: generate + profile-directory: ${{ runner.temp }}/fpp-pgo - if: runner.os == 'Linux' name: Stage instrumented Scala Native tools - working-directory: compiler - run: | - native_binary=$(find tools/fpp -type f -name fpp-out -print -quit) - test -n "$native_binary" - mkdir -p bin - install -m 755 "$native_binary" bin/fpp - while IFS= read -r tool - do - printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \ - > "bin/fpp-$tool" - chmod +x "bin/fpp-$tool" - done < tools.txt - - if: runner.os == 'Linux' - name: Exercise instrumented Scala Native tools - working-directory: compiler - env: - # Each test launches a new fpp process; keep every process's counters. - LLVM_PROFILE_FILE: ${{ runner.temp }}/fpp-pgo/fpp-%p.profraw - run: ./test + uses: ./.github/actions/scala-native-stage - if: runner.os == 'Linux' - name: Merge Scala Native profiles - working-directory: compiler - env: - FPP_PGO_DIR: ${{ runner.temp }}/fpp-pgo - run: | - mkdir -p pgo - llvm-profdata merge -o pgo/fpp.profdata "$FPP_PGO_DIR"/*.profraw - llvm-profdata show pgo/fpp.profdata + name: Profile instrumented Scala Native tools + uses: ./.github/actions/scala-native-profile + with: + fpp-version: ${{ env.FPP_VERSION }} + profile-directory: ${{ runner.temp }}/fpp-pgo + profile-file: ${{ github.workspace }}/compiler/pgo/fpp.profdata + python: ${{ matrix.python }} + fprime-directory: ${{ runner.temp }}/fprime + fprime-venv: ${{ runner.temp }}/fprime-venv - if: runner.os == 'Linux' name: Build profile-guided Scala Native tools - working-directory: compiler - env: - FPP_PGO: apply - # -J forwards the heap limit through sbt to its JVM. The runner has - # 16 GB, leaving 4 GB for Clang, the linker, and other processes. - run: sbt -J-Xmx12G --batch "nativeFpp/nativeLink" + uses: ./.github/actions/scala-native-build + with: + jvm-heap: 12G + pgo-mode: apply + profile-file: ${{ github.workspace }}/compiler/pgo/fpp.profdata - name: Restore Version.scala if: always() working-directory: compiler @@ -156,19 +136,7 @@ jobs: sed -i.restore.bak -e "s/val v = .*/val v = \"[unknown version]\"/" \ $util/Version.scala - name: Stage Scala Native tools - working-directory: compiler - run: | - native_binary=$(find tools/fpp -type f -name fpp-out -print -quit) - test -n "$native_binary" - rm -rf bin - mkdir -p bin - install -m 755 "$native_binary" bin/fpp - while IFS= read -r tool - do - printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \ - > "bin/fpp-$tool" - chmod +x "bin/fpp-$tool" - done < tools.txt + uses: ./.github/actions/scala-native-stage - name: Test Scala Native tools working-directory: compiler run: ./test diff --git a/.gitignore b/.gitignore index 436b5be07..f176eb65f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .* !.gitignore !.gitattributes +!.github/ __SHADOW__ __pycache__ dist/ From c28a7d380af2e77825528f3d8a485b2b57c036f3 Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 22:01:01 -0500 Subject: [PATCH 07/10] Tweak the bash array concat (I'm picky) --- .github/actions/scala-native-build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/scala-native-build/action.yml b/.github/actions/scala-native-build/action.yml index ca2d069a7..0eb876367 100644 --- a/.github/actions/scala-native-build/action.yml +++ b/.github/actions/scala-native-build/action.yml @@ -33,7 +33,7 @@ runs: sbt_args=(--batch) if [[ -n "$SBT_JVM_HEAP" ]] then - sbt_args=("-J-Xmx$SBT_JVM_HEAP" "${sbt_args[@]}") + sbt_args+=("-J-Xmx$SBT_JVM_HEAP") fi if [[ "$FPP_PGO" == generate ]] then From b6698797dc071469b92cd8795a56fab8c7305d19 Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 23:02:30 -0500 Subject: [PATCH 08/10] Use python3.10 --- .github/workflows/build-scala-native.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-scala-native.yml b/.github/workflows/build-scala-native.yml index 7cdadc9de..f2b49ea25 100644 --- a/.github/workflows/build-scala-native.yml +++ b/.github/workflows/build-scala-native.yml @@ -19,12 +19,12 @@ jobs: include: - runner: macos-15 artifact: darwin-arm64 - python: python3 + python: '3.10' wheel_tag: macosx_14_0_arm64 - runner: ubuntu-22.04 artifact: manylinux_2_28_x86_64 container: quay.io/pypa/manylinux_2_28_x86_64 - python: /opt/python/cp39-cp39/bin/python + python: /opt/python/cp310-cp310/bin/python wheel_tag: manylinux_2_28_x86_64 runs-on: ${{ matrix.runner }} container: From 9870fd59c1d8c33b184d544abd17543dcd7d1f5f Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Wed, 12 Aug 2026 23:07:20 -0500 Subject: [PATCH 09/10] They're Graal CE Native images (not scala native) --- .github/workflows/build-native.yml | 2 +- .github/workflows/native-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml index 1c1de107b..a29b19f38 100644 --- a/.github/workflows/build-native.yml +++ b/.github/workflows/build-native.yml @@ -1,4 +1,4 @@ -name: Build Native Images +name: Build Graal CE Native Images on: push: diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 79f254995..fdcfd4839 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -1,4 +1,4 @@ -name: 'fprime-build-native-scala' +name: 'Build Graal CE Native' on: workflow_call: inputs: From e22f52e55222f4e01843d3e3e362cb2b19e3df35 Mon Sep 17 00:00:00 2001 From: Will MacCormack Date: Thu, 13 Aug 2026 00:23:41 -0500 Subject: [PATCH 10/10] Get past Fw/Buffer/Buffer.fpp relative path errors --- .github/workflows/native-build.yml | 2 +- .../scala/codegen/LocateDefsFppWriter.scala | 7 +-- compiler/lib/src/main/scala/util/File.scala | 29 ++++++++++ .../test/defs_parent_dir.ref.txt | 55 +++++++++++++++++++ compiler/tools/fpp-locate-defs/test/run | 5 ++ compiler/tools/fpp-locate-defs/test/tests.sh | 1 + .../tools/fpp-locate-defs/test/update-ref | 5 ++ .../fpp/src/main/scala/fpp-locate-uses.scala | 7 +-- 8 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index fdcfd4839..baa80301a 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -109,7 +109,7 @@ jobs: "tag": "manylinux_2_28_aarch64", "container": "quay.io/pypa/manylinux_2_28_aarch64" } - + ] } tags = {"tag": ["jar"] + [run["tag"] for run in matrix["run"]]} diff --git a/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala b/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala index 9e271f056..79245d9a5 100644 --- a/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala +++ b/compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala @@ -162,10 +162,9 @@ object LocateDefsFppWriter extends AstVisitor with LineUtils { case Some(dir) => dir case None => "" } - // Normalize both operands before relativize (fixes Scala Native bug) - val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath.normalize - val relativePath = baseDirPath.relativize(path.toAbsolutePath.normalize) - val fileNode = AstNode.create(relativePath.normalize.toString) + val baseDirPath = java.nio.file.Paths.get(baseDir) + val relativePath = File.relativize(baseDirPath)(path) + val fileNode = AstNode.create(relativePath.toString) val specLocNode = AstNode.create(Ast.SpecLoc(kind, qualIdentNode, fileNode, isDictionaryDef)) val specLocAnnotatedNode = (Nil, specLocNode, Nil) FppWriter.specLocAnnotatedNode((), specLocAnnotatedNode) diff --git a/compiler/lib/src/main/scala/util/File.scala b/compiler/lib/src/main/scala/util/File.scala index cbe74fe82..83dac9883 100644 --- a/compiler/lib/src/main/scala/util/File.scala +++ b/compiler/lib/src/main/scala/util/File.scala @@ -64,6 +64,35 @@ object File { /** Construct a file from a string representing a file path */ def fromString(s: String): File = Path(getJavaPath(s)) + /** Relativize a path without relying on Path.relativize. + * + * Scala Native 0.4.17 drops parent segments when the target is outside the + * base directory. Its Path.normalize also cancels consecutive leading `..` + * segments, so construct and return the clean lexical path directly. + */ + def relativize(base: JavaPath)(target: JavaPath): JavaPath = { + val normalizedBase = base.toAbsolutePath.normalize + val normalizedTarget = target.toAbsolutePath.normalize + if normalizedBase.getRoot != normalizedTarget.getRoot + then throw new IllegalArgumentException( + s"cannot relativize paths with different roots: $normalizedBase and $normalizedTarget" + ) + + def getNames(path: JavaPath) = + (0 until path.getNameCount).map(path.getName(_).toString) + + val baseNames = getNames(normalizedBase) + val targetNames = getNames(normalizedTarget) + val commonCount = baseNames.zip(targetNames).takeWhile { + case (baseName, targetName) => baseName == targetName + }.length + val relativeNames = + Seq.fill(baseNames.length - commonCount)("..") ++ targetNames.drop(commonCount) + relativeNames.foldLeft(java.nio.file.Paths.get("")) { + case (path, name) => path.resolve(name) + } + } + /** Remove the longest prefix from a Java path */ def removeLongestPrefix(prefixes: List[String])(path: JavaPath): JavaPath = { def removePrefix(s: String) = { diff --git a/compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt b/compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt new file mode 100644 index 000000000..5e885bc5e --- /dev/null +++ b/compiler/tools/fpp-locate-defs/test/defs_parent_dir.ref.txt @@ -0,0 +1,55 @@ +locate component C at "../../defs-1.fpp" +locate component M.C at "../../defs-2.fpp" +locate constant C.a at "../../defs-1.fpp" +locate constant M.C.a at "../../defs-2.fpp" +locate constant M.SM.a at "../../defs-2.fpp" +locate constant M.a at "../../defs-2.fpp" +locate constant SM.a at "../../defs-1.fpp" +locate constant a at "../../defs-1.fpp" +locate dictionary constant a2 at "../../defs-1.fpp" +locate dictionary type Alias2 at "../../defs-1.fpp" +locate dictionary type C.A2 at "../../defs-1.fpp" +locate dictionary type M.C.E2 at "../../defs-2.fpp" +locate dictionary type M.C.S2 at "../../defs-2.fpp" +locate instance M.T at "../../defs-2.fpp" +locate instance M.c at "../../defs-2.fpp" +locate instance T at "../../defs-1.fpp" +locate instance c at "../../defs-1.fpp" +locate interface I at "../../defs-1.fpp" +locate port M.P at "../../defs-2.fpp" +locate port P at "../../defs-1.fpp" +locate state machine M.C.S at "../../defs-2.fpp" +locate state machine M.SM at "../../defs-2.fpp" +locate state machine SM at "../../defs-1.fpp" +locate type A at "../../defs-1.fpp" +locate type Alias at "../../defs-1.fpp" +locate type C.A at "../../defs-1.fpp" +locate type C.Alias at "../../defs-1.fpp" +locate type C.E at "../../defs-1.fpp" +locate type C.S at "../../defs-1.fpp" +locate type C.T at "../../defs-1.fpp" +locate type E at "../../defs-1.fpp" +locate type M.A at "../../defs-2.fpp" +locate type M.Alias at "../../defs-2.fpp" +locate type M.C.A at "../../defs-2.fpp" +locate type M.C.Alias at "../../defs-2.fpp" +locate type M.C.E at "../../defs-2.fpp" +locate type M.C.S at "../../defs-2.fpp" +locate type M.C.T at "../../defs-2.fpp" +locate type M.E at "../../defs-2.fpp" +locate type M.S at "../../defs-2.fpp" +locate type M.SM.A at "../../defs-2.fpp" +locate type M.SM.E at "../../defs-2.fpp" +locate type M.SM.S at "../../defs-2.fpp" +locate type M.SM.State at "../../defs-2.fpp" +locate type M.SM.T at "../../defs-2.fpp" +locate type M.SM.X at "../../defs-2.fpp" +locate type M.T at "../../defs-2.fpp" +locate type S at "../../defs-1.fpp" +locate type SM.A at "../../defs-1.fpp" +locate type SM.E at "../../defs-1.fpp" +locate type SM.S at "../../defs-1.fpp" +locate type SM.State at "../../defs-1.fpp" +locate type SM.T at "../../defs-1.fpp" +locate type SM.X at "../../defs-1.fpp" +locate type T at "../../defs-1.fpp" diff --git a/compiler/tools/fpp-locate-defs/test/run b/compiler/tools/fpp-locate-defs/test/run index 2a88ece40..a326b4e71 100755 --- a/compiler/tools/fpp-locate-defs/test/run +++ b/compiler/tools/fpp-locate-defs/test/run @@ -53,4 +53,9 @@ defs_dir() run_test '-d defs' 'defs/defs-1.fpp defs/defs-2' defs_dir } +defs_parent_dir() +{ + run_test '-d defs/build/subdir' 'defs/defs-1.fpp defs/defs-2' defs_parent_dir +} + run_suite $tests diff --git a/compiler/tools/fpp-locate-defs/test/tests.sh b/compiler/tools/fpp-locate-defs/test/tests.sh index a2423bb00..9d6fc598d 100644 --- a/compiler/tools/fpp-locate-defs/test/tests.sh +++ b/compiler/tools/fpp-locate-defs/test/tests.sh @@ -1,6 +1,7 @@ tests=" defs defs_dir +defs_parent_dir include stdin " diff --git a/compiler/tools/fpp-locate-defs/test/update-ref b/compiler/tools/fpp-locate-defs/test/update-ref index 6b296e32c..c5b359c20 100755 --- a/compiler/tools/fpp-locate-defs/test/update-ref +++ b/compiler/tools/fpp-locate-defs/test/update-ref @@ -45,6 +45,11 @@ defs_dir() update '-d defs' 'defs/defs-1.fpp defs/defs-2' defs_dir } +defs_parent_dir() +{ + update '-d defs/build/subdir' 'defs/defs-1.fpp defs/defs-2' defs_parent_dir +} + stdin() { $fpp_locate_defs < defs/defs-1.fpp > stdin.ref.txt diff --git a/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala b/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala index ff9f21fcb..50d4bd944 100644 --- a/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala +++ b/compiler/tools/fpp/src/main/scala/fpp-locate-uses.scala @@ -55,10 +55,9 @@ object FPPLocateUses { case Some(dir) => dir case None => "" } - // Normalize both operands before relativize (fixes Scala Native bug) - val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath.normalize - val relativePath = baseDirPath.relativize(path.toAbsolutePath.normalize) - val fileNode = AstNode.create(relativePath.normalize.toString) + val baseDirPath = java.nio.file.Paths.get(baseDir) + val relativePath = File.relativize(baseDirPath)(path) + val fileNode = AstNode.create(relativePath.toString) val kind = s match { case _: Symbol.AbsType => Ast.SpecLoc.Type case _: Symbol.AliasType => Ast.SpecLoc.Type