Release/a9 pgo on thin LTO

.github/actions/scala-native-build/action.yml

@@ -0,0 +1,43 @@
+name: Build Scala Native FPP
+description: Link the Scala Native FPP binary with an optional PGO mode
+
+inputs:
+  jvm-heap:
+    description: Maximum sbt JVM heap, such as 12G; empty uses JVM ergonomics
+    required: false
+    default: ''
+  pgo-mode:
+    description: Empty for a normal build, or generate/apply for PGO
+    required: false
+    default: ''
+  profile-directory:
+    description: Directory for raw profiles during PGO generation
+    required: false
+    default: ''
+  profile-file:
+    description: Merged profile file to consume during PGO application
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      working-directory: compiler
+      env:
+        FPP_PGO: ${{ inputs.pgo-mode }}
+        FPP_PGO_DIR: ${{ inputs.profile-directory }}
+        FPP_PGO_PROFILE: ${{ inputs.profile-file }}
+        SBT_JVM_HEAP: ${{ inputs.jvm-heap }}
+      run: |
+        sbt_args=(--batch)
+        if [[ -n "$SBT_JVM_HEAP" ]]
+        then
+          sbt_args+=("-J-Xmx$SBT_JVM_HEAP")
+        fi
+        if [[ "$FPP_PGO" == generate ]]
+        then
+          test -n "$FPP_PGO_DIR"
+          mkdir -p "$FPP_PGO_DIR"
+        fi
+        sbt "${sbt_args[@]}" "nativeFpp/nativeLink"

.github/actions/scala-native-profile/action.yml

@@ -0,0 +1,75 @@
+name: Profile Scala Native FPP
+description: Exercise instrumented FPP, build a matching F Prime revision, and merge profiles
+
+inputs:
+  fpp-version:
+    description: FPP version required by the F Prime revision
+    required: true
+  profile-directory:
+    description: Directory containing raw LLVM profiles
+    required: true
+  profile-file:
+    description: Output path for the merged LLVM profile
+    required: true
+  python:
+    description: Python interpreter used for the F Prime environment
+    required: true
+  fprime-directory:
+    description: Temporary F Prime checkout path
+    required: true
+  fprime-venv:
+    description: Temporary F Prime virtual environment path
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Exercise FPP acceptance suite
+      shell: bash
+      working-directory: compiler
+      env:
+        LLVM_PROFILE_FILE: ${{ inputs.profile-directory }}/fpp-%p.profraw
+      run: ./test
+
+    - name: Build matching F Prime revision
+      shell: bash
+      env:
+        FPP_VERSION: ${{ inputs.fpp-version }}
+        FPRIME_DIR: ${{ inputs.fprime-directory }}
+        FPRIME_VENV: ${{ inputs.fprime-venv }}
+        LLVM_PROFILE_FILE: ${{ inputs.profile-directory }}/fpp-%p.profraw
+      run: |
+        git clone https://github.com/nasa/fprime.git "$FPRIME_DIR"
+        cd "$FPRIME_DIR"
+        pin="fprime-fpp==$FPP_VERSION"
+        if grep -Fqx "$pin" requirements.txt
+        then
+          commit=HEAD
+        else
+          boundary=$(git log -1 -G "^$pin$" --format=%H -- requirements.txt)
+          test -n "$boundary"
+          commit="$boundary^"
+        fi
+        git checkout "$commit"
+        grep -Fqx "$pin" requirements.txt
+        git submodule update --init --recursive
+        git show -s --format='Profiling against F Prime %H (%cs): %s'
+
+        uv venv "$FPRIME_VENV" --python '${{ inputs.python }}'
+        uv pip install --python "$FPRIME_VENV/bin/python" -r requirements.txt
+        export PATH="$GITHUB_WORKSPACE/compiler/bin:$FPRIME_VENV/bin:$PATH"
+        test "$(command -v fpp)" = "$GITHUB_WORKSPACE/compiler/bin/fpp"
+        fpp --version
+        cd TestDeploymentsProject
+        fprime-util generate
+        fprime-util build -j4
+
+    - name: Merge profiles
+      shell: bash
+      env:
+        PROFILE_DIRECTORY: ${{ inputs.profile-directory }}
+        PROFILE_FILE: ${{ inputs.profile-file }}
+      run: |
+        mkdir -p "$(dirname "$PROFILE_FILE")"
+        llvm-profdata merge -o "$PROFILE_FILE" "$PROFILE_DIRECTORY"/*.profraw
+        llvm-profdata show "$PROFILE_FILE"

.github/actions/scala-native-stage/action.yml

@@ -0,0 +1,20 @@
+name: Stage Scala Native FPP
+description: Stage the Scala Native dispatcher and per-tool wrappers
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      working-directory: compiler
+      run: |
+        native_binary=$(find tools/fpp -type f -name fpp-out -print -quit)
+        test -n "$native_binary"
+        rm -rf bin
+        mkdir -p bin
+        install -m 755 "$native_binary" bin/fpp
+        while IFS= read -r tool
+        do
+          printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \
+            > "bin/fpp-$tool"
+          chmod +x "bin/fpp-$tool"
+        done < tools.txt

.github/workflows/build-native.yml

@@ -1,4 +1,4 @@
-name: Build Native Images
+name: Build Graal CE Native Images
 
 on:
   push:

.github/workflows/build-scala-native.yml

@@ -19,12 +19,12 @@ jobs:
         include:
           - runner: macos-15
             artifact: darwin-arm64
-            python: python3
+            python: '3.10'
             wheel_tag: macosx_14_0_arm64
           - runner: ubuntu-22.04
             artifact: manylinux_2_28_x86_64
             container: quay.io/pypa/manylinux_2_28_x86_64
-            python: /opt/python/cp39-cp39/bin/python
+            python: /opt/python/cp310-cp310/bin/python
             wheel_tag: manylinux_2_28_x86_64
     runs-on: ${{ matrix.runner }}
     container:
@@ -53,7 +53,7 @@ jobs:
       - if: runner.os == 'Linux'
         name: Install Linux build tools
         run: |
-          dnf install --assumeyes clang
+          dnf install --assumeyes clang llvm
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
       - name: Show toolchain versions
@@ -92,11 +92,39 @@ jobs:
           fi
           util=lib/src/main/scala/util
           echo "Updating version to $version"
+          echo "FPP_VERSION=${version#v}" >> "$GITHUB_ENV"
           sed -i.update.bak -e "s/val v = .*/val v = \"$version\"/" \
             $util/Version.scala
-      - name: Build Scala Native tools
-        working-directory: compiler
-        run: sbt --batch "nativeFpp/nativeLink"
+      - if: runner.os == 'macOS'
+        name: Build Scala Native tools
+        uses: ./.github/actions/scala-native-build
+      - if: runner.os == 'Linux'
+        name: Build instrumented Scala Native tools
+        uses: ./.github/actions/scala-native-build
+        with:
+          jvm-heap: 12G
+          pgo-mode: generate
+          profile-directory: ${{ runner.temp }}/fpp-pgo
+      - if: runner.os == 'Linux'
+        name: Stage instrumented Scala Native tools
+        uses: ./.github/actions/scala-native-stage
+      - if: runner.os == 'Linux'
+        name: Profile instrumented Scala Native tools
+        uses: ./.github/actions/scala-native-profile
+        with:
+          fpp-version: ${{ env.FPP_VERSION }}
+          profile-directory: ${{ runner.temp }}/fpp-pgo
+          profile-file: ${{ github.workspace }}/compiler/pgo/fpp.profdata
+          python: ${{ matrix.python }}
+          fprime-directory: ${{ runner.temp }}/fprime
+          fprime-venv: ${{ runner.temp }}/fprime-venv
+      - if: runner.os == 'Linux'
+        name: Build profile-guided Scala Native tools
+        uses: ./.github/actions/scala-native-build
+        with:
+          jvm-heap: 12G
+          pgo-mode: apply
+          profile-file: ${{ github.workspace }}/compiler/pgo/fpp.profdata
       - name: Restore Version.scala
         if: always()
         working-directory: compiler
@@ -108,19 +136,7 @@ jobs:
           sed -i.restore.bak -e "s/val v = .*/val v = \"[unknown version]\"/" \
             $util/Version.scala
       - name: Stage Scala Native tools
-        working-directory: compiler
-        run: |
-          native_binary=$(find tools/fpp -type f -name fpp-out -print -quit)
-          test -n "$native_binary"
-          rm -rf bin
-          mkdir -p bin
-          install -m 755 "$native_binary" bin/fpp
-          while IFS= read -r tool
-          do
-            printf '#!/bin/sh\n"$(dirname "$0")/fpp" %s "$@"\n' "$tool" \
-              > "bin/fpp-$tool"
-            chmod +x "bin/fpp-$tool"
-          done < tools.txt
+        uses: ./.github/actions/scala-native-stage
       - name: Test Scala Native tools
         working-directory: compiler
         run: ./test

.github/workflows/native-build.yml

@@ -1,4 +1,4 @@
-name: 'fprime-build-native-scala'
+name: 'Build Graal CE Native'
 on:
   workflow_call:
     inputs:
@@ -109,7 +109,7 @@ jobs:
                 "tag": "manylinux_2_28_aarch64",
                 "container": "quay.io/pypa/manylinux_2_28_aarch64"
               }
-              
+
             ]
           }
           tags = {"tag": ["jar"] + [run["tag"] for run in matrix["run"]]}

.gitignore

@@ -5,6 +5,7 @@
 .*
 !.gitignore
 !.gitattributes
+!.github/
 __SHADOW__
 __pycache__
 dist/

compiler/.gitignore

@@ -1,9 +1,12 @@
 bin
 num_failed.txt
 target/
+target-native/
 test-output.txt
 *.o
 native-fpp-*
 *.class
 # Version is not checked in during typical development
 lib/src/main/scala/util/Version.scala
+# PGO profile
+pgo/fpp.profdata

compiler/build.sbt

@@ -17,24 +17,23 @@ lazy val settings = Seq(
   Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oNCXELOPQRM"),
 )
 
-lazy val jvmDependencies = Seq(
-  "com.github.scopt" %% "scopt" % "4.0.1",
-  "io.circe" %% "circe-core" % "0.14.3",
-  "io.circe" %% "circe-generic" % "0.14.3",
-  "io.circe" %% "circe-parser" % "0.14.3",
-  "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1",
-  "org.scala-lang.modules" %% "scala-xml" % "2.1.0",
-  "org.scalatest" %% "scalatest" % "3.2.12" % "test",
+// Shared (org, artifact, version); JVM cross-builds with %%, Scala Native with %%%.
+lazy val sharedDependencies = Seq(
+  ("com.github.scopt", "scopt", "4.0.1"),
+  ("io.circe", "circe-core", "0.14.3"),
+  ("io.circe", "circe-generic", "0.14.3"),
+  ("io.circe", "circe-parser", "0.14.3"),
+  ("org.scala-lang.modules", "scala-parser-combinators", "2.1.1"),
+  ("org.scala-lang.modules", "scala-xml", "2.1.0"),
 )
 
-lazy val nativeDependencies = Def.setting(Seq(
-  "com.github.scopt" %%% "scopt" % "4.0.1",
-  "io.circe" %%% "circe-core" % "0.14.3",
-  "io.circe" %%% "circe-generic" % "0.14.3",
-  "io.circe" %%% "circe-parser" % "0.14.3",
-  "org.scala-lang.modules" %%% "scala-parser-combinators" % "2.1.1",
-  "org.scala-lang.modules" %%% "scala-xml" % "2.1.0",
-))
+lazy val jvmDependencies =
+  sharedDependencies.map { case (o, a, v) => o %% a % v } :+
+    ("org.scalatest" %% "scalatest" % "3.2.12" % "test")
+
+lazy val nativeDependencies = Def.setting(
+  sharedDependencies.map { case (o, a, v) => o %%% a % v }
+)
 
 lazy val jvmSettings = settings ++ Seq(
   libraryDependencies ++= jvmDependencies,
@@ -77,10 +76,36 @@ lazy val nativeFpp = (project in file("tools/fpp"))
   .settings(nativeSettings)
   .settings(
     name := "fpp",
-    nativeConfig ~= { config =>
-      config.withLTO(LTO.thin).withMode(Mode.releaseFast).withGC(GC.none)
-        .withLinkStubs(true)
-        .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions)
+    nativeConfig := {
+      // FPP_PGO selects an explicit build path:
+      //   unset     normal release/test build (no PGO; releaseFast with full LTO)
+      //   generate  matching build instrumented to write raw profiles
+      //   apply     matching build that consumes FPP_PGO_PROFILE, or the
+      //             local pgo/fpp.profdata when FPP_PGO_PROFILE is unset.
+      val (compile, link) = sys.env.get("FPP_PGO").map(_.trim).filter(_.nonEmpty) match {
+        case Some("generate") => // instrumented
+          val dir = sys.env.getOrElse("FPP_PGO_DIR", "/tmp/fpp-pgo")
+          val rt = sys.env.get("FPP_PGO_RUNTIME").map(_.trim).filter(_.nonEmpty)
+          (Seq("-fprofile-generate=" + dir),
+            Seq("-fprofile-generate=" + dir) ++
+              rt.toSeq.flatMap(a => Seq("-Wl,--whole-archive", a, "-Wl,--no-whole-archive")))
+        case Some("apply") =>
+          val localProf = (ThisBuild / baseDirectory).value / "pgo" / "fpp.profdata"
+          val prof = sys.env.get("FPP_PGO_PROFILE").map(new java.io.File(_)).getOrElse(localProf)
+          if (!prof.exists)
+            sys.error("FPP_PGO=apply requires FPP_PGO_PROFILE or pgo/fpp.profdata")
+          (Seq("-fprofile-use=" + prof.getAbsolutePath,
+              "-Wno-profile-instr-out-of-date", "-Wno-profile-instr-unprofiled"),
+            Seq.empty[String])
+        case None =>
+          (Seq.empty[String], Seq.empty[String])
+        case Some(value) =>
+          sys.error(s"Unsupported FPP_PGO value '$value'; use generate or apply")
+      }
+      val config = nativeConfig.value
+      config.withMode(Mode.releaseFast).withLTO(LTO.full).withGC(GC.none).withLinkStubs(true)
+        .withCompileOptions(config.compileOptions ++ compile)
+        .withLinkingOptions(config.linkingOptions ++ macOSUnwindLinkerOptions ++ link)
     }
   )
   .dependsOn(nativeLib)

compiler/lib/src/main/scala/codegen/LocateDefsFppWriter.scala

@@ -162,9 +162,9 @@ object LocateDefsFppWriter extends AstVisitor with LineUtils {
           case Some(dir) => dir
           case None => ""
         }
-        val baseDirPath = java.nio.file.Paths.get(baseDir).toAbsolutePath
-        val relativePath = baseDirPath.relativize(path)
-        val fileNode = AstNode.create(relativePath.normalize.toString)
+        val baseDirPath = java.nio.file.Paths.get(baseDir)
+        val relativePath = File.relativize(baseDirPath)(path)
+        val fileNode = AstNode.create(relativePath.toString)
         val specLocNode = AstNode.create(Ast.SpecLoc(kind, qualIdentNode, fileNode, isDictionaryDef))
         val specLocAnnotatedNode = (Nil, specLocNode, Nil)
         FppWriter.specLocAnnotatedNode((), specLocAnnotatedNode)