From 0464e70e8c95c8718e02d3e7852a2c040c5beb5c Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Mon, 7 Sep 2026 12:17:43 +0200 Subject: [PATCH 1/9] Add opt-in AGNI shim diagnostics to setup-proteus Log which libSOCRATES_C.so exports PS_real_kind_bytes, which file Julia's Libdl resolves and opens, and any other libSOCRATES_C* copies under the Julia depot or the restored SOCRATES tree that could shadow the intended shim, ahead of building a fix for the precompile-ordering mismatch on macOS Integration CI. The step is gated behind a new shim-diagnostics input (default false) so it runs only where a caller opts in, currently the nightly integration-tier job on both OS legs, rather than on every setup-proteus caller. It also sets JULIA_DEBUG=loading on that job's pytest step. The step can never fail the job: every external command that can plausibly fail falls back to an explanatory echo. --- .github/actions/setup-proteus/action.yml | 39 ++++++++++++++++++++++++ .github/workflows/ci-nightly.yml | 3 ++ 2 files changed, 42 insertions(+) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index 79fdfd0f1..1ba1c2452 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -31,6 +31,13 @@ inputs: data is downloaded (spectral file + minimal stellar spectrum). required: false default: 'false' + shim-diagnostics: + description: > + If 'true', log which libSOCRATES_C.so on disk exports + PS_real_kind_bytes, which file Julia's Libdl resolves, and any + other libSOCRATES_C* copies found on the Julia depot / load path. + required: false + default: 'false' outputs: rad-dir: @@ -565,6 +572,38 @@ runs: julia --project=. -e 'using Pkg; Pkg.instantiate()' julia --project=. deps/build.jl nodata + # ------------------------------------------------------------------ + # 7b. Shim diagnostics + # ------------------------------------------------------------------ + # Prints which libSOCRATES_C.so on disk exports PS_real_kind_bytes, + # which file Julia's Libdl actually resolves and opens, and any other + # libSOCRATES_C* copies on the Julia depot or the restored SOCRATES + # tree that could shadow the intended one. Opt-in per caller via + # shim-diagnostics so it does not run on every setup-proteus job. + - name: Log AGNI shim diagnostics + if: inputs.shim-diagnostics == 'true' + shell: bash + run: | + export RAD_DIR="$GITHUB_WORKSPACE/socrates" + shim="$RAD_DIR/julia/lib/libSOCRATES_C.so" + echo "RAD_DIR=$RAD_DIR" + echo "-- nm -gU $shim | grep PS_real_kind_bytes --" + nm -gU "$shim" 2>&1 | grep PS_real_kind_bytes || echo "(not found / nm failed on $shim)" + echo "-- nm -gU $RAD_DIR/bin/radlib.a | grep PS_real_kind_bytes --" + nm -gU "$RAD_DIR/bin/radlib.a" 2>&1 | grep PS_real_kind_bytes || echo "(not found / nm failed on $RAD_DIR/bin/radlib.a)" + echo "-- Julia-resolved shim path (Libdl.dlopen/dlpath) --" + julia -e ' + using Libdl + h = dlopen(joinpath(ENV["RAD_DIR"], "julia/lib/libSOCRATES_C.so")) + println(dlpath(h)) + ' || echo "(Julia dlopen of the shim failed)" + echo "-- other libSOCRATES_C* files under the Julia depot (~/.julia) --" + find ~/.julia -name 'libSOCRATES_C*' 2>/dev/null || echo "(depot search failed)" + echo "-- other libSOCRATES_C* files under RAD_DIR --" + find "$RAD_DIR" -name 'libSOCRATES_C*' 2>/dev/null || echo "(RAD_DIR search failed)" + echo "-- DYLD_LIBRARY_PATH (macOS only, empty elsewhere) --" + echo "DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:-}" + # ------------------------------------------------------------------ # 8. PROTEUS pip install (resolves PyPI + git deps in pyproject.toml) # ------------------------------------------------------------------ diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index 685a058fc..6bc82152f 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -152,7 +152,10 @@ jobs: - uses: ./.github/actions/setup-proteus with: full-data: 'true' + shim-diagnostics: 'true' - name: Run integration tier (not slow) + env: + JULIA_DEBUG: loading run: | pytest tests/integration \ -m "integration and not slow" \ From f3e150d99accc752ace64869b97393d4ba1bd414 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Mon, 7 Sep 2026 13:12:58 +0200 Subject: [PATCH 2/9] Extend AGNI shim diagnostics with compiler and Mk_cmd checks Log gfortran's version, the SOCRATES tree revision against the tools/get_socrates.sh pin, the content of RAD_DIR/bin/Mk_cmd if present, a direct grep for PS_real_kind_bytes in Utilities_CF.f90, and an unfiltered (no -U) nm scan of Utilities_CF.o and libSOCRATES_C.so for any real_kind symbol under a mangled name. --- .github/actions/setup-proteus/action.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index 1ba1c2452..5003dd886 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -586,7 +586,26 @@ runs: run: | export RAD_DIR="$GITHUB_WORKSPACE/socrates" shim="$RAD_DIR/julia/lib/libSOCRATES_C.so" + obj="$RAD_DIR/julia/lib/Utilities_CF.o" echo "RAD_DIR=$RAD_DIR" + echo "-- gfortran --version --" + gfortran --version | head -n 1 || echo "(gfortran not on PATH)" + echo "-- SOCRATES tree revision vs pin --" + git -C "$RAD_DIR" rev-parse HEAD 2>&1 || echo "(not a git checkout / rev-parse failed)" + cat "$RAD_DIR/version" 2>&1 || echo "(no version file)" + echo "-- RAD_DIR/bin/Mk_cmd (compiler flags SOCRATES's own configure wrote) --" + if [ -f "$RAD_DIR/bin/Mk_cmd" ]; then + echo "present:" + cat "$RAD_DIR/bin/Mk_cmd" + else + echo "(no $RAD_DIR/bin/Mk_cmd; Makefile's -include finds nothing and falls back to bare gfortran)" + fi + echo "-- grep -n PS_real_kind_bytes socrates/julia/src/Utilities_CF.f90 --" + grep -n PS_real_kind_bytes "$RAD_DIR/julia/src/Utilities_CF.f90" || echo "(symbol absent from source)" + echo "-- nm -g $obj | grep -i real_kind --" + nm -g "$obj" 2>&1 | grep -i real_kind || echo "(not found / nm failed on $obj)" + echo "-- nm -g $shim | grep -i real_kind --" + nm -g "$shim" 2>&1 | grep -i real_kind || echo "(not found / nm failed on $shim)" echo "-- nm -gU $shim | grep PS_real_kind_bytes --" nm -gU "$shim" 2>&1 | grep PS_real_kind_bytes || echo "(not found / nm failed on $shim)" echo "-- nm -gU $RAD_DIR/bin/radlib.a | grep PS_real_kind_bytes --" From adfc47b89c3d7b6f514f0b0878b7bc358bed1781 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Mon, 7 Sep 2026 14:48:55 +0200 Subject: [PATCH 3/9] Harden shim diagnostics: errexit-safe, macOS-only, add dlsym check nm_check now captures nm's exit code inside the if condition instead of a bare command substitution, since a failed nm would otherwise abort the whole step under GitHub Actions' default -eo pipefail. The verbose relink dump and each relink_probe's piped tail now tolerate a failing link the same way. Gate the step to runner.os == 'macOS': it currently runs unconditionally on both matrix legs, and GNU nm on ubuntu-latest may not support the same flags as the macOS toolchain. Extend the Julia probe to call dlsym(PS_real_kind_bytes) directly, matching AGNI's own load path instead of only checking dlopen/dlpath. --- .github/actions/setup-proteus/action.yml | 74 +++++++++++++++++++++--- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index 5003dd886..5b025867a 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -581,12 +581,30 @@ runs: # tree that could shadow the intended one. Opt-in per caller via # shim-diagnostics so it does not run on every setup-proteus job. - name: Log AGNI shim diagnostics - if: inputs.shim-diagnostics == 'true' + if: inputs.shim-diagnostics == 'true' && runner.os == 'macOS' shell: bash run: | export RAD_DIR="$GITHUB_WORKSPACE/socrates" shim="$RAD_DIR/julia/lib/libSOCRATES_C.so" obj="$RAD_DIR/julia/lib/Utilities_CF.o" + # nm's own exit status is checked separately from grep's match + # result, so a failed nm (bad file, wrong flag) is distinguishable + # from a clean run that simply found no match. The assignment sits + # in the if condition: this shell runs with GitHub Actions' default + # -eo pipefail, and a bare `out=$(nm ...)` would abort the step. + nm_check() { + local flags="$1" file="$2" label="$3" out rc + if out=$(nm $flags "$file" 2>&1); then + rc=0 + else + rc=$? + fi + if [ "$rc" -ne 0 ]; then + echo "nm $flags $file ($label) exited $rc: $out" + return + fi + echo "$out" | grep -i real_kind || echo "(nm $flags $file ($label) ok, no real_kind match)" + } echo "RAD_DIR=$RAD_DIR" echo "-- gfortran --version --" gfortran --version | head -n 1 || echo "(gfortran not on PATH)" @@ -603,25 +621,67 @@ runs: echo "-- grep -n PS_real_kind_bytes socrates/julia/src/Utilities_CF.f90 --" grep -n PS_real_kind_bytes "$RAD_DIR/julia/src/Utilities_CF.f90" || echo "(symbol absent from source)" echo "-- nm -g $obj | grep -i real_kind --" - nm -g "$obj" 2>&1 | grep -i real_kind || echo "(not found / nm failed on $obj)" + nm_check "-g" "$obj" "obj, global symbols" echo "-- nm -g $shim | grep -i real_kind --" - nm -g "$shim" 2>&1 | grep -i real_kind || echo "(not found / nm failed on $shim)" + nm_check "-g" "$shim" "shim, global symbols" echo "-- nm -gU $shim | grep PS_real_kind_bytes --" - nm -gU "$shim" 2>&1 | grep PS_real_kind_bytes || echo "(not found / nm failed on $shim)" + nm_check "-gU" "$shim" "shim, defined global symbols" echo "-- nm -gU $RAD_DIR/bin/radlib.a | grep PS_real_kind_bytes --" - nm -gU "$RAD_DIR/bin/radlib.a" 2>&1 | grep PS_real_kind_bytes || echo "(not found / nm failed on $RAD_DIR/bin/radlib.a)" - echo "-- Julia-resolved shim path (Libdl.dlopen/dlpath) --" + nm_check "-gU" "$RAD_DIR/bin/radlib.a" "radlib.a, defined global symbols" + echo "-- Julia-resolved shim path and dlsym(PS_real_kind_bytes) (matches AGNI's own load path) --" julia -e ' using Libdl h = dlopen(joinpath(ENV["RAD_DIR"], "julia/lib/libSOCRATES_C.so")) println(dlpath(h)) - ' || echo "(Julia dlopen of the shim failed)" + sym = dlsym(h, :PS_real_kind_bytes; throw_error=false) + println(sym === nothing ? "dlsym(PS_real_kind_bytes) = not found" : "dlsym(PS_real_kind_bytes) = $sym") + ' || echo "(Julia dlopen/dlsym of the shim failed)" echo "-- other libSOCRATES_C* files under the Julia depot (~/.julia) --" find ~/.julia -name 'libSOCRATES_C*' 2>/dev/null || echo "(depot search failed)" echo "-- other libSOCRATES_C* files under RAD_DIR --" find "$RAD_DIR" -name 'libSOCRATES_C*' 2>/dev/null || echo "(RAD_DIR search failed)" echo "-- DYLD_LIBRARY_PATH (macOS only, empty elsewhere) --" echo "DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:-}" + if [ "$(uname -s)" = "Darwin" ]; then + echo "-- nm (no -g, all symbols) $shim | grep -i real_kind --" + nm_check "" "$shim" "shim, all symbols" + echo "-- dyld_info -exports $shim | grep -i real_kind --" + dyld_info -exports "$shim" 2>&1 | grep -i real_kind || echo "(not found / dyld_info failed on $shim)" + echo "-- toolchain identity --" + gfortran -print-prog-name=ld || echo "(gfortran -print-prog-name=ld failed)" + ld -v 2>&1 || echo "(ld -v failed)" + xcrun --show-sdk-version || echo "(xcrun --show-sdk-version failed)" + pkgutil --pkg-info=com.apple.pkg.CLTools_Executables || echo "(pkgutil CLTools_Executables lookup failed, CLT may be an Xcode install instead)" + echo "-- relink diagnostics (reproduce socrates/julia/lib/Makefile's link recipe) --" + objs=$(ls "$RAD_DIR"/julia/lib/*.o 2>/dev/null) + radlib="$RAD_DIR/bin/radlib.a" + base_flags="-O3 -Wall -fPIC -cpp -L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib/" + if [ -z "$objs" ]; then + echo "(no .o files under $RAD_DIR/julia/lib; skipping relink probes)" + else + echo "-- gfortran -shared -v re-link (full driver dump, shows the real ld invocation) --" + gfortran -shared $base_flags -v -o /tmp/relink-verbose.so $objs "$radlib" 2>&1 || echo "(verbose relink failed, see output above)" + relink_probe() { + local label="$1" extra="$2" out="/tmp/relink-${label}.so" + gfortran -shared $base_flags $extra -o "$out" $objs "$radlib" 2>&1 | tail -n 20 || true + if [ -f "$out" ]; then + nm_check "" "$out" "relink $label" + else + echo "(relink $label produced no output file)" + fi + } + echo "-- relink probe: baseline (no extra flag, control) --" + relink_probe "baseline" "" + echo "-- relink probe: -Wl,-ld_classic --" + relink_probe "ld_classic" "-Wl,-ld_classic" + echo "-- relink probe: -Wl,-exported_symbol,_PS_real_kind_bytes --" + relink_probe "exported_symbol" "-Wl,-exported_symbol,_PS_real_kind_bytes" + echo "-- relink probe: -Wl,-export_dynamic --" + relink_probe "export_dynamic" "-Wl,-export_dynamic" + fi + else + echo "(link-step probes are macOS-only; skipped on $(uname -s))" + fi # ------------------------------------------------------------------ # 8. PROTEUS pip install (resolves PyPI + git deps in pyproject.toml) From afa421db707fd9e17afdb8ce25b7fc2d7bfe362b Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Mon, 7 Sep 2026 23:34:12 +0200 Subject: [PATCH 4/9] Add exported_symbols_list relink probe to shim diagnostics Extracts the exact linker symbol name for PS_real_kind_bytes from nm -g output on the object file, with a literal fallback when no match is found, then reproduces the SOCRATES Makefile link recipe with -Wl,-exported_symbols_list, as a fifth relink probe alongside the existing baseline, -ld_classic, -exported_symbol, and -export_dynamic probes. The verbose gfortran -shared -v dump is kept in full for this probe, unlike the others, since it needs full inspection for the auto-injected linker flags GitHub's toolchain adds. The symbol extraction guards its command substitution so a no-match result reaches the fallback branch instead of aborting the step under GitHub Actions' default -eo pipefail. --- .github/actions/setup-proteus/action.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index 5b025867a..1aa4266e1 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -622,6 +622,13 @@ runs: grep -n PS_real_kind_bytes "$RAD_DIR/julia/src/Utilities_CF.f90" || echo "(symbol absent from source)" echo "-- nm -g $obj | grep -i real_kind --" nm_check "-g" "$obj" "obj, global symbols" + sym_name=$(nm -g "$obj" 2>&1 | grep -i real_kind | awk '{print $NF}' | head -n 1) || true + if [ -z "$sym_name" ]; then + sym_name="_PS_real_kind_bytes" + echo "(nm -g found no real_kind symbol in $obj; export-list probe falls back to $sym_name)" + else + echo "exact linker symbol name for the export list: $sym_name" + fi echo "-- nm -g $shim | grep -i real_kind --" nm_check "-g" "$shim" "shim, global symbols" echo "-- nm -gU $shim | grep PS_real_kind_bytes --" @@ -678,6 +685,15 @@ runs: relink_probe "exported_symbol" "-Wl,-exported_symbol,_PS_real_kind_bytes" echo "-- relink probe: -Wl,-export_dynamic --" relink_probe "export_dynamic" "-Wl,-export_dynamic" + echo "-- relink probe: -Wl,-exported_symbols_list (list-file form, using $sym_name, verbose) --" + list_file="/tmp/relink-exported_symbols_list.txt" + printf '%s\n' "$sym_name" > "$list_file" + gfortran -shared $base_flags -Wl,-exported_symbols_list,"$list_file" -v -o /tmp/relink-exported_symbols_list.so $objs "$radlib" 2>&1 || echo "(verbose relink with export list failed, see output above)" + if [ -f /tmp/relink-exported_symbols_list.so ]; then + nm_check "" "/tmp/relink-exported_symbols_list.so" "relink exported_symbols_list" + else + echo "(relink exported_symbols_list produced no output file)" + fi fi else echo "(link-step probes are macOS-only; skipped on $(uname -s))" From 0aa2ea7c4944d219370c675d7e269912780c9b73 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Tue, 8 Sep 2026 02:17:46 +0200 Subject: [PATCH 5/9] Pin macOS CI gcc to 16.1 ahead of AGNI's precompile-time SOCRATES rebuild Homebrew's gcc 16.2 Darwin driver stops exporting SOCRATES's Julia shim symbol PS_real_kind_bytes as external, so any AGNI build step that rebuilds libSOCRATES_C.so breaks Julia's dlsym lookup. 16.1 is the last release without that regression. Formulae must live in a tap, so the historical 16.1.0 formula is fetched into a throwaway local tap and installed by tap-qualified name; pinned before netcdf-fortran so its gcc dependency resolves to the pinned keg instead of upgrading it. Unlink any gcc the runner image already ships before installing the pinned keg, since a pre-existing keg under the same formula name would otherwise conflict with linking the tap-qualified one. --- .github/actions/setup-proteus/action.yml | 27 +++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index 1aa4266e1..95b0057b2 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -187,9 +187,34 @@ runs: run: | set -euo pipefail source "${{ github.action_path }}/retry-with-timeout.sh" + # gcc is pinned to the last release before Homebrew's Darwin driver + # stopped exporting SOCRATES's dlsym-visible shim symbols. Formulae + # must live in a tap, so the formula is fetched into a throwaway tap; + # pinned before netcdf-fortran so its gcc dependency resolves to it. + gcc_formula_url="https://raw.githubusercontent.com/Homebrew/homebrew-core/4509561/Formula/g/gcc.rb" + brew tap-new local/pinned-toolchain --no-git >/dev/null + pinned_tap_formula_dir="$(brew --repository)/Library/Taps/local/homebrew-pinned-toolchain/Formula" + retry_with_timeout "fetch pinned gcc formula" 60 10 "" \ + curl -fsSL -o "$pinned_tap_formula_dir/gcc.rb" "$gcc_formula_url" + # Unlink any gcc the runner image already ships (a different tap, same + # formula name) so linking the pinned keg below cannot hit a conflict. + if brew list --formula gcc >/dev/null 2>&1; then + brew unlink gcc >/dev/null 2>&1 || true + fi brew_rc=0 + retry_with_timeout "brew install gcc (pinned)" 240 10 "" \ + brew install --quiet local/pinned-toolchain/gcc || brew_rc=$? + gcc_version="$(brew list --versions gcc 2>/dev/null || true)" + case "$gcc_version" in + "gcc 16.1."*) : ;; + *) + echo "ERROR: brew install rc=$brew_rc, expected pinned gcc 16.1.x, got: '$gcc_version'" >&2 + exit 1 + ;; + esac + brew pin gcc retry_with_timeout "brew install" 240 10 "" \ - brew install --quiet gcc netcdf netcdf-fortran openssl@3 || brew_rc=$? + brew install --quiet netcdf netcdf-fortran openssl@3 || brew_rc=$? missing=() for pkg in gcc netcdf netcdf-fortran openssl@3; do brew list "$pkg" >/dev/null 2>&1 || missing+=("$pkg") From dcd9415e6ba2d5b307afc6d2397f0efbf895673a Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Tue, 8 Sep 2026 02:36:50 +0200 Subject: [PATCH 6/9] Pin the gcc pin's own resolution to its throwaway tap brew pin took the plain formula name, which two taps now define (homebrew-core and the throwaway pinned-toolchain tap holding the 16.1.0 formula), so it risks a tap-ambiguity error instead of pinning the installed keg. Use the same tap-qualified name already used for the install two lines above. --- .github/actions/setup-proteus/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index 95b0057b2..c93d5a966 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -212,7 +212,7 @@ runs: exit 1 ;; esac - brew pin gcc + brew pin local/pinned-toolchain/gcc retry_with_timeout "brew install" 240 10 "" \ brew install --quiet netcdf netcdf-fortran openssl@3 || brew_rc=$? missing=() From 3eb146c88cccc3b2014a1a44bd2665b010fd4f08 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Tue, 8 Sep 2026 18:47:37 +0200 Subject: [PATCH 7/9] Run the shim symbol-export guard after AGNI builds it, not before Drop the opt-in "Log AGNI shim diagnostics" step, since it was scaffolding for tracking down the PS_real_kind_bytes export bug, and it duplicates what the guard below now checks unconditionally. Add a hard-failing guard that checks libSOCRATES_C.so's export table with nm -gU on every macOS job, placed after "Build AGNI (Pkg.instantiate)" rather than right after the SOCRATES-build cache step. get_socrates.sh only produces radlib.a, never socrates/julia/lib/libSOCRATES_C.so, per the note at get_socrates.sh:204-208; that file is deps/build.jl's own output, and build.jl runs unconditionally, on every job, regardless of any cache hit or miss. Checking for it any earlier would fail every SOCRATES cache-miss run, before AGNI even builds the shim. Remove the now-unused shim-diagnostics input, its ci-nightly.yml pass-through, and the JULIA_DEBUG env var on the integration job that only that step read. --- .github/actions/setup-proteus/action.yml | 145 +++-------------------- .github/workflows/ci-nightly.yml | 3 - 2 files changed, 17 insertions(+), 131 deletions(-) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index c93d5a966..b7c987445 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -31,13 +31,6 @@ inputs: data is downloaded (spectral file + minimal stellar spectrum). required: false default: 'false' - shim-diagnostics: - description: > - If 'true', log which libSOCRATES_C.so on disk exports - PS_real_kind_bytes, which file Julia's Libdl resolves, and any - other libSOCRATES_C* copies found on the Julia depot / load path. - required: false - default: 'false' outputs: rad-dir: @@ -598,131 +591,27 @@ runs: julia --project=. deps/build.jl nodata # ------------------------------------------------------------------ - # 7b. Shim diagnostics + # 7b. Verify the SOCRATES shim's symbol export (macOS only) # ------------------------------------------------------------------ - # Prints which libSOCRATES_C.so on disk exports PS_real_kind_bytes, - # which file Julia's Libdl actually resolves and opens, and any other - # libSOCRATES_C* copies on the Julia depot or the restored SOCRATES - # tree that could shadow the intended one. Opt-in per caller via - # shim-diagnostics so it does not run on every setup-proteus job. - - name: Log AGNI shim diagnostics - if: inputs.shim-diagnostics == 'true' && runner.os == 'macOS' + # deps/build.jl above recompiles socrates/julia/lib/libSOCRATES_C.so on + # every run, regardless of the SOCRATES-build or Julia-depot cache + # state, so this check always sees the shim this job just built and + # cannot pass on a stale cached artifact. Guards against Homebrew's gcc + # Darwin driver dropping this symbol from the shim's external symbol + # table, which breaks Julia's dlsym lookup in AGNI; the gcc pin earlier + # in this action keeps it exported. + - name: Verify SOCRATES shim exports PS_real_kind_bytes (macOS) + if: runner.os == 'macOS' shell: bash run: | - export RAD_DIR="$GITHUB_WORKSPACE/socrates" - shim="$RAD_DIR/julia/lib/libSOCRATES_C.so" - obj="$RAD_DIR/julia/lib/Utilities_CF.o" - # nm's own exit status is checked separately from grep's match - # result, so a failed nm (bad file, wrong flag) is distinguishable - # from a clean run that simply found no match. The assignment sits - # in the if condition: this shell runs with GitHub Actions' default - # -eo pipefail, and a bare `out=$(nm ...)` would abort the step. - nm_check() { - local flags="$1" file="$2" label="$3" out rc - if out=$(nm $flags "$file" 2>&1); then - rc=0 - else - rc=$? - fi - if [ "$rc" -ne 0 ]; then - echo "nm $flags $file ($label) exited $rc: $out" - return - fi - echo "$out" | grep -i real_kind || echo "(nm $flags $file ($label) ok, no real_kind match)" - } - echo "RAD_DIR=$RAD_DIR" - echo "-- gfortran --version --" - gfortran --version | head -n 1 || echo "(gfortran not on PATH)" - echo "-- SOCRATES tree revision vs pin --" - git -C "$RAD_DIR" rev-parse HEAD 2>&1 || echo "(not a git checkout / rev-parse failed)" - cat "$RAD_DIR/version" 2>&1 || echo "(no version file)" - echo "-- RAD_DIR/bin/Mk_cmd (compiler flags SOCRATES's own configure wrote) --" - if [ -f "$RAD_DIR/bin/Mk_cmd" ]; then - echo "present:" - cat "$RAD_DIR/bin/Mk_cmd" - else - echo "(no $RAD_DIR/bin/Mk_cmd; Makefile's -include finds nothing and falls back to bare gfortran)" - fi - echo "-- grep -n PS_real_kind_bytes socrates/julia/src/Utilities_CF.f90 --" - grep -n PS_real_kind_bytes "$RAD_DIR/julia/src/Utilities_CF.f90" || echo "(symbol absent from source)" - echo "-- nm -g $obj | grep -i real_kind --" - nm_check "-g" "$obj" "obj, global symbols" - sym_name=$(nm -g "$obj" 2>&1 | grep -i real_kind | awk '{print $NF}' | head -n 1) || true - if [ -z "$sym_name" ]; then - sym_name="_PS_real_kind_bytes" - echo "(nm -g found no real_kind symbol in $obj; export-list probe falls back to $sym_name)" - else - echo "exact linker symbol name for the export list: $sym_name" - fi - echo "-- nm -g $shim | grep -i real_kind --" - nm_check "-g" "$shim" "shim, global symbols" - echo "-- nm -gU $shim | grep PS_real_kind_bytes --" - nm_check "-gU" "$shim" "shim, defined global symbols" - echo "-- nm -gU $RAD_DIR/bin/radlib.a | grep PS_real_kind_bytes --" - nm_check "-gU" "$RAD_DIR/bin/radlib.a" "radlib.a, defined global symbols" - echo "-- Julia-resolved shim path and dlsym(PS_real_kind_bytes) (matches AGNI's own load path) --" - julia -e ' - using Libdl - h = dlopen(joinpath(ENV["RAD_DIR"], "julia/lib/libSOCRATES_C.so")) - println(dlpath(h)) - sym = dlsym(h, :PS_real_kind_bytes; throw_error=false) - println(sym === nothing ? "dlsym(PS_real_kind_bytes) = not found" : "dlsym(PS_real_kind_bytes) = $sym") - ' || echo "(Julia dlopen/dlsym of the shim failed)" - echo "-- other libSOCRATES_C* files under the Julia depot (~/.julia) --" - find ~/.julia -name 'libSOCRATES_C*' 2>/dev/null || echo "(depot search failed)" - echo "-- other libSOCRATES_C* files under RAD_DIR --" - find "$RAD_DIR" -name 'libSOCRATES_C*' 2>/dev/null || echo "(RAD_DIR search failed)" - echo "-- DYLD_LIBRARY_PATH (macOS only, empty elsewhere) --" - echo "DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:-}" - if [ "$(uname -s)" = "Darwin" ]; then - echo "-- nm (no -g, all symbols) $shim | grep -i real_kind --" - nm_check "" "$shim" "shim, all symbols" - echo "-- dyld_info -exports $shim | grep -i real_kind --" - dyld_info -exports "$shim" 2>&1 | grep -i real_kind || echo "(not found / dyld_info failed on $shim)" - echo "-- toolchain identity --" - gfortran -print-prog-name=ld || echo "(gfortran -print-prog-name=ld failed)" - ld -v 2>&1 || echo "(ld -v failed)" - xcrun --show-sdk-version || echo "(xcrun --show-sdk-version failed)" - pkgutil --pkg-info=com.apple.pkg.CLTools_Executables || echo "(pkgutil CLTools_Executables lookup failed, CLT may be an Xcode install instead)" - echo "-- relink diagnostics (reproduce socrates/julia/lib/Makefile's link recipe) --" - objs=$(ls "$RAD_DIR"/julia/lib/*.o 2>/dev/null) - radlib="$RAD_DIR/bin/radlib.a" - base_flags="-O3 -Wall -fPIC -cpp -L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib/" - if [ -z "$objs" ]; then - echo "(no .o files under $RAD_DIR/julia/lib; skipping relink probes)" - else - echo "-- gfortran -shared -v re-link (full driver dump, shows the real ld invocation) --" - gfortran -shared $base_flags -v -o /tmp/relink-verbose.so $objs "$radlib" 2>&1 || echo "(verbose relink failed, see output above)" - relink_probe() { - local label="$1" extra="$2" out="/tmp/relink-${label}.so" - gfortran -shared $base_flags $extra -o "$out" $objs "$radlib" 2>&1 | tail -n 20 || true - if [ -f "$out" ]; then - nm_check "" "$out" "relink $label" - else - echo "(relink $label produced no output file)" - fi - } - echo "-- relink probe: baseline (no extra flag, control) --" - relink_probe "baseline" "" - echo "-- relink probe: -Wl,-ld_classic --" - relink_probe "ld_classic" "-Wl,-ld_classic" - echo "-- relink probe: -Wl,-exported_symbol,_PS_real_kind_bytes --" - relink_probe "exported_symbol" "-Wl,-exported_symbol,_PS_real_kind_bytes" - echo "-- relink probe: -Wl,-export_dynamic --" - relink_probe "export_dynamic" "-Wl,-export_dynamic" - echo "-- relink probe: -Wl,-exported_symbols_list (list-file form, using $sym_name, verbose) --" - list_file="/tmp/relink-exported_symbols_list.txt" - printf '%s\n' "$sym_name" > "$list_file" - gfortran -shared $base_flags -Wl,-exported_symbols_list,"$list_file" -v -o /tmp/relink-exported_symbols_list.so $objs "$radlib" 2>&1 || echo "(verbose relink with export list failed, see output above)" - if [ -f /tmp/relink-exported_symbols_list.so ]; then - nm_check "" "/tmp/relink-exported_symbols_list.so" "relink exported_symbols_list" - else - echo "(relink exported_symbols_list produced no output file)" - fi - fi - else - echo "(link-step probes are macOS-only; skipped on $(uname -s))" + set -euo pipefail + shim="$GITHUB_WORKSPACE/socrates/julia/lib/libSOCRATES_C.so" + if ! nm -gU "$shim" 2>&1 | grep -q PS_real_kind_bytes; then + echo "PS_real_kind_bytes is not an external symbol in $shim" >&2 + nm -gU "$shim" 2>&1 | grep -i real_kind >&2 || echo "(no real_kind symbol found at all)" >&2 + exit 1 fi + echo "PS_real_kind_bytes confirmed exported in $shim" # ------------------------------------------------------------------ # 8. PROTEUS pip install (resolves PyPI + git deps in pyproject.toml) diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index 6bc82152f..685a058fc 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -152,10 +152,7 @@ jobs: - uses: ./.github/actions/setup-proteus with: full-data: 'true' - shim-diagnostics: 'true' - name: Run integration tier (not slow) - env: - JULIA_DEBUG: loading run: | pytest tests/integration \ -m "integration and not slow" \ From 6a67de25419cca7187a60176d26c7c5b306e880f Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Tue, 8 Sep 2026 19:04:12 +0200 Subject: [PATCH 8/9] fix: avoid SIGPIPE false-negative in SOCRATES shim symbol check The new post-build guard piped nm -gU straight into grep -q under set -o pipefail. Once grep -q found its first match it exited and closed the pipe, so nm received SIGPIPE and exited non-zero; pipefail then reported that as the pipeline's own status, so the guard failed even when the symbol was present. A dispatched CI run on macOS hit this today: the guard reported PS_real_kind_bytes missing, but its own diagnostic fallback line, printed one step later from a second nm call, showed the symbol correctly exported as external. That confirmed the failure was the guard's own plumbing, not a real regression in the shim. Capture nm's output into a variable first, then grep the variable with a here-string instead of a live pipe. Nothing downstream can close early on nm anymore, so there is no writer left to receive SIGPIPE, and the check can only fail when the symbol is truly absent. Verified locally: the original form reproduces a false failure against a finite fixture with the symbol present, and the revised form reports it found under the same fixture. --- .github/actions/setup-proteus/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index b7c987445..abb8b8f13 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -606,9 +606,10 @@ runs: run: | set -euo pipefail shim="$GITHUB_WORKSPACE/socrates/julia/lib/libSOCRATES_C.so" - if ! nm -gU "$shim" 2>&1 | grep -q PS_real_kind_bytes; then + symbols="$(nm -gU "$shim" 2>&1)" + if ! grep -q PS_real_kind_bytes <<<"$symbols"; then echo "PS_real_kind_bytes is not an external symbol in $shim" >&2 - nm -gU "$shim" 2>&1 | grep -i real_kind >&2 || echo "(no real_kind symbol found at all)" >&2 + grep -i real_kind <<<"$symbols" >&2 || echo "(no real_kind symbol found at all)" >&2 exit 1 fi echo "PS_real_kind_bytes confirmed exported in $shim" From 6c0cc243e1ba06272142a81c80c2432566a26d4d Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Wed, 9 Sep 2026 10:17:15 +0200 Subject: [PATCH 9/9] Harden the gcc pin: full SHA, current-state comment, robust version check Pin the gcc formula fetch to the full 40-char homebrew-core commit SHA instead of a 7-char short SHA, so the fetch stays unambiguous, and note in the comment that this commit provides gcc 16.1.0. Reword the pin comment to describe the current reason for the pin (this driver exports SOCRATES's dlsym-visible shim symbols, a newer gcc drops them) instead of narrating the migration that produced it. Replace the prefix-match version gate with a token-and-linkage check. `brew unlink` does not remove a runner image's pre-existing gcc keg, so once a second keg coexists, `brew list --versions gcc` prints both versions on one line in Cellar order and the old prefix match fails even though the pinned version is installed. Check the version as its own token and confirm it is the linked keg. Separate the nm capture from its failure check in the shim symbol-export guard. Under set -euo pipefail, a failing nm previously aborted the step before the custom diagnostic could print, so a real nm failure looked identical to an empty result. The existing SIGPIPE-safe here-string grep for the symbol-absent case is unchanged. --- .github/actions/setup-proteus/action.yml | 44 +++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/.github/actions/setup-proteus/action.yml b/.github/actions/setup-proteus/action.yml index abb8b8f13..c76f34975 100644 --- a/.github/actions/setup-proteus/action.yml +++ b/.github/actions/setup-proteus/action.yml @@ -180,11 +180,13 @@ runs: run: | set -euo pipefail source "${{ github.action_path }}/retry-with-timeout.sh" - # gcc is pinned to the last release before Homebrew's Darwin driver - # stopped exporting SOCRATES's dlsym-visible shim symbols. Formulae - # must live in a tap, so the formula is fetched into a throwaway tap; - # pinned before netcdf-fortran so its gcc dependency resolves to it. - gcc_formula_url="https://raw.githubusercontent.com/Homebrew/homebrew-core/4509561/Formula/g/gcc.rb" + # This gcc revision's Darwin driver exports SOCRATES's dlsym-visible + # shim symbols; a newer gcc drops them, so gcc stays pinned here. + # Formulae must live in a tap, so the formula is fetched into a + # throwaway tap; pinned before netcdf-fortran so its gcc dependency + # resolves to it. Commit 4509561fedf07807c6b806f82551e5f8dac5e678 + # provides gcc 16.1.0. + gcc_formula_url="https://raw.githubusercontent.com/Homebrew/homebrew-core/4509561fedf07807c6b806f82551e5f8dac5e678/Formula/g/gcc.rb" brew tap-new local/pinned-toolchain --no-git >/dev/null pinned_tap_formula_dir="$(brew --repository)/Library/Taps/local/homebrew-pinned-toolchain/Formula" retry_with_timeout "fetch pinned gcc formula" 60 10 "" \ @@ -197,14 +199,26 @@ runs: brew_rc=0 retry_with_timeout "brew install gcc (pinned)" 240 10 "" \ brew install --quiet local/pinned-toolchain/gcc || brew_rc=$? - gcc_version="$(brew list --versions gcc 2>/dev/null || true)" - case "$gcc_version" in - "gcc 16.1."*) : ;; - *) - echo "ERROR: brew install rc=$brew_rc, expected pinned gcc 16.1.x, got: '$gcc_version'" >&2 - exit 1 - ;; + # A prefix match on `brew list --versions gcc` false-fails once a + # second gcc keg coexists (unlink above does not remove the runner + # image's pre-existing keg), since the versions then print on one + # line in Cellar order, e.g. "gcc 15.2.0 16.1.0". Check the pinned + # version is present as its own token, and that it is the one + # actually linked. + gcc_versions="$(brew list --versions gcc 2>/dev/null || true)" + gcc_linked_keg="$(readlink "$(brew --prefix)/opt/gcc" 2>/dev/null || true)" + version_present=0 + if tr ' ' '\n' <<<"$gcc_versions" | grep -qx '16\.1\..*'; then + version_present=1 + fi + version_linked=0 + case "$gcc_linked_keg" in + */gcc/16.1.*) version_linked=1 ;; esac + if [ "$version_present" -ne 1 ] || [ "$version_linked" -ne 1 ]; then + echo "ERROR: brew install rc=$brew_rc, expected pinned gcc 16.1.x installed and linked, got versions: '$gcc_versions', linked keg: '$gcc_linked_keg'" >&2 + exit 1 + fi brew pin local/pinned-toolchain/gcc retry_with_timeout "brew install" 240 10 "" \ brew install --quiet netcdf netcdf-fortran openssl@3 || brew_rc=$? @@ -606,7 +620,11 @@ runs: run: | set -euo pipefail shim="$GITHUB_WORKSPACE/socrates/julia/lib/libSOCRATES_C.so" - symbols="$(nm -gU "$shim" 2>&1)" + if ! symbols="$(nm -gU "$shim" 2>&1)"; then + echo "nm failed on $shim:" >&2 + echo "$symbols" >&2 + exit 1 + fi if ! grep -q PS_real_kind_bytes <<<"$symbols"; then echo "PS_real_kind_bytes is not an external symbol in $shim" >&2 grep -i real_kind <<<"$symbols" >&2 || echo "(no real_kind symbol found at all)" >&2