From 32f4c73c7df8ab4aed8cb80aef8baf6c81e7b8dd Mon Sep 17 00:00:00 2001 From: filoferra <66474175+filoferra@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:22:14 +0200 Subject: [PATCH 1/5] Forward -blas_num_threads to Julia's BLAS pool; initialize no longer mutates options --- CHANGELOG.md | 2 ++ scripts/renames.jl | 1 + src/init.jl | 61 +++++++++++++++++++++++++++++--------------- test/blas_threads.jl | 39 ++++++++++++++++++++++++++++ test/runtests.jl | 1 + 5 files changed, 83 insertions(+), 21 deletions(-) create mode 100644 test/blas_threads.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index a54cbeb8a..67015ed50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ - On an Int32 PETSc build (the `PetscInt = "Int32"` preference), `LibPETSc.DMStagStencil`, `MatStencil` and the other C structs had `Int64` fields, so PETSc read them wrong. They now take the integer width of the loaded library, and the two stencil constructors convert their indices to it. - `LibPETSc.DMStagRestoreProductCoordinateArraysRead` restored the arrays through the writable restore, which does not match the read-only get. - The `KSP` and `SNES` docstrings did not say when the options given to the constructor are applied: once at construction for `KSP`, at every `solve!` for `SNES` (and `TS`). +- `-blas_num_threads` had no effect. Every PETSc_jll build calls BLAS through libblastrampoline, so PETSc's BLAS runs in Julia's OpenBLAS pool, which PETSc cannot size. `initialize` now forwards the option to `LinearAlgebra.BLAS.set_num_threads`. Under MPI, pass `-blas_num_threads 1` (or set `OPENBLAS_NUM_THREADS=1`): several ranks on a node otherwise each run a pool of busy-waiting threads, which made a 4-rank DMStag Stokes solve 25× slower. +- `initialize(petsclib; options)` appended `-no_signal_handler` to the caller's `options` vector. ### Added diff --git a/scripts/renames.jl b/scripts/renames.jl index ae0c7c39d..9849c2b05 100644 --- a/scripts/renames.jl +++ b/scripts/renames.jl @@ -402,6 +402,7 @@ const INTERNAL = Set{Symbol}([ :call_with_views, :field_views, :TSStepHookFn, + :apply_blas_num_threads, :_unsafe_local_array, :_local_arrays, :_restore_local_arrays!, diff --git a/src/init.jl b/src/init.jl index 5fef01322..59c81d0dc 100644 --- a/src/init.jl +++ b/src/init.jl @@ -66,6 +66,15 @@ Additionally: - `options::Vector{String} = String[]`: Additional PETSc command-line options. These are passed via the `PETSC_OPTIONS` environment variable. +# BLAS threads +PETSc's BLAS calls run in Julia's OpenBLAS thread pool, the one +`LinearAlgebra.BLAS.set_num_threads` sizes, for every library in `petsclibs`. +`-blas_num_threads n`, given in `options`, in `PETSC_OPTIONS` or on the command +line, sets that pool to `n` threads. Under MPI, several ranks on one node each +own such a pool, and its busy-waiting threads compete with the ranks for cores: +a solve can run many times slower with the right answer. Pass +`-blas_num_threads 1` there, or set `OPENBLAS_NUM_THREADS=1`. + # Examples ```julia # Basic initialization @@ -82,6 +91,9 @@ PETSc.initialize(petsclib; log_view = true, options = [":logfile.txt", "-log_vie # Pass custom PETSc options without logging PETSc.initialize(petsclib; options = ["-malloc_debug", "-on_error_abort"]) + +# One BLAS thread per rank under MPI +PETSc.initialize(petsclib; options = ["-blas_num_threads", "1"]) ``` # External Links @@ -94,31 +106,23 @@ end function initialize(petsclib; log_view::Bool = false, options = String[]) if !isinitialized(petsclib) - - # deactivate the signal handler to avoid conflicts with Julia's own handlers when using multithreading - push!(options, " -no_signal_handler ") - - if log_view || !isempty(options) - cli_opts = build_petsc_options(log_view, options) - prev_opts = get(ENV, "PETSC_OPTIONS", "") - ENV["PETSC_OPTIONS"] = isempty(prev_opts) ? cli_opts : "$prev_opts $cli_opts" - try - ensure_mpi_initialized() - petsclib.age += 1 - LibPETSc.PetscInitializeNoArguments(petsclib) - post_initialize(petsclib) - finally - if isempty(prev_opts) - delete!(ENV, "PETSC_OPTIONS") - else - ENV["PETSC_OPTIONS"] = prev_opts - end - end - else + # PETSc's signal handler conflicts with Julia's own handlers when using multithreading + # Appended to a copy: the caller's vector stays as it was. + options = [String.(options); "-no_signal_handler"] + cli_opts = build_petsc_options(log_view, options) + prev_opts = get(ENV, "PETSC_OPTIONS", "") + ENV["PETSC_OPTIONS"] = isempty(prev_opts) ? cli_opts : "$prev_opts $cli_opts" + try ensure_mpi_initialized() petsclib.age += 1 LibPETSc.PetscInitializeNoArguments(petsclib) post_initialize(petsclib) + finally + if isempty(prev_opts) + delete!(ENV, "PETSC_OPTIONS") + else + ENV["PETSC_OPTIONS"] = prev_opts + end end end return nothing @@ -212,6 +216,21 @@ function post_initialize(petsclib) LibPETSc.PetscPopSignalHandler(petsclib) _reset_stale_register_flags(petsclib) atexit(() -> finalize(petsclib)) + apply_blas_num_threads(petsclib) + return nothing +end + +# Every PETSc_jll build calls BLAS through libblastrampoline, so PETSc's BLAS runs +# in Julia's own OpenBLAS thread pool. PETSc cannot size that pool: it was built +# against a generic BLAS, so `PetscBLASSetNumThreads` only records the number. +# `-blas_num_threads` is therefore forwarded here. The pool is process-wide and +# is left as it is at `finalize`: it serves all Julia code, not PETSc objects. +function apply_blas_num_threads(petsclib) + global_options = LibPETSc.PetscOptions{typeof(petsclib)}(C_NULL, petsclib.age; own = false) + n, set = LibPETSc.PetscOptionsGetInt(petsclib, global_options, "", "-blas_num_threads") + Bool(set) || return nothing + n >= 1 || throw(ArgumentError("-blas_num_threads must be at least 1, got $n")) + LinearAlgebra.BLAS.set_num_threads(Int(n)) return nothing end diff --git a/test/blas_threads.jl b/test/blas_threads.jl new file mode 100644 index 000000000..c9e2f3e92 --- /dev/null +++ b/test/blas_threads.jl @@ -0,0 +1,39 @@ +# test/blas_threads.jl +# PETSc's BLAS runs in Julia's OpenBLAS pool, so `-blas_num_threads` has to size +# that pool: PETSc's own setter cannot reach it. + +using Test +using PETSc +using MPI +using LinearAlgebra: BLAS + +MPI.Initialized() || MPI.Init() + +@testset "BLAS threads" begin + petsclib = PETSc.petsclibs[1] + PETSc.isinitialized(petsclib) && PETSc.finalize(petsclib) + n0 = BLAS.get_num_threads() + + @testset "-blas_num_threads sizes Julia's pool" begin + options = ["-blas_num_threads", "2"] + PETSc.initialize(petsclib; options) + @test BLAS.get_num_threads() == 2 + @test options == ["-blas_num_threads", "2"] # the caller's vector is untouched + PETSc.finalize(petsclib) + @test BLAS.get_num_threads() == 2 # finalize leaves the pool alone + end + + @testset "without the option the pool keeps its size" begin + BLAS.set_num_threads(3) + PETSc.initialize(petsclib) + @test BLAS.get_num_threads() == 3 + PETSc.finalize(petsclib) + end + + @testset "a count below one is rejected" begin + @test_throws ArgumentError PETSc.initialize(petsclib; options = ["-blas_num_threads", "0"]) + PETSc.finalize(petsclib) + end + + BLAS.set_num_threads(n0) +end diff --git a/test/runtests.jl b/test/runtests.jl index fe17ac61f..34b5bc107 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,6 +25,7 @@ mpi_tests = ("mpivec.jl", "mpimat.jl", "ksp.jl", "dmstag.jl", "mpi_dmstag_halos. do_mpi = true include("init.jl") +include("blas_threads.jl") # -blas_num_threads sizes the BLAS pool PETSc runs in include("lib.jl") include("vec.jl") # autowrapped include("mat.jl") # autowrapped From 6ea8957f20c47b33334f5291e526fb5dbcd79e00 Mon Sep 17 00:00:00 2001 From: filoferra <66474175+filoferra@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:42:58 +0200 Subject: [PATCH 2/5] When several MPI ranks share a node, default to one BLAS thread per rank --- CHANGELOG.md | 3 ++- scripts/renames.jl | 2 ++ src/init.jl | 42 +++++++++++++++++++++++++++++++--------- test/mpi_blas_threads.jl | 28 +++++++++++++++++++++++++++ test/runtests.jl | 2 +- 5 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 test/mpi_blas_threads.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 67015ed50..62923ef6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ - On an Int32 PETSc build (the `PetscInt = "Int32"` preference), `LibPETSc.DMStagStencil`, `MatStencil` and the other C structs had `Int64` fields, so PETSc read them wrong. They now take the integer width of the loaded library, and the two stencil constructors convert their indices to it. - `LibPETSc.DMStagRestoreProductCoordinateArraysRead` restored the arrays through the writable restore, which does not match the read-only get. - The `KSP` and `SNES` docstrings did not say when the options given to the constructor are applied: once at construction for `KSP`, at every `solve!` for `SNES` (and `TS`). -- `-blas_num_threads` had no effect. Every PETSc_jll build calls BLAS through libblastrampoline, so PETSc's BLAS runs in Julia's OpenBLAS pool, which PETSc cannot size. `initialize` now forwards the option to `LinearAlgebra.BLAS.set_num_threads`. Under MPI, pass `-blas_num_threads 1` (or set `OPENBLAS_NUM_THREADS=1`): several ranks on a node otherwise each run a pool of busy-waiting threads, which made a 4-rank DMStag Stokes solve 25× slower. +- `-blas_num_threads` had no effect. Every PETSc_jll build calls BLAS through libblastrampoline, so PETSc's BLAS runs in Julia's OpenBLAS pool, which PETSc cannot size. `initialize` now forwards the option to `LinearAlgebra.BLAS.set_num_threads`. - `initialize(petsclib; options)` appended `-no_signal_handler` to the caller's `options` vector. ### Added @@ -37,6 +37,7 @@ ### Changed +- When several MPI ranks share a node, `initialize` sets Julia's BLAS pool to one thread, unless `-blas_num_threads`, `OPENBLAS_NUM_THREADS` or `OMP_NUM_THREADS` says otherwise. Each rank otherwise ran a pool of busy-waiting threads, which made a 4-rank DMStag Stokes solve 25× slower. Serial runs and one rank per node are unaffected. - `solution(ksp)`, `solution(ts)`, `local_coordinates(dm)` and the `vatol`/`vrtol` of `tolerances(ts)` return a borrowed `PetscVec`, as `solution(snes)` already did, instead of a `VecPtr`. Both are `AbstractPetscVec`s with the same ownership, so only code that checks for `VecPtr` by type notices. - `LibPETSc` release functions that free a C array of handles (`VecDestroyVecs`, `MatDestroyMatrices`, `DMPlexRestoreConeRecursive`, …) no longer accept a Julia array of raw handles, which PETSc would have freed as its own memory. They take the pointer PETSc returned; `MatDestroySubMatrices`, `MatDestroyMatrices`, `VecNestRestoreSubVecsRead` and the two subdomain destroyers also take the vector the matching wrapper returned. diff --git a/scripts/renames.jl b/scripts/renames.jl index 9849c2b05..30c58eb56 100644 --- a/scripts/renames.jl +++ b/scripts/renames.jl @@ -403,6 +403,8 @@ const INTERNAL = Set{Symbol}([ :field_views, :TSStepHookFn, :apply_blas_num_threads, + :BLAS_THREAD_VARIABLES, + :ranks_on_node, :_unsafe_local_array, :_local_arrays, :_restore_local_arrays!, diff --git a/src/init.jl b/src/init.jl index 59c81d0dc..4cb399f91 100644 --- a/src/init.jl +++ b/src/init.jl @@ -70,10 +70,15 @@ Additionally: PETSc's BLAS calls run in Julia's OpenBLAS thread pool, the one `LinearAlgebra.BLAS.set_num_threads` sizes, for every library in `petsclibs`. `-blas_num_threads n`, given in `options`, in `PETSC_OPTIONS` or on the command -line, sets that pool to `n` threads. Under MPI, several ranks on one node each -own such a pool, and its busy-waiting threads compete with the ranks for cores: -a solve can run many times slower with the right answer. Pass -`-blas_num_threads 1` there, or set `OPENBLAS_NUM_THREADS=1`. +line, sets that pool to `n` threads. + +Without that option, when several MPI ranks share a node, `initialize` sets the +pool to one thread. Each rank owns a pool, and its busy-waiting threads would +otherwise compete with the ranks for cores: a solve can run many times slower +with the right answer. A set `OPENBLAS_NUM_THREADS` or `OMP_NUM_THREADS` also +leaves the pool as it is. Serial runs, and runs with one rank per node, are +unaffected. This changes Julia's process-wide BLAS setting, which `finalize` +does not restore. # Examples ```julia @@ -223,17 +228,36 @@ end # Every PETSc_jll build calls BLAS through libblastrampoline, so PETSc's BLAS runs # in Julia's own OpenBLAS thread pool. PETSc cannot size that pool: it was built # against a generic BLAS, so `PetscBLASSetNumThreads` only records the number. -# `-blas_num_threads` is therefore forwarded here. The pool is process-wide and -# is left as it is at `finalize`: it serves all Julia code, not PETSc objects. +# `-blas_num_threads` is therefore forwarded here. Without it, ranks sharing a +# node get one thread each, unless the environment already sizes the pool. The +# pool is process-wide and is left as it is at `finalize`: it serves all Julia +# code, not PETSc objects. function apply_blas_num_threads(petsclib) global_options = LibPETSc.PetscOptions{typeof(petsclib)}(C_NULL, petsclib.age; own = false) n, set = LibPETSc.PetscOptionsGetInt(petsclib, global_options, "", "-blas_num_threads") - Bool(set) || return nothing - n >= 1 || throw(ArgumentError("-blas_num_threads must be at least 1, got $n")) - LinearAlgebra.BLAS.set_num_threads(Int(n)) + if Bool(set) + n >= 1 || throw(ArgumentError("-blas_num_threads must be at least 1, got $n")) + LinearAlgebra.BLAS.set_num_threads(Int(n)) + elseif !any(v -> haskey(ENV, v), BLAS_THREAD_VARIABLES) && ranks_on_node() > 1 + LinearAlgebra.BLAS.set_num_threads(1) + LibPETSc.PetscBLASSetNumThreads(petsclib, petsclib.PetscInt(1)) + end return nothing end +# The variables OpenBLAS and PETSc read for a thread count +const BLAS_THREAD_VARIABLES = ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS") + +# How many ranks of MPI.COMM_WORLD share this rank's node. Collective. +function ranks_on_node() + world = MPI.COMM_WORLD + MPI.Comm_size(world) == 1 && return 1 + node = MPI.Comm_split_type(world, MPI.COMM_TYPE_SHARED, MPI.Comm_rank(world)) + n = MPI.Comm_size(node) + MPI.free(node) + return n +end + # PETSc 3.25.x: `TaoFinalizePackage` destroys the `TaoTerm` type list but never resets # `TaoTermRegisterAllCalled`, so after a finalize/initialize cycle `TaoCreate` fails with # "Unable to find requested TaoTerm type callbacks". Reset the flag so the list is rebuilt. diff --git a/test/mpi_blas_threads.jl b/test/mpi_blas_threads.jl new file mode 100644 index 000000000..3fab355e4 --- /dev/null +++ b/test/mpi_blas_threads.jl @@ -0,0 +1,28 @@ +# test/mpi_blas_threads.jl +# Run on several ranks of one node: `initialize` gives each rank one BLAS +# thread, unless `-blas_num_threads` or the environment sizes the pool. + +using Test +using MPI +MPI.Initialized() || MPI.Init() +using PETSc +using LinearAlgebra: BLAS + +@testset "BLAS threads under MPI" begin + petsclib = PETSc.petsclibs[1] + PETSc.isinitialized(petsclib) && PETSc.finalize(petsclib) + environment_sets_it = any(v -> haskey(ENV, v), PETSc.BLAS_THREAD_VARIABLES) + node = MPI.Comm_split_type(MPI.COMM_WORLD, MPI.COMM_TYPE_SHARED, MPI.Comm_rank(MPI.COMM_WORLD)) + shared = MPI.Comm_size(node) > 1 + MPI.free(node) + + BLAS.set_num_threads(4) + PETSc.initialize(petsclib) + @test BLAS.get_num_threads() == (shared && !environment_sets_it ? 1 : 4) + PETSc.finalize(petsclib) + + # the option wins over the default + PETSc.initialize(petsclib; options = ["-blas_num_threads", "2"]) + @test BLAS.get_num_threads() == 2 + PETSc.finalize(petsclib) +end diff --git a/test/runtests.jl b/test/runtests.jl index 34b5bc107..d941b1d2b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,7 +19,7 @@ else end # Do the MPI tests first so we do not have mpi running inside MPI -mpi_tests = ("mpivec.jl", "mpimat.jl", "ksp.jl", "dmstag.jl", "mpi_dmstag_halos.jl") +mpi_tests = ("mpivec.jl", "mpimat.jl", "ksp.jl", "dmstag.jl", "mpi_dmstag_halos.jl", "mpi_blas_threads.jl") # PETSc_jll >= 3.25.4 has MPI-enabled Windows binaries (MicrosoftMPI), so the MPI tests run everywhere do_mpi = true From 3916ae1f419d59eeac2749c685dc75a02b414d96 Mon Sep 17 00:00:00 2001 From: filoferra <66474175+filoferra@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:31:34 +0200 Subject: [PATCH 3/5] initialize passes options as PETSc's command line, so they reach PETSc on Windows --- CHANGELOG.md | 1 + src/autowrapped/Sys_wrappers.jl | 140 ++++-------------- src/init.jl | 48 +++--- test/blas_threads.jl | 1 + test/init.jl | 15 ++ .../generator/overrides/PetscInitialize.jl | 44 ++++++ 6 files changed, 111 insertions(+), 138 deletions(-) create mode 100644 wrapping/generator/overrides/PetscInitialize.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 62923ef6f..2bf96981e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - The `KSP` and `SNES` docstrings did not say when the options given to the constructor are applied: once at construction for `KSP`, at every `solve!` for `SNES` (and `TS`). - `-blas_num_threads` had no effect. Every PETSc_jll build calls BLAS through libblastrampoline, so PETSc's BLAS runs in Julia's OpenBLAS pool, which PETSc cannot size. `initialize` now forwards the option to `LinearAlgebra.BLAS.set_num_threads`. - `initialize(petsclib; options)` appended `-no_signal_handler` to the caller's `options` vector. +- On Windows, the `options` and `log_view` given to `initialize` never reached PETSc: they went through `ENV["PETSC_OPTIONS"]`, which PETSc there reads from its own copy of the environment. They are now PETSc's command line (`LibPETSc.PetscInitialize` takes a `Vector{String}`), and still override a `PETSC_OPTIONS` set before Julia starts. ### Added diff --git a/src/autowrapped/Sys_wrappers.jl b/src/autowrapped/Sys_wrappers.jl index 093c32e21..4646f3495 100644 --- a/src/autowrapped/Sys_wrappers.jl +++ b/src/autowrapped/Sys_wrappers.jl @@ -5981,130 +5981,50 @@ end return nothing end +# override for PetscInitialize; C signature: PetscInitialize(int* argc, char*** args, const char file[], const char help[]) """ - argc::Cint = PetscInitialize(petsclib::PetscLibType, args::Cchar, file::String, help::String) -Initializes the PETSc database and MPI. -`PetscInitialize()` calls MPI_Init() if that has yet to be called, -so this routine should always be called near the beginning of -your program -- usually the very first line! + PetscInitialize(petsclib::PetscLibType, args::Vector{String}, file::Union{Nothing, String} = nothing, help::Union{Nothing, String} = nothing) +Initializes the PETSc database and MPI, with `args` as the command line: `args[1]` is the program name and the rest are +options, one token per entry, such as `["prog", "-ksp_type", "cg"]`. Options given here override those in the +`PETSC_OPTIONS` environment variable. + +PETSc keeps pointers to `args` until `PetscFinalize()`, so the strings are held here until the next call. Collective on `MPI_COMM_WORLD` or `PETSC_COMM_WORLD` if it has been set Input Parameters: -- `argc` - count of number of command line arguments -- `args` - the command line arguments -- `file` - [optional] PETSc database file, append ":yaml" to filename to specify YAML options format. -Use `NULL` or empty string to not check for code specific file. -Also checks `~/.petscrc`, `.petscrc` and `petscrc`. -Use `-skip_petscrc` in the code specific file (or command line) to skip `~/.petscrc`, `.petscrc` and `petscrc` files. -- `help` - [optional] Help message to print, use `NULL` for no message - -If you wish PETSc code to run ONLY on a subcommunicator of `MPI_COMM_WORLD`, create that -communicator first and assign it to `PETSC_COMM_WORLD` BEFORE calling `PetscInitialize()`. -then do this. If ALL processes in the job are using `PetscInitialize()` and `PetscFinalize()` then you don't need to do this, even -if different subcommunicators of the job are doing different things with PETSc. - -Options Database Keys: -- `-help [intro]` - prints help method for each option; if `intro` is given the program stops after printing the introductory help message -- `-start_in_debugger [(noxterm)],[(gdb|lldb|...)]` - Starts program in debugger -- `-on_error_attach_debugger [(noxterm)],[(gdb|lldb|...)]` - Starts debugger when error detected -- `-on_error_emacs machinename` - causes `emacsclient` to jump to error file if an error is detected -- `-on_error_abort` - calls `abort()` when error detected (no traceback) -- `-on_error_mpiabort` - calls `MPI_abort()` when error detected -- `-error_output_stdout` - prints PETSc error messages to `stdout` instead of the default `stderr` -- `-error_output_none` - does not print the error messages (but handles errors in the same way as if this was not called) -- `-debugger_ranks rank1,rank2,...` - Indicates MPI ranks to start in debugger -- `-debugger_pause secs` - Pauses debugger, use if it takes a long time for the debugger to start up on your system, `sleeptime` is number of seconds to sleep -- `-stop_for_debugger` - Print message on how to attach debugger manually to -process and wait (`-debugger_pause`) seconds for attachment -- `-malloc_dump` - prints a list of all unfreed memory at the end of the run -- `-malloc_test` - like `-malloc_dump` `-malloc_debug`, only active for debugging build, ignored in optimized build. Often set in `PETSC_OPTIONS` environmental variable -- `-malloc_view [filename]` - show a list of all allocated memory during `PetscFinalize()` -- `-malloc_view_threshold t` - only list memory allocations of size greater than t with `-malloc_view` -- `-malloc_requested_size` - malloc logging will record the requested size rather than (possibly large) size after alignment -- `-fp_trap` - Stops on floating point exceptions -- `-no_signal_handler` - Indicates not to trap error signals -- `-python exe` - Initializes Python, and optionally takes a Python executable name -- `-mpiuni-allow-multiprocess-launch` - allow `mpiexec` to launch multiple independent MPI-Uni jobs, otherwise a sanity check error is invoked to prevent misuse of MPI-Uni - -Options Database Keys for Option Database: -- `-skip_petscrc` - skip the default option files `~/.petscrc`, `.petscrc`, `petscrc` -- `-options_monitor` - monitor all set options to standard output for the whole program run -- `-options_monitor_cancel` - cancel options monitoring hard-wired using `PetscOptionsMonitorSet()` - -Options -options_monitor_{all,cancel} are -position-independent and apply to all options set since the PETSc start. -They can be used also in option files. - -See `PetscOptionsMonitorSet()` to do monitoring programmatically. - -Options Database Keys for Profiling: -See Users-Manual: ch_profiling for details. -- `-info [filename][:[~]c1,c2,...[:[~]self]]` - Prints verbose information for classes c1, c2, etc. See `PetscInfo()`. -- `-log_sync` - Enable barrier synchronization for all events. This option is useful to debug imbalance within each event, -however it slows things down and gives a distorted view of the overall runtime. -- `-log_trace [filename]` - Print traces of all PETSc calls to the screen (useful to determine where a program -hangs without running in the debugger). See `PetscLogTraceBegin()`. -- `-log_view [:filename:format][,[:filename:format]...]` - Prints summary of flop and timing information to screen or file, see `PetscLogView()` (up to 4 viewers) -- `-log_view_memory` - Includes in the summary from -log_view the memory used in each event, see `PetscLogView()`. -- `-log_view_gpu_time` - Includes in the summary from -log_view the time used in each GPU kernel, see `PetscLogView(). -- `-log_view_gpu_energy` - Includes in the summary from -log_view the energy (estimated with power*gtime) consumed in each GPU kernel, see `PetscLogView()`. -- `-log_view_gpu_energy_meter` - Includes in the summary from -log_view the energy (readings from meters) consumed in each GPU kernel, see `PetscLogView()`. -- `-log_exclude: c1,c2,...` - excludes subset of object classes from logging, for example vec,ksp would exclude the `Vec` and `KSP` classes -- `-log [filename]` - Logs profiling information in a dump file, see `PetscLogDump()`. -- `-log_all [filename]` - Same as `-log`. -- `-log_mpe [filename]` - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution) -- `-log_perfstubs` - Starts a log handler with the perfstubs interface (which is used by TAU) -- `-log_nvtx` - Starts an nvtx log handler for use with Nsight -- `-log_roctx` - Starts an roctx log handler for use with rocprof on AMD GPUs -- `-viewfromoptions on,off` - Enable or disable `XXXSetFromOptions()` calls, for applications with many small solves turn this off -- `-get_total_flops` - Returns total flops done by all processors -- `-memory_view` - Print memory usage at end of run -- `-check_pointer_intensity 0,1,2` - if pointers are checked for validity (debug version only), using 0 will result in faster code - -Options Database Keys for SAWs: -- `-saws_port portnumber` - port number to publish SAWs data, default is 8080 -- `-saws_port_auto_select` - have SAWs select a new unique port number where it publishes the data, the URL is printed to the screen -this is useful when you are running many jobs that utilize SAWs at the same time -- `-saws_log filename` - save a log of all SAWs communication -- `-saws_https certificate_file` - have SAWs use HTTPS instead of HTTP -- `-saws_root directory` - allow SAWs to have access to the given directory to search for requested resources and files - -Environmental Variables: -- `PETSC_TMP` - alternative directory to use instead of `/tmp` -- `PETSC_SHARED_TMP` - `/tmp` is shared by all processes -- `PETSC_NOT_SHARED_TMP` - each process has its own private `/tmp` -- `PETSC_OPTIONS` - a string containing additional options for PETSc in the form of command line "-key value" pairs -- `PETSC_OPTIONS_YAML` - (requires configuring PETSc to use libyaml with `--download-yaml`) a string containing additional options for PETSc in the form of a YAML document -- `PETSC_VIEWER_SOCKET_PORT` - socket number to use for socket viewer -- `PETSC_VIEWER_SOCKET_MACHINE` - machine to use for socket viewer to connect to +- `args` - the command line, program name first +- `file` - an optional options file, or `nothing` +- `help` - an optional help message printed by `-help`, or `nothing` Level: beginner -See also: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscInitializeNoArguments()`, `PetscLogGpuTime()` +See also: `PetscFinalize()`, `PetscInitializeNoArguments()`, `PetscGetArgs()` # External Links $(_doc_external("Sys/PetscInitialize")) """ -function PetscInitialize(petsclib::PetscLibType, args::Cchar, file::String, help::String) - error("PetscInitialize: no generated method for these argument types") -end +function PetscInitialize(petsclib::PetscLibType, args::Vector{String}, file::Union{Nothing, String} = nothing, help::Union{Nothing, String} = nothing) end -@for_petsc function PetscInitialize(petsclib::$UnionPetscLib, args::Cchar, file::String, help::String ) - argc_ = Ref{Cint}() - - @chk ccall( - (:PetscInitialize, $petsc_library), - PetscErrorCode, - (Ptr{Cint}, Ptr{Ptr{Cchar}}, Ptr{Cchar}, Ptr{Cchar}), - argc_, args, file, help, - ) - - argc = argc_[] - - return argc -end +# the command line each library was last initialized with, kept alive for PETSc +const _petsc_initialize_args = IdDict{Any, Any}() +@for_petsc function PetscInitialize(petsclib::$UnionPetscLib, args::Vector{String}, file::Union{Nothing, String} = nothing, help::Union{Nothing, String} = nothing) + isempty(args) && throw(ArgumentError("args must start with a program name")) + args = copy(args) + argv = Ptr{Cchar}[pointer(a) for a in args] + push!(argv, C_NULL) + _petsc_initialize_args[petsclib] = (args, argv) + argc_ = Ref{Cint}(length(args)) + args_ = Ref{Ptr{Ptr{Cchar}}}(pointer(argv)) + @chk ccall( + (:PetscInitialize, $petsc_library), + PetscErrorCode, + (Ptr{Cint}, Ptr{Ptr{Ptr{Cchar}}}, Ptr{Cchar}, Ptr{Cchar}), + argc_, args_, something(file, C_NULL), something(help, C_NULL), + ) + return nothing +end """ PetscInitializeFortran(petsclib::PetscLibType) Routine that should be called soon AFTER diff --git a/src/init.jl b/src/init.jl index 4cb399f91..c0612d6d7 100644 --- a/src/init.jl +++ b/src/init.jl @@ -63,14 +63,15 @@ Additionally: # Arguments - `log_view::Bool = false`: Enable PETSc's `-log_view` performance logging. When enabled, PETSc will output performance statistics at finalization. -- `options::Vector{String} = String[]`: Additional PETSc command-line options. - These are passed via the `PETSC_OPTIONS` environment variable. +- `options::Vector{String} = String[]`: Additional PETSc command-line options, + handed to `PetscInitialize` as its command line. They override the + `PETSC_OPTIONS` environment variable. # BLAS threads PETSc's BLAS calls run in Julia's OpenBLAS thread pool, the one `LinearAlgebra.BLAS.set_num_threads` sizes, for every library in `petsclibs`. -`-blas_num_threads n`, given in `options`, in `PETSC_OPTIONS` or on the command -line, sets that pool to `n` threads. +`-blas_num_threads n`, given in `options` or in `PETSC_OPTIONS`, sets that pool +to `n` threads. Without that option, when several MPI ranks share a node, `initialize` sets the pool to one thread. Each rank owns a pool, and its busy-waiting threads would @@ -102,7 +103,7 @@ PETSc.initialize(petsclib; options = ["-blas_num_threads", "1"]) ``` # External Links -$(doc_external("Sys/PetscInitializeNoArguments")) +$(doc_external("Sys/PetscInitialize")) """ function initialize(; log_view::Bool = false, options = String[]) map(petsclib -> initialize(petsclib; log_view, options), petsclibs) @@ -111,24 +112,10 @@ end function initialize(petsclib; log_view::Bool = false, options = String[]) if !isinitialized(petsclib) - # PETSc's signal handler conflicts with Julia's own handlers when using multithreading - # Appended to a copy: the caller's vector stays as it was. - options = [String.(options); "-no_signal_handler"] - cli_opts = build_petsc_options(log_view, options) - prev_opts = get(ENV, "PETSC_OPTIONS", "") - ENV["PETSC_OPTIONS"] = isempty(prev_opts) ? cli_opts : "$prev_opts $cli_opts" - try - ensure_mpi_initialized() - petsclib.age += 1 - LibPETSc.PetscInitializeNoArguments(petsclib) - post_initialize(petsclib) - finally - if isempty(prev_opts) - delete!(ENV, "PETSC_OPTIONS") - else - ENV["PETSC_OPTIONS"] = prev_opts - end - end + ensure_mpi_initialized() + petsclib.age += 1 + LibPETSc.PetscInitialize(petsclib, build_petsc_options(log_view, options)) + post_initialize(petsclib) end return nothing end @@ -202,13 +189,18 @@ the caller destroys the result), and the objects a callback receives. """ owns(obj) = obj.own +# The command line PETSc is initialized with: the options go in as arguments, +# not through `PETSC_OPTIONS`, which PETSc on Windows cannot see once set from +# Julia. An entry holding several words, such as "-ksp_type cg", is split. +# PETSc's signal handler conflicts with Julia's own, notably under multithreading. function build_petsc_options(log_view::Bool, options) - opts = String[] - if log_view - push!(opts, "-log_view") + args = [petsc_program_name] + log_view && push!(args, "-log_view") + for opt in options + append!(args, split(String(opt))) end - append!(opts, [String(opt) for opt in options]) - return join(opts, " ") + push!(args, "-no_signal_handler") + return args end function ensure_mpi_initialized() diff --git a/test/blas_threads.jl b/test/blas_threads.jl index c9e2f3e92..6ba2eca3c 100644 --- a/test/blas_threads.jl +++ b/test/blas_threads.jl @@ -15,6 +15,7 @@ MPI.Initialized() || MPI.Init() n0 = BLAS.get_num_threads() @testset "-blas_num_threads sizes Julia's pool" begin + BLAS.set_num_threads(1) # so that reaching 2 shows the option arrived options = ["-blas_num_threads", "2"] PETSc.initialize(petsclib; options) @test BLAS.get_num_threads() == 2 diff --git a/test/init.jl b/test/init.jl index 7eedbdab7..3b650431d 100644 --- a/test/init.jl +++ b/test/init.jl @@ -218,6 +218,21 @@ using PETSc end end + # `options` reach PETSc on every platform: they are handed to PetscInitialize + # as its command line, where PETSC_OPTIONS set from Julia is invisible on Windows + @testset "options reach PETSc" begin + petsclib = PETSc.petsclibs[1] + PETSc.isinitialized(petsclib) && PETSc.finalize(petsclib) + options = ["-petscjl_test_int", "7", "-petscjl_test_pair 8"] + PETSc.initialize(petsclib; options) + global_options = PETSc.LibPETSc.PetscOptions{typeof(petsclib)}(C_NULL, petsclib.age; own = false) + value(name) = PETSc.LibPETSc.PetscOptionsGetInt(petsclib, global_options, "", name) + @test value("-petscjl_test_int") == (7, PETSc.LibPETSc.PETSC_TRUE) + @test value("-petscjl_test_pair") == (8, PETSc.LibPETSc.PETSC_TRUE) # one entry, two words + @test options == ["-petscjl_test_int", "7", "-petscjl_test_pair 8"] # left as given + PETSc.finalize(petsclib) + end + # naming.md §14: user input raises `ArgumentError`, not a bare `error`. @testset "ArgumentError on user input" begin @test_throws ArgumentError PETSc.set_library!( diff --git a/wrapping/generator/overrides/PetscInitialize.jl b/wrapping/generator/overrides/PetscInitialize.jl new file mode 100644 index 000000000..3e834df8e --- /dev/null +++ b/wrapping/generator/overrides/PetscInitialize.jl @@ -0,0 +1,44 @@ +# override for PetscInitialize; C signature: PetscInitialize(int* argc, char*** args, const char file[], const char help[]) +""" + PetscInitialize(petsclib::PetscLibType, args::Vector{String}, file::Union{Nothing, String} = nothing, help::Union{Nothing, String} = nothing) +Initializes the PETSc database and MPI, with `args` as the command line: `args[1]` is the program name and the rest are +options, one token per entry, such as `["prog", "-ksp_type", "cg"]`. Options given here override those in the +`PETSC_OPTIONS` environment variable. + +PETSc keeps pointers to `args` until `PetscFinalize()`, so the strings are held here until the next call. + +Collective on `MPI_COMM_WORLD` or `PETSC_COMM_WORLD` if it has been set + +Input Parameters: +- `args` - the command line, program name first +- `file` - an optional options file, or `nothing` +- `help` - an optional help message printed by `-help`, or `nothing` + +Level: beginner + +See also: `PetscFinalize()`, `PetscInitializeNoArguments()`, `PetscGetArgs()` + +# External Links +$(_doc_external("Sys/PetscInitialize")) +""" +function PetscInitialize(petsclib::PetscLibType, args::Vector{String}, file::Union{Nothing, String} = nothing, help::Union{Nothing, String} = nothing) end + +# the command line each library was last initialized with, kept alive for PETSc +const _petsc_initialize_args = IdDict{Any, Any}() + +@for_petsc function PetscInitialize(petsclib::$UnionPetscLib, args::Vector{String}, file::Union{Nothing, String} = nothing, help::Union{Nothing, String} = nothing) + isempty(args) && throw(ArgumentError("args must start with a program name")) + args = copy(args) + argv = Ptr{Cchar}[pointer(a) for a in args] + push!(argv, C_NULL) + _petsc_initialize_args[petsclib] = (args, argv) + argc_ = Ref{Cint}(length(args)) + args_ = Ref{Ptr{Ptr{Cchar}}}(pointer(argv)) + @chk ccall( + (:PetscInitialize, $petsc_library), + PetscErrorCode, + (Ptr{Cint}, Ptr{Ptr{Ptr{Cchar}}}, Ptr{Cchar}, Ptr{Cchar}), + argc_, args_, something(file, C_NULL), something(help, C_NULL), + ) + return nothing +end From f4df6553b4274c504c9680c5747f9fefba631bce Mon Sep 17 00:00:00 2001 From: filoferra <66474175+filoferra@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:05:36 +0200 Subject: [PATCH 4/5] Log tests use relative paths: PETSc splits an absolute Windows path at its drive letter --- src/init.jl | 3 ++- test/init.jl | 36 +++++++++++++----------------------- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/init.jl b/src/init.jl index c0612d6d7..d25698bb9 100644 --- a/src/init.jl +++ b/src/init.jl @@ -89,7 +89,8 @@ PETSc.initialize(petsclib) # Enable performance logging to stdout PETSc.initialize(petsclib; log_view = true) -# Write log to a file +# Write log to a file. PETSc reads `:file:format`, so on Windows give a path +# without a drive letter, such as a relative one PETSc.initialize(petsclib; log_view = true, options = [":logfile.txt"]) # Enable memory logging diff --git a/test/init.jl b/test/init.jl index 3b650431d..c72130c2c 100644 --- a/test/init.jl +++ b/test/init.jl @@ -37,9 +37,10 @@ using PETSc # Store original PETSC_OPTIONS if it exists original_opts = get(ENV, "PETSC_OPTIONS", nothing) - # Initialize with log view and a test file - # Use current directory to avoid Windows temp directory issues with short paths - test_log_file = joinpath(pwd(), "test_log_$(hash(petsclib)).txt") + # Initialize with log view and a test file. The name is relative, so the + # file lands in the current directory: PETSc reads `-log_view :file:format`, + # and would split an absolute Windows path at its drive letter. + test_log_file = "test_log_$(hash(petsclib)).txt" try PETSc.initialize(petsclib; log_view = true, options = [":$test_log_file"]) @@ -57,15 +58,9 @@ using PETSc @test !(PETSc.isinitialized(petsclib)) @test PETSc.isfinalized(petsclib) - # Check that log file was created - # Note: On some platforms/configurations, PETSc may not create the log file - if isfile(test_log_file) - # Verify log file contains expected content - log_content = read(test_log_file, String) - @test occursin("PETSc Performance Summary", log_content) - else - @warn "Log file not created on this platform: $test_log_file" - end + # The options reached PETSc, so the log file exists + @test isfile(test_log_file) + @test occursin("PETSc Performance Summary", read(test_log_file, String)) finally # Clean up test log file even if tests fail rm(test_log_file; force = true) @@ -106,7 +101,7 @@ using PETSc # Initialize with multiple options # Use current directory to avoid Windows temp directory issues - test_log_file = joinpath(pwd(), "test_log_multi_$(hash(petsclib)).txt") + test_log_file = "test_log_multi_$(hash(petsclib)).txt" try PETSc.initialize(petsclib; log_view = true, options = [":$test_log_file", "-log_view_memory"]) @@ -115,15 +110,10 @@ using PETSc PETSc.finalize(petsclib) - # Check log file was created and contains memory info - # Note: On some platforms/configurations, PETSc may not create the log file - if isfile(test_log_file) - log_content = read(test_log_file, String) - @test occursin("PETSc Performance Summary", log_content) - # Memory summary may not always be present depending on PETSc build - else - @warn "Log file not created on this platform: $test_log_file" - end + # The options reached PETSc, so the log file exists. The memory summary + # depends on the PETSc build, so only the performance summary is checked. + @test isfile(test_log_file) + @test occursin("PETSc Performance Summary", read(test_log_file, String)) finally # Clean up even if tests fail rm(test_log_file; force = true) @@ -138,7 +128,7 @@ using PETSc ENV["PETSC_OPTIONS"] = "-malloc_debug 0" # Use current directory to avoid Windows temp directory issues - test_log_file = joinpath(pwd(), "test_log_preserved_$(hash(petsclib)).txt") + test_log_file = "test_log_preserved_$(hash(petsclib)).txt" try PETSc.initialize(petsclib; log_view = true, options = [":$test_log_file"]) From ba2f36c78fa4f44f3b3ed1ed58c44e5754c489a4 Mon Sep 17 00:00:00 2001 From: filoferra <66474175+filoferra@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:58:31 +0200 Subject: [PATCH 5/5] Document that BLAS.set_num_threads after initialize overrides the default --- src/init.jl | 7 ++++--- test/mpi_blas_threads.jl | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/init.jl b/src/init.jl index d25698bb9..d47d5cb58 100644 --- a/src/init.jl +++ b/src/init.jl @@ -79,7 +79,8 @@ otherwise compete with the ranks for cores: a solve can run many times slower with the right answer. A set `OPENBLAS_NUM_THREADS` or `OMP_NUM_THREADS` also leaves the pool as it is. Serial runs, and runs with one rank per node, are unaffected. This changes Julia's process-wide BLAS setting, which `finalize` -does not restore. +does not restore. `initialize` sets the pool only once, so a later +`BLAS.set_num_threads(n)` takes effect as usual. # Examples ```julia @@ -99,8 +100,8 @@ PETSc.initialize(petsclib; log_view = true, options = [":logfile.txt", "-log_vie # Pass custom PETSc options without logging PETSc.initialize(petsclib; options = ["-malloc_debug", "-on_error_abort"]) -# One BLAS thread per rank under MPI -PETSc.initialize(petsclib; options = ["-blas_num_threads", "1"]) +# Four BLAS threads per rank, overriding the one-thread default for ranks sharing a node +PETSc.initialize(petsclib; options = ["-blas_num_threads", "4"]) ``` # External Links diff --git a/test/mpi_blas_threads.jl b/test/mpi_blas_threads.jl index 3fab355e4..0fb7df58b 100644 --- a/test/mpi_blas_threads.jl +++ b/test/mpi_blas_threads.jl @@ -19,6 +19,9 @@ using LinearAlgebra: BLAS BLAS.set_num_threads(4) PETSc.initialize(petsclib) @test BLAS.get_num_threads() == (shared && !environment_sets_it ? 1 : 4) + # the user can still resize the pool after initialize + BLAS.set_num_threads(3) + @test BLAS.get_num_threads() == 3 PETSc.finalize(petsclib) # the option wins over the default