From 6329e066bc92a616dd454dffdf243e23af390a5f Mon Sep 17 00:00:00 2001 From: Boris Kaus Date: Fri, 18 Sep 2026 22:40:37 +0200 Subject: [PATCH 1/5] Make CI robust against transient GMT download failures The recurring CI failures (test_WaterFlow, test_GMT and the LaPalma tutorial) all raised the same error: "Could not download GMT topography data", triggered by GMT reporting gmtread [ERROR]: Remote download is currently deactivated gmtread [ERROR]: Unable to obtain remote file @earth_relief_03s When the GMT data server is briefly unreachable, GMT sets auto_download = GMT_NO_DOWNLOAD and reports it with that misleading wording. The state clears once the server responds again, so retrying in-process does recover - but the existing backoff (5 s flat, ~20 s total) was too short to outlast the outage, so all attempts failed. This is not a version issue: the passing and failing jobs ran identical GMT (v1.43.2 / GMT_jll 6.7.2) and the same Julia versions appear on both sides. * ext/GMT_utils.jl: use progressive backoff (~100 s total), fix the attempt counter (warnings reported "attempt 0/5") and report the file and underlying error when giving up. * src/IO.jl: same off-by-one in download_data, plus a truncated download could return a non-empty path and break out of the retry loop, showing up later as a confusing load error. * test/test_paraview_collection.jl: replace exact filesize assertions with structural checks. The .pvd embeds absolute paths and OS-dependent separators, so the byte count varies per machine (verified: 435 bytes from a longer path, not the asserted 317). * CI.yml: test Julia 1.13 explicitly and track 'pre'. Also wire up continue-on-error, as allow_failure was set on every matrix entry but never referenced, so the allow-failure job still failed the run. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/CI.yml | 13 +++++++++++++ ext/GMT_utils.jl | 24 ++++++++++++++++-------- src/IO.jl | 20 +++++++++++++------- test/test_paraview_collection.jl | 20 +++++++++++++++----- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ad3c7a7a9..92b37200b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,6 +20,7 @@ jobs: run_tests: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.allow_failure }} strategy: fail-fast: false matrix: @@ -47,6 +48,18 @@ jobs: arch: aarch64 version: '1.12' allow_failure: true + # Julia 1.13 is the current stable release; test it explicitly on Linux + # so a regression there is visible independently of the '1' alias. + - os: ubuntu-latest + arch: x64 + version: '1.13' + allow_failure: false + # Track the upcoming release so breakage is caught early, without + # blocking CI when an unreleased Julia is temporarily broken. + - os: ubuntu-latest + arch: x64 + version: 'pre' + allow_failure: true steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 diff --git a/ext/GMT_utils.jl b/ext/GMT_utils.jl index 98998b75e..b94769508 100644 --- a/ext/GMT_utils.jl +++ b/ext/GMT_utils.jl @@ -75,20 +75,28 @@ function import_topo(limits; file::String = "@earth_relief_01m", maxattempts = 5 limits[1:2] = sort(limits[1:2]) end - # Download topo file - add a few attempts to do so + # Download topo file - add a few attempts to do so. + # When the GMT data server is briefly unreachable, GMT turns off + # auto-downloading and reports "Remote download is currently deactivated". + # That state clears once the server responds again, so retrying in-process + # does recover -- but only with a long enough backoff to outlast the outage. local G - attempt = 0 - while attempt < maxattempts + local last_err = nothing + for attempt in 1:maxattempts try G = gmtread(file, limits = limits, grid = true) break - catch - @warn "Failed downloading GMT topography on attempt $attempt/$maxattempts" - sleep(5) # wait a few sec + catch e + last_err = e + @warn "Failed downloading GMT topography ($file) on attempt $attempt/$maxattempts" + if attempt < maxattempts + sleep(10 * attempt) # back off progressively + end end - attempt += 1 end - (@isdefined G) || error("Could not download GMT topography data") + if !(@isdefined G) + error("Could not download GMT topography \"$file\" after $maxattempts attempts. Last error: $last_err") + end # Transfer to GeoData nx, ny = size(G.z, 2), size(G.z, 1) diff --git a/src/IO.jl b/src/IO.jl index cbf946ffa..5e026a6bf 100644 --- a/src/IO.jl +++ b/src/IO.jl @@ -102,19 +102,25 @@ function download_data(url::String, local_filename = "temp.dat"; dir = pwd(), ma #download remote file to a local temporary directory file_ext = [] - attempt = 0 - while attempt < maxattempts + for attempt in 1:maxattempts try file_ext = Downloads.download(url, joinpath(dir, local_filename)) + # A truncated transfer can still return a path, so only accept a + # download that actually produced a non-empty file. + if filesize(file_ext) == 0 + error("downloaded file is empty") + end break - catch - @warn "Failed downloading data on attempt $attempt/$maxattempts" - sleep(5) # wait a few sec + catch e + file_ext = [] + @warn "Failed downloading $url on attempt $attempt/$maxattempts: $e" + if attempt < maxattempts + sleep(5 * attempt) # back off progressively + end end - attempt += 1 end if isempty(file_ext) - error("Could not download GMT topography data") + error("Could not download data from $url after $maxattempts attempts") end return file_ext diff --git a/test/test_paraview_collection.jl b/test/test_paraview_collection.jl index 645dd28a9..86f88329f 100644 --- a/test/test_paraview_collection.jl +++ b/test/test_paraview_collection.jl @@ -1,5 +1,16 @@ using Test using GeophysicalModelGenerator, WriteVTK + +# Check the structure of a `*.pvd` collection rather than its exact byte size: +# the file embeds absolute paths and OS-dependent separators, so `filesize` varies +# between machines/platforms and is not a portable assertion. +function check_pvd(fname, n_datasets) + isfile(fname) || return false + str = read(fname, String) + occursin("type=\"Collection\"", str) || return false + return count(_ -> true, eachmatch(r" Date: Sat, 19 Sep 2026 12:23:07 +0200 Subject: [PATCH 2/5] Try different GMT mirrors if a topography download fails The CI failures in test_WaterFlow, test_GMT and the LaPalma tutorial all came from GMT being unable to reach its data server: gmtread [ERROR]: Remote download is currently deactivated gmtread [ERROR]: Unable to obtain remote file @earth_relief_03s Retrying the same server did not help, since the server was unreachable for the whole retry window. Instead, cycle through the official mirrors (https://www.generic-mapping-tools.org/mirrors) so that a single server being offline no longer fails the download. The download is still genuinely exercised - only *which* mirror serves it changes - so the tests keep checking that downloading works. Verified against a cold GMT cache: with two unreachable mirrors in front, the download correctly falls through to a working one (GMT reports "Remote data courtesy of GMT data server oceania"), and when every mirror is unreachable import_topo still errors, so a real regression is not masked. Note that GMT reads GMT_DATA_SERVER when the session starts, so setting the environment variable at runtime has no effect; `gmtset` is used instead. All six configured mirrors were verified to serve the data (singapore is omitted as it was unreachable). Co-Authored-By: Claude Opus 5 (1M context) --- ext/GMT_utils.jl | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ext/GMT_utils.jl b/ext/GMT_utils.jl index b94769508..8dbe8bdbb 100644 --- a/ext/GMT_utils.jl +++ b/ext/GMT_utils.jl @@ -16,6 +16,12 @@ using GeophysicalModelGenerator: lonlatdepth_grid, GeoData, UTMData, km, remove_ println("Loading GMT routines within GMG") +# Official GMT data server mirrors, see https://www.generic-mapping-tools.org/mirrors. +# `import_topo` cycles through these if a download fails, so that a single +# mirror being offline does not break the download (or the tests that check it). +# Ordered by measured response; "singapore" is omitted as it was unreachable. +const GMT_DATA_SERVERS = ["oceania", "australia", "sdsc-opentopography", "noaa", "brasil", "china"] + """ Topo = import_topo(limits; file::String="@earth_relief_01m", maxattempts=5) @@ -75,27 +81,32 @@ function import_topo(limits; file::String = "@earth_relief_01m", maxattempts = 5 limits[1:2] = sort(limits[1:2]) end - # Download topo file - add a few attempts to do so. - # When the GMT data server is briefly unreachable, GMT turns off + # Download topo file. If the GMT data server is unreachable, GMT stops # auto-downloading and reports "Remote download is currently deactivated". - # That state clears once the server responds again, so retrying in-process - # does recover -- but only with a long enough backoff to outlast the outage. + # Rather than only retrying the same server, cycle through the official + # mirrors (see https://www.generic-mapping-tools.org/mirrors), so a single + # server being down no longer fails the download. The download itself is + # still exercised -- we only change *which* mirror serves it. local G local last_err = nothing for attempt in 1:maxattempts + server = GMT_DATA_SERVERS[mod1(attempt, length(GMT_DATA_SERVERS))] try + # Note: GMT reads GMT_DATA_SERVER when the session starts, so setting + # the environment variable here has no effect; `gmtset` changes it at runtime. + gmtset(GMT_DATA_SERVER = server) G = gmtread(file, limits = limits, grid = true) break catch e last_err = e - @warn "Failed downloading GMT topography ($file) on attempt $attempt/$maxattempts" + @warn "Failed downloading GMT topography ($file) from mirror \"$server\" on attempt $attempt/$maxattempts" if attempt < maxattempts - sleep(10 * attempt) # back off progressively + sleep(5) # brief pause before trying the next mirror end end end if !(@isdefined G) - error("Could not download GMT topography \"$file\" after $maxattempts attempts. Last error: $last_err") + error("Could not download GMT topography \"$file\" after $maxattempts attempts across mirrors $(GMT_DATA_SERVERS). Last error: $last_err") end # Transfer to GeoData From 3575d196b078b864b3128526716318743f59c9bc Mon Sep 17 00:00:00 2001 From: Boris Kaus Date: Sat, 19 Sep 2026 13:42:16 +0200 Subject: [PATCH 3/5] Only switch GMT mirrors after a failure, and restore the config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous version called `gmtset(GMT_DATA_SERVER=...)` before *every* download attempt, including the first. That was wrong in two ways: `gmtset` changes GMT's global configuration by writing a gmt.conf in the working directory. The test suite runs with ParallelTestRunner, whose workers share that directory, so one test switching mirrors also changed which server the other tests downloaded from. On macOS-aarch64 this made test_WaterFlow build its grid from a different mirror than intended and the values no longer matched the references: maximum(Topo_water.fields.area) ≈ 9.309204547276944e8 Evaluated: 2.6336469421851286e6 ≈ 9.309204547276944e8 The failing jobs are exactly those that logged mirror-fallback warnings (3 warnings on the failing macOS job, 0 on the passing ubuntu one). Switching the server unconditionally also meant the configured default was never used, so the mirrors were exercised even when nothing was wrong. Now the first attempt uses whatever server GMT is already configured with and touches no global state; mirrors are only tried after a failure, and the original gmt.conf (or its absence) is restored afterwards. Verified that the happy path leaves no gmt.conf behind, and that with an unreachable default the download still falls back to a working mirror ("Remote data courtesy of GMT data server oceania") and cleans up after itself. Co-Authored-By: Claude Opus 5 (1M context) --- ext/GMT_utils.jl | 63 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/ext/GMT_utils.jl b/ext/GMT_utils.jl index 8dbe8bdbb..f88f69beb 100644 --- a/ext/GMT_utils.jl +++ b/ext/GMT_utils.jl @@ -83,30 +83,59 @@ function import_topo(limits; file::String = "@earth_relief_01m", maxattempts = 5 # Download topo file. If the GMT data server is unreachable, GMT stops # auto-downloading and reports "Remote download is currently deactivated". - # Rather than only retrying the same server, cycle through the official - # mirrors (see https://www.generic-mapping-tools.org/mirrors), so a single - # server being down no longer fails the download. The download itself is - # still exercised -- we only change *which* mirror serves it. + # + # First try the server the user/GMT is already configured with, so that the + # normal case does not touch any global state. Only if that fails do we fall + # back to the official mirrors (https://www.generic-mapping-tools.org/mirrors), + # so a single server being offline does not fail the download. The download + # itself is still exercised -- only *which* mirror serves it changes. + # + # Note: `gmtset` changes GMT's *global* configuration (it writes a gmt.conf in + # the working directory), so switching mirrors is visible to anything else + # using GMT concurrently. We therefore restore the original setting afterwards + # and only switch when a download has actually failed. local G local last_err = nothing - for attempt in 1:maxattempts - server = GMT_DATA_SERVERS[mod1(attempt, length(GMT_DATA_SERVERS))] + + # `gmtset` writes/updates a gmt.conf in the working directory. Remember whether + # one already existed so we can leave the directory as we found it. + conf_file = joinpath(pwd(), "gmt.conf") + had_conf = isfile(conf_file) + conf_backup = had_conf ? read(conf_file) : nothing + + try + for attempt in 1:maxattempts + if attempt > 1 + # Previous attempt failed -> try the next mirror in the list + server = GMT_DATA_SERVERS[mod1(attempt - 1, length(GMT_DATA_SERVERS))] + gmtset(GMT_DATA_SERVER = server) + @info "Retrying GMT topography download from mirror \"$server\"" + end + try + G = gmtread(file, limits = limits, grid = true) + break + catch e + last_err = e + @warn "Failed downloading GMT topography ($file) on attempt $attempt/$maxattempts" + if attempt < maxattempts + sleep(5) # brief pause before trying the next mirror + end + end + end + finally + # Restore the original configuration, so our mirror choice does not leak + # into other code (or other tests) that use GMT afterwards. try - # Note: GMT reads GMT_DATA_SERVER when the session starts, so setting - # the environment variable here has no effect; `gmtset` changes it at runtime. - gmtset(GMT_DATA_SERVER = server) - G = gmtread(file, limits = limits, grid = true) - break - catch e - last_err = e - @warn "Failed downloading GMT topography ($file) from mirror \"$server\" on attempt $attempt/$maxattempts" - if attempt < maxattempts - sleep(5) # brief pause before trying the next mirror + if had_conf + conf_backup !== nothing && write(conf_file, conf_backup) + elseif isfile(conf_file) + rm(conf_file, force = true) end + catch end end if !(@isdefined G) - error("Could not download GMT topography \"$file\" after $maxattempts attempts across mirrors $(GMT_DATA_SERVERS). Last error: $last_err") + error("Could not download GMT topography \"$file\" after $maxattempts attempts (mirrors tried: $(GMT_DATA_SERVERS)). Last error: $last_err") end # Transfer to GeoData From 6640dfc7167a2df8149988cae4a813ef17c8f8af Mon Sep 17 00:00:00 2001 From: Boris Kaus Date: Sat, 19 Sep 2026 13:54:21 +0200 Subject: [PATCH 4/5] Initialize gmsh explicitly before reading a mesh test_Gmsh intermittently failed on Windows with [ Error : Gmsh has not been initialized `GmshDiscreteModel` initializes gmsh itself, but gmsh keeps global state that is occasionally not in place yet on a freshly started worker process under the parallel test runner. Initializing explicitly first makes that state well-defined. Note we deliberately do not finalize afterwards: GmshDiscreteModel already finalizes the session it used, and a second finalize reproduces the very same "Gmsh has not been initialized" error (confirmed locally - an earlier version of this fix that finalized emitted exactly that message twice). Co-Authored-By: Claude Opus 5 (1M context) --- ext/Gmsh_utils.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ext/Gmsh_utils.jl b/ext/Gmsh_utils.jl index 2984326b3..fa27f55fb 100644 --- a/ext/Gmsh_utils.jl +++ b/ext/Gmsh_utils.jl @@ -3,6 +3,7 @@ module Gmsh_utils using GridapGmsh, StaticArrays +using GridapGmsh: gmsh import GeophysicalModelGenerator: import_Gmsh, FEData, CartData @@ -14,6 +15,22 @@ Reads a Gmsh file and returns a `FEData` object with info about the mesh. `tag_n """ function import_Gmsh(fname::String) + # `GmshDiscreteModel` initializes and finalizes gmsh itself. That normally + # works, but on a freshly started worker process gmsh's global state is + # occasionally not in place yet and it fails with "Gmsh has not been + # initialized" (seen on Windows under the parallel test runner). + # + # Initializing here first makes that state explicit. gmsh reference-counts + # nothing, so we must NOT finalize afterwards: GmshDiscreteModel already + # finalizes the session it used, and a second finalize errors. + try + if gmsh.isInitialized() == 0 + gmsh.initialize() + end + catch e + @warn "Could not initialize gmsh, letting GmshDiscreteModel handle it" exception = e + end + mesh = GmshDiscreteModel(fname, renumber = false, has_affine_map = false) # Extract vertices From 712aef58f42e9f4a6686b9b19b20bafc8a2dee06 Mon Sep 17 00:00:00 2001 From: Boris Kaus Date: Sat, 19 Sep 2026 14:49:02 +0200 Subject: [PATCH 5/5] Prefetch GMT topography tiles before the parallel tests, and cache them in CI The remaining CI failures were all GMT tile downloads timing out: grdblend [ERROR]: Libcurl Error: Timeout was reached grdblend [ERROR]: Unable to obtain remote file @N30E000.earth_relief_01m_g.nc This is not the server blocking us (there is not a single 403/429 in the logs; the server is contacted fine and the transfer then stalls). It is contention: test_GMT, test_WaterFlow and the LaPalma tutorial run concurrently under ParallelTestRunner and all pull tiles from the GMT data server at the same time, each with GMT's own retries on top. Download the tiles serially, once, before the workers are spawned (test/topo_prefetch.jl, included from runtests.jl). The workers then find them in GMT's cache (~/.gmt/server) and never touch the network. Verified locally: after the prefetch, test_GMT and test_WaterFlow pass with the data server *and* all mirrors unreachable. On CI, ~/.gmt is additionally cached across runs (keyed on the hash of topo_prefetch.jl, so changing the tile list invalidates it). The scheduled run deliberately bypasses that cache, so the download itself is still exercised regularly rather than only served from cache. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/CI.yml | 15 +++++++++++++ test/runtests.jl | 7 ++++++ test/topo_prefetch.jl | 46 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 test/topo_prefetch.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 92b37200b..0d51c0971 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -70,7 +70,22 @@ jobs: with: include-matrix: false - uses: julia-actions/julia-buildpkg@latest + # Cache the GMT topography tiles that the tests download (see + # test/topo_prefetch.jl), so PR runs do not depend on the GMT data + # server. The scheduled run skips the cache on purpose, so that the + # download itself is still tested regularly. + - uses: actions/cache/restore@v4 + id: gmt-cache + if: github.event_name != 'schedule' + with: + path: ~/.gmt + key: gmt-topo-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('test/topo_prefetch.jl') }} - uses: julia-actions/julia-runtest@latest + - uses: actions/cache/save@v4 + if: always() && github.event_name != 'schedule' && steps.gmt-cache.outputs.cache-hit != 'true' + with: + path: ~/.gmt + key: gmt-topo-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('test/topo_prefetch.jl') }} - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v7 with: diff --git a/test/runtests.jl b/test/runtests.jl index d681790cf..5474eac48 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,14 @@ using GeophysicalModelGenerator using ParallelTestRunner +# Download the topography tiles the tests need *before* spawning the parallel +# workers, so they are served from GMT's cache instead of being downloaded +# concurrently (which times out regularly in CI). See topo_prefetch.jl. +include("topo_prefetch.jl") + testsuite = find_tests(@__DIR__) +# Not a test file, only a helper +delete!(testsuite, "topo_prefetch") # Add `using GeophysicalModelGenerator` to each test for (name, expr) in testsuite diff --git a/test/topo_prefetch.jl b/test/topo_prefetch.jl new file mode 100644 index 000000000..280dffb29 --- /dev/null +++ b/test/topo_prefetch.jl @@ -0,0 +1,46 @@ +# Warm the GMT topography cache before the parallel test workers start. +# +# Several tests (test_GMT, test_WaterFlow, the LaPalma tutorial) download +# topography tiles through GMT. Under ParallelTestRunner they run concurrently, +# so they all hit the GMT data server at the same time, and the transfers then +# regularly time out in CI ("Libcurl Error: Timeout was reached"). +# +# Downloading the tiles here, one at a time and with generous retries, means the +# workers later find them in GMT's cache (~/.gmt/server) and never touch the +# network. The download itself is still exercised: it happens right here, and +# on CI the cache is bypassed for the scheduled run so it is tested regularly. +# +# NOTE: .github/workflows/CI.yml caches ~/.gmt keyed on the hash of this file, +# so changing the list below invalidates that cache automatically. + +using GeophysicalModelGenerator, GMT + +# (region, file) pairs, matching the calls made in the tests/tutorials +const TOPO_PREFETCH = [ + ([8.0, 9.0, 50.0, 51.0], "@earth_relief_01m"), # test_GMT + ([6.5, 7.3, 50.2, 50.6], "@earth_relief_03s"), # test_WaterFlow + ([-18.2, -17.5, 28.4, 29.0], "@earth_relief_15s.grd"), # LaPalma tutorial +] + +function prefetch_topography() + gmt_cache = joinpath(get(ENV, "GMT_USERDIR", joinpath(homedir(), ".gmt")), "server") + n_before = isdir(gmt_cache) ? count(f -> endswith(f, ".nc"), collect(Iterators.flatten(map(t -> t[3], walkdir(gmt_cache))))) : 0 + + for (limits, file) in TOPO_PREFETCH + t = @elapsed try + # copy: import_topo may modify `limits` in place (negative longitudes) + import_topo(copy(limits); file = file, maxattempts = 8) + catch e + # Do not abort the test run: the test that needs this tile will try + # again itself and report a proper error if the server really is down. + @warn "Could not prefetch topography $file for $limits; the test will retry" exception = (e, catch_backtrace()) + end + @info "Prefetched topography $file for $limits in $(round(t, digits = 1)) s" + end + + n_after = isdir(gmt_cache) ? count(f -> endswith(f, ".nc"), collect(Iterators.flatten(map(t -> t[3], walkdir(gmt_cache))))) : 0 + @info "GMT topography cache: $n_after tiles in $gmt_cache ($(n_after - n_before) newly downloaded)" + return nothing +end + +prefetch_topography()