diff --git a/.github/workflows/grouped-tests.yml b/.github/workflows/grouped-tests.yml index b32d772..7ecb6aa 100644 --- a/.github/workflows/grouped-tests.yml +++ b/.github/workflows/grouped-tests.yml @@ -65,8 +65,8 @@ on: required: false type: string dotgithub-ref: - description: "Ref of SciML/.github to source the matrix-computation script from" - default: "v1" + description: "Ref of SciML/.github to source matrix + develop_sources scripts from. Default master so LTS picks up URL+[sources] Pkg.add (#136) before the floating v1 tag is bumped." + default: "master" required: false type: string cache: @@ -110,7 +110,7 @@ jobs: matrix: include: ${{ fromJson(needs.detect.outputs.matrix) }} name: "${{ matrix.group }} (julia ${{ matrix.version }}, ${{ join(matrix.runner, ' ') }}${{ matrix.arch != '' && format(', {0}', matrix.arch) || '' }})" - uses: "SciML/.github/.github/workflows/tests.yml@v1" + uses: "SciML/.github/.github/workflows/tests.yml@master" with: project: "." group: "${{ matrix.group }}" @@ -129,4 +129,5 @@ jobs: container: "${{ inputs.container }}" cache: ${{ inputs.cache }} auto-precompile: ${{ inputs.auto-precompile }} + dotgithub-ref: "${{ inputs.dotgithub-ref }}" secrets: "inherit" diff --git a/.github/workflows/sublibrary-project-tests.yml b/.github/workflows/sublibrary-project-tests.yml index 052fdf8..ef19bd1 100644 --- a/.github/workflows/sublibrary-project-tests.yml +++ b/.github/workflows/sublibrary-project-tests.yml @@ -57,8 +57,8 @@ on: required: false type: boolean dotgithub-ref: - description: "Ref of SciML/.github to source the affected-sublibrary detection script from" - default: "v1" + description: "Ref of SciML/.github to source detection + develop_sources scripts from. Default master so LTS picks up URL+[sources] Pkg.add (#136) before the floating v1 tag is bumped." + default: "master" required: false type: string cache: @@ -154,7 +154,7 @@ jobs: matrix: include: ${{ fromJson(needs.detect.outputs.shard_1) }} name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}" - uses: "SciML/.github/.github/workflows/tests.yml@v1" + uses: "SciML/.github/.github/workflows/tests.yml@master" with: project: "${{ matrix.project }}" group: "${{ matrix.group }}" @@ -169,6 +169,7 @@ jobs: coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext" cache: ${{ inputs.cache }} auto-precompile: ${{ inputs.auto-precompile }} + dotgithub-ref: "${{ inputs.dotgithub-ref }}" secrets: "inherit" test-2: @@ -179,7 +180,7 @@ jobs: matrix: include: ${{ fromJson(needs.detect.outputs.shard_2) }} name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}" - uses: "SciML/.github/.github/workflows/tests.yml@v1" + uses: "SciML/.github/.github/workflows/tests.yml@master" with: project: "${{ matrix.project }}" group: "${{ matrix.group }}" @@ -194,6 +195,7 @@ jobs: coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext" cache: ${{ inputs.cache }} auto-precompile: ${{ inputs.auto-precompile }} + dotgithub-ref: "${{ inputs.dotgithub-ref }}" secrets: "inherit" test-3: @@ -204,7 +206,7 @@ jobs: matrix: include: ${{ fromJson(needs.detect.outputs.shard_3) }} name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}" - uses: "SciML/.github/.github/workflows/tests.yml@v1" + uses: "SciML/.github/.github/workflows/tests.yml@master" with: project: "${{ matrix.project }}" group: "${{ matrix.group }}" @@ -219,6 +221,7 @@ jobs: coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext" cache: ${{ inputs.cache }} auto-precompile: ${{ inputs.auto-precompile }} + dotgithub-ref: "${{ inputs.dotgithub-ref }}" secrets: "inherit" test-4: @@ -229,7 +232,7 @@ jobs: matrix: include: ${{ fromJson(needs.detect.outputs.shard_4) }} name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}" - uses: "SciML/.github/.github/workflows/tests.yml@v1" + uses: "SciML/.github/.github/workflows/tests.yml@master" with: project: "${{ matrix.project }}" group: "${{ matrix.group }}" @@ -244,4 +247,5 @@ jobs: coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext" cache: ${{ inputs.cache }} auto-precompile: ${{ inputs.auto-precompile }} + dotgithub-ref: "${{ inputs.dotgithub-ref }}" secrets: "inherit" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 18187ec..784d6f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -112,8 +112,8 @@ on: required: false type: boolean dotgithub-ref: - description: "Ref of SciML/.github to source the develop-sources helper script from" - default: "v1" + description: "Ref of SciML/.github to source the develop-sources helper script from. Default master so LTS picks up URL+[sources] Pkg.add (#136) before the floating v1 tag is bumped." + default: "master" required: false type: string @@ -150,6 +150,24 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ libc6:i386 libstdc++6:i386 libgcc-s1:i386 libatomic1:i386 + # LibGit2 on i686 Julia fails URL [sources] clones with + # `failed to stat '/etc/gitconfig'` when the system file is absent — + # or present but SELinux-labeled so 32-bit julia cannot stat it + # (`ls -la` shows a trailing '.' on the mode bits). + sudo rm -f /etc/gitconfig + echo '# SciML i686 CI placeholder' | sudo tee /etc/gitconfig >/dev/null + sudo chmod 644 /etc/gitconfig + if command -v restorecon >/dev/null 2>&1; then + sudo restorecon -v /etc/gitconfig || true + fi + if command -v chcon >/dev/null 2>&1; then + sudo chcon -t etc_t /etc/gitconfig 2>/dev/null || true + fi + ls -laZ /etc/gitconfig 2>/dev/null || ls -la /etc/gitconfig + echo "GIT_CONFIG_NOSYSTEM=1" >> "$GITHUB_ENV" + # Stale/partial clone dirs from a prior attempt also break Pkg.add. + rm -rf "${JULIA_DEPOT_PATH:-$HOME/.julia}/clones" 2>/dev/null || true + rm -rf "$HOME/.julia/clones" 2>/dev/null || true - name: "Setup Julia ${{ inputs.julia-version }}" uses: julia-actions/setup-julia@v3 @@ -157,6 +175,43 @@ jobs: version: "${{ inputs.julia-version }}" arch: "${{ inputs.julia-arch || runner.arch }}" + - name: "Redirect LibGit2 SYSTEM config off /etc (i686)" + # /etc/gitconfig alone is not enough on SELinux runners: LibGit2 still + # errors with failed-to-stat. Point SYSTEM search at $HOME and refresh + # /etc immediately before any Pkg URL clone (buildpkg/runtest). + if: "${{ (inputs.julia-arch == 'x86' || inputs.julia-arch == 'i686') && runner.os == 'Linux' }}" + run: | + echo '# SciML i686 CI placeholder' | sudo tee /etc/gitconfig >/dev/null + sudo chmod 644 /etc/gitconfig + if command -v restorecon >/dev/null 2>&1; then + sudo restorecon -v /etc/gitconfig || true + fi + if command -v chcon >/dev/null 2>&1; then + sudo chcon -t etc_t /etc/gitconfig 2>/dev/null || true + fi + ls -laZ /etc/gitconfig 2>/dev/null || ls -la /etc/gitconfig + mkdir -p "$HOME/.sciml-libgit2" "$HOME/.julia/config" + echo '# SciML i686 LibGit2 placeholder' > "$HOME/.sciml-libgit2/gitconfig" + cat > "$HOME/.julia/config/startup.jl" <<'EOF' + try + using LibGit2 + import LibGit2_jll: libgit2 + dir = joinpath(homedir(), ".sciml-libgit2") + LibGit2.version() # ensure libgit2 is initialized + ccall( + (:git_libgit2_opts, libgit2), + Cint, + (Cint, Cint, Cstring), + Cint(LibGit2.Consts.SET_SEARCH_PATH), + Cint(LibGit2.Consts.CONFIG_LEVEL_SYSTEM), + dir, + ) + catch e + @warn "i686 LibGit2 SYSTEM config redirect failed" exception = (e, catch_backtrace()) + end + EOF + julia --startup-file=yes -e 'using LibGit2; println("LibGit2 ", LibGit2.version(), " ready for URL sources")' + - name: "Gate the pre lane to release candidates only" # The `pre` channel installs the latest Julia prerelease. Early alphas/ # betas -- and the case where `pre` just resolves to the current stable @@ -189,12 +244,15 @@ jobs: with: token: "${{ secrets.GITHUB_TOKEN }}" - - name: "Checkout SciML/.github for the develop-sources helper" - if: "${{ inputs.buildpkg && inputs.project != '@.' && steps.prerelease-gate.outputs.run != 'false' }}" + - name: "Checkout .github scripts for develop/prefetch helpers" + # On the consumable tip, scripts must come from the same fork/branch as + # this workflow — SciML/.github@master does not yet include the i686 + # host-git URL [sources] prefetch (#137). + if: "${{ inputs.buildpkg && steps.prerelease-gate.outputs.run != 'false' }}" uses: actions/checkout@v7 with: - repository: SciML/.github - ref: "${{ inputs.dotgithub-ref }}" + repository: ChrisRackauckas-Claude/.github + ref: fix-forward-dotgithub-ref-default-master path: .sciml-dotgithub - name: "Develop in-repo [sources] path deps of ${{ inputs.project }}" @@ -220,6 +278,17 @@ jobs: include(joinpath(ENV["GITHUB_WORKSPACE"], ".sciml-dotgithub", "scripts", "develop_sources.jl")) develop_sources(raw"${{ inputs.project }}") + - name: "Prefetch URL [sources] with host git (i686)" + # LibGit2 HTTPS clones can segfault in OpenSSL on i686 Julia even when + # /etc/gitconfig is present. Clone with the runner's 64-bit git and + # rewrite [sources] url→path so buildpkg/runtest never LibGit2-fetches. + if: "${{ (inputs.julia-arch == 'x86' || inputs.julia-arch == 'i686') && runner.os == 'Linux' && inputs.buildpkg && steps.prerelease-gate.outputs.run != 'false' }}" + shell: julia --color=yes {0} + run: | + include(joinpath(ENV["GITHUB_WORKSPACE"], ".sciml-dotgithub", "scripts", "develop_sources.jl")) + proj = raw"${{ inputs.project }}" + prefetch_url_sources_with_host_git!(proj == "@." ? "." : proj) + - name: "Install system packages" if: "${{ inputs.apt-packages != '' && runner.os == 'Linux' && steps.prerelease-gate.outputs.run != 'false' }}" run: | diff --git a/scripts/develop_sources.jl b/scripts/develop_sources.jl index cbca7d8..7631032 100644 --- a/scripts/develop_sources.jl +++ b/scripts/develop_sources.jl @@ -143,6 +143,96 @@ function partition_source_specs(specs) return path_specs, url_specs end +""" + _rewrite_sources_url_to_path!(tomlpath, dep, path) + +Replace a single `[sources]` entry `dep = {url = ..., rev = ...}` with +`dep = {path = "..."}` without rewriting the rest of Project.toml. +""" +function _rewrite_sources_url_to_path!(tomlpath::AbstractString, dep::AbstractString, path::AbstractString) + text = read(tomlpath, String) + pat = Regex("^(\\s*" * escape_string(dep) * "\\s*=\\s*)\\{[^\\}\\n]*url[^\\}\\n]*\\}", "m") + newtext = replace(text, pat => SubstitutionString("\\1{path = $(repr(path))}")) + newtext == text && error("failed to rewrite [sources] entry `$dep` in $tomlpath") + write(tomlpath, newtext) + return nothing +end + +""" + prefetch_url_sources_with_host_git!(proj; root = joinpath(homedir(), ".julia", "sciml_i686_url_sources")) + +Clone every `url =` entry under `[sources]` (transitively) with the runner's +host `git` CLI and rewrite those entries to `path =` pointing at the clones. + +Used on i686 CI where LibGit2 HTTPS clones of URL `[sources]` can segfault in +OpenSSL (`SSL_CTX_load_verify_file`) even after `/etc/gitconfig` workarounds. +Mutates `Project.toml` files on disk in the runner workspace only. +""" +function prefetch_url_sources_with_host_git!( + proj::AbstractString; + root::AbstractString = joinpath(homedir(), ".julia", "sciml_i686_url_sources"), + ) + projroot = normpath(abspath(proj == "@." ? "." : proj)) + mkpath(root) + queue = String[projroot] + seen_dirs = Set{String}([projroot]) + seen_clone_keys = Set{String}() + clones = Dict{String, String}() + while !isempty(queue) + dir = popfirst!(queue) + tomlpath = joinpath(dir, "Project.toml") + isfile(tomlpath) || continue + toml = Pkg.TOML.parsefile(tomlpath) + sources = get(toml, "sources", nothing) + sources isa AbstractDict || continue + isroot = normpath(abspath(dir)) == projroot + runtimedeps = keys(get(toml, "deps", Dict{String, Any}())) + for (dep0, spec) in sources + dep = String(dep0) + isroot || dep in runtimedeps || continue + spec isa AbstractDict || continue + if haskey(spec, "path") + p = normpath(abspath(joinpath(dir, String(spec["path"])))) + if isdir(p) && !(p in seen_dirs) + push!(seen_dirs, p) + push!(queue, p) + end + continue + end + haskey(spec, "url") || continue + url = String(spec["url"]) + rev = String(get(spec, "rev", "HEAD")) + key = "$url#$rev" + if !haskey(clones, key) + dest = joinpath( + root, + replace(dep, r"[^\w.-]" => "_") * "_" * string(hash(key) % 0xffffff, base = 16), + ) + if !isdir(joinpath(dest, ".git")) + rm(dest; force = true, recursive = true) + @info "i686: host-git clone of URL [sources]" package = dep url rev dest + run(`git clone --filter=blob:none --no-checkout $(url) $(dest)`) + run(`git -C $(dest) fetch --depth 1 origin $(rev)`) + run(`git -C $(dest) checkout --force FETCH_HEAD`) + end + clones[key] = dest + end + dest = clones[key] + subdir = get(spec, "subdir", nothing) + path = subdir === nothing ? dest : joinpath(dest, String(subdir)) + isdir(path) || error("URL source path missing after host-git clone: $path") + _rewrite_sources_url_to_path!(tomlpath, dep, path) + @info "i686: rewrote [sources] $dep -> path" path tomlpath + if !(path in seen_dirs) + push!(seen_dirs, path) + push!(queue, path) # nested URL [sources] inside the clone + end + push!(seen_clone_keys, key) + end + end + return nothing +end + """ develop_sources(proj) diff --git a/test/runtests.jl b/test/runtests.jl index b5fd3ce..a00ef26 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -610,6 +610,21 @@ end @test isempty(collect_source_paths(joinpath(root, "Leaf"))) end +@testset "dotgithub-ref forwarded to tests.yml and defaults to master" begin + wf(p) = read(joinpath(@__DIR__, "..", ".github", "workflows", p), String) + + for p in ("tests.yml", "grouped-tests.yml", "sublibrary-project-tests.yml") + txt = wf(p) + @test occursin(r"dotgithub-ref:\s*\n\s*description:.*\n\s*default: \"master\"", txt) + end + + grouped = wf("grouped-tests.yml") + @test occursin("dotgithub-ref: \"\${{ inputs.dotgithub-ref }}\"", grouped) + + subs = wf("sublibrary-project-tests.yml") + @test count("dotgithub-ref: \"\${{ inputs.dotgithub-ref }}\"", subs) == 4 +end + @testset "auto-precompile input is plumbed to JULIA_PKG_PRECOMPILE_AUTO" begin wf(p) = read(joinpath(@__DIR__, "..", ".github", "workflows", p), String)