Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .github/workflows/grouped-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ on:
required: false
type: string
dotgithub-ref:
description: "Ref of SciML/.github to source the matrix-computation script from"
default: "v1"
description: "Ref of SciML/.github to source matrix + develop_sources scripts from. Default master so LTS picks up URL+[sources] Pkg.add (#136) before the floating v1 tag is bumped."
default: "master"
required: false
type: string
cache:
Expand Down Expand Up @@ -110,7 +110,7 @@ jobs:
matrix:
include: ${{ fromJson(needs.detect.outputs.matrix) }}
name: "${{ matrix.group }} (julia ${{ matrix.version }}, ${{ join(matrix.runner, ' ') }}${{ matrix.arch != '' && format(', {0}', matrix.arch) || '' }})"
uses: "SciML/.github/.github/workflows/tests.yml@v1"
uses: "SciML/.github/.github/workflows/tests.yml@master"
with:
project: "."
group: "${{ matrix.group }}"
Expand All @@ -129,4 +129,5 @@ jobs:
container: "${{ inputs.container }}"
cache: ${{ inputs.cache }}
auto-precompile: ${{ inputs.auto-precompile }}
dotgithub-ref: "${{ inputs.dotgithub-ref }}"
secrets: "inherit"
16 changes: 10 additions & 6 deletions .github/workflows/sublibrary-project-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ on:
required: false
type: boolean
dotgithub-ref:
description: "Ref of SciML/.github to source the affected-sublibrary detection script from"
default: "v1"
description: "Ref of SciML/.github to source detection + develop_sources scripts from. Default master so LTS picks up URL+[sources] Pkg.add (#136) before the floating v1 tag is bumped."
default: "master"
required: false
type: string
cache:
Expand Down Expand Up @@ -154,7 +154,7 @@ jobs:
matrix:
include: ${{ fromJson(needs.detect.outputs.shard_1) }}
name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}"
uses: "SciML/.github/.github/workflows/tests.yml@v1"
uses: "SciML/.github/.github/workflows/tests.yml@master"
with:
project: "${{ matrix.project }}"
group: "${{ matrix.group }}"
Expand All @@ -169,6 +169,7 @@ jobs:
coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext"
cache: ${{ inputs.cache }}
auto-precompile: ${{ inputs.auto-precompile }}
dotgithub-ref: "${{ inputs.dotgithub-ref }}"
secrets: "inherit"

test-2:
Expand All @@ -179,7 +180,7 @@ jobs:
matrix:
include: ${{ fromJson(needs.detect.outputs.shard_2) }}
name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}"
uses: "SciML/.github/.github/workflows/tests.yml@v1"
uses: "SciML/.github/.github/workflows/tests.yml@master"
with:
project: "${{ matrix.project }}"
group: "${{ matrix.group }}"
Expand All @@ -194,6 +195,7 @@ jobs:
coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext"
cache: ${{ inputs.cache }}
auto-precompile: ${{ inputs.auto-precompile }}
dotgithub-ref: "${{ inputs.dotgithub-ref }}"
secrets: "inherit"

test-3:
Expand All @@ -204,7 +206,7 @@ jobs:
matrix:
include: ${{ fromJson(needs.detect.outputs.shard_3) }}
name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}"
uses: "SciML/.github/.github/workflows/tests.yml@v1"
uses: "SciML/.github/.github/workflows/tests.yml@master"
with:
project: "${{ matrix.project }}"
group: "${{ matrix.group }}"
Expand All @@ -219,6 +221,7 @@ jobs:
coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext"
cache: ${{ inputs.cache }}
auto-precompile: ${{ inputs.auto-precompile }}
dotgithub-ref: "${{ inputs.dotgithub-ref }}"
secrets: "inherit"

test-4:
Expand All @@ -229,7 +232,7 @@ jobs:
matrix:
include: ${{ fromJson(needs.detect.outputs.shard_4) }}
name: "${{ matrix.project }}${{ matrix.group != 'Core' && format(' [{0}]', matrix.group) || '' }} / Julia ${{ matrix.version }}"
uses: "SciML/.github/.github/workflows/tests.yml@v1"
uses: "SciML/.github/.github/workflows/tests.yml@master"
with:
project: "${{ matrix.project }}"
group: "${{ matrix.group }}"
Expand All @@ -244,4 +247,5 @@ jobs:
coverage-directories: "${{ matrix.project }}/src,${{ matrix.project }}/ext"
cache: ${{ inputs.cache }}
auto-precompile: ${{ inputs.auto-precompile }}
dotgithub-ref: "${{ inputs.dotgithub-ref }}"
secrets: "inherit"
81 changes: 75 additions & 6 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,8 @@ on:
required: false
type: boolean
dotgithub-ref:
description: "Ref of SciML/.github to source the develop-sources helper script from"
default: "v1"
description: "Ref of SciML/.github to source the develop-sources helper script from. Default master so LTS picks up URL+[sources] Pkg.add (#136) before the floating v1 tag is bumped."
default: "master"
required: false
type: string

Expand Down Expand Up @@ -150,13 +150,68 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libc6:i386 libstdc++6:i386 libgcc-s1:i386 libatomic1:i386
# LibGit2 on i686 Julia fails URL [sources] clones with
# `failed to stat '/etc/gitconfig'` when the system file is absent —
# or present but SELinux-labeled so 32-bit julia cannot stat it
# (`ls -la` shows a trailing '.' on the mode bits).
sudo rm -f /etc/gitconfig
echo '# SciML i686 CI placeholder' | sudo tee /etc/gitconfig >/dev/null
sudo chmod 644 /etc/gitconfig
if command -v restorecon >/dev/null 2>&1; then
sudo restorecon -v /etc/gitconfig || true
fi
if command -v chcon >/dev/null 2>&1; then
sudo chcon -t etc_t /etc/gitconfig 2>/dev/null || true
fi
ls -laZ /etc/gitconfig 2>/dev/null || ls -la /etc/gitconfig
echo "GIT_CONFIG_NOSYSTEM=1" >> "$GITHUB_ENV"
# Stale/partial clone dirs from a prior attempt also break Pkg.add.
rm -rf "${JULIA_DEPOT_PATH:-$HOME/.julia}/clones" 2>/dev/null || true
rm -rf "$HOME/.julia/clones" 2>/dev/null || true

- name: "Setup Julia ${{ inputs.julia-version }}"
uses: julia-actions/setup-julia@v3
with:
version: "${{ inputs.julia-version }}"
arch: "${{ inputs.julia-arch || runner.arch }}"

- name: "Redirect LibGit2 SYSTEM config off /etc (i686)"
# /etc/gitconfig alone is not enough on SELinux runners: LibGit2 still
# errors with failed-to-stat. Point SYSTEM search at $HOME and refresh
# /etc immediately before any Pkg URL clone (buildpkg/runtest).
if: "${{ (inputs.julia-arch == 'x86' || inputs.julia-arch == 'i686') && runner.os == 'Linux' }}"
run: |
echo '# SciML i686 CI placeholder' | sudo tee /etc/gitconfig >/dev/null
sudo chmod 644 /etc/gitconfig
if command -v restorecon >/dev/null 2>&1; then
sudo restorecon -v /etc/gitconfig || true
fi
if command -v chcon >/dev/null 2>&1; then
sudo chcon -t etc_t /etc/gitconfig 2>/dev/null || true
fi
ls -laZ /etc/gitconfig 2>/dev/null || ls -la /etc/gitconfig
mkdir -p "$HOME/.sciml-libgit2" "$HOME/.julia/config"
echo '# SciML i686 LibGit2 placeholder' > "$HOME/.sciml-libgit2/gitconfig"
cat > "$HOME/.julia/config/startup.jl" <<'EOF'
try
using LibGit2
import LibGit2_jll: libgit2
dir = joinpath(homedir(), ".sciml-libgit2")
LibGit2.version() # ensure libgit2 is initialized
ccall(
(:git_libgit2_opts, libgit2),
Cint,
(Cint, Cint, Cstring),
Cint(LibGit2.Consts.SET_SEARCH_PATH),
Cint(LibGit2.Consts.CONFIG_LEVEL_SYSTEM),
dir,
)
catch e
@warn "i686 LibGit2 SYSTEM config redirect failed" exception = (e, catch_backtrace())
end
EOF
julia --startup-file=yes -e 'using LibGit2; println("LibGit2 ", LibGit2.version(), " ready for URL sources")'

- name: "Gate the pre lane to release candidates only"
# The `pre` channel installs the latest Julia prerelease. Early alphas/
# betas -- and the case where `pre` just resolves to the current stable
Expand Down Expand Up @@ -189,12 +244,15 @@ jobs:
with:
token: "${{ secrets.GITHUB_TOKEN }}"

- name: "Checkout SciML/.github for the develop-sources helper"
if: "${{ inputs.buildpkg && inputs.project != '@.' && steps.prerelease-gate.outputs.run != 'false' }}"
- name: "Checkout .github scripts for develop/prefetch helpers"
# On the consumable tip, scripts must come from the same fork/branch as
# this workflow — SciML/.github@master does not yet include the i686
# host-git URL [sources] prefetch (#137).
if: "${{ inputs.buildpkg && steps.prerelease-gate.outputs.run != 'false' }}"
uses: actions/checkout@v7
with:
repository: SciML/.github
ref: "${{ inputs.dotgithub-ref }}"
repository: ChrisRackauckas-Claude/.github
ref: fix-forward-dotgithub-ref-default-master
path: .sciml-dotgithub

- name: "Develop in-repo [sources] path deps of ${{ inputs.project }}"
Expand All @@ -220,6 +278,17 @@ jobs:
include(joinpath(ENV["GITHUB_WORKSPACE"], ".sciml-dotgithub", "scripts", "develop_sources.jl"))
develop_sources(raw"${{ inputs.project }}")

- name: "Prefetch URL [sources] with host git (i686)"
# LibGit2 HTTPS clones can segfault in OpenSSL on i686 Julia even when
# /etc/gitconfig is present. Clone with the runner's 64-bit git and
# rewrite [sources] url→path so buildpkg/runtest never LibGit2-fetches.
if: "${{ (inputs.julia-arch == 'x86' || inputs.julia-arch == 'i686') && runner.os == 'Linux' && inputs.buildpkg && steps.prerelease-gate.outputs.run != 'false' }}"
shell: julia --color=yes {0}
run: |
include(joinpath(ENV["GITHUB_WORKSPACE"], ".sciml-dotgithub", "scripts", "develop_sources.jl"))
proj = raw"${{ inputs.project }}"
prefetch_url_sources_with_host_git!(proj == "@." ? "." : proj)

- name: "Install system packages"
if: "${{ inputs.apt-packages != '' && runner.os == 'Linux' && steps.prerelease-gate.outputs.run != 'false' }}"
run: |
Expand Down
90 changes: 90 additions & 0 deletions scripts/develop_sources.jl
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,96 @@ function partition_source_specs(specs)
return path_specs, url_specs
end

"""
_rewrite_sources_url_to_path!(tomlpath, dep, path)

Replace a single `[sources]` entry `dep = {url = ..., rev = ...}` with
`dep = {path = "..."}` without rewriting the rest of Project.toml.
"""
function _rewrite_sources_url_to_path!(tomlpath::AbstractString, dep::AbstractString, path::AbstractString)
text = read(tomlpath, String)
pat = Regex("^(\\s*" * escape_string(dep) * "\\s*=\\s*)\\{[^\\}\\n]*url[^\\}\\n]*\\}", "m")
newtext = replace(text, pat => SubstitutionString("\\1{path = $(repr(path))}"))
newtext == text && error("failed to rewrite [sources] entry `$dep` in $tomlpath")
write(tomlpath, newtext)
return nothing
end

"""
prefetch_url_sources_with_host_git!(proj; root = joinpath(homedir(), ".julia", "sciml_i686_url_sources"))

Clone every `url =` entry under `[sources]` (transitively) with the runner's
host `git` CLI and rewrite those entries to `path =` pointing at the clones.

Used on i686 CI where LibGit2 HTTPS clones of URL `[sources]` can segfault in
OpenSSL (`SSL_CTX_load_verify_file`) even after `/etc/gitconfig` workarounds.
Mutates `Project.toml` files on disk in the runner workspace only.
"""
function prefetch_url_sources_with_host_git!(
proj::AbstractString;
root::AbstractString = joinpath(homedir(), ".julia", "sciml_i686_url_sources"),
)
projroot = normpath(abspath(proj == "@." ? "." : proj))
mkpath(root)
queue = String[projroot]
seen_dirs = Set{String}([projroot])
seen_clone_keys = Set{String}()
clones = Dict{String, String}()
while !isempty(queue)
dir = popfirst!(queue)
tomlpath = joinpath(dir, "Project.toml")
isfile(tomlpath) || continue
toml = Pkg.TOML.parsefile(tomlpath)
sources = get(toml, "sources", nothing)
sources isa AbstractDict || continue
isroot = normpath(abspath(dir)) == projroot
runtimedeps = keys(get(toml, "deps", Dict{String, Any}()))
for (dep0, spec) in sources
dep = String(dep0)
isroot || dep in runtimedeps || continue
spec isa AbstractDict || continue
if haskey(spec, "path")
p = normpath(abspath(joinpath(dir, String(spec["path"]))))
if isdir(p) && !(p in seen_dirs)
push!(seen_dirs, p)
push!(queue, p)
end
continue
end
haskey(spec, "url") || continue
url = String(spec["url"])
rev = String(get(spec, "rev", "HEAD"))
key = "$url#$rev"
if !haskey(clones, key)
dest = joinpath(
root,
replace(dep, r"[^\w.-]" => "_") * "_" * string(hash(key) % 0xffffff, base = 16),
)
if !isdir(joinpath(dest, ".git"))
rm(dest; force = true, recursive = true)
@info "i686: host-git clone of URL [sources]" package = dep url rev dest
run(`git clone --filter=blob:none --no-checkout $(url) $(dest)`)
run(`git -C $(dest) fetch --depth 1 origin $(rev)`)
run(`git -C $(dest) checkout --force FETCH_HEAD`)
end
clones[key] = dest
end
dest = clones[key]
subdir = get(spec, "subdir", nothing)
path = subdir === nothing ? dest : joinpath(dest, String(subdir))
isdir(path) || error("URL source path missing after host-git clone: $path")
_rewrite_sources_url_to_path!(tomlpath, dep, path)
@info "i686: rewrote [sources] $dep -> path" path tomlpath
if !(path in seen_dirs)
push!(seen_dirs, path)
push!(queue, path) # nested URL [sources] inside the clone
end
push!(seen_clone_keys, key)
end
end
return nothing
end

"""
develop_sources(proj)

Expand Down
15 changes: 15 additions & 0 deletions test/runtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,21 @@ end
@test isempty(collect_source_paths(joinpath(root, "Leaf")))
end

@testset "dotgithub-ref forwarded to tests.yml and defaults to master" begin
wf(p) = read(joinpath(@__DIR__, "..", ".github", "workflows", p), String)

for p in ("tests.yml", "grouped-tests.yml", "sublibrary-project-tests.yml")
txt = wf(p)
@test occursin(r"dotgithub-ref:\s*\n\s*description:.*\n\s*default: \"master\"", txt)
end

grouped = wf("grouped-tests.yml")
@test occursin("dotgithub-ref: \"\${{ inputs.dotgithub-ref }}\"", grouped)

subs = wf("sublibrary-project-tests.yml")
@test count("dotgithub-ref: \"\${{ inputs.dotgithub-ref }}\"", subs) == 4
end

@testset "auto-precompile input is plumbed to JULIA_PKG_PRECOMPILE_AUTO" begin
wf(p) = read(joinpath(@__DIR__, "..", ".github", "workflows", p), String)

Expand Down
Loading