From 3b77f3be9729cd88c493b425db87308abfe3efd5 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Mon, 9 Nov 2026 20:58:05 -0500 Subject: [PATCH 1/8] Test componentwise operations on wide StructArrays Covers 32 and 64 components, same and mixed types, Tuple and NamedTuple. Operations not yet inferrable at these widths are marked broken. Co-authored-by: Kristoffer Carlsson --- test/runtests.jl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 9ec78d51..c2df8e84 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -790,6 +790,36 @@ end @inferred map_params_infer() end +@testset "wide components" begin + # Base.map loses inference on tuples with 32 or more elements, + # so componentwise operations must not rely on it + widecols(n, het) = ntuple(i -> het && iseven(i) ? [1.0, 2.0] : [1, 2], n) + @testset "$n columns, $(het ? "mixed" : "same") types, $(named ? "NamedTuple" : "Tuple")" for n in (32, 64), het in (false, true), named in (false, true) + cols = named ? NamedTuple{ntuple(i -> Symbol(:x, i), n)}(widecols(n, het)) : widecols(n, het) + t = StructArray(cols) + row = t[1] + + @test (@inferred StructArray(cols)) == t broken=true + @test (@inferred StructArrays.eltypes(cols)) == eltype(t) broken=true + + @test (@inferred t[1]) == row + @test (@inferred t[1:2]) == t broken=true + @test (@inferred view(t, 1:2)) == t broken=true + + @test (@inferred similar(t)) isa typeof(t) broken=true + @test (@inferred similar(t, 3)) isa typeof(t) broken=true + @test (@inferred copy(t)) == t broken=true + @test (@inferred reshape(t, 1, 2)) == reshape(collect(t), 1, 2) broken=true + @test (@inferred deleteat!(copy(t), 1)) == t[2:2] broken=true + @test (@inferred keepat!(copy(t), [1])) == t[1:1] broken=true + @test (@inferred pop!(copy(t))) == t[2] broken=!named + @test (@inferred popfirst!(copy(t))) == row broken=!named + @test (@inferred refarray(t)) == t broken=true + @test (@inferred StructArrays.replace_storage(identity, t)) == t broken=true + @test (@inferred StructArrays.colmetadata(t)) == map(_ -> nothing, Tables.columns(t)) broken=true + end +end + @testset "propertynames" begin a = StructArray{ComplexF64}((Float64[], Float64[])) @test sort(collect(propertynames(a))) == [:im, :re] From 9ff9377a39a8dc6000b8a4e9312e65feaca587da Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Mon, 9 Nov 2026 21:01:08 -0500 Subject: [PATCH 2/8] Add _mapcols; use it for range getindex, view and get_ith _mapcols is a generated componentwise map that stays inferrable for any number of components, unlike Base.map on 32+ element tuples. Co-authored-by: Kristoffer Carlsson --- src/structarray.jl | 6 +++--- src/utils.jl | 9 +++++++++ test/runtests.jl | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/structarray.jl b/src/structarray.jl index 2b07a517..1da64aa9 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -342,7 +342,7 @@ to map(c -> c[I...], Tuple(cols)) ``` """ -@inline @generated get_ith(cols::Tup, I...) = :(Base.Cartesian.@ntuple $(fieldcount(cols)) i -> @inbounds cols[i][I...]) +@inline get_ith(cols::Tup, I...) = _mapcols(c -> @inbounds(c[I...]), cols) Base.@propagate_inbounds Base.getindex(x::StructArray, I...) = _getindex(x, to_indices(x, I)...) @@ -354,12 +354,12 @@ end @inline function _getindex(s::StructArray{T}, I...) where {T} @boundscheck checkbounds(s, I...) - StructArray{T}(map(v -> @inbounds(getindex(v, I...)), components(s))) + StructArray{T}(_mapcols(v -> @inbounds(getindex(v, I...)), components(s))) end @inline function Base.view(s::StructArray{T, N, C}, I...) where {T, N, C} @boundscheck checkbounds(s, I...) - StructArray{T}(map(v -> @inbounds(view(v, I...)), components(s))) + StructArray{T}(_mapcols(v -> @inbounds(view(v, I...)), components(s))) end function Base.parentindices(s::StructArray) diff --git a/src/utils.jl b/src/utils.jl index 00458e1b..eff0ae96 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -32,6 +32,15 @@ end map_params_as_tuple_fallback(f, ::Type{T}) where {T<:Tup} = map(f, fieldtypes(T)) +""" + StructArrays._mapcols(f, cols::Union{Tuple,NamedTuple}) + +Apply `f` to each element of `cols`, returning a `Tuple` or `NamedTuple` with the same names. +Unlike `map`, stays inferrable for any number of elements: `map` gives up on tuples with 32 or more. +""" +@inline @generated _mapcols(f::F, cols::Tup) where {F} = + :($(strip_params(cols))(Base.Cartesian.@ntuple $(fieldcount(cols)) i -> f(cols[i]))) + buildfromschema(initializer::F, ::Type{T}) where {F, T} = buildfromschema(initializer, T, staticschema(T)) """ diff --git a/test/runtests.jl b/test/runtests.jl index c2df8e84..58f0b404 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -803,8 +803,8 @@ end @test (@inferred StructArrays.eltypes(cols)) == eltype(t) broken=true @test (@inferred t[1]) == row - @test (@inferred t[1:2]) == t broken=true - @test (@inferred view(t, 1:2)) == t broken=true + @test (@inferred t[1:2]) == t + @test (@inferred view(t, 1:2)) == t @test (@inferred similar(t)) isa typeof(t) broken=true @test (@inferred similar(t, 3)) isa typeof(t) broken=true From 803ee0cbff82a0e91a40170be42bc26edb349eeb Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Mon, 9 Nov 2026 21:03:15 -0500 Subject: [PATCH 3/8] Use _mapcols in eltypes Makes StructArray(cols) inferrable for 32+ components. Co-authored-by: Kristoffer Carlsson --- src/utils.jl | 2 +- test/runtests.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index eff0ae96..8700d787 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -3,7 +3,7 @@ argtail(_, args...) = args split_tuple_type(T) = fieldtype(T, 1), Tuple{argtail(T.parameters...)...} eltypes(nt::NamedTuple{names}) where {names} = NamedTuple{names, eltypes(values(nt))} -eltypes(t::Tuple) = Tuple{map(eltype, t)...} +eltypes(t::Tuple) = Tuple{_mapcols(eltype, t)...} alwaysfalse(t) = false diff --git a/test/runtests.jl b/test/runtests.jl index 58f0b404..1e85653a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -799,8 +799,8 @@ end t = StructArray(cols) row = t[1] - @test (@inferred StructArray(cols)) == t broken=true - @test (@inferred StructArrays.eltypes(cols)) == eltype(t) broken=true + @test (@inferred StructArray(cols)) == t + @test (@inferred StructArrays.eltypes(cols)) == eltype(t) @test (@inferred t[1]) == row @test (@inferred t[1:2]) == t From 6c59d15c4f96c4eefb08a279fc9d96a69d6751a7 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Mon, 9 Nov 2026 21:05:25 -0500 Subject: [PATCH 4/8] Use _mapcols for componentwise maps similar, copy, reshape, deleteat!, keepat!, pop!, popfirst!, refarray, replace_storage and colmetadata become inferrable for 32+ components. Co-authored-by: Kristoffer Carlsson --- src/StructArrays.jl | 4 ++-- src/structarray.jl | 12 ++++++------ src/utils.jl | 2 +- test/runtests.jl | 22 +++++++++++----------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/StructArrays.jl b/src/StructArrays.jl index bef1167c..67e46925 100644 --- a/src/StructArrays.jl +++ b/src/StructArrays.jl @@ -21,7 +21,7 @@ include("tables.jl") import DataAPI: refarray, refvalue using DataAPI: defaultarray -refarray(s::StructArray) = StructArray(map(refarray, components(s))) +refarray(s::StructArray) = StructArray(_mapcols(refarray, components(s))) function refvalue(s::StructArray{T}, v::Tup) where {T} createinstance(T, map(refvalue, components(s), v)...) @@ -36,7 +36,7 @@ colmetadatasupport(::Type{T}) where {T<:StructArray} = ( ) colmetadata(sa::StructArray, col::Symbol) = metadata(getproperty(sa, col)) colmetadata(sa::StructArray) = - map(Tables.columns(sa)) do col + _mapcols(Tables.columns(sa)) do col metadatasupport(typeof(col)).read ? metadata(col) : nothing end diff --git a/src/structarray.jl b/src/structarray.jl index 1da64aa9..40422686 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -273,7 +273,7 @@ Base.convert(::Type{StructVector}, v::StructVector) = v # Helper function to avoid adding too many dispatches to `Base.similar` function _similar(s::StructArray{T}, ::Type{T}, sz) where {T} - return StructArray{T}(map(typ -> similar(typ, sz), components(s))) + return StructArray{T}(_mapcols(typ -> similar(typ, sz), components(s))) end function _similar(s::StructArray{T}, S::Type, sz) where {T} @@ -407,18 +407,18 @@ end for f in (:pop!, :popfirst!) @eval function Base.$f(s::StructVector{T}) where T - t = map($f, components(s)) + t = _mapcols($f, components(s)) return createinstance(T, t...) end end function Base.deleteat!(s::StructVector{T}, idxs) where T - t = map(Base.Fix2(deleteat!, idxs), components(s)) + t = _mapcols(Base.Fix2(deleteat!, idxs), components(s)) return StructVector{T}(t) end function Base.keepat!(s::StructVector{T}, idxs) where T - t = map(Base.Fix2(keepat!, idxs), components(s)) + t = _mapcols(Base.Fix2(keepat!, idxs), components(s)) return StructVector{T}(t) end @@ -484,7 +484,7 @@ end Base.reduce(::typeof(vcat), A::AbstractVector{<:StructArray}) = _reducecat_structarray(vcat, A) Base.reduce(::typeof(hcat), A::AbstractVector{<:StructArray}) = _reducecat_structarray(hcat, A) -Base.copy(s::StructArray{T}) where {T} = StructArray{T}(map(copy, components(s))) +Base.copy(s::StructArray{T}) where {T} = StructArray{T}(_mapcols(copy, components(s))) for type in ( :Dims, @@ -500,7 +500,7 @@ for type in ( :(Tuple{Colon}), ) @eval function Base.reshape(s::StructArray{T}, d::$(type)) where {T} - StructArray{T}(map(x -> reshape(x, d), components(s))) + StructArray{T}(_mapcols(x -> reshape(x, d), components(s))) end end diff --git a/src/utils.jl b/src/utils.jl index 8700d787..48d493ae 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -160,7 +160,7 @@ julia> s_pooled = StructArrays.replace_storage(s) do v """ function replace_storage(f, s::StructArray{T}) where T cols = components(s) - newcols = map(t -> replace_storage(f, t), cols) + newcols = _mapcols(t -> replace_storage(f, t), cols) StructArray{T}(newcols) end diff --git a/test/runtests.jl b/test/runtests.jl index 1e85653a..75f63f61 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -806,17 +806,17 @@ end @test (@inferred t[1:2]) == t @test (@inferred view(t, 1:2)) == t - @test (@inferred similar(t)) isa typeof(t) broken=true - @test (@inferred similar(t, 3)) isa typeof(t) broken=true - @test (@inferred copy(t)) == t broken=true - @test (@inferred reshape(t, 1, 2)) == reshape(collect(t), 1, 2) broken=true - @test (@inferred deleteat!(copy(t), 1)) == t[2:2] broken=true - @test (@inferred keepat!(copy(t), [1])) == t[1:1] broken=true - @test (@inferred pop!(copy(t))) == t[2] broken=!named - @test (@inferred popfirst!(copy(t))) == row broken=!named - @test (@inferred refarray(t)) == t broken=true - @test (@inferred StructArrays.replace_storage(identity, t)) == t broken=true - @test (@inferred StructArrays.colmetadata(t)) == map(_ -> nothing, Tables.columns(t)) broken=true + @test (@inferred similar(t)) isa typeof(t) + @test (@inferred similar(t, 3)) isa typeof(t) + @test (@inferred copy(t)) == t + @test (@inferred reshape(t, 1, 2)) == reshape(collect(t), 1, 2) + @test (@inferred deleteat!(copy(t), 1)) == t[2:2] + @test (@inferred keepat!(copy(t), [1])) == t[1:1] + @test (@inferred pop!(copy(t))) == t[2] + @test (@inferred popfirst!(copy(t))) == row + @test (@inferred refarray(t)) == t + @test (@inferred StructArrays.replace_storage(identity, t)) == t + @test (@inferred StructArrays.colmetadata(t)) == map(_ -> nothing, Tables.columns(t)) end end From 5d33fafff62fc3a526028af21ab7b9619ed52e88 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Tue, 10 Nov 2026 09:40:24 -0500 Subject: [PATCH 5/8] Use _mapcols in findconsistentvalue Co-authored-by: Kristoffer Carlsson --- src/utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.jl b/src/utils.jl index 48d493ae..d402004b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -210,6 +210,6 @@ If not all values are equal, return `nothing`. Otherwise, return the unique valu """ function findconsistentvalue(f::F, cols::Tup) where F val = f(first(cols)) - isconsistent = all(map(isequal(val) ∘ f, values(cols))) + isconsistent = all(_mapcols(isequal(val) ∘ f, cols)) return ifelse(isconsistent, val, nothing) end From 81a5b618b4149a1b0bf3db053f9d7d04a3a7d3d8 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Thu, 12 Nov 2026 09:56:50 -0500 Subject: [PATCH 6/8] Use _mapcols in StaticArrays and SparseArrays extensions Co-authored-by: Kristoffer Carlsson --- ext/SparseArraysExt.jl | 3 ++- ext/StaticArraysExt.jl | 2 +- test/runtests.jl | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/SparseArraysExt.jl b/ext/SparseArraysExt.jl index 00dc998f..fda77f62 100644 --- a/ext/SparseArraysExt.jl +++ b/ext/SparseArraysExt.jl @@ -1,10 +1,11 @@ module SparseArraysExt using StructArrays: StructArray, components, createinstance +import StructArrays import SparseArrays: sparse, issparse function sparse(S::StructArray{T}) where {T} - sparse_components = map(sparse, components(S)) + sparse_components = StructArrays._mapcols(sparse, components(S)) return createinstance.(T, sparse_components...) end diff --git a/ext/StaticArraysExt.jl b/ext/StaticArraysExt.jl index ca8a1ea6..52a947fd 100644 --- a/ext/StaticArraysExt.jl +++ b/ext/StaticArraysExt.jl @@ -46,7 +46,7 @@ let shape = isdefined(Base, :AbstractOneTo) && SOneTo <: Base.AbstractOneTo ? @eval Base.similar(s::StructArray, ::Type{T}, sz::$shape) where T = StructArrays._similar(s, T, sz) @eval Base.similar(::Type{<:StructArray{T, N, C}}, sz::$shape) where {T, N, C} = StructArrays.buildfromschema(typ -> similar(typ, sz), T, C) end -Base.reshape(s::StructArray{T}, d::Tuple{SOneTo, Vararg{SOneTo}}) where {T} = StructArray{T}(map(x -> reshape(x, d), StructArrays.components(s))) +Base.reshape(s::StructArray{T}, d::Tuple{SOneTo, Vararg{SOneTo}}) where {T} = StructArray{T}(StructArrays._mapcols(x -> reshape(x, d), StructArrays.components(s))) # Broadcast overload using StaticArrays: StaticArrayStyle, similar_type, Size, SOneTo diff --git a/test/runtests.jl b/test/runtests.jl index 75f63f61..bab2f23a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -817,6 +817,7 @@ end @test (@inferred refarray(t)) == t @test (@inferred StructArrays.replace_storage(identity, t)) == t @test (@inferred StructArrays.colmetadata(t)) == map(_ -> nothing, Tables.columns(t)) + @test (@inferred reshape(t, (SOneTo(2),))) == t end end From 335aba1a67f28460a1bbbbbd4df6103d4dd09001 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Thu, 12 Nov 2026 10:03:40 -0500 Subject: [PATCH 7/8] Use _mapcols in constructorof Co-authored-by: Kristoffer Carlsson --- src/constructionbase.jl | 2 +- test/runtests.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/constructionbase.jl b/src/constructionbase.jl index 24b4f5b4..5171be8c 100644 --- a/src/constructionbase.jl +++ b/src/constructionbase.jl @@ -7,7 +7,7 @@ ConstructionBase.constructorof(::Type{<:StructArray{<:Union{Tuple, NamedTuple}}} ConstructionBase.constructorof(::Type{<:StructArray{T}}) where {T} = function(comps::CT) where {CT} # the resulting eltype is like T, but potentially with different type parameters, eg Complex{Int} -> Complex{Float64} # probe its constructorof to get the right concrete type - ET = Base.promote_op(constructorof(T), map(eltype, fieldtypes(CT))...) + ET = Base.promote_op(constructorof(T), _mapcols(eltype, comps)...) StructArray{ET}(comps) end diff --git a/test/runtests.jl b/test/runtests.jl index bab2f23a..8e58ade1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -790,6 +790,10 @@ end @inferred map_params_infer() end +@eval struct Wide64 + $([:($(Symbol(:x, i))::$(iseven(i) ? Float64 : Int)) for i in 1:64]...) +end + @testset "wide components" begin # Base.map loses inference on tuples with 32 or more elements, # so componentwise operations must not rely on it @@ -819,6 +823,9 @@ end @test (@inferred StructArrays.colmetadata(t)) == map(_ -> nothing, Tables.columns(t)) @test (@inferred reshape(t, (SOneTo(2),))) == t end + + s = StructArray{Wide64}(widecols(64, true)) + @test (@inferred constructorof(typeof(s))(components(s))) == s end @testset "propertynames" begin From 89b78552acc3d19457077cb2c5dd990a512e6d04 Mon Sep 17 00:00:00 2001 From: Alexander Plavin Date: Sun, 15 Nov 2026 21:34:54 -0500 Subject: [PATCH 8/8] address comment --- src/structarray.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/structarray.jl b/src/structarray.jl index 40422686..611b29be 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -336,10 +336,10 @@ Base.axes(s::StructArray) = axes(components(s)[1]) """ StructArrays.get_ith(cols::Union{Tuple,NamedTuple}, I...) -Form a `Tuple` of the `I`th index of each element of `cols`, i.e. is equivalent -to +Form a `Tuple` or `NamedTuple` (matching `cols`) of the `I`th index of each element +of `cols`, i.e. is equivalent to ```julia -map(c -> c[I...], Tuple(cols)) +map(c -> c[I...], cols) ``` """ @inline get_ith(cols::Tup, I...) = _mapcols(c -> @inbounds(c[I...]), cols)