diff --git a/benchmark/append_rows.jl b/benchmark/append_rows.jl new file mode 100644 index 00000000..a12dde4f --- /dev/null +++ b/benchmark/append_rows.jl @@ -0,0 +1,19 @@ +using BenchmarkTools +using StructArrays + +n = 1_000_000 +rows = [(a=i, b=2i, c=3.0i, d=4.0i) for i in 1:n] +base = StructArray((a=Int[], b=Int[], c=Float64[], d=Float64[])) + +probe = copy(base) +append!(probe, rows) +@assert probe == rows + +trial = @benchmark append!(dest, $rows) setup=(dest=copy($base)) samples=10 evals=1 seconds=60 +estimate = minimum(trial) +println( + "append! rows: ", + BenchmarkTools.prettytime(estimate.time), ", ", + BenchmarkTools.prettymemory(estimate.memory), ", ", + estimate.allocs, " allocations", +) diff --git a/benchmark/group_wide.jl b/benchmark/group_wide.jl new file mode 100644 index 00000000..7484061c --- /dev/null +++ b/benchmark/group_wide.jl @@ -0,0 +1,26 @@ +using BenchmarkTools +using Random +using StructArrays + +function countgroups(keys, permutation) + count = 0 + for _ in StructArrays.GroupPerm(keys, permutation) + count += 1 + end + return count +end + +rng = MersenneTwister(42) +columns = ntuple(_ -> rand(rng, 1:16, 100_000), 64) +table = StructArray(columns) +permutation = sortperm(table) +@assert countgroups(table, permutation) == length(unique(table)) + +trial = @benchmark countgroups($table, $permutation) samples=10 evals=1 seconds=60 +estimate = minimum(trial) +println( + "64-column grouping: ", + BenchmarkTools.prettytime(estimate.time), ", ", + BenchmarkTools.prettymemory(estimate.memory), ", ", + estimate.allocs, " allocations", +) diff --git a/benchmark/sortperm_recursive.jl b/benchmark/sortperm_recursive.jl new file mode 100644 index 00000000..395f544b --- /dev/null +++ b/benchmark/sortperm_recursive.jl @@ -0,0 +1,23 @@ +using BenchmarkTools +using Random +using StructArrays + +function nested_table(::Type{T}; n=100_000, width=6, cardinality=16, seed=1234) where {T} + rng = MersenneTwister(seed) + columns = ntuple(_ -> T.(rand(rng, 1:cardinality, n)), width) + return StructArray(columns) +end + +for T in (Float64, Int) + table = nested_table(T) + @assert issorted(table[sortperm(table)]) + + trial = @benchmark sortperm($table) samples=20 evals=1 seconds=60 + estimate = minimum(trial) + println( + T, " keys: ", + BenchmarkTools.prettytime(estimate.time), ", ", + BenchmarkTools.prettymemory(estimate.memory), ", ", + estimate.allocs, " allocations", + ) +end diff --git a/benchmark/wide_constructor.jl b/benchmark/wide_constructor.jl new file mode 100644 index 00000000..3c477ae6 --- /dev/null +++ b/benchmark/wide_constructor.jl @@ -0,0 +1,18 @@ +using BenchmarkTools +using StructArrays + +construct_wide(columns) = StructArray(columns) + +names = ntuple(i -> Symbol(:x, i), 128) +columns = NamedTuple{names}(ntuple(_ -> rand(1), 128)) +probe = construct_wide(columns) +@assert size(probe) == (1,) + +trial = @benchmark construct_wide($columns) +estimate = minimum(trial) +println( + "128-column constructor: ", + BenchmarkTools.prettytime(estimate.time), ", ", + BenchmarkTools.prettymemory(estimate.memory), ", ", + estimate.allocs, " allocations", +) diff --git a/benchmark/wide_getindex.jl b/benchmark/wide_getindex.jl new file mode 100644 index 00000000..7d23924d --- /dev/null +++ b/benchmark/wide_getindex.jl @@ -0,0 +1,15 @@ +using BenchmarkTools +using StructArrays + +table = StructArray(ntuple(_ -> rand(100), 128)) +probe = table[50] +@assert probe == ntuple(i -> components(table)[i][50], 128) + +trial = @benchmark $table[50] +estimate = minimum(trial) +println( + "128-column getindex: ", + BenchmarkTools.prettytime(estimate.time), ", ", + BenchmarkTools.prettymemory(estimate.memory), ", ", + estimate.allocs, " allocations", +) diff --git a/src/sort.jl b/src/sort.jl index 64316d91..c8f0385b 100644 --- a/src/sort.jl +++ b/src/sort.jl @@ -35,9 +35,25 @@ Base.eltype(::Type{<:GroupPerm}) = UnitRange{Int} return eq end -roweq(t::Tuple{}, i, j) = true -roweq(t::Tuple, i, j) = roweq(t[1], i, j) ? roweq(tail(t), i, j) : false -roweq(s::StructArray, i, j) = roweq(Tuple(components(s)), i, j) +_roweq(t::Tuple{}, i, j) = true +_roweq(t::Tuple, i, j) = roweq(t[1], i, j) ? _roweq(tail(t), i, j) : false +function roweq(t::T, i, j) where {T<:Tup} + if @generated + types = fieldtypes(T) + if length(types) > WIDE_TUPLE_LENGTH && all(==(types[1]), types) && isconcretetype(types[1]) + return quote + for k in 1:$(length(types)) + roweq(getfield(t, k), i, j) || return false + end + return true + end + end + return :(_roweq(Tuple(t), i, j)) + else + return _roweq(Tuple(t), i, j) + end +end +roweq(s::StructArray, i, j) = roweq(components(s), i, j) function uniquesorted(keys, perm=sortperm(keys)) (keys[perm[idxs[1]]] for idxs in GroupPerm(keys, perm)) @@ -72,30 +88,29 @@ forward_vec(::Ordering) = nothing # Methods from IndexedTables to refine sorting: # # assuming x[p] is sorted, sort by remaining columns where x[p] is constant -function refine_perm!(p, cols, c, x, y′, lo, hi) - temp = similar(p, 0) +function refine_perm!(p, cols, c, x, y′, lo, hi, temp=similar(p, 0), counts=Int[]) order = Perm(Forward, y′) y = something(forward_vec(order), y′) nc = length(cols) for idxs in GroupPerm(x, p, lo:hi) i, i1 = extrema(idxs) if i1 > i - sort_sub_by!(p, i, i1, y, order, temp) + sort_sub_by!(p, i, i1, y, order, temp, counts) if c < nc-1 z = cols[c+2] - refine_perm!(p, cols, c+1, y, z, i, i1) + refine_perm!(p, cols, c+1, y, z, i, i1, temp, counts) end end end end # sort the values in v[i0:i1] in place, by array `by` -Base.@noinline function sort_sub_by!(v, i0, i1, by, order, temp) +Base.@noinline function sort_sub_by!(v, i0, i1, by, order, temp, counts=Int[]) empty!(temp) sort!(v, i0, i1, MergeSort, order, temp) end -Base.@noinline function sort_sub_by!(v, i0, i1, by::AbstractVector{T}, order, temp) where T<:Integer +Base.@noinline function sort_sub_by!(v, i0, i1, by::AbstractVector{T}, order, temp, counts=Int[]) where T<:Integer min = max = by[v[i0]] @inbounds for i = i0+1:i1 val = by[v[i]] @@ -108,7 +123,7 @@ Base.@noinline function sort_sub_by!(v, i0, i1, by::AbstractVector{T}, order, te rangelen = max-min+1 n = i1-i0+1 if rangelen <= n - sort_int_range_sub_by!(v, i0-1, n, by, rangelen, min, temp) + sort_int_range_sub_by!(v, i0-1, n, by, rangelen, min, temp, counts) else empty!(temp) sort!(v, i0, i1, MergeSort, order, temp) @@ -117,10 +132,11 @@ Base.@noinline function sort_sub_by!(v, i0, i1, by::AbstractVector{T}, order, te end # in-place counting sort of x[ioffs+1:ioffs+n] by values in `by` -function sort_int_range_sub_by!(x, ioffs, n, by, rangelen, minval, temp) +function sort_int_range_sub_by!(x, ioffs, n, by, rangelen, minval, temp, where=Int[]) offs = 1 - minval - where = fill(0, rangelen+1) + resize!(where, rangelen+1) + fill!(where, 0) where[1] = 1 @inbounds for i = 1:n where[by[x[i+ioffs]] + offs + 1] += 1 diff --git a/src/structarray.jl b/src/structarray.jl index 2b07a517..1cccdada 100644 --- a/src/structarray.jl +++ b/src/structarray.jl @@ -352,6 +352,12 @@ Base.@propagate_inbounds function _getindex(x::StructArray{T}, I::Vararg{Int}) w return createinstance(T, get_ith(cols, I...)...) end +Base.@propagate_inbounds function _getindex(x::StructArray{T}, I::Vararg{Int}) where {T<:Tup} + cols = components(x) + @boundscheck checkbounds(x, I...) + return T(get_ith(cols, I...)) +end + @inline function _getindex(s::StructArray{T}, I...) where {T} @boundscheck checkbounds(s, I...) StructArray{T}(map(v -> @inbounds(getindex(v, I...)), components(s))) diff --git a/src/tables.jl b/src/tables.jl index 432a75b2..58c79277 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -31,6 +31,16 @@ end try_compatible_columns(rows::StructArray{T}, s::StructArray{T}) where {T} = Tables.columntable(rows) try_compatible_columns(rows::StructArray{R}, s::StructArray{S}) where {R,S} = nothing +function _prepare_rows!(s, rows, ::typeof(push!)) + _sizehint_rows!(s, rows, Base.IteratorSize(rows)) +end +_prepare_rows!(s, rows, ::typeof(pushfirst!)) = s + +function _sizehint_rows!(s, rows, ::Union{Base.HasLength, Base.HasShape}) + sizehint!(s, length(s) + length(rows)) +end +_sizehint_rows!(s, rows, ::Any) = s + for (f, g) in zip((:append!, :prepend!), (:push!, :pushfirst!)) @eval function Base.$f(s::StructVector, rows) table = try_compatible_columns(rows, s) @@ -42,6 +52,7 @@ for (f, g) in zip((:append!, :prepend!), (:push!, :pushfirst!)) else # Otherwise, fallback to a generic implementation expecting # that `rows` is an iterator: + _prepare_rows!(s, rows, $g) return foldl($g, rows; init = s) end end diff --git a/src/utils.jl b/src/utils.jl index 00458e1b..afeafab1 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -2,8 +2,23 @@ argtail(_, args...) = args split_tuple_type(T) = fieldtype(T, 1), Tuple{argtail(T.parameters...)...} +# Above this many components, recursive/splatting tuple code hits Julia's +# specialization limits, so homogeneous component tuples use a loop instead. +const WIDE_TUPLE_LENGTH = 32 + eltypes(nt::NamedTuple{names}) where {names} = NamedTuple{names, eltypes(values(nt))} -eltypes(t::Tuple) = Tuple{map(eltype, t)...} +_eltypes(t::Tuple) = Tuple{map(eltype, t)...} +function eltypes(t::T) where {T<:Tuple} + if @generated + types = fieldtypes(T) + if !isempty(types) && all(==(types[1]), types) && isconcretetype(types[1]) && types[1] <: AbstractArray + return :(NTuple{$(length(types)), $(eltype(types[1]))}) + end + return :(_eltypes(t)) + else + return _eltypes(t) + end +end alwaysfalse(t) = false @@ -193,14 +208,33 @@ maybe_convert_elt(::Type{T}, vals) where T = convert(T, vals) maybe_convert_elt(::Type{T}, vals::Tuple) where T = T <: Tuple ? convert(T, vals) : vals # assignment of fields by position maybe_convert_elt(::Type{T}, vals::NamedTuple) where T = T<:NamedTuple ? convert(T, vals) : vals # assignment of fields by name +function _findconsistentvalue(f, cols) + val = f(first(cols)) + isconsistent = all(map(isequal(val) ∘ f, values(cols))) + return ifelse(isconsistent, val, nothing) +end + """ findconsistentvalue(f, components::Union{Tuple, NamedTuple}) Compute the unique value that `f` takes on each `component ∈ components`. If not all values are equal, return `nothing`. Otherwise, return the unique value. """ -function findconsistentvalue(f::F, cols::Tup) where F - val = f(first(cols)) - isconsistent = all(map(isequal(val) ∘ f, values(cols))) - return ifelse(isconsistent, val, nothing) +function findconsistentvalue(f::F, cols::T) where {F, T<:Tup} + if @generated + types = fieldtypes(T) + if length(types) > WIDE_TUPLE_LENGTH && all(==(types[1]), types) && isconcretetype(types[1]) + # Unrolled with constant field indices: looping over `cols` would + # index it dynamically, which can force a wide tuple onto the heap. + checks = [:(isequal(val, f(getfield(cols, $i))) || return nothing) for i in 2:length(types)] + return quote + val = f(getfield(cols, 1)) + $(checks...) + return val + end + end + return :(_findconsistentvalue(f, cols)) + else + return _findconsistentvalue(f, cols) + end end diff --git a/test/runtests.jl b/test/runtests.jl index 9ec78d51..199121e7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1660,6 +1660,111 @@ end end end +@testset "wide component tuples" begin + # These widths are above `StructArrays.WIDE_TUPLE_LENGTH`, exercising the + # loop-based code paths for homogeneous component tuples. + N = 2 * StructArrays.WIDE_TUPLE_LENGTH + names = ntuple(i -> Symbol(:x, i), N) + # Measure allocations behind a function barrier (and without returning a + # wide tuple) so that only the code under test is counted. + allocs(f::F, args...) where {F} = (f(args...); @allocated f(args...)) + construct(c) = StructArray(c) + storerow!(out, s, i) = (out[1] = s[i]; nothing) + roweq(s, i, j) = StructArrays.roweq(s, i, j) + + @testset "construction" begin + cols = ntuple(_ -> rand(10), N) + s = @inferred StructArray(cols) + @test eltype(s) == NTuple{N, Float64} + @test size(s) == (10,) + @test allocs(construct, cols) == 0 + + ncols = NamedTuple{names}(cols) + ns = @inferred StructArray(ncols) + @test eltype(ns) == NamedTuple{names, NTuple{N, Float64}} + @test allocs(construct, ncols) == 0 + + bad = ntuple(i -> i == N ? rand(9) : rand(10), N) + @test_throws ArgumentError StructArray(bad) + @test_throws ArgumentError StructArray(NamedTuple{names}(bad)) + bad2d = ntuple(i -> i == N ? rand(10, 1) : rand(10), N) + @test_throws ArgumentError StructArray(bad2d) + + # heterogeneous wide tuples still go through the generic path + mixed = ntuple(i -> isodd(i) ? rand(10) : rand(1:3, 10), N) + ms = StructArray(mixed) + @test eltype(ms) == Tuple{ntuple(i -> isodd(i) ? Float64 : Int, N)...} + end + + @testset "getindex" begin + cols = ntuple(_ -> rand(10), N) + s = StructArray(cols) + @test @inferred(s[3]) == map(c -> c[3], cols) + @test allocs(storerow!, Vector{eltype(s)}(undef, 1), s, 3) == 0 + @test_throws BoundsError s[11] + + ns = StructArray(NamedTuple{names}(cols)) + @test @inferred(ns[3]) == NamedTuple{names}(map(c -> c[3], cols)) + @test allocs(storerow!, Vector{eltype(ns)}(undef, 1), ns, 3) == 0 + + # conversion to the declared eltype is preserved + cs = StructArray{NTuple{2, Float64}}(([1, 2], [3, 4])) + @test cs[1] === (1.0, 3.0) + cns = StructArray{@NamedTuple{a::Float64, b::Int}}(([1, 2], [3, 4])) + @test cns[2] === (a = 2.0, b = 4) + end + + @testset "roweq and grouping" begin + cols = ntuple(i -> rand(1:2, 200), N) + s = StructArray(cols) + for (i, j) in ((1, 1), (1, 2), (3, 7)) + @test StructArrays.roweq(s, i, j) == (s[i] == s[j]) + end + t = StructArray(ntuple(_ -> [1, 1, 2], N)) + @test StructArrays.roweq(t, 1, 2) + @test !StructArrays.roweq(t, 1, 3) + @test allocs(roweq, t, 1, 2) == 0 + nt = StructArray(NamedTuple{names}(components(t))) + @test StructArrays.roweq(nt, 1, 2) + @test !StructArrays.roweq(nt, 1, 3) + @test allocs(roweq, nt, 1, 2) == 0 + + k = StructArray(ntuple(i -> i == N ? [1, 2, 1, 2] : [1, 1, 1, 1], N)) + groups = collect(StructArrays.GroupPerm(k, sortperm(k))) + @test length(groups) == 2 + @test all(g -> allequal(k[sortperm(k)[g]]), groups) + end +end + +@testset "sortperm workspace reuse" begin + # Many subgroups on several levels, so the merge-sort scratch vector and the + # counting-sort histogram are reused (and resized) across recursive calls. + for T in (Int, Float64) + n = 2_000 + cols = (T.(rand(1:4, n)), T.(rand(1:8, n)), T.(rand(1:3, n)), T.(rand(1:50, n)), T.(rand(1:2, n))) + s = StructArray(cols) + p = sortperm(s) + @test p == sortperm(collect(s)) + @test issorted(s[p]) + end + # integer sub-keys whose ranges shrink and grow between groups + a = repeat(1:3, inner = 4) + b = [5, 1, 3, 2, 100, 90, 95, 99, 0, -3, 7, 7] + s = StructArray((a, b)) + @test sortperm(s) == sortperm(collect(s)) +end + +@testset "append! with iterator" begin + s = StructArray(a = [1], b = [1.0]) + itr = ((a = i, b = Float64(i)) for i in 2:100) + append!(s, itr) + @test s.a == 1:100 + @test s.b == 1.0:100.0 + # unknown length + append!(s, ((a = i, b = Float64(i)) for i in 101:200 if iseven(i))) + @test s.a == [1:100; 102:2:200] +end + @testset "project quality" begin Aqua.test_all(StructArrays) end