From d2e839b0f8a366d7ed4b5be49202968ab5abb6fd Mon Sep 17 00:00:00 2001 From: Martin Kirk Bonde Date: Tue, 5 May 2026 12:33:20 +0200 Subject: [PATCH] Use Tables.jl for GDX record storage Make GDX symbols store generic Tables.jl-compatible records, preserve table metadata descriptions when writing, and document the new default read behavior. --- Project.toml | 11 +- README.md | 27 +++-- src/GDXFile.jl | 280 ++++++++++++++++++++++--------------------- src/GDXInterface.jl | 3 +- test/runtests.jl | 3 +- test/test_gdxfile.jl | 254 ++++++++++++++++++++++----------------- 6 files changed, 312 insertions(+), 266 deletions(-) diff --git a/Project.toml b/Project.toml index 14da5ed..bfa6963 100644 --- a/Project.toml +++ b/Project.toml @@ -1,19 +1,22 @@ name = "GDXInterface" uuid = "b8352055-3412-4df1-864e-ee7edae71aec" -version = "0.1.1" +version = "0.2.0" authors = ["Martin Kirk Bonde", "James Daniel Foster"] [deps] -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" gdx_jll = "aa021fa7-5c7d-5dc1-9fa7-c90cd282ac20" [compat] -DataFrames = "1" +DataAPI = "1" +Tables = "1" gdx_jll = "7.11.20" julia = "1.6" [extras] +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["Test", "DataFrames"] diff --git a/README.md b/README.md index 4694cc1..1ad45ca 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ list_parameters(gdx) list_variables(gdx) list_equations(gdx) -# Access data as DataFrames +# Access data as Tables.jl-compatible column tables demand = gdx[:demand] # bracket syntax demand = gdx.demand # property syntax (with tab completion) @@ -57,17 +57,21 @@ sym = get_symbol(gdx, :demand) sym.name # "demand" sym.description # explanatory text from GAMS sym.domain # ["j"] -sym.records # the DataFrame +sym.records # the records table + +# Pass DataFrame as a sink to materialize DataFrames while reading +using DataFrames +gdx = read_gdx("transport.gdx", DataFrame) ``` ### Writing GDX files ```julia -using GDXInterface, DataFrames +using GDXInterface -# Write DataFrames as parameters -supply = DataFrame(i = ["seattle", "san-diego"], value = [350.0, 600.0]) -demand = DataFrame(j = ["new-york", "chicago", "topeka"], value = [325.0, 300.0, 275.0]) +# Write Tables.jl-compatible tables as parameters +supply = (; i = ["seattle", "san-diego"], value = [350.0, 600.0]) +demand = (; j = ["new-york", "chicago", "topeka"], value = [325.0, 300.0, 275.0]) write_gdx("output.gdx", "supply" => supply, "demand" => demand) # Round-trip: read a GDX file and write it back (preserves all symbol types) @@ -98,17 +102,18 @@ gdx = read_gdx("big_model.gdx", only=[:x, :demand]) ### Reading ```julia -read_gdx(filepath; parse_integers=true, only=nothing) -> GDXFile +read_gdx(filepath[, sink]; parse_integers=true, only=nothing) -> GDXFile ``` +- `sink`: callable that materializes a column table, defaulting to `Tables.columntable` - `parse_integers`: convert set elements like `"2020"` to `Int` - `only`: vector of symbol names to load (e.g. `[:x, :demand]`) ### Writing ```julia -# Write DataFrames as parameters (convenience) -write_gdx(filepath, "name" => DataFrame, ...) +# Write tables as parameters (convenience) +write_gdx(filepath, "name" => table, ...) # Write a full GDXFile (sets, parameters, variables, equations) write_gdx(filepath, gdxfile::GDXFile) @@ -117,8 +122,8 @@ write_gdx(filepath, gdxfile::GDXFile) ### Querying a GDXFile ```julia -gdx[:name] # records DataFrame (bracket access) -gdx.name # records DataFrame (property access) +gdx[:name] # records table (bracket access) +gdx.name # records table (property access) get_symbol(gdx, :name) # full GDXSymbol object list_sets(gdx) # list set names diff --git a/src/GDXFile.jl b/src/GDXFile.jl index f44da18..4dbbc27 100644 --- a/src/GDXFile.jl +++ b/src/GDXFile.jl @@ -1,7 +1,5 @@ # High-level GDX file API for GDXInterface.jl -# requires `import DataFrames` - # ============================================================================= # Enums # ============================================================================= @@ -33,63 +31,63 @@ end # Symbol types # ============================================================================= -abstract type GDXSymbol end +abstract type GDXSymbol{T} end """ - GDXSet + GDXSet{T} A GAMS set with its elements and optional explanatory text. Records may include an `element_text` column. """ -struct GDXSet <: GDXSymbol +struct GDXSet{T} <: GDXSymbol{T} name::String description::String domain::Vector{String} - records::DataFrames.DataFrame + records::T end """ - GDXParameter + GDXParameter{T} A GAMS parameter with domain and values. """ -struct GDXParameter <: GDXSymbol +struct GDXParameter{T} <: GDXSymbol{T} name::String description::String domain::Vector{String} - records::DataFrames.DataFrame + records::T end """ - GDXVariable + GDXVariable{T} A GAMS variable with level, marginal, lower, upper, and scale values. """ -struct GDXVariable <: GDXSymbol +struct GDXVariable{T} <: GDXSymbol{T} name::String description::String domain::Vector{String} vartype::VariableType - records::DataFrames.DataFrame + records::T end -GDXVariable(name::String, desc::String, domain::Vector{String}, vartype::Integer, records::DataFrames.DataFrame) = +GDXVariable(name::String, desc::String, domain::Vector{String}, vartype::Integer, records) = GDXVariable(name, desc, domain, VariableType(vartype), records) """ - GDXEquation + GDXEquation{T} A GAMS equation with level, marginal, lower, upper, and scale values. """ -struct GDXEquation <: GDXSymbol +struct GDXEquation{T} <: GDXSymbol{T} name::String description::String domain::Vector{String} equtype::EquationType - records::DataFrames.DataFrame + records::T end -GDXEquation(name::String, desc::String, domain::Vector{String}, equtype::Integer, records::DataFrames.DataFrame) = +GDXEquation(name::String, desc::String, domain::Vector{String}, equtype::Integer, records) = GDXEquation(name, desc, domain, EquationType(equtype), records) """ @@ -97,7 +95,7 @@ GDXEquation(name::String, desc::String, domain::Vector{String}, equtype::Integer A GAMS alias referencing another set by name. """ -struct GDXAlias <: GDXSymbol +struct GDXAlias <: GDXSymbol{Nothing} name::String description::String alias_for::String @@ -105,6 +103,17 @@ end Base.show(io::IO, a::GDXAlias) = print(io, "GDXAlias: $(a.name) -> $(a.alias_for)") +# ============================================================================= +# Tables.jl interface for GDXSymbol types +# ============================================================================= + +const GDXRecordSymbol = Union{GDXSet, GDXParameter, GDXVariable, GDXEquation} + +Tables.istable(::Type{<:GDXRecordSymbol}) = true +Tables.columnaccess(::Type{<:GDXRecordSymbol}) = true +Tables.columns(sym::GDXRecordSymbol) = Tables.columns(sym.records) +Tables.schema(sym::GDXRecordSymbol) = Tables.schema(sym.records) + # ============================================================================= # Case-insensitive key helpers # ============================================================================= @@ -126,7 +135,7 @@ is preserved in the symbol's `name` field. # Example ```julia gdx = read_gdx("model.gdx") -gdx[:demand] # Access records as DataFrame +gdx[:demand] # Access records get_symbol(gdx, :demand) # Access full GDXSymbol object list_parameters(gdx) # List all parameters ``` @@ -183,12 +192,12 @@ list_symbols(gdx::GDXFile) = Symbol[Symbol(gdx._symbols[k].name) for k in gdx._o get_symbol(gdx::GDXFile, sym) -> GDXSymbol Return the full GDXSymbol object (with name, description, domain, etc.), -not just the records DataFrame. Lookup is case-insensitive. +not just the records. Lookup is case-insensitive. """ get_symbol(gdx::GDXFile, sym::Symbol) = gdx._symbols[_symkey(sym)] get_symbol(gdx::GDXFile, sym::String) = gdx._symbols[_symkey(sym)] -# Resolve alias chains to get the underlying records DataFrame +# Resolve alias chains to get the underlying records function _get_records(gdx::GDXFile, sym::GDXSymbol, seen::Set{Symbol}=Set{Symbol}()) sym isa GDXAlias || return sym.records key = _symkey(sym.alias_for) @@ -197,7 +206,7 @@ function _get_records(gdx::GDXFile, sym::GDXSymbol, seen::Set{Symbol}=Set{Symbol return _get_records(gdx, gdx._symbols[key], seen) end -# Dictionary-like access (returns records DataFrame, resolving aliases) +# Dictionary-like access (returns records, resolving aliases) Base.getindex(gdx::GDXFile, sym::Symbol) = _get_records(gdx, gdx._symbols[_symkey(sym)]) Base.getindex(gdx::GDXFile, sym::String) = gdx[Symbol(sym)] Base.haskey(gdx::GDXFile, sym::Symbol) = haskey(gdx._symbols, _symkey(sym)) @@ -239,12 +248,19 @@ end # ============================================================================= """ - read_gdx(filepath::String; parse_integers=true, only=nothing) -> GDXFile + read_gdx(filepath::String[, sink]; parse_integers=true, only=nothing) -> GDXFile Read a GDX file and return a GDXFile container with all symbols. +The optional `sink` argument is a callable that materializes the intermediate +columnar data (a `NamedTuple` of vectors) into the desired table type. It +defaults to `Tables.columntable` (returns `NamedTuple` of vectors). Pass e.g. +`DataFrame` to get DataFrames. + # Arguments - `filepath`: Path to the GDX file +- `sink`: A callable `sink(::NamedTuple)` that produces the table storage type. + Defaults to `Tables.columntable`. - `parse_integers`: If true, attempt to parse set elements that look like integers as Int - `only`: Optional collection of symbol names (Strings or Symbols) to read. When provided, only the specified symbols are loaded from the file. @@ -252,13 +268,16 @@ Read a GDX file and return a GDXFile container with all symbols. # Example ```julia gdx = read_gdx("transport.gdx") +demand = gdx[:demand] # Get parameter as NamedTuple of vectors + +using DataFrames +gdx = read_gdx("transport.gdx", DataFrame) demand = gdx[:demand] # Get parameter as DataFrame -# Read only specific symbols from a large file gdx = read_gdx("big_model.gdx", only=[:x, :demand]) ``` """ -function read_gdx(filepath::String; parse_integers::Bool=true, only=nothing) +function read_gdx(filepath::String, sink=Tables.columntable; parse_integers::Bool=true, only=nothing) gdx = GDXHandle() gdx_create(gdx) only_filter = only === nothing ? nothing : Set{Symbol}(_symkey.(only)) @@ -280,13 +299,13 @@ function read_gdx(filepath::String; parse_integers::Bool=true, only=nothing) sym_count, sym_user_info, sym_description = gdx_symbol_info_x(gdx, sym_nr) if sym_type == GMS_DT_SET - _insert!(gdxfile, sym_key, _read_set(gdx, sym_nr, sym_name, sym_dim, sym_description)) + _insert!(gdxfile, sym_key, _read_set(gdx, sym_nr, sym_name, sym_dim, sym_description, sink)) elseif sym_type == GMS_DT_PAR - _insert!(gdxfile, sym_key, _read_parameter(gdx, sym_nr, sym_name, sym_dim, sym_description, parse_integers)) + _insert!(gdxfile, sym_key, _read_parameter(gdx, sym_nr, sym_name, sym_dim, sym_description, parse_integers, sink)) elseif sym_type == GMS_DT_VAR - _insert!(gdxfile, sym_key, _read_variable(gdx, sym_nr, sym_name, sym_dim, sym_description, sym_user_info, parse_integers)) + _insert!(gdxfile, sym_key, _read_variable(gdx, sym_nr, sym_name, sym_dim, sym_description, sym_user_info, parse_integers, sink)) elseif sym_type == GMS_DT_EQU - _insert!(gdxfile, sym_key, _read_equation(gdx, sym_nr, sym_name, sym_dim, sym_description, sym_user_info, parse_integers)) + _insert!(gdxfile, sym_key, _read_equation(gdx, sym_nr, sym_name, sym_dim, sym_description, sym_user_info, parse_integers, sink)) elseif sym_type == GMS_DT_ALIAS aliased_name = sym_user_info > 0 ? gdx_symbol_info(gdx, sym_user_info)[1] : "*" _insert!(gdxfile, sym_key, GDXAlias(sym_name, sym_description, aliased_name)) @@ -300,7 +319,7 @@ function read_gdx(filepath::String; parse_integers::Bool=true, only=nothing) end end -function _read_set(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String) +function _read_set(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, sink) domains = dim > 0 ? gdx_symbol_get_domain_x(gdx, sym_nr, dim) : String[] n_recs = gdx_data_read_str_start(gdx, sym_nr) @@ -319,11 +338,7 @@ function _read_set(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, descript end gdx_data_read_done(gdx) - df = DataFrames.DataFrame() - for (d, domain) in enumerate(domains) - col_name = domain == "*" ? "dim$d" : domain - df[!, col_name] = columns[d] - end + col_names = Symbol[Symbol(domain == "*" ? "dim$d" : domain) for (d, domain) in enumerate(domains)] has_text = any(>(0), text_nrs) if has_text @@ -336,13 +351,15 @@ function _read_set(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, descript element_text[i] = "" end end - df[!, :element_text] = element_text + nt = NamedTuple{(col_names..., :element_text)}((columns..., element_text)) + else + nt = NamedTuple{Tuple(col_names)}(Tuple(columns)) end - return GDXSet(name, description, domains, df) + return GDXSet(name, description, domains, sink(nt)) end -function _read_parameter(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, parse_integers::Bool) +function _read_parameter(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, parse_integers::Bool, sink) domains = dim > 0 ? gdx_symbol_get_domain_x(gdx, sym_nr, dim) : String[] n_recs = gdx_data_read_str_start(gdx, sym_nr) @@ -361,24 +378,14 @@ function _read_parameter(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, de end gdx_data_read_done(gdx) - df = DataFrames.DataFrame() - for (d, domain) in enumerate(domains) - col_name = domain == "*" ? "dim$d" : domain - col_data = columns[d] - if parse_integers - col_data = _try_parse_integers(col_data) - end - df[!, col_name] = col_data - end - df[!, :value] = values - - DataFrames.metadata!(df, "name", name, style=:default) - DataFrames.metadata!(df, "description", description, style=:default) + col_names = Symbol[Symbol(domain == "*" ? "dim$d" : domain) for (d, domain) in enumerate(domains)] + col_data = Any[parse_integers ? _try_parse_integers(c) : c for c in columns] - return GDXParameter(name, description, domains, df) + nt = NamedTuple{(col_names..., :value)}((col_data..., values)) + return GDXParameter(name, description, domains, sink(nt)) end -function _read_variable(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, user_info::Int, parse_integers::Bool) +function _read_variable(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, user_info::Int, parse_integers::Bool, sink) domains = dim > 0 ? gdx_symbol_get_domain_x(gdx, sym_nr, dim) : String[] n_recs = gdx_data_read_str_start(gdx, sym_nr) @@ -405,28 +412,14 @@ function _read_variable(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, des end gdx_data_read_done(gdx) - df = DataFrames.DataFrame() - for (d, domain) in enumerate(domains) - col_name = domain == "*" ? "dim$d" : domain - col_data = columns[d] - if parse_integers - col_data = _try_parse_integers(col_data) - end - df[!, col_name] = col_data - end - df[!, :level] = level - df[!, :marginal] = marginal - df[!, :lower] = lower - df[!, :upper] = upper - df[!, :scale] = scale - - DataFrames.metadata!(df, "name", name, style=:default) - DataFrames.metadata!(df, "description", description, style=:default) + col_names = Symbol[Symbol(domain == "*" ? "dim$d" : domain) for (d, domain) in enumerate(domains)] + col_data = Any[parse_integers ? _try_parse_integers(c) : c for c in columns] - return GDXVariable(name, description, domains, VariableType(user_info), df) + nt = NamedTuple{(col_names..., :level, :marginal, :lower, :upper, :scale)}((col_data..., level, marginal, lower, upper, scale)) + return GDXVariable(name, description, domains, VariableType(user_info), sink(nt)) end -function _read_equation(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, user_info::Int, parse_integers::Bool) +function _read_equation(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, description::String, user_info::Int, parse_integers::Bool, sink) domains = dim > 0 ? gdx_symbol_get_domain_x(gdx, sym_nr, dim) : String[] n_recs = gdx_data_read_str_start(gdx, sym_nr) @@ -453,25 +446,11 @@ function _read_equation(gdx::GDXHandle, sym_nr::Int, name::String, dim::Int, des end gdx_data_read_done(gdx) - df = DataFrames.DataFrame() - for (d, domain) in enumerate(domains) - col_name = domain == "*" ? "dim$d" : domain - col_data = columns[d] - if parse_integers - col_data = _try_parse_integers(col_data) - end - df[!, col_name] = col_data - end - df[!, :level] = level - df[!, :marginal] = marginal - df[!, :lower] = lower - df[!, :upper] = upper - df[!, :scale] = scale + col_names = Symbol[Symbol(domain == "*" ? "dim$d" : domain) for (d, domain) in enumerate(domains)] + col_data = Any[parse_integers ? _try_parse_integers(c) : c for c in columns] - DataFrames.metadata!(df, "name", name, style=:default) - DataFrames.metadata!(df, "description", description, style=:default) - - return GDXEquation(name, description, domains, EquationType(user_info), df) + nt = NamedTuple{(col_names..., :level, :marginal, :lower, :upper, :scale)}((col_data..., level, marginal, lower, upper, scale)) + return GDXEquation(name, description, domains, EquationType(user_info), sink(nt)) end # ============================================================================= @@ -517,27 +496,32 @@ function write_gdx(filepath::String, gdxfile::GDXFile; producer::String="GDXInte end """ - write_gdx(filepath::String, symbols::Pair{String, DataFrame}...; producer="GDXInterface.jl") + write_gdx(filepath::String, symbols::Pair{String}...; producer="GDXInterface.jl") -Write DataFrames to a GDX file as parameters. Each pair maps a symbol name to its DataFrame. -The DataFrame must have a `:value` column; all other columns are treated as domain dimensions. +Write tables to a GDX file as parameters. Each pair maps a symbol name to a +Tables.jl-compatible table. The table must have a `:value` column; all other +columns are treated as domain dimensions. # Example ```julia +using DataFrames df = DataFrame(i=["a", "b", "c"], value=[1.0, 2.0, 3.0]) write_gdx("output.gdx", "demand" => df) + +# Or with NamedTuples directly: +nt = (; i=["a", "b", "c"], value=[1.0, 2.0, 3.0]) +write_gdx("output.gdx", "demand" => nt) ``` """ -function write_gdx(filepath::String, symbols::Pair{String, DataFrames.DataFrame}...; producer::String="GDXInterface.jl") +function write_gdx(filepath::String, symbols::Pair{String}...; producer::String="GDXInterface.jl") gdx = GDXHandle() gdx_create(gdx) try gdx_open_write(gdx, filepath, producer) - for (name, df) in symbols - desc = get(DataFrames.metadata(df), "description", "") - _write_parameter_df(gdx, name, df, desc) + for (name, tbl) in symbols + _write_parameter_table(gdx, name, tbl, _table_description(tbl)) end gdx_close(gdx) @@ -553,6 +537,11 @@ function _set_domain_x(gdx::GDXHandle, name::String, domain::Vector{String}, dim found && gdx_symbol_set_domain_x(gdx, sym_nr, domain) end +function _table_description(tbl) + DataAPI.metadatasupport(typeof(tbl)).read || return "" + return DataAPI.metadata(tbl, "description", "") +end + # Type dispatch for writing symbols _write_symbol(gdx::GDXHandle, sym::GDXSet) = _write_set(gdx, sym) _write_symbol(gdx::GDXHandle, sym::GDXParameter) = _write_parameter(gdx, sym) @@ -560,27 +549,25 @@ _write_symbol(gdx::GDXHandle, sym::GDXVariable) = _write_variable(gdx, sym) _write_symbol(gdx::GDXHandle, sym::GDXEquation) = _write_equation(gdx, sym) function _write_set(gdx::GDXHandle, sym::GDXSet) - df = sym.records - has_text = "element_text" in names(df) - cols = [n for n in names(df) if n != "element_text"] - dim = length(cols) + tbl = Tables.columns(sym.records) + col_names = collect(Tables.columnnames(tbl)) + has_text = :element_text in col_names + dim_cols = has_text ? filter(!=(Symbol("element_text")), col_names) : col_names + dim = length(dim_cols) gdx_data_write_str_start(gdx, sym.name, sym.description, dim, GMS_DT_SET) keys = Vector{String}(undef, dim) vals = zeros(Float64, GMS_VAL_MAX) - for row in eachrow(df) - for (i, col) in enumerate(cols) - keys[i] = string(row[col]) + n_rows = length(Tables.getcolumn(tbl, first(col_names))) + for i in 1:n_rows + for (j, col) in enumerate(dim_cols) + keys[j] = string(Tables.getcolumn(tbl, col)[i]) end if has_text - text = string(row[:element_text]) - if !isempty(text) - vals[GAMS_VALUE_LEVEL] = Float64(gdx_add_set_text(gdx, text)) - else - vals[GAMS_VALUE_LEVEL] = 0.0 - end + text = string(Tables.getcolumn(tbl, :element_text)[i]) + vals[GAMS_VALUE_LEVEL] = isempty(text) ? 0.0 : Float64(gdx_add_set_text(gdx, text)) end gdx_data_write_str(gdx, keys, vals) end @@ -590,23 +577,26 @@ function _write_set(gdx::GDXHandle, sym::GDXSet) end function _write_parameter(gdx::GDXHandle, sym::GDXParameter) - _write_parameter_df(gdx, sym.name, sym.records, sym.description, sym.domain) + _write_parameter_table(gdx, sym.name, sym.records, sym.description, sym.domain) end -function _write_parameter_df(gdx::GDXHandle, name::String, df::DataFrames.DataFrame, description::String="", domain::Vector{String}=String[]) - dim_cols = [n for n in names(df) if n != "value"] +function _write_parameter_table(gdx::GDXHandle, name::String, tbl, description::String="", domain::Vector{String}=String[]) + cols = Tables.columns(tbl) + col_names = collect(Tables.columnnames(cols)) + dim_cols = filter(!=(:value), col_names) dim = length(dim_cols) gdx_data_write_str_start(gdx, name, description, dim, GMS_DT_PAR) keys = Vector{String}(undef, dim) vals = zeros(Float64, GMS_VAL_MAX) + value_col = Tables.getcolumn(cols, :value) - for row in eachrow(df) - for (i, col) in enumerate(dim_cols) - keys[i] = string(row[col]) + for i in 1:length(value_col) + for (j, col) in enumerate(dim_cols) + keys[j] = string(Tables.getcolumn(cols, col)[i]) end - vals[GAMS_VALUE_LEVEL] = _to_gdx_value(row[:value]) + vals[GAMS_VALUE_LEVEL] = _to_gdx_value(value_col[i]) gdx_data_write_str(gdx, keys, vals) end @@ -614,11 +604,12 @@ function _write_parameter_df(gdx::GDXHandle, name::String, df::DataFrames.DataFr _set_domain_x(gdx, name, domain, dim) end -const _VAR_EQU_COLS = Set(["level", "marginal", "lower", "upper", "scale"]) +const _VAR_EQU_COLS = Set([:level, :marginal, :lower, :upper, :scale]) function _write_variable(gdx::GDXHandle, sym::GDXVariable) - df = sym.records - dim_cols = [n for n in names(df) if !(n in _VAR_EQU_COLS)] + tbl = Tables.columns(sym.records) + col_names = collect(Tables.columnnames(tbl)) + dim_cols = filter(c -> !(c in _VAR_EQU_COLS), col_names) dim = length(dim_cols) gdx_data_write_str_start(gdx, sym.name, sym.description, dim, GMS_DT_VAR, Int(sym.vartype)) @@ -626,15 +617,21 @@ function _write_variable(gdx::GDXHandle, sym::GDXVariable) keys = Vector{String}(undef, dim) vals = zeros(Float64, GMS_VAL_MAX) - for row in eachrow(df) - for (i, col) in enumerate(dim_cols) - keys[i] = string(row[col]) + level_col = Tables.getcolumn(tbl, :level) + marginal_col = Tables.getcolumn(tbl, :marginal) + lower_col = Tables.getcolumn(tbl, :lower) + upper_col = Tables.getcolumn(tbl, :upper) + scale_col = Tables.getcolumn(tbl, :scale) + + for i in 1:length(level_col) + for (j, col) in enumerate(dim_cols) + keys[j] = string(Tables.getcolumn(tbl, col)[i]) end - vals[GAMS_VALUE_LEVEL] = _to_gdx_value(row[:level]) - vals[GAMS_VALUE_MARGINAL] = _to_gdx_value(row[:marginal]) - vals[GAMS_VALUE_LOWER] = _to_gdx_value(row[:lower]) - vals[GAMS_VALUE_UPPER] = _to_gdx_value(row[:upper]) - vals[GAMS_VALUE_SCALE] = _to_gdx_value(row[:scale]) + vals[GAMS_VALUE_LEVEL] = _to_gdx_value(level_col[i]) + vals[GAMS_VALUE_MARGINAL] = _to_gdx_value(marginal_col[i]) + vals[GAMS_VALUE_LOWER] = _to_gdx_value(lower_col[i]) + vals[GAMS_VALUE_UPPER] = _to_gdx_value(upper_col[i]) + vals[GAMS_VALUE_SCALE] = _to_gdx_value(scale_col[i]) gdx_data_write_str(gdx, keys, vals) end @@ -643,8 +640,9 @@ function _write_variable(gdx::GDXHandle, sym::GDXVariable) end function _write_equation(gdx::GDXHandle, sym::GDXEquation) - df = sym.records - dim_cols = [n for n in names(df) if !(n in _VAR_EQU_COLS)] + tbl = Tables.columns(sym.records) + col_names = collect(Tables.columnnames(tbl)) + dim_cols = filter(c -> !(c in _VAR_EQU_COLS), col_names) dim = length(dim_cols) gdx_data_write_str_start(gdx, sym.name, sym.description, dim, GMS_DT_EQU, Int(sym.equtype)) @@ -652,15 +650,21 @@ function _write_equation(gdx::GDXHandle, sym::GDXEquation) keys = Vector{String}(undef, dim) vals = zeros(Float64, GMS_VAL_MAX) - for row in eachrow(df) - for (i, col) in enumerate(dim_cols) - keys[i] = string(row[col]) + level_col = Tables.getcolumn(tbl, :level) + marginal_col = Tables.getcolumn(tbl, :marginal) + lower_col = Tables.getcolumn(tbl, :lower) + upper_col = Tables.getcolumn(tbl, :upper) + scale_col = Tables.getcolumn(tbl, :scale) + + for i in 1:length(level_col) + for (j, col) in enumerate(dim_cols) + keys[j] = string(Tables.getcolumn(tbl, col)[i]) end - vals[GAMS_VALUE_LEVEL] = _to_gdx_value(row[:level]) - vals[GAMS_VALUE_MARGINAL] = _to_gdx_value(row[:marginal]) - vals[GAMS_VALUE_LOWER] = _to_gdx_value(row[:lower]) - vals[GAMS_VALUE_UPPER] = _to_gdx_value(row[:upper]) - vals[GAMS_VALUE_SCALE] = _to_gdx_value(row[:scale]) + vals[GAMS_VALUE_LEVEL] = _to_gdx_value(level_col[i]) + vals[GAMS_VALUE_MARGINAL] = _to_gdx_value(marginal_col[i]) + vals[GAMS_VALUE_LOWER] = _to_gdx_value(lower_col[i]) + vals[GAMS_VALUE_UPPER] = _to_gdx_value(upper_col[i]) + vals[GAMS_VALUE_SCALE] = _to_gdx_value(scale_col[i]) gdx_data_write_str(gdx, keys, vals) end diff --git a/src/GDXInterface.jl b/src/GDXInterface.jl index a016379..4ed8e39 100644 --- a/src/GDXInterface.jl +++ b/src/GDXInterface.jl @@ -1,6 +1,7 @@ module GDXInterface -import DataFrames +import DataAPI +import Tables import gdx_jll const LIBGDX = gdx_jll.libgdx diff --git a/test/runtests.jl b/test/runtests.jl index cb64953..95972a5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ import Test: @testset, @test, @test_throws -import DataFrames: DataFrame +import Tables +import DataFrames: DataFrame, names, metadata! using GDXInterface const TEST_DATA_DIR = joinpath(@__DIR__, "test_data") diff --git a/test/test_gdxfile.jl b/test/test_gdxfile.jl index 9cc8935..a6d100a 100644 --- a/test/test_gdxfile.jl +++ b/test/test_gdxfile.jl @@ -27,8 +27,8 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test :p in list_parameters(gdxfile) p = gdxfile[:p] - @test "value" in names(p) - @test p.value == [1.5, 2.5, 3.5] + @test :value in Tables.columnnames(p) + @test collect(Tables.getcolumn(p, :value)) == [1.5, 2.5, 3.5] end @testset "Reading variables" begin @@ -38,28 +38,22 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test :y in list_variables(gdxfile) x = gdxfile[:x] - @test "level" in names(x) - @test "marginal" in names(x) - @test "lower" in names(x) - @test "upper" in names(x) - @test x.level == [10.0, 20.0, 30.0] - @test x.marginal ≈ [0.1, 0.2, 0.3] + @test :level in Tables.columnnames(x) + @test :marginal in Tables.columnnames(x) + @test :lower in Tables.columnnames(x) + @test :upper in Tables.columnnames(x) + @test collect(Tables.getcolumn(x, :level)) == [10.0, 20.0, 30.0] + @test collect(Tables.getcolumn(x, :marginal)) ≈ [0.1, 0.2, 0.3] y = gdxfile[:y] - @test y.level == [5.0, 10.0, 15.0] - @test all(y.lower .== 0.0) - @test all(y.upper .== 100.0) + @test collect(Tables.getcolumn(y, :level)) == [5.0, 10.0, 15.0] + @test all(Tables.getcolumn(y, :lower) .== 0.0) + @test all(Tables.getcolumn(y, :upper) .== 100.0) end @testset "Write and read round-trip" begin - supply = DataFrame( - i = ["seattle", "san-diego"], - value = [350.0, 600.0] - ) - demand = DataFrame( - j = ["new-york", "chicago", "topeka"], - value = [325.0, 300.0, 275.0] - ) + supply = (; i = ["seattle", "san-diego"], value = [350.0, 600.0]) + demand = (; j = ["new-york", "chicago", "topeka"], value = [325.0, 300.0, 275.0]) outfile = joinpath(tempdir(), "gdx_jl_write_test.gdx") write_gdx(outfile, "supply" => supply, "demand" => demand) @@ -68,8 +62,8 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test :supply in list_parameters(gdxfile) @test :demand in list_parameters(gdxfile) - @test gdxfile[:supply].value == [350.0, 600.0] - @test gdxfile[:demand].value == [325.0, 300.0, 275.0] + @test collect(Tables.getcolumn(gdxfile[:supply], :value)) == [350.0, 600.0] + @test collect(Tables.getcolumn(gdxfile[:demand], :value)) == [325.0, 300.0, 275.0] @test gdxfile.supply == gdxfile[:supply] @test gdxfile.demand == gdxfile[:demand] @@ -78,7 +72,7 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; end @testset "Multi-dimensional parameters" begin - cost = DataFrame( + cost = (; i = ["seattle", "seattle", "san-diego", "san-diego"], j = ["new-york", "chicago", "new-york", "chicago"], value = [2.5, 1.7, 2.5, 1.8] @@ -90,32 +84,33 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdxfile = read_gdx(outfile) result = gdxfile[:cost] - @test size(result, 1) == 4 - @test length(names(result)) == 3 - @test "value" in names(result) + @test length(Tables.getcolumn(result, :value)) == 4 + col_names = collect(Tables.columnnames(result)) + @test length(col_names) == 3 + @test :value in col_names rm(outfile, force=true) end @testset "Integer parsing" begin - df = DataFrame(year = ["2020", "2021", "2022"], value = [1.0, 2.0, 3.0]) + df = (; year = ["2020", "2021", "2022"], value = [1.0, 2.0, 3.0]) outfile = joinpath(tempdir(), "gdx_jl_int_test.gdx") write_gdx(outfile, "data" => df) gdxfile = read_gdx(outfile, parse_integers=true) - @test eltype(gdxfile[:data].dim1) == Int + @test eltype(Tables.getcolumn(gdxfile[:data], :dim1)) == Int gdxfile = read_gdx(outfile, parse_integers=false) - @test eltype(gdxfile[:data].dim1) == String + @test eltype(Tables.getcolumn(gdxfile[:data], :dim1)) == String rm(outfile, force=true) end @testset "GDXFile show and propertynames" begin - df = DataFrame(i = ["a", "b"], value = [1.0, 2.0]) + tbl = (; i = ["a", "b"], value = [1.0, 2.0]) outfile = joinpath(tempdir(), "gdx_jl_show_test.gdx") - write_gdx(outfile, "param" => df) + write_gdx(outfile, "param" => tbl) gdxfile = read_gdx(outfile) @@ -132,11 +127,11 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; end @testset "Symbol listing" begin - df1 = DataFrame(i = ["a"], value = [1.0]) - df2 = DataFrame(j = ["x"], value = [2.0]) + t1 = (; i = ["a"], value = [1.0]) + t2 = (; j = ["x"], value = [2.0]) outfile = joinpath(tempdir(), "gdx_jl_list_test.gdx") - write_gdx(outfile, "param1" => df1, "param2" => df2) + write_gdx(outfile, "param1" => t1, "param2" => t2) gdxfile = read_gdx(outfile) @@ -160,46 +155,45 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test sort(list_symbols(gdx1)) == sort(list_symbols(gdx2)) - # Parameters match - @test gdx1[:p].value == gdx2[:p].value + @test collect(Tables.getcolumn(gdx1[:p], :value)) == collect(Tables.getcolumn(gdx2[:p], :value)) - # Variables match - @test gdx1[:x].level == gdx2[:x].level - @test gdx1[:x].marginal ≈ gdx2[:x].marginal - @test gdx1[:y].level == gdx2[:y].level - @test gdx1[:y].upper == gdx2[:y].upper + @test collect(Tables.getcolumn(gdx1[:x], :level)) == collect(Tables.getcolumn(gdx2[:x], :level)) + @test collect(Tables.getcolumn(gdx1[:x], :marginal)) ≈ collect(Tables.getcolumn(gdx2[:x], :marginal)) + @test collect(Tables.getcolumn(gdx1[:y], :level)) == collect(Tables.getcolumn(gdx2[:y], :level)) + @test collect(Tables.getcolumn(gdx1[:y], :upper)) == collect(Tables.getcolumn(gdx2[:y], :upper)) - # Sets match (compare first column values) - @test sort(Vector(gdx1[:i][!, 1])) == sort(Vector(gdx2[:i][!, 1])) + i1_col = collect(Tables.getcolumn(gdx1[:i], first(Tables.columnnames(gdx1[:i])))) + i2_col = collect(Tables.getcolumn(gdx2[:i], first(Tables.columnnames(gdx2[:i])))) + @test sort(i1_col) == sort(i2_col) rm(outfile, force=true) end @testset "Special values round-trip" begin - df = DataFrame(i = ["a", "b", "c", "d", "e"], value = [NaN, Inf, -Inf, 42.0, -0.0]) + tbl = (; i = ["a", "b", "c", "d", "e"], value = [NaN, Inf, -Inf, 42.0, -0.0]) outfile = joinpath(tempdir(), "gdx_jl_special.gdx") - write_gdx(outfile, "special" => df) + write_gdx(outfile, "special" => tbl) gdxfile = read_gdx(outfile) - result = gdxfile[:special] - @test isnan(result.value[1]) - @test result.value[2] == Inf - @test result.value[3] == -Inf - @test result.value[4] == 42.0 - @test result.value[5] === -0.0 + result = Tables.getcolumn(gdxfile[:special], :value) + @test isnan(result[1]) + @test result[2] == Inf + @test result[3] == -Inf + @test result[4] == 42.0 + @test result[5] === -0.0 rm(outfile, force=true) end @testset "Scalar (0-dim) parameters" begin - df = DataFrame(value = [42.0]) + tbl = (; value = [42.0]) outfile = joinpath(tempdir(), "gdx_jl_scalar.gdx") - write_gdx(outfile, "scalar_param" => df) + write_gdx(outfile, "scalar_param" => tbl) gdxfile = read_gdx(outfile) @test :scalar_param in list_parameters(gdxfile) - @test gdxfile[:scalar_param].value == [42.0] - @test size(gdxfile[:scalar_param], 2) == 1 # only the value column + @test collect(Tables.getcolumn(gdxfile[:scalar_param], :value)) == [42.0] + @test length(collect(Tables.columnnames(gdxfile[:scalar_param]))) == 1 rm(outfile, force=true) end @@ -213,9 +207,8 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test :x in list_variables(gdx_partial) @test !haskey(gdx_partial, :i) @test !haskey(gdx_partial, :y) - @test gdx_partial[:p].value == gdx_full[:p].value + @test collect(Tables.getcolumn(gdx_partial[:p], :value)) == collect(Tables.getcolumn(gdx_full[:p], :value)) - # String names should also work gdx_str = read_gdx(test_gdx, only=["i"]) @test length(gdx_str) == 1 @test :i in list_sets(gdx_str) @@ -254,7 +247,7 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; end @testset "Writing equations" begin - eq_df = DataFrame( + eq_tbl = (; i = ["a", "b"], level = [1.0, 2.0], marginal = [0.5, 0.6], @@ -262,7 +255,7 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; upper = [Inf, Inf], scale = [1.0, 1.0] ) - eq = GDXEquation("myeq", "test equation", ["i"], 0, eq_df) + eq = GDXEquation("myeq", "test equation", ["i"], 0, eq_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:myeq => eq)) outfile = joinpath(tempdir(), "gdx_jl_eq_test.gdx") @@ -270,15 +263,15 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) @test :myeq in list_equations(gdx2) - @test gdx2[:myeq].level == [1.0, 2.0] - @test gdx2[:myeq].marginal == [0.5, 0.6] + @test collect(Tables.getcolumn(gdx2[:myeq], :level)) == [1.0, 2.0] + @test collect(Tables.getcolumn(gdx2[:myeq], :marginal)) == [0.5, 0.6] rm(outfile, force=true) end @testset "Writing sets standalone" begin - set_df = DataFrame(dim1 = ["x", "y", "z"]) - s = GDXSet("myset", "test set", ["*"], set_df) + set_tbl = (; dim1 = ["x", "y", "z"]) + s = GDXSet("myset", "test set", ["*"], set_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:myset => s)) outfile = joinpath(tempdir(), "gdx_jl_set_test.gdx") @@ -286,7 +279,8 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) @test :myset in list_sets(gdx2) - @test sort(Vector(gdx2[:myset][!, 1])) == ["x", "y", "z"] + col = collect(Tables.getcolumn(gdx2[:myset], first(Tables.columnnames(gdx2[:myset])))) + @test sort(col) == ["x", "y", "z"] rm(outfile, force=true) end @@ -296,16 +290,15 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; sym_x = get_symbol(gdxfile, :x) @test sym_x.vartype isa VariableType - @test sym_x.vartype == VarFree # Free Variable x(i) + @test sym_x.vartype == VarFree sym_y = get_symbol(gdxfile, :y) @test sym_y.vartype == VarPositive - # Integer constructor still works - v = GDXVariable("test", "", String[], 3, DataFrame(level=[0.0], marginal=[0.0], lower=[0.0], upper=[0.0], scale=[1.0])) + v = GDXVariable("test", "", String[], 3, (; level=[0.0], marginal=[0.0], lower=[0.0], upper=[0.0], scale=[1.0])) @test v.vartype == VarPositive - e = GDXEquation("test", "", String[], 0, DataFrame(level=[0.0], marginal=[0.0], lower=[0.0], upper=[0.0], scale=[1.0])) + e = GDXEquation("test", "", String[], 0, (; level=[0.0], marginal=[0.0], lower=[0.0], upper=[0.0], scale=[1.0])) @test e.equtype == EqE end @@ -320,11 +313,9 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test get_symbol(gdxfile, :P) === get_symbol(gdxfile, :p) @test get_symbol(gdxfile, "P") === get_symbol(gdxfile, "p") - # Original case is preserved in the name field sym = get_symbol(gdxfile, :p) @test sym.name == "p" - # Selective read is also case-insensitive gdx2 = read_gdx(test_gdx, only=[:P, :X]) @test length(gdx2) == 2 @test :p in list_parameters(gdx2) @@ -335,11 +326,9 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdxfile = read_gdx(test_gdx) syms = list_symbols(gdxfile) - # Iteration order matches list_symbols order iter_syms = Symbol[k for (k, _) in gdxfile] @test iter_syms == syms - # Round-trip preserves order outfile = joinpath(tempdir(), "gdx_jl_order_test.gdx") write_gdx(outfile, gdxfile) gdx2 = read_gdx(outfile) @@ -349,11 +338,11 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; end @testset "Set element text round-trip" begin - set_df = DataFrame( + set_tbl = (; dim1 = ["seattle", "san-diego", "topeka"], element_text = ["rainy city", "sunny city", ""] ) - s = GDXSet("cities", "transport cities", ["*"], set_df) + s = GDXSet("cities", "transport cities", ["*"], set_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:cities => s)) outfile = joinpath(tempdir(), "gdx_jl_elemtext_test.gdx") @@ -361,31 +350,32 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) result = gdx2[:cities] - @test "element_text" in names(result) - @test result.element_text[1] == "rainy city" - @test result.element_text[2] == "sunny city" - @test result.element_text[3] == "" + @test :element_text in Tables.columnnames(result) + et = Tables.getcolumn(result, :element_text) + @test et[1] == "rainy city" + @test et[2] == "sunny city" + @test et[3] == "" rm(outfile, force=true) end @testset "Set without element text has no extra column" begin - set_df = DataFrame(dim1 = ["a", "b", "c"]) - s = GDXSet("simple", "no text", ["*"], set_df) + set_tbl = (; dim1 = ["a", "b", "c"]) + s = GDXSet("simple", "no text", ["*"], set_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:simple => s)) outfile = joinpath(tempdir(), "gdx_jl_notext_test.gdx") write_gdx(outfile, gdxfile) gdx2 = read_gdx(outfile) - @test !("element_text" in names(gdx2[:simple])) + @test !(:element_text in Tables.columnnames(gdx2[:simple])) rm(outfile, force=true) end @testset "Alias round-trip" begin - set_df = DataFrame(dim1 = ["a", "b", "c"]) - s = GDXSet("i", "original set", ["*"], set_df) + set_tbl = (; dim1 = ["a", "b", "c"]) + s = GDXSet("i", "original set", ["*"], set_tbl) a = GDXAlias("j", "alias for i", "i") gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:i => s, :j => a)) @@ -400,7 +390,6 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; @test alias_sym isa GDXAlias @test alias_sym.alias_for == "i" - # Accessing alias records resolves to the aliased set's records @test gdx2[:j] == gdx2[:i] rm(outfile, force=true) @@ -414,10 +403,10 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; end @testset "Domain preservation on round-trip (issue #3)" begin - set_df = DataFrame(i = ["a", "b", "c"]) - s = GDXSet("i", "index set", ["*"], set_df) - par_df = DataFrame(i = ["a", "b", "c"], value = [10.0, 20.0, 30.0]) - p = GDXParameter("x", "A parameter over i", ["i"], par_df) + set_tbl = (; i = ["a", "b", "c"]) + s = GDXSet("i", "index set", ["*"], set_tbl) + par_tbl = (; i = ["a", "b", "c"], value = [10.0, 20.0, 30.0]) + p = GDXParameter("x", "A parameter over i", ["i"], par_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:i => s, :x => p)) outfile = joinpath(tempdir(), "gdx_jl_domain_test.gdx") @@ -426,15 +415,15 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) x2 = get_symbol(gdx2, :x) @test x2.domain == ["i"] - @test names(gdx2[:x])[1] == "i" + @test first(Tables.columnnames(gdx2[:x])) == :i rm(outfile, force=true) end @testset "Domain preservation for variables (issue #3)" begin - set_df = DataFrame(i = ["a", "b", "c"]) - s = GDXSet("i", "index set", ["*"], set_df) - var_df = DataFrame( + set_tbl = (; i = ["a", "b", "c"]) + s = GDXSet("i", "index set", ["*"], set_tbl) + var_tbl = (; i = ["a", "b", "c"], level = [1.0, 2.0, 3.0], marginal = [0.0, 0.0, 0.0], @@ -442,7 +431,7 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; upper = [Inf, Inf, Inf], scale = [1.0, 1.0, 1.0] ) - v = GDXVariable("y", "A variable over i", ["i"], VarFree, var_df) + v = GDXVariable("y", "A variable over i", ["i"], VarFree, var_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:i => s, :y => v)) outfile = joinpath(tempdir(), "gdx_jl_domain_var_test.gdx") @@ -451,15 +440,15 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) y2 = get_symbol(gdx2, :y) @test y2.domain == ["i"] - @test names(gdx2[:y])[1] == "i" + @test first(Tables.columnnames(gdx2[:y])) == :i rm(outfile, force=true) end @testset "Domain preservation for equations (issue #3)" begin - set_df = DataFrame(i = ["a", "b"]) - s = GDXSet("i", "index set", ["*"], set_df) - eq_df = DataFrame( + set_tbl = (; i = ["a", "b"]) + s = GDXSet("i", "index set", ["*"], set_tbl) + eq_tbl = (; i = ["a", "b"], level = [1.0, 2.0], marginal = [0.5, 0.6], @@ -467,7 +456,7 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; upper = [Inf, Inf], scale = [1.0, 1.0] ) - eq = GDXEquation("myeq", "test eq", ["i"], EqE, eq_df) + eq = GDXEquation("myeq", "test eq", ["i"], EqE, eq_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:i => s, :myeq => eq)) outfile = joinpath(tempdir(), "gdx_jl_domain_eq_test.gdx") @@ -476,20 +465,20 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) eq2 = get_symbol(gdx2, :myeq) @test eq2.domain == ["i"] - @test names(gdx2[:myeq])[1] == "i" + @test first(Tables.columnnames(gdx2[:myeq])) == :i rm(outfile, force=true) end @testset "Multi-dimensional domain preservation (issue #3)" begin - si = GDXSet("i", "rows", ["*"], DataFrame(i = ["a", "b"])) - sj = GDXSet("j", "cols", ["*"], DataFrame(j = ["x", "y"])) - par_df = DataFrame( + si = GDXSet("i", "rows", ["*"], (; i = ["a", "b"])) + sj = GDXSet("j", "cols", ["*"], (; j = ["x", "y"])) + par_tbl = (; i = ["a", "a", "b", "b"], j = ["x", "y", "x", "y"], value = [1.0, 2.0, 3.0, 4.0] ) - p = GDXParameter("cost", "transport cost", ["i", "j"], par_df) + p = GDXParameter("cost", "transport cost", ["i", "j"], par_tbl) gdxfile = GDXFile("", Dict{Symbol,GDXSymbol}(:i => si, :j => sj, :cost => p)) outfile = joinpath(tempdir(), "gdx_jl_domain_2d_test.gdx") @@ -498,7 +487,8 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; gdx2 = read_gdx(outfile) cost2 = get_symbol(gdx2, :cost) @test cost2.domain == ["i", "j"] - @test names(gdx2[:cost])[1:2] == ["i", "j"] + cnames = collect(Tables.columnnames(gdx2[:cost])) + @test cnames[1:2] == [:i, :j] rm(outfile, force=true) end @@ -523,23 +513,65 @@ execute_unload "gams_gdx_test.gdx", i, p, x, y; rm(outfile, force=true) end - @testset "Setting symbols via indexing" begin gdxfile = GDXFile("") - df = DataFrame(i = ["a", "b"], value = [1.0, 2.0]) - p = GDXParameter("p", "test param", ["i"], df) + tbl = (; i = ["a", "b"], value = [1.0, 2.0]) + p = GDXParameter("p", "test param", ["i"], tbl) gdxfile[:p] = p @test :p in list_parameters(gdxfile) - @test gdxfile[:p].value == [1.0, 2.0] + @test collect(Tables.getcolumn(gdxfile[:p], :value)) == [1.0, 2.0] - # String keys should also work - df2 = DataFrame(j = ["x", "y"], value = [3.0, 4.0]) - p2 = GDXParameter("q", "another param", ["j"], df2) + tbl2 = (; j = ["x", "y"], value = [3.0, 4.0]) + p2 = GDXParameter("q", "another param", ["j"], tbl2) gdxfile["q"] = p2 @test :q in list_parameters(gdxfile) - @test gdxfile[:q].value == [3.0, 4.0] + @test collect(Tables.getcolumn(gdxfile[:q], :value)) == [3.0, 4.0] + end + + @testset "DataFrame sink" begin + gdxfile = read_gdx(test_gdx, DataFrame) + + p = gdxfile[:p] + @test p isa DataFrame + @test "value" in names(p) + @test p.value == [1.5, 2.5, 3.5] + + x = gdxfile[:x] + @test x isa DataFrame + @test x.level == [10.0, 20.0, 30.0] + end + + @testset "DataFrame metadata description" begin + df = DataFrame(i = ["a", "b"], value = [1.0, 2.0]) + metadata!(df, "description", "from metadata", style=:default) + + outfile = joinpath(tempdir(), "gdx_jl_metadata_desc.gdx") + write_gdx(outfile, "meta_param" => df) + + gdxfile = read_gdx(outfile) + @test get_symbol(gdxfile, :meta_param).description == "from metadata" + + rm(outfile, force=true) + end + + @testset "Tables.jl interface on GDXSymbol" begin + gdxfile = read_gdx(test_gdx) + sym_p = get_symbol(gdxfile, :p) + + @test Tables.istable(typeof(sym_p)) + @test Tables.columnaccess(typeof(sym_p)) + + cols = Tables.columns(sym_p) + @test :value in Tables.columnnames(cols) + + schema = Tables.schema(sym_p) + @test schema !== nothing + @test :value in schema.names + + alias = GDXAlias("j", "", "i") + @test !Tables.istable(typeof(alias)) end end