access DimArray at `missing` column value

[filchristou]

is there a workaround to access the missing value of a named column ?

da = rand(DD.X([:a, :b, missing]), DD.Y(100.0:50:200.0))

da[X=At(:a)]

da[X=At(missing)]
TypeError: non-boolean (Missing) used in boolean context
Stacktrace:
  [1] findnext
    @ ./array.jl:2393 [inlined]
  [2] findfirst
    @ ./array.jl:2444 [inlined]
  [3] at(::DimensionalData.Dimensions.Lookups.Unordered, ::DimensionalData.Dimensions.Lookups.NoSpan, lookup::DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}, selval::Missing, atol::Nothing, rtol::Nothing; err::DimensionalData.Dimensions.Lookups._True)
    @ DimensionalData.Dimensions.Lookups ~/.julia/packages/DimensionalData/FWnw9/src/Lookups/selector.jl:214
  [4] at(::DimensionalData.Dimensions.Lookups.Unordered, ::DimensionalData.Dimensions.Lookups.NoSpan, lookup::DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}, selval::Missing, atol::Nothing, rtol::Nothing)
    @ DimensionalData.Dimensions.Lookups ~/.julia/packages/DimensionalData/FWnw9/src/Lookups/selector.jl:213
  [5] at(lookup::DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}, sel::DimensionalData.Dimensions.Lookups.At{Missing, Nothing, Nothing}; kw::@Kwargs{})
    @ DimensionalData.Dimensions.Lookups ~/.julia/packages/DimensionalData/FWnw9/src/Lookups/selector.jl:158
  [6] at(lookup::DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}, sel::DimensionalData.Dimensions.Lookups.At{Missing, Nothing, Nothing})
    @ DimensionalData.Dimensions.Lookups ~/.julia/packages/DimensionalData/FWnw9/src/Lookups/selector.jl:157
  [7] _selectindices
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Lookups/selector.jl:1099 [inlined]
  [8] selectindices(l::DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}, sel::DimensionalData.Dimensions.Lookups.At{Missing, Nothing, Nothing}; kw::@Kwargs{})
    @ DimensionalData.Dimensions.Lookups ~/.julia/packages/DimensionalData/FWnw9/src/Lookups/selector.jl:1075
  [9] _dims2indices
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:144 [inlined]
 [10] _dims2indices
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:136 [inlined]
 [11] dims2indices
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:31 [inlined]
 [12] macro expansion
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:89 [inlined]
 [13] split_alignments(fa::typeof(DimensionalData.Dimensions.dims2indices), fu::typeof(DimensionalData.Dimensions.unalligned_dims2indices), lookups::Tuple{DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}, DimensionalData.Dimensions.Lookups.Sampled{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, DimensionalData.Dimensions.Lookups.ForwardOrdered, DimensionalData.Dimensions.Lookups.Regular{Float64}, DimensionalData.Dimensions.Lookups.Points, DimensionalData.Dimensions.Lookups.NoMetadata}}, dims::Tuple{DimensionalData.Dimensions.X{DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}}, DimensionalData.Dimensions.Y{DimensionalData.Dimensions.Lookups.Sampled{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, DimensionalData.Dimensions.Lookups.ForwardOrdered, DimensionalData.Dimensions.Lookups.Regular{Float64}, DimensionalData.Dimensions.Lookups.Points, DimensionalData.Dimensions.Lookups.NoMetadata}}}, I::Tuple{DimensionalData.Dimensions.X{DimensionalData.Dimensions.Lookups.At{Missing, Nothing, Nothing}}, Nothing})
    @ DimensionalData.Dimensions ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:89
 [14] split_alignments(fa::Function, fu::Function, dims::Tuple{DimensionalData.Dimensions.X{DimensionalData.Dimensions.Lookups.Categorical{Union{Missing, Symbol}, Vector{Union{Missing, Symbol}}, DimensionalData.Dimensions.Lookups.Unordered, DimensionalData.Dimensions.Lookups.NoMetadata}}, DimensionalData.Dimensions.Y{DimensionalData.Dimensions.Lookups.Sampled{Float64, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, DimensionalData.Dimensions.Lookups.ForwardOrdered, DimensionalData.Dimensions.Lookups.Regular{Float64}, DimensionalData.Dimensions.Lookups.Points, DimensionalData.Dimensions.Lookups.NoMetadata}}}, I::Tuple{DimensionalData.Dimensions.X{DimensionalData.Dimensions.Lookups.At{Missing, Nothing, Nothing}}, Nothing})
    @ DimensionalData.Dimensions ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:87
 [15] dims2indices
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:56 [inlined]
 [16] dims2indices
    @ ~/.julia/packages/DimensionalData/FWnw9/src/Dimensions/indexing.jl:32 [inlined]
 [17] _dim_getindex
    @ ~/.julia/packages/DimensionalData/FWnw9/src/array/indexing.jl:106 [inlined]
 [18] getindex
    @ ~/.julia/packages/DimensionalData/FWnw9/src/array/indexing.jl:75 [inlined]
 [19] macro expansion
    @ ~/IKR/Workspace/DissStudy/studies/FinalizingParams/metana.jl#==#f0329cea-10c2-402c-9f5a-3caf2f1ebacf:1 [inlined]

[filchristou]

workaround:

function getdim(da::DimArray, val, name::Symbol)
  if ismissing(val)
    lup = lookup(da, name)
    ff = findfirst(ismissing, lup)
    return Dim{name}(ff)
  else
    return Dim{name}(At(val))
  end
end

then this works

da[getdim(da, missing, :X)]

However, I still think this should be treated from the package.

[felixcremer]

How do you end up with a missing value in your lookup? Dimensional Data assumes that all lookups are non missing, because otherwise a lot wont work anymore.

[filchristou]

All columns are simulation parameters. Some simulation parameters might be instantiated as missing because they are not applicable or because they will be assigned a default value based on other parameters.

Dimensional Data assumes that all lookups are non missing

I didn't know about that. But why not treat missing is as a specific value ?
Otherwise I guess it's better if I give it a string or symbol to signify a missing value otherwise..

[asinghvi17]

A sentinel value is probably best. A lot of issues are caused by the fact that missing is viral:

julia> missing == missing
missing

so you'd have to build in protection against missings everywhere...

[rafaqz]

Yeah its essentially undefined behavior.

[rafaqz]

Yeah its essentially undefined behavior.