Switch to the DataScienceTraits.jl package (#210)

* Switch to the SciTypes.jl package

* SciTypes.jl is now DataScienceTraits.jl

* Update Coerce docstring

* Update Coerce code

* Update Coerce tests

Author: eliascarv

Project.toml

@@ -7,12 +7,12 @@ version = "1.15.4"
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 ColumnSelectors = "9cc86067-7e36-4c61-b350-1ac9833d277f"
+DataScienceTraits = "6cb2f572-2d2b-4ba6-bdb3-e710fa044d6c"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 NelderMead = "2f6b4ddb-b4ff-44c0-b59b-2ab99302f970"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
@@ -24,10 +24,10 @@ Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
 AbstractTrees = "0.4"
 CategoricalArrays = "0.10"
 ColumnSelectors = "0.1"
+DataScienceTraits = "0.1"
 Distributions = "0.25"
 NelderMead = "0.4"
 PrettyTables = "2"
-ScientificTypes = "3.0"
 StatsBase = "0.33, 0.34"
 Tables = "1.6"
 Transducers = "0.4"

src/TableTransforms.jl

@@ -6,7 +6,7 @@ module TableTransforms
 
 using Tables
 using Unitful
-using ScientificTypes
+using DataScienceTraits
 using Distributions: Normal
 using Transducers: tcollect
 using StatsBase: AbstractWeights
@@ -19,6 +19,7 @@ using CategoricalArrays
 using Random
 using NelderMead: optimise
 
+using DataScienceTraits: SciType, Continuous, coerce
 using ColumnSelectors: ColumnSelector, SingleColumnSelector
 using ColumnSelectors: AllSelector, Column, selector, selectsingle
 using Unitful: AbstractQuantity, AffineQuantity, AffineUnits, Units

src/assertions.jl

@@ -5,9 +5,9 @@
 """
     SciTypeAssertion{T}(selector = AllSelector())
 
-Asserts that the columns in the `selector` have a scientific type `T`.
+Asserts that the elements of the columns in the `selector` have a scientific type `T`.
 """
-struct SciTypeAssertion{T,S<:ColumnSelector}
+struct SciTypeAssertion{T<:SciType,S<:ColumnSelector}
   selector::S
 end
 
@@ -22,6 +22,30 @@ function (assertion::SciTypeAssertion{T})(table) where {T}
 
   for nm in snames
     x = Tables.getcolumn(cols, nm)
-    @assert elscitype(x) <: T "The column '$nm' is not of scientific type $T"
+    @assert elscitype(x) <: T "the elements of the column '$nm' are not of scientific type $T"
+  end
+end
+
+"""
+    ColumnTypeAssertion{T}(selector = AllSelector())
+
+Asserts that the columns in the `selector` have a type `T`.
+"""
+struct ColumnTypeAssertion{T,S<:ColumnSelector}
+  selector::S
+end
+
+ColumnTypeAssertion{T}(selector::S) where {T,S<:ColumnSelector} = ColumnTypeAssertion{T,S}(selector)
+
+ColumnTypeAssertion{T}() where {T} = ColumnTypeAssertion{T}(AllSelector())
+
+function (assertion::ColumnTypeAssertion{T})(table) where {T}
+  cols = Tables.columns(table)
+  names = Tables.columnnames(cols)
+  snames = assertion.selector(names)
+
+  for nm in snames
+    x = Tables.getcolumn(cols, nm)
+    @assert typeof(x) <: T "the column '$nm' is not of type $T"
   end
 end

src/transforms/coerce.jl

@@ -3,47 +3,59 @@
 # ------------------------------------------------------------------
 
 """
-    Coerce(pairs, tight=false, verbosity=1)
+    Coerce(col₁ => S₁, col₂ => S₂, ..., colₙ => Sₙ)
 
 Return a copy of the table, ensuring that the scientific types of the columns match the new specification.
 
-This transform wraps the ScientificTypes.coerce function. Please see their docstring for more details.
+This transform uses the `DataScienceTraits.coerce` function. Please see their docstring for more details.
 
 # Examples
 
 ```julia
-using ScientificTypes
-Coerce(:col1 => Continuous, :col2 => Count)
+import DataScienceTraits as DST
+Coerce(1 => DST.Continuous, 2 => DST.Continuous)
+Coerce(:a => DST.Continuous, :b => DST.Continuous)
+Coerce("a" => DST.Continuous, "b" => DST.Continuous)
 ```
 """
-struct Coerce{P} <: FeatureTransform
-  pairs::P
-  tight::Bool
-  verbosity::Int
+struct Coerce{S<:ColumnSelector} <: StatelessFeatureTransform
+  selector::S
+  scitypes::Vector{DataType}
 end
 
-Coerce(pairs::Pair{Symbol,<:Type}...; tight=false, verbosity=1) = Coerce(pairs, tight, verbosity)
-Coerce(pairs::Pair{<:AbstractString,<:Type}...; kwargs...) = Coerce((Symbol(k) => v for (k, v) in pairs)...; kwargs...)
+Coerce() = throw(ArgumentError("cannot create Coerce transform without arguments"))
+
+Coerce(pairs::Pair{C,DataType}...) where {C<:Column} = Coerce(selector(first.(pairs)), collect(last.(pairs)))
 
 isrevertible(::Type{<:Coerce}) = true
 
 function applyfeat(transform::Coerce, feat, prep)
-  newtable = coerce(feat, transform.pairs...; tight=transform.tight, verbosity=transform.verbosity)
-
+  cols = Tables.columns(feat)
+  names = Tables.columnnames(cols)
   types = Tables.schema(feat).types
+  snames = transform.selector(names)
+  typedict = Dict(zip(snames, transform.scitypes))
+
+  columns = map(names) do name
+    x = Tables.getcolumn(cols, name)
+    name ∈ snames ? coerce(typedict[name], x) : x
+  end
+
+  𝒯 = (; zip(names, columns)...)
+  newfeat = 𝒯 |> Tables.materializer(feat)
 
-  newtable, types
+  newfeat, types
 end
 
 function revertfeat(::Coerce, newfeat, fcache)
   cols = Tables.columns(newfeat)
   names = Tables.columnnames(cols)
 
-  oldcols = map(zip(fcache, names)) do (T, n)
+  columns = map(fcache, names) do T, n
     x = Tables.getcolumn(cols, n)
     collect(T, x)
   end
 
-  𝒯 = (; zip(names, oldcols)...)
+  𝒯 = (; zip(names, columns)...)
   𝒯 |> Tables.materializer(newfeat)
 end

src/transforms/levels.jl

@@ -27,7 +27,7 @@ Levels(pairs::Pair{C}...; ordered=nothing) where {C<:Column} =
 
 Levels(; kwargs...) = throw(ArgumentError("cannot create Levels transform without arguments"))
 
-assertions(transform::Levels) = [SciTypeAssertion{Finite}(transform.selector)]
+assertions(transform::Levels) = [ColumnTypeAssertion{CategoricalArray}(transform.selector)]
 
 isrevertible(::Type{<:Levels}) = true
 

src/transforms/onehot.jl

@@ -26,7 +26,7 @@ end
 
 OneHot(col::Column; categ=false) = OneHot(selector(col), categ)
 
-assertions(transform::OneHot) = [SciTypeAssertion{Finite}(transform.selector)]
+assertions(transform::OneHot) = [ColumnTypeAssertion{CategoricalArray}(transform.selector)]
 
 isrevertible(::Type{<:OneHot}) = true
 

test/Project.toml

@@ -1,6 +1,7 @@
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 ColumnSelectors = "9cc86067-7e36-4c61-b350-1ac9833d277f"
+DataScienceTraits = "6cb2f572-2d2b-4ba6-bdb3-e710fa044d6c"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 GR = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
 ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19"
@@ -9,7 +10,6 @@ PairPlots = "43a3c2be-4208-490b-832a-a21dcd55d7da"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ReferenceTests = "324d217c-45ce-50fc-942e-d289b448e8cf"
-ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"

test/assertions.jl

@@ -8,23 +8,30 @@
   table = Table(; a, b, c, d, e, f)
 
   selector = CS.selector([:a, :b])
-  assertion = TT.SciTypeAssertion{Continuous}(selector)
+  assertion = TT.SciTypeAssertion{DST.Continuous}(selector)
   @test isnothing(assertion(table))
   selector = CS.selector([:a, :b, :c])
-  assertion = TT.SciTypeAssertion{Continuous}(selector)
+  assertion = TT.SciTypeAssertion{DST.Continuous}(selector)
   @test_throws AssertionError assertion(table)
 
   selector = CS.selector([:c, :d])
-  assertion = TT.SciTypeAssertion{Count}(selector)
+  assertion = TT.SciTypeAssertion{DST.Categorical}(selector)
   @test isnothing(assertion(table))
-  selector = CS.selector([:c, :d, :e])
-  assertion = TT.SciTypeAssertion{Count}(selector)
+  selector = CS.selector([:a, :c, :d])
+  assertion = TT.SciTypeAssertion{DST.Categorical}(selector)
   @test_throws AssertionError assertion(table)
 
   selector = CS.selector([:e, :f])
-  assertion = TT.SciTypeAssertion{Finite}(selector)
+  assertion = TT.SciTypeAssertion{DST.Categorical}(selector)
   @test isnothing(assertion(table))
-  selector = CS.selector([:d, :e, :f])
-  assertion = TT.SciTypeAssertion{Finite}(selector)
+  selector = CS.selector([:b, :e, :f])
+  assertion = TT.SciTypeAssertion{DST.Categorical}(selector)
+  @test_throws AssertionError assertion(table)
+
+  selector = CS.selector([:e, :f])
+  assertion = TT.ColumnTypeAssertion{CategoricalArray}(selector)
+  @test isnothing(assertion(table))
+  selector = CS.selector([:b, :e, :f])
+  assertion = TT.ColumnTypeAssertion{CategoricalArray}(selector)
   @test_throws AssertionError assertion(table)
 end

test/runtests.jl

@@ -4,14 +4,15 @@ using Tables
 using Unitful
 using TypedTables
 using CategoricalArrays
-using ScientificTypes: Continuous, Count, Finite, Multiclass
 using LinearAlgebra
 using Statistics
 using Test, Random, Plots
 using ReferenceTests, ImageIO
 using StatsBase
 using PairPlots
+
 import ColumnSelectors as CS
+import DataScienceTraits as DST
 
 const TT = TableTransforms
 

test/shows.jl

@@ -200,19 +200,18 @@
   end
 
   @testset "Coerce" begin
-    T = Coerce(:a => Continuous, :b => Count)
+    T = Coerce(:a => DST.Continuous, :b => DST.Categorical)
 
     # compact mode
     iostr = sprint(show, T)
-    @test iostr == "Coerce((:a => Continuous, :b => Count), false, 1)"
+    @test iostr == "Coerce([:a, :b], DataType[DataScienceTraits.Continuous, DataScienceTraits.Categorical])"
 
     # full mode
     iostr = sprint(show, MIME("text/plain"), T)
     @test iostr == """
     Coerce transform
-    ├─ pairs = (:a => Continuous, :b => Count)
-    ├─ tight = false
-    └─ verbosity = 1"""
+    ├─ selector = [:a, :b]
+    └─ scitypes = DataType[DataScienceTraits.Continuous, DataScienceTraits.Categorical]"""
   end
 
   @testset "Levels" begin

test/transforms/coerce.jl

@@ -1,39 +1,38 @@
 @testset "Coerce" begin
-  x1 = [1.0, 2.0, 3.0, 4.0, 5.0]
-  x2 = [1.0, 2.0, 3.0, 4.0, 5.0]
-  x3 = [5.0, 5.0, 5.0, 5.0, 5.0]
-  t = Table(; x1, x2, x3)
+  a = [1, 2, 3, 4, 5]
+  b = [1.0, 2.0, 3.0, 4.0, 5.0]
+  t = Table(; a, b)
 
-  T = Coerce(:x1 => Count, :x2 => Count)
+  T = Coerce(1 => DST.Continuous, 2 => DST.Categorical)
   n, c = apply(T, t)
-  @test eltype(n.x1) == Int
-  @test eltype(n.x2) == Int
+  @test eltype(n.a) <: Float64
+  @test eltype(n.b) <: Int
   n, c = apply(T, t)
   tₒ = revert(T, n, c)
-  @test eltype(tₒ.x1) == eltype(t.x1)
-  @test eltype(tₒ.x2) == eltype(t.x2)
+  @test eltype(tₒ.a) == eltype(t.a)
+  @test eltype(tₒ.b) == eltype(t.b)
 
-  T = Coerce(:x1 => Multiclass, :x2 => Multiclass)
+  T = Coerce(:a => DST.Continuous, :b => DST.Categorical)
   n, c = apply(T, t)
-  @test eltype(n.x1) <: CategoricalValue
-  @test eltype(n.x2) <: CategoricalValue
+  @test eltype(n.a) <: Float64
+  @test eltype(n.b) <: Int
   n, c = apply(T, t)
   tₒ = revert(T, n, c)
-  @test eltype(tₒ.x1) == eltype(t.x1)
-  @test eltype(tₒ.x2) == eltype(t.x2)
+  @test eltype(tₒ.a) == eltype(t.a)
+  @test eltype(tₒ.b) == eltype(t.b)
 
-  T = Coerce("x1" => Multiclass, "x2" => Multiclass)
+  T = Coerce("a" => DST.Continuous, "b" => DST.Categorical)
   n, c = apply(T, t)
-  @test eltype(n.x1) <: CategoricalValue
-  @test eltype(n.x2) <: CategoricalValue
+  @test eltype(n.a) <: Float64
+  @test eltype(n.b) <: Int
   n, c = apply(T, t)
   tₒ = revert(T, n, c)
-  @test eltype(tₒ.x1) == eltype(t.x1)
-  @test eltype(tₒ.x2) == eltype(t.x2)
+  @test eltype(tₒ.a) == eltype(t.a)
+  @test eltype(tₒ.b) == eltype(t.b)
 
   # row table
   rt = Tables.rowtable(t)
-  T = Coerce(:x1 => Count, :x2 => Count)
+  T = Coerce(:a => DST.Continuous, :b => DST.Categorical)
   n, c = apply(T, rt)
   @test Tables.isrowtable(n)
   rtₒ = revert(T, n, c)