Remove reversibility from transforms: Coalesce, DropExtrema, DropMissing, Filter, Satisfies, Select and Reject (#257)

* Remove reversibility from transforms: 'Coalesce', 'DropExtrema', 'DropMissing', 'Filter', 'Satisfies', 'Select' and 'Reject'

* Fix tests

Author: eliascarv

src/transforms/coalesce.jl

@@ -42,10 +42,8 @@ Coalesce(; value) = Coalesce(AllSelector(), value)
 Coalesce(cols; value) = Coalesce(selector(cols), value)
 Coalesce(cols::C...; value) where {C<:Column} = Coalesce(selector(cols), value)
 
-isrevertible(::Type{<:Coalesce}) = true
+isrevertible(::Type{<:Coalesce}) = false
 
-colcache(::Coalesce, x) = findall(ismissing, x)
+colcache(::Coalesce, x) = nothing
 
 colapply(transform::Coalesce, x, c) = coalesce.(x, transform.value)
-
-colrevert(::Coalesce, y, c) = [i ∈ c ? missing : y[i] for i in 1:length(y)]

src/transforms/dropextrema.jl

@@ -50,7 +50,7 @@ DropExtrema(; low=0.25, high=0.75) = DropExtrema(AllSelector(), low, high)
 DropExtrema(cols; low=0.25, high=0.75) = DropExtrema(selector(cols), low, high)
 DropExtrema(cols::C...; low=0.25, high=0.75) where {C<:Column} = DropExtrema(selector(cols), low, high)
 
-isrevertible(::Type{<:DropExtrema}) = true
+isrevertible(::Type{<:DropExtrema}) = false
 
 function preprocess(transform::DropExtrema, feat)
   cols = Tables.columns(feat)
@@ -71,11 +71,6 @@ end
 
 function applyfeat(::DropExtrema, feat, prep)
   ftrans, fprep = prep
-  newfeat, ffcache = applyfeat(ftrans, feat, fprep)
-  newfeat, (ftrans, ffcache)
-end
-
-function revertfeat(::DropExtrema, newfeat, fcache)
-  ftrans, ffcache = fcache
-  revertfeat(ftrans, newfeat, ffcache)
+  newfeat, _ = applyfeat(ftrans, feat, fprep)
+  newfeat, nothing
 end

src/transforms/dropmissing.jl

@@ -41,7 +41,7 @@ DropMissing() = DropMissing(AllSelector())
 DropMissing(cols) = DropMissing(selector(cols))
 DropMissing(cols::C...) where {C<:Column} = DropMissing(selector(cols))
 
-isrevertible(::Type{<:DropMissing}) = true
+isrevertible(::Type{<:DropMissing}) = false
 
 function preprocess(transform::DropMissing, feat)
   cols = Tables.columns(feat)
@@ -60,7 +60,7 @@ _nonmissing(::Type{Union{Missing,T}}, x) where {T} = collect(T, x)
 function applyfeat(::DropMissing, feat, prep)
   # apply filter transform
   ftrans, fprep, snames = prep
-  newfeat, ffcache = applyfeat(ftrans, feat, fprep)
+  newfeat, _ = applyfeat(ftrans, feat, fprep)
 
   # drop Missing type
   cols = Tables.columns(newfeat)
@@ -72,25 +72,5 @@ function applyfeat(::DropMissing, feat, prep)
   𝒯 = (; zip(names, columns)...)
   newfeat = 𝒯 |> Tables.materializer(feat)
 
-  # original column types
-  types = Tables.schema(feat).types
-
-  newfeat, (ftrans, ffcache, snames, types)
-end
-
-function revertfeat(::DropMissing, newfeat, fcache)
-  ftrans, ffcache, snames, types = fcache
-
-  # reintroduce Missing type
-  cols = Tables.columns(newfeat)
-  names = Tables.columnnames(cols)
-  columns = map(types, names) do T, name
-    x = Tables.getcolumn(cols, name)
-    name ∈ snames ? collect(T, x) : x
-  end
-  𝒯 = (; zip(names, columns)...)
-  ofeat = 𝒯 |> Tables.materializer(newfeat)
-
-  # revert filter transform
-  revertfeat(ftrans, ofeat, ffcache)
+  newfeat, nothing
 end

src/transforms/filter.jl

@@ -27,58 +27,28 @@ struct Filter{F} <: StatelessFeatureTransform
   pred::F
 end
 
-isrevertible(::Type{<:Filter}) = true
+isrevertible(::Type{<:Filter}) = false
 
 function preprocess(transform::Filter, feat)
   # lazy row iterator
   rows = tablerows(feat)
 
   # selected indices
-  sinds, nrows = Int[], 0
+  sinds = Int[]
   for (i, row) in enumerate(rows)
     transform.pred(row) && push!(sinds, i)
-    nrows += 1
   end
 
-  # rejected indices
-  rinds = setdiff(1:nrows, sinds)
-
-  sinds, rinds
+  sinds
 end
 
 function applyfeat(::Filter, feat, prep)
   # preprocessed indices
-  sinds, rinds = prep
+  sinds = prep
 
-  # selected/rejected rows
+  # selected rows
   srows = Tables.subset(feat, sinds, viewhint=true)
-  rrows = Tables.subset(feat, rinds, viewhint=true)
 
   newfeat = srows |> Tables.materializer(feat)
-
-  newfeat, (rinds, rrows)
-end
-
-function revertfeat(::Filter, newfeat, fcache)
-  cols = Tables.columns(newfeat)
-  names = Tables.columnnames(cols)
-
-  rinds, rrows = fcache
-
-  # columns with selected rows
-  columns = map(names) do name
-    collect(Tables.getcolumn(cols, name))
-  end
-
-  # insert rejected rows into columns
-  rrcols = Tables.columns(rrows)
-  for (name, x) in zip(names, columns)
-    r = Tables.getcolumn(rrcols, name)
-    for (i, v) in zip(rinds, r)
-      insert!(x, i, v)
-    end
-  end
-
-  𝒯 = (; zip(names, columns)...)
-  𝒯 |> Tables.materializer(newfeat)
+  newfeat, nothing
 end

src/transforms/satisfies.jl

@@ -19,7 +19,7 @@ struct Satisfies{F} <: StatelessFeatureTransform
   pred::F
 end
 
-isrevertible(::Type{<:Satisfies}) = true
+isrevertible(::Type{<:Satisfies}) = false
 
 function applyfeat(transform::Satisfies, feat, prep)
   pred = transform.pred
@@ -30,13 +30,8 @@ function applyfeat(transform::Satisfies, feat, prep)
     pred(x)
   end
   strans = Select(snames)
-  newfeat, sfcache = applyfeat(strans, feat, prep)
-  newfeat, (strans, sfcache)
-end
-
-function revertfeat(::Satisfies, newfeat, fcache)
-  strans, sfcache = fcache
-  revertfeat(strans, newfeat, sfcache)
+  newfeat, _ = applyfeat(strans, feat, prep)
+  newfeat, nothing
 end
 
 """

src/transforms/select.jl

@@ -45,7 +45,7 @@ Select(pairs::Pair{C,S}...) where {C<:Column,S<:AbstractString} =
 
 Select() = throw(ArgumentError("cannot create Select transform without arguments"))
 
-isrevertible(::Type{<:Select}) = true
+isrevertible(::Type{<:Select}) = false
 
 # utils
 _newnames(::Nothing, select) = select
@@ -54,50 +54,12 @@ _newnames(names::Vector{Symbol}, select) = names
 function applyfeat(transform::Select, feat, prep)
   cols = Tables.columns(feat)
   names = collect(Tables.columnnames(cols))
-
-  # retrieve relevant column names
   select = transform.selector(names)
-  reject = setdiff(names, select)
   newnames = _newnames(transform.newnames, select)
-
-  # keep track of indices to revert later
-  sinds = indexin(select, names)
-  rinds = indexin(reject, names)
-
-  # sort indices to facilitate reinsertion
-  sperm = sortperm(sinds)
-
-  # rejected columns
-  rcolumns = [Tables.getcolumn(cols, name) for name in reject]
-
-  fcache = (select, sperm, reject, rcolumns, rinds)
   newfeat = TableSelection(feat, newnames, select)
-  newfeat, fcache
-end
-
-function revertfeat(::Select, newfeat, fcache)
-  cols = Tables.columns(newfeat)
-  names = Tables.columnnames(cols)
-  # https://github.com/JuliaML/TableTransforms.jl/issues/76
-  columns = Any[Tables.getcolumn(cols, name) for name in names]
-
-  select, sperm, reject, rcolumns, rinds = fcache
-
-  # restore rejected columns
-  onames = select[sperm]
-  ocolumns = columns[sperm]
-  for (i, rind) in enumerate(rinds)
-    insert!(onames, rind, reject[i])
-    insert!(ocolumns, rind, rcolumns[i])
-  end
-
-  𝒯 = (; zip(onames, ocolumns)...)
-  𝒯 |> Tables.materializer(newfeat)
+  newfeat, nothing
 end
 
-# reverting a single TableSelection is trivial
-revertfeat(::Select, newfeat::TableSelection, fcache) = newfeat.table
-
 """
     Reject(col₁, col₂, ..., colₙ)
     Reject([col₁, col₂, ..., colₙ])
@@ -129,19 +91,14 @@ Reject(cols::C...) where {C<:Column} = Reject(selector(cols))
 Reject() = throw(ArgumentError("cannot create Reject transform without arguments"))
 Reject(::AllSelector) = throw(ArgumentError("cannot reject all columns"))
 
-isrevertible(::Type{<:Reject}) = true
+isrevertible(::Type{<:Reject}) = false
 
 function applyfeat(transform::Reject, feat, prep)
   cols = Tables.columns(feat)
   names = Tables.columnnames(cols)
   reject = transform.selector(names)
   select = setdiff(names, reject)
   strans = Select(select)
-  newfeat, sfcache = applyfeat(strans, feat, prep)
-  newfeat, (strans, sfcache)
-end
-
-function revertfeat(::Reject, newfeat, fcache)
-  strans, sfcache = fcache
-  revertfeat(strans, newfeat, sfcache)
+  newfeat, _ = applyfeat(strans, feat, prep)
+  newfeat, nothing
 end

test/metadata.jl

@@ -13,8 +13,6 @@
     tn, tc = apply(T, t)
     @test mn.meta == m
     @test mn.table == tn
-    mtₒ = revert(T, mn, mc)
-    @test mtₒ == mt
 
     T = Rename(:a => :x, :c => :y)
     mn, mc = apply(T, mt)
@@ -57,8 +55,6 @@
     tn, tc = apply(T, t)
     @test mn.meta == VarMeta(m.data .+ 2)
     @test mn.table == tn
-    mtₒ = revert(T, mn, mc)
-    @test mtₒ == mt
 
     T = Rename(:b => :x, :d => :y)
     mn, mc = apply(T, mt)

test/transforms/coalesce.jl

@@ -1,4 +1,6 @@
 @testset "Coalesce" begin
+  @test !isrevertible(Coalesce(value=0))
+
   a = [3, 2, missing, 4, 5, 3]
   b = [missing, 4, 4, 5, 8, 5]
   c = [1, 1, 6, 2, 4, missing]
@@ -16,41 +18,22 @@
   @test n.e == [0, 5, 2, 6, 5, 2]
   @test n.f == [4, 0, 3, 4, 5, 2]
 
-  # revert test
-  @test isrevertible(T)
-  tₒ = revert(T, n, c)
-  cols = Tables.columns(t)
-  colsₒ = Tables.columns(tₒ)
-  colnames = Tables.columnnames(t)
-  for n in colnames
-    col = Tables.getcolumn(cols, n)
-    colₒ = Tables.getcolumn(colsₒ, n)
-    @test isequal(col, colₒ)
-  end
-
-  # table schema after apply and revert
+  # table schema after apply
   T = Coalesce(value=0)
   n, c = apply(T, t)
-  tₒ = revert(T, n, c)
-  ttypes = Tables.schema(t).types
   ntypes = Tables.schema(n).types
   @test ntypes[1] == Int
   @test ntypes[2] == Int
   @test ntypes[3] == Int
   @test ntypes[4] == Int
   @test ntypes[5] == Int
   @test ntypes[6] == Int
-  @test ttypes == Tables.schema(tₒ).types
 
   # row table
   rt = Tables.rowtable(t)
   T = Coalesce(value=0)
   n, c = apply(T, rt)
   @test Tables.isrowtable(n)
-  rtₒ = revert(T, n, c)
-  for (row, rowₒ) in zip(rt, rtₒ)
-    @test isequal(row, rowₒ)
-  end
 
   # colspec
   T = Coalesce(1, 3, 5, value=0)

test/transforms/dropextrema.jl

@@ -1,5 +1,5 @@
 @testset "DropExtrema" begin
-  @test isrevertible(DropExtrema(:a))
+  @test !isrevertible(DropExtrema(:a))
 
   a = [6.9, 9.0, 7.8, 0.0, 5.1, 4.8, 1.1, 8.0, 5.4, 7.9]
   b = [7.7, 4.2, 6.3, 1.4, 4.4, 0.5, 3.0, 6.1, 1.9, 1.5]
@@ -17,8 +17,6 @@
   @test n.d == [1.0, 6.2, 8.1, 4.1]
   @test n.e == [1.5, 4.1, 5.9, 2.4]
   @test n.f == [1.9, 9.0, 1.3, 5.1]
-  tₒ = revert(T, n, c)
-  @test t == tₒ
 
   T = DropExtrema(1, 2)
   n, c = apply(T, t)
@@ -28,8 +26,6 @@
   @test n.d == [8.1, 4.1]
   @test n.e == [5.9, 2.4]
   @test n.f == [1.3, 5.1]
-  tₒ = revert(T, n, c)
-  @test t == tₒ
 
   T = DropExtrema(:c, low=0.3, high=0.7)
   n, c = apply(T, t)
@@ -39,8 +35,6 @@
   @test n.d == [1.0, 6.2, 6.2, 6.9]
   @test n.e == [1.5, 4.1, 1.3, 3.5]
   @test n.f == [1.9, 9.0, 8.9, 3.8]
-  tₒ = revert(T, n, c)
-  @test t == tₒ
 
   T = DropExtrema([:c, :d], low=0.3, high=0.7)
   n, c = apply(T, t)
@@ -50,8 +44,6 @@
   @test n.d == [6.2, 6.2]
   @test n.e == [4.1, 1.3]
   @test n.f == [9.0, 8.9]
-  tₒ = revert(T, n, c)
-  @test t == tₒ
 
   T = DropExtrema("e", low=0.2, high=0.8)
   n, c = apply(T, t)
@@ -61,8 +53,6 @@
   @test n.d == [6.2, 1.9, 8.1, 4.0, 6.9, 4.1]
   @test n.e == [4.1, 1.6, 5.9, 4.9, 3.5, 2.4]
   @test n.f == [9.0, 6.2, 1.3, 6.2, 3.8, 5.1]
-  tₒ = revert(T, n, c)
-  @test t == tₒ
 
   T = DropExtrema(("e", "f"), low=0.2, high=0.8)
   n, c = apply(T, t)
@@ -72,8 +62,6 @@
   @test n.d == [1.9, 4.0, 6.9, 4.1]
   @test n.e == [1.6, 4.9, 3.5, 2.4]
   @test n.f == [6.2, 6.2, 3.8, 5.1]
-  tₒ = revert(T, n, c)
-  @test t == tₒ
 
   # error: invalid quantiles
   @test_throws AssertionError DropExtrema(:a, low=0, high=1.4)