Skip to content

Commit 63ec04f

Browse files
committed
Fix issue with Parallel reapply
1 parent 7d6c578 commit 63ec04f

File tree

3 files changed

+11
-3
lines changed

3 files changed

+11
-3
lines changed

src/transforms.jl

+1-1
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ function reapply(transform::Colwise, table, cache)
201201
end
202202

203203
# parallel map with multiple threads
204-
vals = foldxt(vcat, Map(colfunc), 1:length(names))
204+
vals = tcollect(colfunc(i) for i in 1:length(names))
205205

206206
# new table with transformed columns
207207
newtable = (; vals...) |> Tables.materializer(table)

src/transforms/parallel.jl

+2-2
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,9 @@ function reapply(p::Parallel, table, cache)
7878
caches = cache[2]
7979

8080
# reapply transforms in parallel
81-
f((t,c)) = reapply(t, table, c) |> first
81+
f(t, c) = reapply(t, table, c) |> first
8282
itr = zip(p.transforms, caches)
83-
tables = foldxt(vcat, Map(f), itr)
83+
tables = tcollect(f(t, c) for (t, c) in itr)
8484

8585
# table with concatenated columns
8686
newtable = tablehcat(tables)

test/transforms.jl

+8
Original file line numberDiff line numberDiff line change
@@ -406,5 +406,13 @@
406406
@test isapprox(std(n.x), 1.0, atol=1e-8)
407407
@test isapprox(mean(r.x), mean(t.x), atol=1e-8)
408408
@test isapprox(std(r.x), std(t.x), atol=1e-8)
409+
410+
# reapply with Parallel transform
411+
t = Table(x=rand(1000))
412+
T = ZScore() Quantile()
413+
n1, c1 = apply(T, t)
414+
n2, c2 = reapply(T, t, c1)
415+
@test n1 == n2
416+
@test c1 == c2
409417
end
410418
end

0 commit comments

Comments
 (0)