Skip to content

Commit 316f3a7

Browse files
committed
fix: Keeping unique values AND dedupe field in Deduplicate::Table
Fixes the fact that, when run with `compile_uniq_fieldvals: true` and `delete_field: false`, the field value on which a table was deduplicated was deleted.
1 parent 4f2ec6b commit 316f3a7

1 file changed

Lines changed: 33 additions & 1 deletion

File tree

  • lib/kiba/extend/transforms/deduplicate

lib/kiba/extend/transforms/deduplicate/table.rb

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,37 @@ module Deduplicate
158158
# {foo: "e", bar: "f", baz: ""}
159159
# ]
160160
# expect(result).to eq(expected)
161+
#
162+
# @example Compiling unique field values keeping dedupe field
163+
# # Used in pipeline as:
164+
# # transform Deduplicate::Table,
165+
# # field: :combine,
166+
# # delete_field: false,
167+
# # compile_uniq_fieldvals: true,
168+
# # compile_delim: ", "
169+
# xform = Deduplicate::Table.new(field: :combine, delete_field: false,
170+
# compile_uniq_fieldvals: true, compile_delim: ", ")
171+
# input = [
172+
# {baz: "f", combine: "a b"},
173+
# {baz: "f", combine: "a b"},
174+
# {baz: "g", combine: "c d"},
175+
# {baz: "", combine: "c d"},
176+
# {baz: "h", combine: "c e"},
177+
# {baz: "i", combine: "c d"},
178+
# {baz: "j", combine: "c d"},
179+
# {baz: nil, combine: "c d"},
180+
# {baz: "k", combine: "c d"},
181+
# {baz: nil, combine: "e f"}
182+
# ]
183+
# result = Kiba::StreamingRunner.transform_stream(input, xform)
184+
# .map{ |row| row }
185+
# expected = [
186+
# {baz: "f", combine: "a b"},
187+
# {baz: "g, i, j, k", combine: "c d"},
188+
# {baz: "h", combine: "c e"},
189+
# {baz: "", combine: "e f"}
190+
# ]
191+
# expect(result).to eq(expected)
161192
# @since 2.2.0
162193
class Table
163194
# @param field [Symbol] name of field on which to deduplicate
@@ -211,7 +242,7 @@ def process(row)
211242
end
212243

213244
def close
214-
deduper.values.each do |hash|
245+
deduper.each do |val, hash|
215246
row = hash[:row]
216247
add_example_field(row, hash) if example
217248
row[occ_target] = hash[:occs] if occs
@@ -222,6 +253,7 @@ def close
222253

223254
[fld, hash[:fieldvals][fld].join(compile_delim)]
224255
end.compact.to_h
256+
row[field] = val unless delete
225257
end
226258
yield row
227259
end

0 commit comments

Comments
 (0)