|
9 | 9 | #include <cudf_test/iterator_utilities.hpp> |
10 | 10 |
|
11 | 11 | #include <cudf/column/column.hpp> |
| 12 | +#include <cudf/copying.hpp> |
| 13 | +#include <cudf/lists/lists_column_view.hpp> |
12 | 14 | #include <cudf/strings/strings_column_view.hpp> |
13 | 15 |
|
14 | 16 | #include <nvtext/minhash.hpp> |
15 | 17 |
|
16 | 18 | #include <cuda/iterator> |
17 | 19 |
|
| 20 | +#include <algorithm> |
| 21 | +#include <string> |
18 | 22 | #include <vector> |
19 | 23 |
|
20 | 24 | struct MinHashTest : public cudf::test::BaseFixture {}; |
21 | 25 |
|
| 26 | +namespace { |
| 27 | +// Builds a deterministic string of at least min_bytes using 1-4 byte UTF-8 characters |
| 28 | +std::string make_multibyte_string(std::size_t min_bytes) |
| 29 | +{ |
| 30 | + char const* pieces[] = {"a", "b", " ", "é", "€", "😀", "z", "q"}; |
| 31 | + std::string result; |
| 32 | + uint32_t x = 1; |
| 33 | + while (result.size() < min_bytes) { |
| 34 | + x = x * 1103515245u + 12345u; |
| 35 | + result += pieces[(x >> 16) % 8]; |
| 36 | + } |
| 37 | + return result; |
| 38 | +} |
| 39 | + |
| 40 | +// Verifies the minhash results for count parameters match the results computed |
| 41 | +// separately for each slice of 16 parameters |
| 42 | +template <typename T, typename MinHashFn> |
| 43 | +void verify_parameter_slices(cudf::strings_column_view const& input, |
| 44 | + cudf::size_type count, |
| 45 | + MinHashFn minhash_fn) |
| 46 | +{ |
| 47 | + auto first = cuda::counting_iterator<uint32_t>{10}; |
| 48 | + auto params = cudf::test::fixed_width_column_wrapper<T, uint32_t>(first, first + count); |
| 49 | + auto pview = cudf::column_view(params); |
| 50 | + auto results = minhash_fn(input, pview); |
| 51 | + auto actual = cudf::test::to_host<T>(cudf::lists_column_view(results->view()).child()).first; |
| 52 | + |
| 53 | + constexpr cudf::size_type slice_size = 16; |
| 54 | + auto expected = std::vector<T>(static_cast<std::size_t>(input.size()) * count); |
| 55 | + for (cudf::size_type s = 0; s < count; s += slice_size) { |
| 56 | + auto const size = std::min(slice_size, count - s); |
| 57 | + auto const sliced = cudf::slice(pview, {s, s + size}).front(); |
| 58 | + auto const part = minhash_fn(input, sliced); |
| 59 | + auto const values = cudf::test::to_host<T>(cudf::lists_column_view(part->view()).child()).first; |
| 60 | + for (cudf::size_type row = 0; row < input.size(); ++row) { |
| 61 | + std::copy_n(values.begin() + row * size, size, expected.begin() + row * count + s); |
| 62 | + } |
| 63 | + } |
| 64 | + EXPECT_EQ(std::vector<T>(actual.begin(), actual.end()), expected); |
| 65 | +} |
| 66 | +} // namespace |
| 67 | + |
22 | 68 | TEST_F(MinHashTest, Permuted) |
23 | 69 | { |
24 | 70 | auto input = |
@@ -80,8 +126,8 @@ TEST_F(MinHashTest, Permuted) |
80 | 126 |
|
81 | 127 | TEST_F(MinHashTest, PermutedWide) |
82 | 128 | { |
83 | | - std::string const small(2 << 10, 'x'); // below wide_string_threshold |
84 | | - std::string const wide(2 << 19, 'y'); // above wide_string_threshold |
| 129 | + std::string const small(2 << 10, 'x'); // processed by a single block |
| 130 | + std::string const wide(2 << 19, 'y'); // processed by multiple blocks |
85 | 131 | auto input = cudf::test::strings_column_wrapper({small, wide}); |
86 | 132 | auto view = cudf::strings_column_view(input); |
87 | 133 |
|
@@ -168,6 +214,131 @@ TEST_F(MinHashTest, PermutedManyParameters) |
168 | 214 | CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64); |
169 | 215 | } |
170 | 216 |
|
| 217 | +TEST_F(MinHashTest, PermutedWithNulls) |
| 218 | +{ |
| 219 | + auto validity = cudf::test::iterators::nulls_at({2, 6}); |
| 220 | + auto input = cudf::test::strings_column_wrapper({"ignored row", |
| 221 | + "doc 1", |
| 222 | + "", |
| 223 | + "", |
| 224 | + "abc", // fewer characters than width |
| 225 | + "héllo wörld €😀 ok", |
| 226 | + "", |
| 227 | + "this is doc 2", |
| 228 | + "ignored row"}, |
| 229 | + validity); |
| 230 | + auto view = cudf::strings_column_view(cudf::slice(input, {1, 8}).front()); |
| 231 | + |
| 232 | + auto first = cuda::counting_iterator<uint32_t>{10}; |
| 233 | + auto params = cudf::test::fixed_width_column_wrapper<uint32_t>(first, first + 3); |
| 234 | + auto results = nvtext::minhash(view, 0, cudf::column_view(params), cudf::column_view(params), 4); |
| 235 | + |
| 236 | + using LCW32 = cudf::test::lists_column_wrapper<uint32_t>; |
| 237 | + // clang-format off |
| 238 | + LCW32 expected{{ |
| 239 | + {1392101586u, 394869177u, 811528444u}, |
| 240 | + {}, |
| 241 | + { 0u, 0u, 0u}, |
| 242 | + { 111658958u, 3129301961u, 1851977668u}, |
| 243 | + { 26214848u, 887829792u, 89411880u}, |
| 244 | + {}, |
| 245 | + { 211415830u, 187088503u, 130291444u} |
| 246 | + }, cudf::test::iterators::nulls_at({1, 5})}; |
| 247 | + // clang-format on |
| 248 | + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); |
| 249 | + |
| 250 | + auto params64 = cudf::test::fixed_width_column_wrapper<uint64_t, uint32_t>(first, first + 3); |
| 251 | + auto results64 = |
| 252 | + nvtext::minhash64(view, 0, cudf::column_view(params64), cudf::column_view(params64), 4); |
| 253 | + |
| 254 | + using LCW64 = cudf::test::lists_column_wrapper<uint64_t>; |
| 255 | + // clang-format off |
| 256 | + LCW64 expected64{{ |
| 257 | + { 827364888116975697ul, 1601854279692781452ul, 70500662054893256ul}, |
| 258 | + {}, |
| 259 | + { 0ul, 0ul, 0ul}, |
| 260 | + { 999368626207634448ul, 176968285142920318ul, 1660410953291900131ul}, |
| 261 | + { 20596815676767043ul, 371077111640517574ul, 133315142570913187ul}, |
| 262 | + {}, |
| 263 | + { 18312093741021833ul, 133793446674258329ul, 21974512489226198ul} |
| 264 | + }, cudf::test::iterators::nulls_at({1, 5})}; |
| 265 | + // clang-format on |
| 266 | + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64); |
| 267 | +} |
| 268 | + |
| 269 | +TEST_F(MinHashTest, PermutedMultibyteWide) |
| 270 | +{ |
| 271 | + // multibyte characters span the tile and block boundaries of the wide row |
| 272 | + auto input = cudf::test::strings_column_wrapper({make_multibyte_string(40000), "short"}); |
| 273 | + auto view = cudf::strings_column_view(input); |
| 274 | + |
| 275 | + auto first = cuda::counting_iterator<uint32_t>{20}; |
| 276 | + auto params = cudf::test::fixed_width_column_wrapper<uint32_t>(first, first + 3); |
| 277 | + auto params64 = cudf::test::fixed_width_column_wrapper<uint64_t, uint32_t>(first, first + 3); |
| 278 | + |
| 279 | + using LCW32 = cudf::test::lists_column_wrapper<uint32_t>; |
| 280 | + using LCW64 = cudf::test::lists_column_wrapper<uint64_t>; |
| 281 | + |
| 282 | + auto results = nvtext::minhash(view, 0, cudf::column_view(params), cudf::column_view(params), 4); |
| 283 | + // clang-format off |
| 284 | + LCW32 expected({ |
| 285 | + { 614084u, 199764u, 10836u}, |
| 286 | + {1169017180u, 153726215u, 3433402546u} |
| 287 | + }); |
| 288 | + // clang-format on |
| 289 | + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); |
| 290 | + |
| 291 | + auto results64 = |
| 292 | + nvtext::minhash64(view, 0, cudf::column_view(params64), cudf::column_view(params64), 4); |
| 293 | + // clang-format off |
| 294 | + LCW64 expected64({ |
| 295 | + { 631838257067200ul, 397872287627084ul, 603345717124258ul}, |
| 296 | + {1487620210039902932ul, 1467307088494363002ul, 252876425515676857ul} |
| 297 | + }); |
| 298 | + // clang-format on |
| 299 | + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64); |
| 300 | + |
| 301 | + // a large width where the characters may not all fit in the staged shared memory |
| 302 | + results = nvtext::minhash(view, 0, cudf::column_view(params), cudf::column_view(params), 1100); |
| 303 | + // clang-format off |
| 304 | + LCW32 expected_large({ |
| 305 | + { 983380u, 8459u, 65568u}, |
| 306 | + { 487932736u, 1371322832u, 2254712928u} |
| 307 | + }); |
| 308 | + // clang-format on |
| 309 | + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected_large); |
| 310 | + |
| 311 | + results64 = |
| 312 | + nvtext::minhash64(view, 0, cudf::column_view(params64), cudf::column_view(params64), 1100); |
| 313 | + // clang-format off |
| 314 | + LCW64 expected64_large({ |
| 315 | + { 14118113140685ul, 180992485396952ul, 133096206945288ul}, |
| 316 | + {497504788972762502ul, 752964329342770017ul, 1008423869712777540ul} |
| 317 | + }); |
| 318 | + // clang-format on |
| 319 | + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64_large); |
| 320 | +} |
| 321 | + |
| 322 | +TEST_F(MinHashTest, PermutedParameterChunks) |
| 323 | +{ |
| 324 | + auto input = |
| 325 | + cudf::test::strings_column_wrapper({make_multibyte_string(20000), "doc 1", "", "line two"}); |
| 326 | + auto view = cudf::strings_column_view(input); |
| 327 | + |
| 328 | + auto mh32 = [](cudf::strings_column_view const& in, cudf::column_view const& p) { |
| 329 | + return nvtext::minhash(in, 0, p, p, 5); |
| 330 | + }; |
| 331 | + auto mh64 = [](cudf::strings_column_view const& in, cudf::column_view const& p) { |
| 332 | + return nvtext::minhash64(in, 0, p, p, 5); |
| 333 | + }; |
| 334 | + // parameter groups that do not evenly divide a warp |
| 335 | + verify_parameter_slices<uint32_t>(view, 260, mh32); |
| 336 | + verify_parameter_slices<uint64_t>(view, 260, mh64); |
| 337 | + // more parameters than a block processes at once |
| 338 | + verify_parameter_slices<uint32_t>(view, 5000, mh32); |
| 339 | + verify_parameter_slices<uint64_t>(view, 5000, mh64); |
| 340 | +} |
| 341 | + |
171 | 342 | TEST_F(MinHashTest, EmptyTest) |
172 | 343 | { |
173 | 344 | auto input = cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); |
|
0 commit comments