Skip to content

Commit 25e0ba6

Browse files
davidwendtclaude
andcommitted
Add minhash gtests for the fused kernel code paths
Cover nulls, empty strings and strings shorter than the width in a sliced column, multibyte characters across tile and block boundaries, widths too large to stage in shared memory, and parameter counts that require multiple chunks or do not evenly divide a warp. Expected values were generated with the previous implementation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent fc8cc8b commit 25e0ba6

1 file changed

Lines changed: 173 additions & 2 deletions

File tree

‎cpp/tests/text/minhash_tests.cpp‎

Lines changed: 173 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,62 @@
99
#include <cudf_test/iterator_utilities.hpp>
1010

1111
#include <cudf/column/column.hpp>
12+
#include <cudf/copying.hpp>
13+
#include <cudf/lists/lists_column_view.hpp>
1214
#include <cudf/strings/strings_column_view.hpp>
1315

1416
#include <nvtext/minhash.hpp>
1517

1618
#include <cuda/iterator>
1719

20+
#include <algorithm>
21+
#include <string>
1822
#include <vector>
1923

2024
struct MinHashTest : public cudf::test::BaseFixture {};
2125

26+
namespace {
27+
// Builds a deterministic string of at least min_bytes using 1-4 byte UTF-8 characters
28+
std::string make_multibyte_string(std::size_t min_bytes)
29+
{
30+
char const* pieces[] = {"a", "b", " ", "é", "€", "😀", "z", "q"};
31+
std::string result;
32+
uint32_t x = 1;
33+
while (result.size() < min_bytes) {
34+
x = x * 1103515245u + 12345u;
35+
result += pieces[(x >> 16) % 8];
36+
}
37+
return result;
38+
}
39+
40+
// Verifies the minhash results for count parameters match the results computed
41+
// separately for each slice of 16 parameters
42+
template <typename T, typename MinHashFn>
43+
void verify_parameter_slices(cudf::strings_column_view const& input,
44+
cudf::size_type count,
45+
MinHashFn minhash_fn)
46+
{
47+
auto first = cuda::counting_iterator<uint32_t>{10};
48+
auto params = cudf::test::fixed_width_column_wrapper<T, uint32_t>(first, first + count);
49+
auto pview = cudf::column_view(params);
50+
auto results = minhash_fn(input, pview);
51+
auto actual = cudf::test::to_host<T>(cudf::lists_column_view(results->view()).child()).first;
52+
53+
constexpr cudf::size_type slice_size = 16;
54+
auto expected = std::vector<T>(static_cast<std::size_t>(input.size()) * count);
55+
for (cudf::size_type s = 0; s < count; s += slice_size) {
56+
auto const size = std::min(slice_size, count - s);
57+
auto const sliced = cudf::slice(pview, {s, s + size}).front();
58+
auto const part = minhash_fn(input, sliced);
59+
auto const values = cudf::test::to_host<T>(cudf::lists_column_view(part->view()).child()).first;
60+
for (cudf::size_type row = 0; row < input.size(); ++row) {
61+
std::copy_n(values.begin() + row * size, size, expected.begin() + row * count + s);
62+
}
63+
}
64+
EXPECT_EQ(std::vector<T>(actual.begin(), actual.end()), expected);
65+
}
66+
} // namespace
67+
2268
TEST_F(MinHashTest, Permuted)
2369
{
2470
auto input =
@@ -80,8 +126,8 @@ TEST_F(MinHashTest, Permuted)
80126

81127
TEST_F(MinHashTest, PermutedWide)
82128
{
83-
std::string const small(2 << 10, 'x'); // below wide_string_threshold
84-
std::string const wide(2 << 19, 'y'); // above wide_string_threshold
129+
std::string const small(2 << 10, 'x'); // processed by a single block
130+
std::string const wide(2 << 19, 'y'); // processed by multiple blocks
85131
auto input = cudf::test::strings_column_wrapper({small, wide});
86132
auto view = cudf::strings_column_view(input);
87133

@@ -168,6 +214,131 @@ TEST_F(MinHashTest, PermutedManyParameters)
168214
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64);
169215
}
170216

217+
TEST_F(MinHashTest, PermutedWithNulls)
218+
{
219+
auto validity = cudf::test::iterators::nulls_at({2, 6});
220+
auto input = cudf::test::strings_column_wrapper({"ignored row",
221+
"doc 1",
222+
"",
223+
"",
224+
"abc", // fewer characters than width
225+
"héllo wörld €😀 ok",
226+
"",
227+
"this is doc 2",
228+
"ignored row"},
229+
validity);
230+
auto view = cudf::strings_column_view(cudf::slice(input, {1, 8}).front());
231+
232+
auto first = cuda::counting_iterator<uint32_t>{10};
233+
auto params = cudf::test::fixed_width_column_wrapper<uint32_t>(first, first + 3);
234+
auto results = nvtext::minhash(view, 0, cudf::column_view(params), cudf::column_view(params), 4);
235+
236+
using LCW32 = cudf::test::lists_column_wrapper<uint32_t>;
237+
// clang-format off
238+
LCW32 expected{{
239+
{1392101586u, 394869177u, 811528444u},
240+
{},
241+
{ 0u, 0u, 0u},
242+
{ 111658958u, 3129301961u, 1851977668u},
243+
{ 26214848u, 887829792u, 89411880u},
244+
{},
245+
{ 211415830u, 187088503u, 130291444u}
246+
}, cudf::test::iterators::nulls_at({1, 5})};
247+
// clang-format on
248+
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
249+
250+
auto params64 = cudf::test::fixed_width_column_wrapper<uint64_t, uint32_t>(first, first + 3);
251+
auto results64 =
252+
nvtext::minhash64(view, 0, cudf::column_view(params64), cudf::column_view(params64), 4);
253+
254+
using LCW64 = cudf::test::lists_column_wrapper<uint64_t>;
255+
// clang-format off
256+
LCW64 expected64{{
257+
{ 827364888116975697ul, 1601854279692781452ul, 70500662054893256ul},
258+
{},
259+
{ 0ul, 0ul, 0ul},
260+
{ 999368626207634448ul, 176968285142920318ul, 1660410953291900131ul},
261+
{ 20596815676767043ul, 371077111640517574ul, 133315142570913187ul},
262+
{},
263+
{ 18312093741021833ul, 133793446674258329ul, 21974512489226198ul}
264+
}, cudf::test::iterators::nulls_at({1, 5})};
265+
// clang-format on
266+
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64);
267+
}
268+
269+
TEST_F(MinHashTest, PermutedMultibyteWide)
270+
{
271+
// multibyte characters span the tile and block boundaries of the wide row
272+
auto input = cudf::test::strings_column_wrapper({make_multibyte_string(40000), "short"});
273+
auto view = cudf::strings_column_view(input);
274+
275+
auto first = cuda::counting_iterator<uint32_t>{20};
276+
auto params = cudf::test::fixed_width_column_wrapper<uint32_t>(first, first + 3);
277+
auto params64 = cudf::test::fixed_width_column_wrapper<uint64_t, uint32_t>(first, first + 3);
278+
279+
using LCW32 = cudf::test::lists_column_wrapper<uint32_t>;
280+
using LCW64 = cudf::test::lists_column_wrapper<uint64_t>;
281+
282+
auto results = nvtext::minhash(view, 0, cudf::column_view(params), cudf::column_view(params), 4);
283+
// clang-format off
284+
LCW32 expected({
285+
{ 614084u, 199764u, 10836u},
286+
{1169017180u, 153726215u, 3433402546u}
287+
});
288+
// clang-format on
289+
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
290+
291+
auto results64 =
292+
nvtext::minhash64(view, 0, cudf::column_view(params64), cudf::column_view(params64), 4);
293+
// clang-format off
294+
LCW64 expected64({
295+
{ 631838257067200ul, 397872287627084ul, 603345717124258ul},
296+
{1487620210039902932ul, 1467307088494363002ul, 252876425515676857ul}
297+
});
298+
// clang-format on
299+
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64);
300+
301+
// a large width where the characters may not all fit in the staged shared memory
302+
results = nvtext::minhash(view, 0, cudf::column_view(params), cudf::column_view(params), 1100);
303+
// clang-format off
304+
LCW32 expected_large({
305+
{ 983380u, 8459u, 65568u},
306+
{ 487932736u, 1371322832u, 2254712928u}
307+
});
308+
// clang-format on
309+
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected_large);
310+
311+
results64 =
312+
nvtext::minhash64(view, 0, cudf::column_view(params64), cudf::column_view(params64), 1100);
313+
// clang-format off
314+
LCW64 expected64_large({
315+
{ 14118113140685ul, 180992485396952ul, 133096206945288ul},
316+
{497504788972762502ul, 752964329342770017ul, 1008423869712777540ul}
317+
});
318+
// clang-format on
319+
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results64, expected64_large);
320+
}
321+
322+
TEST_F(MinHashTest, PermutedParameterChunks)
323+
{
324+
auto input =
325+
cudf::test::strings_column_wrapper({make_multibyte_string(20000), "doc 1", "", "line two"});
326+
auto view = cudf::strings_column_view(input);
327+
328+
auto mh32 = [](cudf::strings_column_view const& in, cudf::column_view const& p) {
329+
return nvtext::minhash(in, 0, p, p, 5);
330+
};
331+
auto mh64 = [](cudf::strings_column_view const& in, cudf::column_view const& p) {
332+
return nvtext::minhash64(in, 0, p, p, 5);
333+
};
334+
// parameter groups that do not evenly divide a warp
335+
verify_parameter_slices<uint32_t>(view, 260, mh32);
336+
verify_parameter_slices<uint64_t>(view, 260, mh64);
337+
// more parameters than a block processes at once
338+
verify_parameter_slices<uint32_t>(view, 5000, mh32);
339+
verify_parameter_slices<uint64_t>(view, 5000, mh64);
340+
}
341+
171342
TEST_F(MinHashTest, EmptyTest)
172343
{
173344
auto input = cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING});

0 commit comments

Comments
 (0)