Skip to content

Commit c40ab53

Browse files
committed
Merge branch 'main' into nightly-debug
2 parents d89c380 + aa9e398 commit c40ab53

14 files changed

Lines changed: 91 additions & 62 deletions

File tree

‎cpp/src/binaryop/compiled/binary_ops.cu‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
#include <cudf/utilities/concepts.hpp>
1818
#include <cudf/utilities/memory_resource.hpp>
1919

20-
#include <rmm/device_uvector.hpp>
2120
#include <rmm/exec_policy.hpp>
2221

22+
#include <cuda/buffer>
2323
#include <cuda/functional>
2424
#include <cuda/iterator>
2525
#include <cuda/std/concepts>
@@ -234,7 +234,8 @@ struct null_considering_binop {
234234
"Output column type should match input column type");
235235

236236
// Shallow copy of the resultant strings
237-
rmm::device_uvector<cudf::string_view> out_col_strings(col_size, stream);
237+
cuda::device_buffer<cudf::string_view> out_col_strings(
238+
stream, cudf::get_current_device_resource_ref(), col_size, cuda::no_init);
238239

239240
// Invalid output column strings - null rows
240241
cudf::string_view const invalid_str{nullptr, 0};

‎cpp/src/bitmask/null_mask.cu‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <cudf/detail/null_mask.cuh>
88
#include <cudf/detail/null_mask.hpp>
99
#include <cudf/detail/nvtx/ranges.hpp>
10+
#include <cudf/detail/utilities/buffer_factories.hpp>
1011
#include <cudf/detail/utilities/grid_1d.cuh>
1112
#include <cudf/detail/utilities/integer_utils.hpp>
1213
#include <cudf/detail/utilities/vector_factories.hpp>
@@ -25,6 +26,7 @@
2526
#include <cub/block/block_reduce.cuh>
2627
#include <cuda/atomic>
2728
#include <cuda/bit>
29+
#include <cuda/buffer>
2830
#include <cuda/memory_resource>
2931
#include <cuda/numeric>
3032
#include <cuda/std/execution>
@@ -500,7 +502,7 @@ std::vector<size_type> batch_count_set_bits(host_span<bitmask_type const* const>
500502
auto const d_bitmasks = cudf::detail::make_device_uvector_async(bitmasks, stream, tmp_mr);
501503
auto const num_words = num_bitmask_words(num_bits_to_count);
502504
auto d_non_zero_count =
503-
cudf::detail::make_zeroed_device_uvector_async<size_type>(num_bitmasks, stream, tmp_mr);
505+
cudf::detail::make_zeroed_device_buffer_async<size_type>(num_bitmasks, stream, tmp_mr);
504506

505507
constexpr size_type block_size{256};
506508
// We use a 2D grid to launch the kernel, where the first dimension is to access elements in each

‎cpp/src/column/column_device_view.cu‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
#include <cudf/detail/utilities/vector_factories.hpp>
99
#include <cudf/types.hpp>
1010
#include <cudf/utilities/error.hpp>
11+
#include <cudf/utilities/memory_resource.hpp>
1112

13+
#include <cuda/buffer>
1214
#include <cuda/stream>
1315

1416
#include <functional>
@@ -75,19 +77,26 @@ create_device_view_from_view(ColumnView const& source,
7577
// Each ColumnDeviceView instance may have child objects that
7678
// require setting some internal device pointers before being copied
7779
// from CPU to device.
78-
auto const descendant_storage =
79-
new rmm::device_uvector<char>(descendant_storage_bytes, stream, mr);
80-
81-
auto deleter = [descendant_storage](ColumnDeviceView* v) {
80+
auto descendant_storage = std::make_unique<cuda::device_buffer<char>>(
81+
stream,
82+
mr,
83+
descendant_storage_bytes,
84+
cuda::no_init,
85+
cuda::std::execution::prop{cuda::allocation_alignment, alignof(ColumnDeviceView)});
86+
cudf::device_span<char> descendant_storage_data = *descendant_storage;
87+
88+
auto tmp_result = std::make_unique<ColumnDeviceView>(
89+
source, staging_buffer.data(), descendant_storage_data.data());
90+
91+
auto deleter = [descendant_storage = descendant_storage.release()](ColumnDeviceView* v) {
8292
v->destroy();
8393
delete descendant_storage;
8494
};
8595

86-
std::unique_ptr<ColumnDeviceView, decltype(deleter)> result{
87-
new ColumnDeviceView(source, staging_buffer.data(), descendant_storage->data()), deleter};
96+
std::unique_ptr<ColumnDeviceView, decltype(deleter)> result{tmp_result.release(), deleter};
8897

8998
// copy the CPU memory with all the children into device memory
90-
detail::cuda_memcpy<char>(*descendant_storage, staging_buffer, stream);
99+
detail::cuda_memcpy<char>(descendant_storage_data, staging_buffer, stream);
91100
return result;
92101
}
93102

‎cpp/src/column/column_factories.cu‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <cudf/strings/detail/strings_column_factories.cuh>
1515
#include <cudf/utilities/memory_resource.hpp>
1616

17+
#include <cuda/buffer>
1718
#include <cuda/iterator>
1819
#include <thrust/uninitialized_fill.h>
1920

@@ -67,7 +68,8 @@ std::unique_ptr<cudf::column> column_from_scalar_dispatch::operator()<cudf::stri
6768
auto const d_str = ss.value(stream); // no actual data is copied
6869

6970
// fill the column with the scalar
70-
rmm::device_uvector<cudf::strings::detail::string_index_pair> indices(size, stream);
71+
cuda::device_buffer<cudf::strings::detail::string_index_pair> indices(
72+
stream, cudf::get_current_device_resource_ref(), size, cuda::no_init);
7173
auto const row_value =
7274
d_str.empty() ? cudf::strings::detail::string_index_pair{"", 0}
7375
: cudf::strings::detail::string_index_pair{d_str.data(), d_str.size_bytes()};

‎cpp/src/jit/helpers.hpp‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
#include <cudf/column/scalar_column_view.hpp>
1010
#include <cudf/detail/utilities/cuda_memcpy.hpp>
1111
#include <cudf/detail/utilities/vector_factories.hpp>
12+
#include <cudf/utilities/memory_resource.hpp>
1213

14+
#include <cuda/buffer>
1315
#include <cuda/iterator>
1416

1517
#include <jit/cache.hpp>
@@ -42,19 +44,9 @@ std::map<uint32_t, std::string> build_ptx_params(std::span<std::string const> ou
4244
std::span<std::string const> input_typenames,
4345
bool has_user_data);
4446

45-
template <typename T>
46-
rmm::device_uvector<T> to_device_vector(std::vector<T> const& host,
47-
cuda::stream_ref stream,
48-
rmm::device_async_resource_ref mr)
49-
{
50-
rmm::device_uvector<T> device{host.size(), stream, mr};
51-
cudf::detail::cuda_memcpy_async<T>(device, host, stream);
52-
return device;
53-
}
54-
5547
template <typename DeviceView, typename ColumnView>
5648
std::tuple<std::vector<std::unique_ptr<DeviceView, std::function<void(DeviceView*)>>>,
57-
rmm::device_uvector<DeviceView>>
49+
cuda::device_buffer<DeviceView>>
5850
column_views_to_device(std::span<ColumnView const> views,
5951
cuda::stream_ref stream,
6052
rmm::device_async_resource_ref mr)
@@ -72,7 +64,7 @@ column_views_to_device(std::span<ColumnView const> views,
7264
host_array.push_back(*h);
7365
}
7466

75-
rmm::device_uvector<DeviceView> device_array{handles.size(), stream, mr};
67+
cuda::device_buffer<DeviceView> device_array{stream, mr, handles.size(), cuda::no_init};
7668
cudf::detail::cuda_memcpy<DeviceView>(device_array, host_array, stream);
7769

7870
return std::make_tuple(std::move(handles), std::move(device_array));

‎cpp/src/partitioning/partitioning.cu‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <cudf/detail/nvtx/ranges.hpp>
1111
#include <cudf/detail/row_operator/hashing.cuh>
1212
#include <cudf/detail/scatter.hpp>
13+
#include <cudf/detail/utilities/buffer_factories.hpp>
1314
#include <cudf/detail/utilities/cuda.cuh>
1415
#include <cudf/detail/utilities/grid_1d.cuh>
1516
#include <cudf/detail/utilities/integer_utils.hpp>
@@ -368,7 +369,7 @@ void copy_block_partitions_impl(InputIter const input,
368369
CUDF_CUDA_TRY(cudaGetLastError());
369370
}
370371

371-
rmm::device_uvector<size_type> compute_gather_map(size_type num_rows,
372+
cuda::device_buffer<size_type> compute_gather_map(size_type num_rows,
372373
size_type num_partitions,
373374
size_type const* row_partition_numbers,
374375
size_type const* row_partition_offset,
@@ -378,10 +379,11 @@ rmm::device_uvector<size_type> compute_gather_map(size_type num_rows,
378379
cuda::stream_ref stream)
379380
{
380381
auto sequence = cuda::counting_iterator<cudf::size_type>{0};
381-
rmm::device_uvector<size_type> gather_map(num_rows, stream);
382+
cuda::device_buffer<size_type> gather_map(
383+
stream, cudf::get_current_device_resource_ref(), num_rows, cuda::no_init);
382384

383385
copy_block_partitions_impl(sequence,
384-
gather_map.begin(),
386+
gather_map.data(),
385387
num_rows,
386388
num_partitions,
387389
row_partition_numbers,
@@ -480,7 +482,8 @@ std::pair<std::unique_ptr<table>, std::vector<size_type>> hash_partition_table_g
480482
CUDF_EXPECTS(num_partitions < std::numeric_limits<size_type>::max(),
481483
"num_partitions exceeds cudf's supported limit");
482484

483-
auto row_partition_numbers = rmm::device_uvector<size_type>(num_rows, stream);
485+
cuda::device_buffer<size_type> row_partition_numbers(
486+
stream, cudf::get_current_device_resource_ref(), num_rows, cuda::no_init);
484487

485488
// Compute partition number for each row
486489
if (is_power_two(num_partitions)) {
@@ -506,7 +509,7 @@ std::pair<std::unique_ptr<table>, std::vector<size_type>> hash_partition_table_g
506509
// Build histogram via cub::DeviceHistogram::HistogramEven.
507510
// HistogramEven writes num_partitions bins; the extra element is used by the exclusive scan
508511
// below to produce the total row count as the last offset. Zero-initialize to avoid UB.
509-
auto histogram = cudf::detail::make_zeroed_device_uvector_async<size_type>(
512+
auto histogram = cudf::detail::make_zeroed_device_buffer_async<size_type>(
510513
num_partitions + 1, stream, cudf::get_current_device_resource_ref());
511514
{
512515
auto const num_levels = num_partitions + 1;
@@ -547,7 +550,8 @@ std::pair<std::unique_ptr<table>, std::vector<size_type>> hash_partition_table_g
547550
auto const pinned_offsets = cudf::detail::make_pinned_vector_async(histogram, stream);
548551

549552
// Build scatter map: atomically increment partition offsets
550-
rmm::device_uvector<size_type> scatter_map(num_rows, stream);
553+
cuda::device_buffer<size_type> scatter_map(
554+
stream, cudf::get_current_device_resource_ref(), num_rows, cuda::no_init);
551555
thrust::transform(
552556
rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()),
553557
row_partition_numbers.begin(),
@@ -606,7 +610,7 @@ std::pair<std::unique_ptr<table>, std::vector<size_type>> hash_partition_table(
606610
std::size_t const grid_size = util::div_rounding_up_safe(num_rows, rows_per_block);
607611

608612
// Allocate array to hold which partition each row belongs to
609-
auto row_partition_numbers = rmm::device_uvector<size_type>(num_rows, stream, temp_mr);
613+
cuda::device_buffer<size_type> row_partition_numbers(stream, temp_mr, num_rows, cuda::no_init);
610614

611615
// Array to hold the size of each partition computed by each block
612616
// i.e., { {block0 partition0 size, block1 partition0 size, ...},
@@ -615,17 +619,17 @@ std::pair<std::unique_ptr<table>, std::vector<size_type>> hash_partition_table(
615619
// {block0 partition(num_partitions-1) size, block1
616620
// partition(num_partitions -1) size, ...} }
617621
auto block_partition_sizes =
618-
rmm::device_uvector<size_type>(grid_size * num_partitions, stream, temp_mr);
622+
cuda::device_buffer<size_type>(stream, temp_mr, grid_size * num_partitions, cuda::no_init);
619623

620624
auto scanned_block_partition_sizes =
621-
rmm::device_uvector<size_type>(grid_size * num_partitions, stream, temp_mr);
625+
cuda::device_buffer<size_type>(stream, temp_mr, grid_size * num_partitions, cuda::no_init);
622626

623627
// Holds the total number of rows in each partition
624628
auto global_partition_sizes =
625-
cudf::detail::make_zeroed_device_uvector_async<size_type>(num_partitions, stream, temp_mr);
629+
cudf::detail::make_zeroed_device_buffer_async<size_type>(num_partitions, stream, temp_mr);
626630

627631
auto row_partition_offset =
628-
cudf::detail::make_zeroed_device_uvector_async<size_type>(num_rows, stream, temp_mr);
632+
cudf::detail::make_zeroed_device_buffer_async<size_type>(num_rows, stream, temp_mr);
629633

630634
// If the number of partitions is a power of two, we can compute the partition
631635
// number of each row more efficiently with bitwise operations
@@ -784,7 +788,8 @@ struct dispatch_map_type {
784788
requires(is_index_type<MapType>())
785789
{
786790
// Build a histogram of the number of rows in each partition
787-
rmm::device_uvector<size_type> histogram(num_partitions + 1, stream);
791+
cuda::device_buffer<size_type> histogram(
792+
stream, cudf::get_current_device_resource_ref(), num_partitions + 1, cuda::no_init);
788793
std::size_t temp_storage_bytes{};
789794
std::size_t const num_levels = num_partitions + 1;
790795
size_type const lower_level = 0;
@@ -824,7 +829,8 @@ struct dispatch_map_type {
824829

825830
// Unfortunately need to materialize the scatter map because
826831
// `detail::scatter` requires multiple passes through the iterator
827-
rmm::device_uvector<size_type> scatter_map(partition_map.size(), stream);
832+
cuda::device_buffer<size_type> scatter_map(
833+
stream, cudf::get_current_device_resource_ref(), partition_map.size(), cuda::no_init);
828834

829835
// For each `partition_map[i]`, atomically increment the corresponding
830836
// partition offset to determine `i`s location in the output

‎cpp/src/partitioning/round_robin.cu‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <rmm/device_uvector.hpp>
2323
#include <rmm/exec_policy.hpp>
2424

25+
#include <cuda/buffer>
2526
#include <cuda/functional>
2627
#include <cuda/iterator>
2728
#include <cuda/stream>
@@ -84,7 +85,8 @@ std::pair<std::unique_ptr<cudf::table>, std::vector<cudf::size_type>> degenerate
8485
}));
8586

8687
if (num_partitions == nrows) {
87-
rmm::device_uvector<cudf::size_type> partition_offsets(num_partitions + 1, stream);
88+
cuda::device_buffer<cudf::size_type> partition_offsets(
89+
stream, cudf::get_current_device_resource_ref(), num_partitions + 1, cuda::no_init);
8890
thrust::sequence(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()),
8991
partition_offsets.begin(),
9092
partition_offsets.end());
@@ -98,7 +100,8 @@ std::pair<std::unique_ptr<cudf::table>, std::vector<cudf::size_type>> degenerate
98100

99101
return std::pair{std::move(uniq_tbl), cudf::detail::make_std_vector(partition_offsets, stream)};
100102
} else { //( num_partitions > nrows )
101-
rmm::device_uvector<cudf::size_type> d_row_indices(nrows, stream);
103+
cuda::device_buffer<cudf::size_type> d_row_indices(
104+
stream, cudf::get_current_device_resource_ref(), nrows, cuda::no_init);
102105

103106
// copy rotated right partition indexes that
104107
// fall in the interval [0, nrows):
@@ -128,7 +131,8 @@ std::pair<std::unique_ptr<cudf::table>, std::vector<cudf::size_type>> degenerate
128131
[nrows] __device__(auto index) { return (index < nrows ? 1 : 0); }));
129132

130133
// offsets (part 2: compute partition offsets):
131-
rmm::device_uvector<cudf::size_type> partition_offsets(num_partitions + 1, stream);
134+
cuda::device_buffer<cudf::size_type> partition_offsets(
135+
stream, cudf::get_current_device_resource_ref(), num_partitions + 1, cuda::no_init);
132136
thrust::exclusive_scan(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()),
133137
nedges_iter_begin,
134138
nedges_iter_begin + num_partitions,

‎cpp/src/rolling/detail/lead_lag_nested.cuh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
#include <rmm/exec_policy.hpp>
2222

23+
#include <cuda/buffer>
2324
#include <cuda/functional>
2425
#include <cuda/iterator>
2526
#include <cuda/std/iterator>
@@ -158,19 +159,23 @@ std::unique_ptr<column> compute_lead_lag_for_nested(aggregation::Kind op,
158159
if (default_outputs.is_empty()) { return std::move(output_with_nulls->release()[0]); }
159160

160161
// Must scatter defaults.
161-
auto scatter_map = rmm::device_uvector<size_type>(input.size(), stream);
162+
// TODO: Use a resizable buffer when available: https://github.com/NVIDIA/cccl/issues/12051
163+
auto scatter_map_storage = cuda::device_buffer<size_type>(
164+
stream, cudf::get_current_device_resource_ref(), input.size(), cuda::no_init);
162165

163166
// Find all indices at which LEAD/LAG computed nulls previously.
164167
auto scatter_map_end =
165168
cudf::detail::copy_if(cuda::counting_iterator<size_type>{0},
166169
cuda::counting_iterator<size_type>{input.size()},
167-
scatter_map.begin(),
170+
scatter_map_storage.begin(),
168171
is_null_index_predicate(input.size(), gather_map.begin<size_type>()),
169172
stream);
170173

171-
scatter_map.resize(cuda::std::distance(scatter_map.begin(), scatter_map_end), stream);
174+
auto scatter_map = cudf::device_span<size_type>{
175+
scatter_map_storage.data(),
176+
static_cast<std::size_t>(cuda::std::distance(scatter_map_storage.begin(), scatter_map_end))};
172177
// Bail early, if all LEAD/LAG computations succeeded. No defaults need be substituted.
173-
if (scatter_map.is_empty()) { return std::move(output_with_nulls->release()[0]); }
178+
if (scatter_map.empty()) { return std::move(output_with_nulls->release()[0]); }
174179

175180
// Gather only those default values that are to be substituted.
176181
auto gathered_defaults =

‎cpp/src/rolling/detail/nth_element.cuh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include <rmm/exec_policy.hpp>
1616

17+
#include <cuda/buffer>
1718
#include <cuda/iterator>
1819
#include <cuda/std/iterator>
1920
#include <cuda/std/limits>
@@ -146,7 +147,8 @@ std::unique_ptr<column> nth_element(size_type n,
146147
gather_index_calculator<null_handling, PrecedingIter, FollowingIter>{
147148
n, input, preceding, following, min_periods, stream});
148149

149-
auto gather_map = rmm::device_uvector<size_type>(input.size(), stream);
150+
auto gather_map = cuda::device_buffer<size_type>(
151+
stream, cudf::get_current_device_resource_ref(), input.size(), cuda::no_init);
150152
thrust::copy(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()),
151153
gather_iter,
152154
gather_iter + input.size(),

‎cpp/src/search/contains_scalar.cu‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include <rmm/exec_policy.hpp>
2424

25+
#include <cuda/buffer>
2526
#include <cuda/stream>
2627
#include <thrust/transform.h>
2728

@@ -114,7 +115,7 @@ struct contains_scalar_dispatch {
114115
// Using a temporary buffer for intermediate transform results from the lambda containing
115116
// the comparator speeds up compile-time significantly without much degradation in
116117
// runtime performance over using the comparator in a transform iterator with thrust::count_if.
117-
auto d_results = rmm::device_uvector<bool>(haystack.size(), stream, temp_mr);
118+
auto d_results = cuda::device_buffer<bool>(stream, temp_mr, haystack.size(), cuda::no_init);
118119
thrust::transform(
119120
rmm::exec_policy_nosync(stream, temp_mr),
120121
begin,

0 commit comments

Comments
 (0)