diff --git a/velox/dwio/nimble/encodings/common/Encoding.h b/velox/dwio/nimble/encodings/common/Encoding.h index 4d74d0f9a99..f081ad7e854 100644 --- a/velox/dwio/nimble/encodings/common/Encoding.h +++ b/velox/dwio/nimble/encodings/common/Encoding.h @@ -687,6 +687,20 @@ void readWithVisitorFast( auto numNonNulls = velox::simd::indicesOfSetBits( nulls, visitor.rowIndex(), visitor.numRows(), outerRows.data()); outerRows.resize(numNonNulls); + if constexpr (kOutputNulls) { + if (numNonNulls != numRows) { + if (!visitor.reader().returnReaderNulls()) { + params.prepareResultNulls(); + velox::bits::copyBits( + nulls, + visitor.rowIndex(), + visitor.reader().rawResultNulls(), + visitor.rowIndex(), + numRows); + } + visitor.setHasNulls(); + } + } if (outerRows.empty()) { if constexpr (kOutputNulls) { visitor.addNumValues(numRows); diff --git a/velox/dwio/nimble/encodings/legacy/FixedBitWidthEncoding.h b/velox/dwio/nimble/encodings/legacy/FixedBitWidthEncoding.h index 1e42b332b5c..5acdf29bd5f 100644 --- a/velox/dwio/nimble/encodings/legacy/FixedBitWidthEncoding.h +++ b/velox/dwio/nimble/encodings/legacy/FixedBitWidthEncoding.h @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include @@ -64,6 +65,14 @@ class FixedBitWidthEncoding final template void readWithVisitor(DecoderVisitor& visitor, ReadWithVisitorParams& params); + template + void bulkScan( + Visitor& visitor, + vector_size_t currentRow, + const vector_size_t* selectedRows, + vector_size_t numSelected, + const vector_size_t* scatterRows); + static std::string_view encode( EncodingSelection& selection, std::span values, @@ -131,6 +140,25 @@ template void FixedBitWidthEncoding::readWithVisitor( V& visitor, ReadWithVisitorParams& params) { + using OutputType = detail::ValueType; + constexpr bool kIsSuitableWidth = + (isFourByteIntegralType() || + isEightByteIntegralType()); + constexpr bool kIsFluidCast = sizeof(OutputType) >= sizeof(physicalType) && + std::is_integral_v && std::is_integral_v; + // Limit bulk decoding to supported integral conversions. + if constexpr ( + kIsSuitableWidth && + std::is_same_v< + typename V::Extract, + velox::dwio::common::ExtractToReader> && + kIsFluidCast) { + auto* nulls = visitor.reader().rawNullsInReadRange(); + if (velox::dwio::common::useFastPath(visitor, nulls)) { + detail::readWithVisitorFast(*this, visitor, params, nulls); + return; + } + } detail::readWithVisitorSlow( visitor, params, @@ -141,6 +169,106 @@ void FixedBitWidthEncoding::readWithVisitor( }); } +template +template +void FixedBitWidthEncoding::bulkScan( + V& visitor, + vector_size_t currentRow, + const vector_size_t* selectedRows, + vector_size_t numSelected, + const vector_size_t* scatterRows) { + using OutputType = detail::ValueType; + static_assert( + isFourByteIntegralType() || + isEightByteIntegralType(), + "bulkScan only supports 4-byte or 8-byte integral types"); + + if (numSelected == 0) { + return; + } + + const auto numRows = visitor.numRows() - visitor.rowIndex(); + + // Translate logical rows to the encoding's non-null row space. + const auto offset = + static_cast(row_) - static_cast(currentRow); + + auto* values = detail::mutableValues(visitor, numRows); + + constexpr bool kSameSize = sizeof(physicalType) == sizeof(OutputType); + constexpr bool kIsUpcast = sizeof(OutputType) > sizeof(physicalType) && + std::is_integral_v && std::is_integral_v; + + if constexpr (V::dense) { + if constexpr (isFourByteIntegralType()) { + if constexpr (kSameSize) { + buffer_.resize(numSelected); + fixedBitArray_.bulkGetWithBaseline( + selectedRows[0] + offset, numSelected, buffer_.data(), baseline_); + std::memcpy(values, buffer_.data(), numSelected * sizeof(physicalType)); + } else if constexpr (kIsUpcast) { + static_assert(isEightByteIntegralType()); + // Decode directly into the widened output. + fixedBitArray_.bulkGetWithBaseline32Into64( + selectedRows[0] + offset, + numSelected, + reinterpret_cast(values), + baseline_); + } + } else { + static_assert(isEightByteIntegralType()); + static_assert(kSameSize, "8-byte bulkScan requires same-size output"); + fixedBitArray_.bulkGetWithBaseline( + selectedRows[0] + offset, + numSelected, + reinterpret_cast(values), + baseline_); + } + } else { + for (vector_size_t i = 0; i < numSelected; ++i) { + values[i] = static_cast( + fixedBitArray_.get(selectedRows[i] + offset) + baseline_); + } + } + + row_ += selectedRows[numSelected - 1] - currentRow + 1; + + if constexpr (!kScatter && !V::kHasFilter && !V::kHasHook) { + visitor.addNumValues(numRows); + visitor.setRowIndex(visitor.numRows()); + return; + } + + // Apply scattering, filtering, or hooks after decoding. + if constexpr (!V::kHasHook) { + values = reinterpret_cast(visitor.reader().rawValues()); + } + + auto numValues = visitor.reader().numValues(); + int32_t* filterHits = nullptr; + if constexpr (V::kHasFilter) { + filterHits = visitor.outputRows(numSelected) - numValues; + } + + velox::dwio::common:: + processFixedWidthRun( + velox::RowSet(selectedRows, numSelected), + 0, + numSelected, + scatterRows, + values, + filterHits, + numValues, + visitor.filter(), + visitor.hook()); + + if constexpr (!V::kHasHook) { + visitor.addNumValues( + V::kHasFilter ? numValues - visitor.reader().numValues() : numRows); + } + visitor.setRowIndex(visitor.numRows()); +} + template std::string_view FixedBitWidthEncoding::encode( EncodingSelection& selection, diff --git a/velox/dwio/nimble/encodings/tests/ReadWithVisitorTest.cpp b/velox/dwio/nimble/encodings/tests/ReadWithVisitorTest.cpp index 131610a4812..a84dfa67618 100644 --- a/velox/dwio/nimble/encodings/tests/ReadWithVisitorTest.cpp +++ b/velox/dwio/nimble/encodings/tests/ReadWithVisitorTest.cpp @@ -748,6 +748,14 @@ TEST_P(ReadWithVisitorTest, denseNoFilterWithNulls) { // Every row is "output" since there is no filter. EXPECT_EQ(child->numValues(), kRows); EXPECT_TRUE(child->hasNulls()); + + std::vector rowNumbers(kRows); + std::iota(rowNumbers.begin(), rowNumbers.end(), 0); + VectorPtr result; + child->getValues(RowSet(rowNumbers.data(), rowNumbers.size()), &result); + for (int i = 0; i < kRows; ++i) { + EXPECT_EQ(result->isNullAt(i), i % 7 == 0) << "row " << i; + } } // ===========================================================================