diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc index eb8416872da1..5b23cc792070 100644 --- a/cpp/src/gandiva/function_registry_string.cc +++ b/cpp/src/gandiva/function_registry_string.cc @@ -559,6 +559,24 @@ std::vector GetStringFunctionRegistry() { kResultNullIfNull, "gdv_mask_show_last_n_utf8_int32", NativeFunction::kNeedsContext), + // Single entry point for all five Hive masking modes with caller-supplied + // replacements, for engines that normalise MASK / MASK_FIRST_N / MASK_LAST_N / + // MASK_SHOW_FIRST_N / MASK_SHOW_LAST_N into one call. + NativeFunction("mask_internal", {}, + DataTypeVector{utf8() /*text*/, utf8() /*mode*/, + int32() /*char_count*/, utf8() /*upper*/, + utf8() /*lower*/, utf8() /*digit*/, utf8() /*other*/}, + utf8(), kResultNullIfNull, "gdv_fn_mask_internal", + NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + + // The five-argument form takes the Hive otherChar: the replacement for every + // character that is neither an uppercase letter, a lowercase letter nor a decimal + // digit. The shorter forms leave those characters unchanged, which is Hive's + // default. + NativeFunction("mask", {}, DataTypeVector{utf8(), utf8(), utf8(), utf8(), utf8()}, + utf8(), kResultNullIfNull, "mask_utf8_utf8_utf8_utf8_utf8", + NativeFunction::kNeedsContext), + NativeFunction("mask", {}, DataTypeVector{utf8(), utf8(), utf8(), utf8()}, utf8(), kResultNullIfNull, "mask_utf8_utf8_utf8_utf8", NativeFunction::kNeedsContext), diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index cc5e09284d85..a835181f2012 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -387,20 +387,23 @@ const char* gdv_mask_first_n_utf8_int32(int64_t context, const char* data, return nullptr; } + // Only uppercase letters (Lu), lowercase letters (Ll) and decimal digits (Nd) + // are masked, per the Hive MASK specification (A-Z : X, a-z : x, 0-9 : n). + // Every other Unicode general category passes through unchanged -- including + // titlecase letters (Lt), other letters (Lo, e.g. CJK and Hangul), letter + // numbers (Nl, e.g. Roman numerals) and other numbers (No, e.g. fractions), + // which have no case or decimal-digit meaning to map onto X/x/n. This matches + // the ASCII mask_array above and the `mask` function below. switch (utf8proc_category(utf8_char)) { - case 1: + case UTF8PROC_CATEGORY_LU: out[out_idx] = 'X'; out_idx++; break; - case 2: + case UTF8PROC_CATEGORY_LL: out[out_idx] = 'x'; out_idx++; break; - case 9: - out[out_idx] = 'n'; - out_idx++; - break; - case 10: + case UTF8PROC_CATEGORY_ND: out[out_idx] = 'n'; out_idx++; break; @@ -502,20 +505,17 @@ const char* gdv_mask_last_n_utf8_int32(int64_t context, const char* data, auto char_len = utf8proc_iterate(reinterpret_cast(data + bytes_read), data_len, &utf8_char); + // Only Lu / Ll / Nd are masked; see gdv_mask_first_n_utf8_int32 above. switch (utf8proc_category(utf8_char)) { - case 1: + case UTF8PROC_CATEGORY_LU: out[out_idx] = 'X'; out_idx++; break; - case 2: + case UTF8PROC_CATEGORY_LL: out[out_idx] = 'x'; out_idx++; break; - case 9: - out[out_idx] = 'n'; - out_idx++; - break; - case 10: + case UTF8PROC_CATEGORY_ND: out[out_idx] = 'n'; out_idx++; break; @@ -532,19 +532,51 @@ const char* gdv_mask_last_n_utf8_int32(int64_t context, const char* data, return out; } -GANDIVA_EXPORT -const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t data_len, - const char* upper, int32_t upper_length, - const char* lower, int32_t lower_length, - const char* num, int32_t num_length, - int32_t* out_len) { +/// Shared implementation of the mask() overloads. +/// +/// \p other is the replacement for every character that is neither an uppercase letter, +/// a lowercase letter nor a decimal digit. A null \p other leaves those characters +/// unchanged, which is the Hive default and what the one- to four-argument overloads +/// pass; the five-argument overload passes the caller's replacement instead. +static const char* mask_impl(int64_t context, const char* data, int32_t data_len, + const char* upper, int32_t upper_length, const char* lower, + int32_t lower_length, const char* num, int32_t num_length, + const char* other, int32_t other_length, int32_t* out_len) { if (data_len <= 0) { *out_len = 0; return nullptr; } + // An empty replacement argument means "use the default for this class", matching + // Hive's GenericUDFMaskBase, rather than deleting the character. The default for + // `other` is to leave the character alone, so an empty `other` becomes a null one. + // + // This also keeps every replacement length >= 1, which the output size bound + // below depends on: with max_repl >= 1 and every input character at least one + // byte wide, max_repl * data_len >= sum(max(max_repl, char_len_i)), so the + // allocation covers both replaced and passed-through characters. If an empty + // argument reached the bound, max_repl would be 0 and the allocation would be + // too small for the characters that pass through unmasked. + if (upper_length <= 0) { + upper = "X"; + upper_length = 1; + } + if (lower_length <= 0) { + lower = "x"; + lower_length = 1; + } + if (num_length <= 0) { + num = "n"; + num_length = 1; + } + if (other != nullptr && other_length <= 0) { + other = nullptr; + } + int32_t max_length = - std::max(upper_length, std::max(lower_length, num_length)) * data_len; + std::max(other == nullptr ? 0 : other_length, + std::max(upper_length, std::max(lower_length, num_length))) * + data_len; char* out = reinterpret_cast(gdv_fn_context_arena_malloc(context, max_length)); if (out == nullptr) { gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); @@ -575,6 +607,9 @@ const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t } else if (isdigit(char_single_byte)) { memcpy(out + out_index, num, num_length); out_index += num_length; + } else if (other != nullptr) { + memcpy(out + out_index, other, other_length); + out_index += other_length; } else { out[out_index] = char_single_byte; out_index++; @@ -591,38 +626,36 @@ const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t auto char_len = utf8proc_iterate(reinterpret_cast(data + bytes_read), data_len, &utf8_char); + // Only uppercase letters (Lu), lowercase letters (Ll) and decimal digits (Nd) + // are masked, per the Hive MASK specification (A-Z : X, a-z : x, 0-9 : n). + // Every other Unicode general category passes through unchanged -- including + // titlecase letters (Lt), other letters (Lo, e.g. CJK and Hangul), letter + // numbers (Nl, e.g. Roman numerals) and other numbers (No, e.g. fractions). + // Those categories have no case or decimal-digit meaning to map onto X/x/n: + // scripts in Lo have no case at all, so masking them as "lowercase" would + // assert something untrue about the input. This matches the ASCII fast path + // above and gdv_mask_first_n_utf8_int32 / gdv_mask_last_n_utf8_int32. switch (utf8proc_category(utf8_char)) { case UTF8PROC_CATEGORY_LU: memcpy(out + out_index, upper, upper_length); out_index += upper_length; break; - case UTF8PROC_CATEGORY_LT: - memcpy(out + out_index, upper, upper_length); - out_index += upper_length; - break; case UTF8PROC_CATEGORY_LL: memcpy(out + out_index, lower, lower_length); out_index += lower_length; break; - case UTF8PROC_CATEGORY_LO: - memcpy(out + out_index, lower, lower_length); - out_index += lower_length; - break; case UTF8PROC_CATEGORY_ND: memcpy(out + out_index, num, num_length); out_index += num_length; break; - case UTF8PROC_CATEGORY_NL: - memcpy(out + out_index, num, num_length); - out_index += num_length; - break; - case UTF8PROC_CATEGORY_NO: - memcpy(out + out_index, num, num_length); - out_index += num_length; - break; default: - memcpy(out + out_index, data + bytes_read, char_len); - out_index += static_cast(char_len); + if (other != nullptr) { + memcpy(out + out_index, other, other_length); + out_index += other_length; + } else { + memcpy(out + out_index, data + bytes_read, char_len); + out_index += static_cast(char_len); + } break; } bytes_read += static_cast(char_len); @@ -631,24 +664,341 @@ const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t return out; } +GANDIVA_EXPORT +const char* mask_utf8_utf8_utf8_utf8_utf8(int64_t context, const char* data, + int32_t data_len, const char* upper, + int32_t upper_length, const char* lower, + int32_t lower_length, const char* num, + int32_t num_length, const char* other, + int32_t other_length, int32_t* out_len) { + return mask_impl(context, data, data_len, upper, upper_length, lower, lower_length, num, + num_length, other, other_length, out_len); +} + +GANDIVA_EXPORT +const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t data_len, + const char* upper, int32_t upper_length, + const char* lower, int32_t lower_length, + const char* num, int32_t num_length, + int32_t* out_len) { + return mask_impl(context, data, data_len, upper, upper_length, lower, lower_length, num, + num_length, nullptr, 0, out_len); +} + GANDIVA_EXPORT const char* mask_utf8_utf8_utf8(int64_t context, const char* in, int32_t length, const char* upper, int32_t upper_len, const char* lower, int32_t lower_len, int32_t* out_len) { - return mask_utf8_utf8_utf8_utf8(context, in, length, upper, upper_len, lower, lower_len, - "n", 1, out_len); + return mask_impl(context, in, length, upper, upper_len, lower, lower_len, "n", 1, + nullptr, 0, out_len); } GANDIVA_EXPORT const char* mask_utf8_utf8(int64_t context, const char* in, int32_t length, const char* upper, int32_t upper_len, int32_t* out_len) { - return mask_utf8_utf8_utf8_utf8(context, in, length, upper, upper_len, "x", 1, "n", 1, - out_len); + return mask_impl(context, in, length, upper, upper_len, "x", 1, "n", 1, nullptr, 0, + out_len); } GANDIVA_EXPORT const char* mask_utf8(int64_t context, const char* in, int32_t length, int32_t* out_len) { - return mask_utf8_utf8_utf8_utf8(context, in, length, "X", 1, "x", 1, "n", 1, out_len); + return mask_impl(context, in, length, "X", 1, "x", 1, "n", 1, nullptr, 0, out_len); +} + +// --------------------------------------------------------------------------- +// mask_internal +// +// mask_internal(text, mode, char_count, upper, lower, digit, other) -> utf8 +// +// A single entry point covering all five Hive masking modes with caller-supplied +// replacements, so an engine that normalises MASK / MASK_FIRST_N / MASK_LAST_N / +// MASK_SHOW_FIRST_N / MASK_SHOW_LAST_N into one call can evaluate every shape +// natively instead of one function per mode and arity. +// +// Classification matches the mask() family above: only uppercase letters (Lu), +// lowercase letters (Ll) and decimal digits (Nd) are masked. Everything else takes +// the `other` replacement, whose default leaves the character unchanged. +// +// Known divergence from a UTF-16 host implementation: character counts here are in +// Unicode codepoints, so a non-BMP character counts once rather than twice. This is +// visible only when a char_count boundary falls inside a surrogate pair. +// --------------------------------------------------------------------------- + +enum MaskInternalSlot { + kMaskSlotUpper = 0, + kMaskSlotLower = 1, + kMaskSlotDigit = 2, + kMaskSlotOther = 3, + kMaskSlotCount = 4 +}; + +enum MaskInternalMode { + kMaskModeFull = 0, + kMaskModeFirstN, + kMaskModeLastN, + kMaskModeShowFirstN, + kMaskModeShowLastN +}; + +// One resolved replacement. A zero length means "leave this class unchanged". +struct MaskInternalRepl { + char bytes[4]; + int32_t len; +}; + +// "Leave unmasked" is spelled as an argument that parses to -1, i.e. '-' followed by +// any number of '0's and a final '1'. Anything else either parses to a different +// value or is not numeric at all, and is used as a replacement character. +static inline bool is_mask_internal_unmasked(const char* arg, int32_t len) { + if (len < 2 || arg[0] != '-') { + return false; + } + int32_t i = 1; + while (i < len - 1 && arg[i] == '0') { + i++; + } + return i == len - 1 && arg[i] == '1'; +} + +// Resolves one replacement argument: absent or empty takes the default, the unmasked +// spelling disables masking for that class, and anything else is truncated to its +// first character. +static inline bool mask_internal_make_repl(int64_t context, const char* arg, + int32_t arg_len, int32_t default_cp, + MaskInternalRepl* out) { + int32_t codepoint = default_cp; + if (arg != nullptr && arg_len > 0) { + if (is_mask_internal_unmasked(arg, arg_len)) { + codepoint = -1; + } else { + utf8proc_int32_t decoded = 0; + auto char_len = utf8proc_iterate(reinterpret_cast(arg), + arg_len, &decoded); + if (char_len < 0) { + gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len)); + return false; + } + codepoint = decoded; + } + } + + if (codepoint < 0) { + out->len = 0; + return true; + } + auto encoded = + utf8proc_encode_char(codepoint, reinterpret_cast(out->bytes)); + if (encoded <= 0) { + gdv_fn_context_set_error_msg(context, "Invalid mask replacement character"); + return false; + } + out->len = static_cast(encoded); + return true; +} + +static inline int mask_internal_slot_of(utf8proc_int32_t codepoint) { + switch (utf8proc_category(codepoint)) { + case UTF8PROC_CATEGORY_LU: + return kMaskSlotUpper; + case UTF8PROC_CATEGORY_LL: + return kMaskSlotLower; + case UTF8PROC_CATEGORY_ND: + return kMaskSlotDigit; + default: + return kMaskSlotOther; + } +} + +// Mode names are matched exactly and case-sensitively. All five have distinct +// lengths, so the dispatch is a switch plus one comparison. +static inline bool mask_internal_parse_mode(const char* mode, int32_t mode_len, + int* out_mode) { + switch (mode_len) { + case 4: + if (memcmp(mode, "FULL", 4) == 0) { + *out_mode = kMaskModeFull; + return true; + } + break; + case 6: + if (memcmp(mode, "LAST_N", 6) == 0) { + *out_mode = kMaskModeLastN; + return true; + } + break; + case 7: + if (memcmp(mode, "FIRST_N", 7) == 0) { + *out_mode = kMaskModeFirstN; + return true; + } + break; + case 11: + if (memcmp(mode, "SHOW_LAST_N", 11) == 0) { + *out_mode = kMaskModeShowLastN; + return true; + } + break; + case 12: + if (memcmp(mode, "SHOW_FIRST_N", 12) == 0) { + *out_mode = kMaskModeShowFirstN; + return true; + } + break; + default: + break; + } + return false; +} + +// Counts codepoints and validates the encoding in one pass. Note the remaining length +// is passed to utf8proc_iterate: passing the full data_len, as the older mask stubs +// do, lets it read past the end of the buffer for a truncated trailing sequence. +static inline bool mask_internal_count_chars(int64_t context, const char* data, + int32_t data_len, int32_t* out_count) { + int32_t count = 0; + int32_t pos = 0; + utf8proc_int32_t codepoint = 0; + while (pos < data_len) { + auto char_len = + utf8proc_iterate(reinterpret_cast(data + pos), + data_len - pos, &codepoint); + if (char_len < 0) { + gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len)); + return false; + } + pos += static_cast(char_len); + count++; + } + *out_count = count; + return true; +} + +GANDIVA_EXPORT +const char* gdv_fn_mask_internal(int64_t context, const char* data, int32_t data_len, + const char* mode, int32_t mode_len, int32_t char_count, + const char* upper, int32_t upper_len, const char* lower, + int32_t lower_len, const char* digit, int32_t digit_len, + const char* other, int32_t other_len, int32_t* out_len) { + int mask_mode = kMaskModeFull; + if (!mask_internal_parse_mode(mode, mode_len, &mask_mode)) { + std::string err = "Unknown mask mode: " + + std::string(mode == nullptr ? "" : mode, std::max(mode_len, 0)); + gdv_fn_context_set_error_msg(context, err.c_str()); + *out_len = 0; + return nullptr; + } + + if (data_len <= 0) { + *out_len = 0; + return nullptr; + } + + MaskInternalRepl repls[kMaskSlotCount]; + if (!mask_internal_make_repl(context, upper, upper_len, 'X', &repls[kMaskSlotUpper]) || + !mask_internal_make_repl(context, lower, lower_len, 'x', &repls[kMaskSlotLower]) || + !mask_internal_make_repl(context, digit, digit_len, 'n', &repls[kMaskSlotDigit]) || + !mask_internal_make_repl(context, other, other_len, -1, &repls[kMaskSlotOther])) { + *out_len = 0; + return nullptr; + } + + int32_t max_repl_len = 1; + bool any_masked = false; + for (int slot = 0; slot < kMaskSlotCount; slot++) { + max_repl_len = std::max(max_repl_len, repls[slot].len); + any_masked = any_masked || repls[slot].len > 0; + } + + // Every class is unmasked, so the result is the input verbatim. + if (!any_masked) { + *out_len = data_len; + return data; + } + + int32_t num_chars = 0; + if (!mask_internal_count_chars(context, data, data_len, &num_chars)) { + *out_len = 0; + return nullptr; + } + + // A negative char_count clamps to zero; FULL ignores it entirely. + const int32_t count = std::max(char_count, 0); + int32_t mask_begin = 0; + int32_t mask_end = 0; + switch (mask_mode) { + case kMaskModeFull: + mask_end = num_chars; + break; + case kMaskModeFirstN: + mask_end = std::min(num_chars, count); + break; + case kMaskModeLastN: + mask_begin = (num_chars <= count) ? 0 : num_chars - count; + mask_end = num_chars; + break; + case kMaskModeShowFirstN: + mask_begin = std::min(num_chars, count); + mask_end = num_chars; + break; + case kMaskModeShowLastN: + mask_end = (num_chars <= count) ? 0 : num_chars - count; + break; + default: + break; + } + + if (mask_begin >= mask_end) { + *out_len = data_len; + return data; + } + + // A masked character contributes at most max_repl_len bytes and an unmasked one its + // own width, and every character is at least one byte, so + // out <= sum(max(char_len_i, max_repl_len)) <= data_len + (max_repl_len - 1) * chars + // which is exactly data_len for the common single-byte replacements. + const int64_t alloc = static_cast(data_len) + + static_cast(max_repl_len - 1) * + static_cast(num_chars); + if (alloc > std::numeric_limits::max()) { + gdv_fn_context_set_error_msg(context, "Mask output would exceed the maximum size"); + *out_len = 0; + return nullptr; + } + + char* out = reinterpret_cast( + gdv_fn_context_arena_malloc(context, static_cast(alloc))); + if (out == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_len = 0; + return nullptr; + } + + int32_t bytes_read = 0; + int32_t out_idx = 0; + int32_t char_idx = 0; + utf8proc_int32_t codepoint = 0; + while (bytes_read < data_len) { + // Already validated by mask_internal_count_chars, so char_len is positive. + auto char_len = + utf8proc_iterate(reinterpret_cast(data + bytes_read), + data_len - bytes_read, &codepoint); + const MaskInternalRepl* repl = nullptr; + if (char_idx >= mask_begin && char_idx < mask_end) { + repl = &repls[mask_internal_slot_of(codepoint)]; + } + if (repl != nullptr && repl->len > 0) { + memcpy(out + out_idx, repl->bytes, repl->len); + out_idx += repl->len; + } else { + memcpy(out + out_idx, data + bytes_read, char_len); + out_idx += static_cast(char_len); + } + bytes_read += static_cast(char_len); + char_idx++; + } + + *out_len = out_idx; + return out; } int64_t gdv_fn_to_date_utf8_utf8(int64_t context_ptr, int64_t holder_ptr, @@ -1482,6 +1832,49 @@ arrow::Status ExportedStubFunctions::AddMappings(Engine* engine) const { "gdv_mask_show_last_n_utf8_int32", types->i8_ptr_type() /*return_type*/, mask_args, reinterpret_cast(gdv_mask_show_last_n_utf8_int32)); + // gdv_fn_mask_internal + args = { + types->i64_type(), // context + types->i8_ptr_type(), // data + types->i32_type(), // data_len + types->i8_ptr_type(), // mode + types->i32_type(), // mode_len + types->i32_type(), // char_count + types->i8_ptr_type(), // upper + types->i32_type(), // upper_len + types->i8_ptr_type(), // lower + types->i32_type(), // lower_len + types->i8_ptr_type(), // digit + types->i32_type(), // digit_len + types->i8_ptr_type(), // other + types->i32_type(), // other_len + types->i32_ptr_type() // out_length + }; + + engine->AddGlobalMappingForFunc("gdv_fn_mask_internal", + types->i8_ptr_type() /*return_type*/, args, + reinterpret_cast(gdv_fn_mask_internal)); + + // mask_utf8_utf8_utf8_utf8_utf8 + args = { + types->i64_type(), // context + types->i8_ptr_type(), // data + types->i32_type(), // data_len + types->i8_ptr_type(), // upper + types->i32_type(), // upper_len + types->i8_ptr_type(), // lower + types->i32_type(), // lower_len + types->i8_ptr_type(), // num + types->i32_type(), // num_len + types->i8_ptr_type(), // other + types->i32_type(), // other_len + types->i32_ptr_type() // out_length + }; + + engine->AddGlobalMappingForFunc("mask_utf8_utf8_utf8_utf8_utf8", + types->i8_ptr_type() /*return_type*/, args, + reinterpret_cast(mask_utf8_utf8_utf8_utf8_utf8)); + // mask_utf8_utf8_utf8_utf8 args = { types->i64_type(), // context diff --git a/cpp/src/gandiva/gdv_function_stubs.h b/cpp/src/gandiva/gdv_function_stubs.h index 54480ac7f6f4..89abe3fbf678 100644 --- a/cpp/src/gandiva/gdv_function_stubs.h +++ b/cpp/src/gandiva/gdv_function_stubs.h @@ -416,6 +416,21 @@ const char* gdv_fn_substring_index(int64_t context, const char* txt, int32_t txt const char* pat, int32_t pat_len, int32_t cnt, int32_t* out_len); +GANDIVA_EXPORT +const char* gdv_fn_mask_internal(int64_t context, const char* data, int32_t data_len, + const char* mode, int32_t mode_len, int32_t char_count, + const char* upper, int32_t upper_len, const char* lower, + int32_t lower_len, const char* digit, int32_t digit_len, + const char* other, int32_t other_len, int32_t* out_len); + +GANDIVA_EXPORT +const char* mask_utf8_utf8_utf8_utf8_utf8(int64_t context, const char* in, int32_t length, + const char* upper, int32_t upper_length, + const char* lower, int32_t lower_length, + const char* num, int32_t num_length, + const char* other, int32_t other_length, + int32_t* out_len); + GANDIVA_EXPORT const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* in, int32_t length, const char* upper, int32_t upper_length, diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index d6d459f62bd5..3831700064d8 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -1388,18 +1388,28 @@ TEST(TestGdvFnStubs, TestMask) { result = mask_utf8(ctx_ptr, data.c_str(), data_len, &out_len); EXPECT_EQ(std::string(result, out_len), expected); + // An empty replacement argument means "use the default for this class", so every + // overload collapses to the default X/x/n masking rather than deleting characters. + // + // Deleting was both a divergence from Hive and an under-allocation bug: the output + // buffer is sized max(upper_len, lower_len, num_len) * data_len, which was 0 when + // all three arguments were empty, even though pass-through characters such as ':' + // and ')' still get written. SimpleArena::Allocate(0) hands back the arena cursor + // without advancing it, so those bytes landed in space the next row's allocation + // would reuse -- silent cross-row corruption in a batch rather than a crash, which + // is why this case previously passed while asserting ":)". data = "QwErTy:4)ß"; - expected = ":)"; + expected = "XxXxXx:n)x"; data_len = static_cast(data.length()); result = mask_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "", 0, "", 0, "", 0, &out_len); EXPECT_EQ(std::string(result, out_len), expected); - expected = ":n)"; result = mask_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "", 0, "", 0, &out_len); EXPECT_EQ(std::string(result, out_len), expected); - expected = "xxx:n)x"; result = mask_utf8_utf8(ctx_ptr, data.c_str(), data_len, "", 0, &out_len); EXPECT_EQ(std::string(result, out_len), expected); + result = mask_utf8(ctx_ptr, data.c_str(), data_len, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); data = "hunny-BEE-5121"; expected = "*****-\?\?\?-####"; @@ -1409,6 +1419,265 @@ TEST(TestGdvFnStubs, TestMask) { EXPECT_EQ(std::string(result, out_len), expected); } +namespace { +// Convenience wrapper mirroring how an engine calls mask_internal: every argument +// supplied, defaults spelled out. +std::string CallMaskInternal(int64_t ctx_ptr, const std::string& data, + const std::string& mode, int32_t char_count, + const std::string& upper = "X", + const std::string& lower = "x", + const std::string& digit = "n", + const std::string& other = "-1") { + int32_t out_len = 0; + const char* result = gdv_fn_mask_internal( + ctx_ptr, data.data(), static_cast(data.length()), mode.data(), + static_cast(mode.length()), char_count, upper.data(), + static_cast(upper.length()), lower.data(), + static_cast(lower.length()), digit.data(), + static_cast(digit.length()), other.data(), + static_cast(other.length()), &out_len); + return result == nullptr ? std::string() : std::string(result, out_len); +} +} // namespace + +TEST(TestGdvFnStubs, TestMaskInternalModes) { + gandiva::ExecutionContext ctx; + int64_t ctx_ptr = reinterpret_cast(&ctx); + + // FULL ignores char_count entirely. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1), "Xxx-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", 2), "Xxx-nnn"); + + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FIRST_N", 4), "Xxx-123"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "LAST_N", 4), "Abc-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "SHOW_FIRST_N", 4), "Abc-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "SHOW_LAST_N", 4), "Xxx-123"); + + // Empty input is a valid empty result, not an error. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "", "FULL", -1), ""); + EXPECT_TRUE(ctx.get_error().empty()) << ctx.get_error(); + + // Mode matching is exact and case-sensitive. + CallMaskInternal(ctx_ptr, "Abc", "full", -1); + EXPECT_THAT(ctx.get_error(), ::testing::HasSubstr("Unknown mask mode: full")); + ctx.Reset(); + CallMaskInternal(ctx_ptr, "Abc", "BOGUS", -1); + EXPECT_THAT(ctx.get_error(), ::testing::HasSubstr("Unknown mask mode")); + ctx.Reset(); +} + +TEST(TestGdvFnStubs, TestMaskInternalCharCountBoundaries) { + gandiva::ExecutionContext ctx; + int64_t ctx_ptr = reinterpret_cast(&ctx); + const std::string in = "Abc-123"; // 7 characters + + for (int32_t n : {0, -5, std::numeric_limits::min()}) { + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "FIRST_N", n), in); + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "LAST_N", n), in); + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "SHOW_FIRST_N", n), "Xxx-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "SHOW_LAST_N", n), "Xxx-nnn"); + } + + for (int32_t n : {7, 99, std::numeric_limits::max()}) { + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "FIRST_N", n), "Xxx-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "LAST_N", n), "Xxx-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "SHOW_FIRST_N", n), in); + EXPECT_EQ(CallMaskInternal(ctx_ptr, in, "SHOW_LAST_N", n), in); + } + + EXPECT_TRUE(ctx.get_error().empty()) << ctx.get_error(); +} + +TEST(TestGdvFnStubs, TestMaskInternalReplacementArguments) { + gandiva::ExecutionContext ctx; + int64_t ctx_ptr = reinterpret_cast(&ctx); + + // Custom replacements, including for the "other" class. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "Y", "y", "8", ":"), + "Yyy:888"); + + // The unmasked spelling disables one class at a time, and "-01" is the same + // sentinel while "-2" is an ordinary replacement character. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "-1"), "Axx-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "-01"), "Axx-nnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "-2"), "-xx-nnn"); + // Everything unmasked returns the input verbatim. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "-1", "-1", "-1", "-1"), + "Abc-123"); + + // Multi-character arguments are truncated to the first character. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "CAP", "low", "19"), + "Cll-111"); + + // An empty argument means "use the default for this class"; the default for + // `other` leaves the character alone. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123", "FULL", -1, "", "", "", ""), "Xxx-nnn"); + + // Ranger's MASK_SHOW_LAST_4 shape: show the last four, mask the rest to 'x'. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc-123456", "SHOW_LAST_N", 4, "x", "x", "x"), + "xxx-xx3456"); + + EXPECT_TRUE(ctx.get_error().empty()) << ctx.get_error(); +} + +TEST(TestGdvFnStubs, TestMaskInternalUnicode) { + gandiva::ExecutionContext ctx; + int64_t ctx_ptr = reinterpret_cast(&ctx); + + // Only Lu / Ll / Nd are masked; other categories take the `other` replacement, + // which by default leaves them alone. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "AaÇéß٣０5", "FULL", -1), "XxXxxnnn"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Dž世ㅏ½²Ⅷ", "FULL", -1), "Dž世ㅏ½²Ⅷ"); + // With an explicit otherChar those categories are masked, which is the only way + // to redact an uncased script. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "王小明", "FULL", -1, "X", "x", "n", "*"), "***"); + + // char_count is in characters, not bytes: A 世 a ½ 5 Ⅷ is 6 characters, 11 bytes. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "A世a½5Ⅷ", "FIRST_N", 3), "X世x½5Ⅷ"); + EXPECT_EQ(CallMaskInternal(ctx_ptr, "A世a½5Ⅷ", "LAST_N", 3), "A世a½nⅧ"); + + // A multi-byte replacement widens the output; the size bound must hold. + EXPECT_EQ(CallMaskInternal(ctx_ptr, "Abc", "FULL", -1, "Ω", "ω", "n"), "Ωωω"); + + EXPECT_TRUE(ctx.get_error().empty()) << ctx.get_error(); + + // Truncated UTF-8 is reported rather than read past the end of the buffer. + CallMaskInternal(ctx_ptr, std::string("\xE4\xB8", 2), "FULL", -1); + EXPECT_FALSE(ctx.get_error().empty()); + ctx.Reset(); +} + +// The five-argument mask takes the Hive otherChar, which replaces every character +// that is neither an uppercase letter, a lowercase letter nor a decimal digit. This +// is the only form that masks uncased scripts such as CJK. +TEST(TestGdvFnStubs, TestMaskOtherChar) { + gandiva::ExecutionContext ctx; + int64_t ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + + // ASCII fast path: punctuation is replaced rather than preserved. + std::string data = "user@dom.com"; + auto data_len = static_cast(data.length()); + const char* result = mask_utf8_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "X", + 1, "x", 1, "n", 1, "*", 1, &out_len); + EXPECT_EQ(std::string(result, out_len), "xxxx*xxx*xxx"); + + // utf8proc path: Lo, No, Nl and punctuation all take the otherChar. + // A(Lu) 世(Lo) a(Ll) ½(No) 5(Nd) Ⅷ(Nl) -(Pd) + data = "A世a½5Ⅷ-"; + data_len = static_cast(data.length()); + result = mask_utf8_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "X", 1, "x", 1, + "n", 1, "*", 1, &out_len); + EXPECT_EQ(std::string(result, out_len), "X*x*n**"); + + // The case this overload exists for: an uncased script is fully masked, where the + // four-argument form would return it verbatim. + data = "王小明"; + data_len = static_cast(data.length()); + result = mask_utf8_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "X", 1, "x", 1, + "n", 1, "*", 1, &out_len); + EXPECT_EQ(std::string(result, out_len), "***"); + result = mask_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "X", 1, "x", 1, "n", + 1, &out_len); + EXPECT_EQ(std::string(result, out_len), data); + + // An empty otherChar means "use the default", and the default is to leave the + // character alone, so it matches the four-argument form. + data = "A世a"; + data_len = static_cast(data.length()); + result = mask_utf8_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "X", 1, "x", 1, + "n", 1, "", 0, &out_len); + EXPECT_EQ(std::string(result, out_len), "X世x"); + + // A multi-byte otherChar widens the output; the size bound must account for it. + result = mask_utf8_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "X", 1, "x", 1, + "n", 1, "--", 2, &out_len); + EXPECT_EQ(std::string(result, out_len), "X--x"); + + EXPECT_TRUE(ctx.get_error().empty()) << ctx.get_error(); +} + +// Per the Hive MASK specification, only uppercase letters (Lu), lowercase letters +// (Ll) and decimal digits (Nd) are masked. Every other Unicode general category +// passes through unchanged. This test pins that contract for every mask variant, +// including the categories the two families used to disagree on: `mask` masked +// Lt/Lo/Nl/No, while mask_first_n and friends masked Nl only. +TEST(TestGdvFnStubs, TestMaskUnicodeCategories) { + gandiva::ExecutionContext ctx; + int64_t ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + + // Non-ASCII Lu / Ll / Nd are masked, exactly like their ASCII counterparts. + // A U+0041 Lu, a U+0061 Ll, Ç U+00C7 Lu, é U+00E9 Ll, ß U+00DF Ll, + // ٣ U+0663 Nd (Arabic-Indic), ０ U+FF10 Nd (fullwidth), 5 U+0035 Nd + std::string data = "AaÇéß٣０5"; + std::string expected = "XxXxxnnn"; + auto data_len = static_cast(data.length()); + const char* result = mask_utf8(ctx_ptr, data.c_str(), data_len, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); + + // Categories with no case and no decimal-digit meaning pass through untouched. + // Dž U+01C5 Lt (titlecase), 世 U+4E16 Lo, ㅏ U+314F Lo (Hangul), + // ½ U+00BD No, ² U+00B2 No, Ⅷ U+2167 Nl (Roman numeral) + data = "Dž世ㅏ½²Ⅷ"; + data_len = static_cast(data.length()); + result = mask_utf8(ctx_ptr, data.c_str(), data_len, &out_len); + EXPECT_EQ(std::string(result, out_len), data); + + // Interleaved masked and pass-through characters of differing byte widths, which + // also exercises the output-length correction when a 2- or 3-byte character is + // replaced by a 1-byte one. + // A(Lu) 世(Lo) a(Ll) ½(No) 5(Nd) Ⅷ(Nl) -- 6 characters, 11 bytes + data = "A世a½5Ⅷ"; + expected = "X世x½nⅧ"; + data_len = static_cast(data.length()); + EXPECT_EQ(data_len, 11); + result = mask_utf8(ctx_ptr, data.c_str(), data_len, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); + + // Cross-family consistency: masking every character via the *_N variants must + // now agree with `mask`. Before the Lu/Ll/Nd narrowing these disagreed on Ⅷ + // (Nl), which mask_first_n masked to 'n' via an undocumented `case 10`. + result = gdv_mask_first_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 6, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); + result = gdv_mask_last_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 6, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); + result = gdv_mask_show_first_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 0, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); + result = gdv_mask_show_last_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 0, &out_len); + EXPECT_EQ(std::string(result, out_len), expected); + + // Partial windows, counted in characters rather than bytes. + // first 3 characters are A 世 a + result = gdv_mask_first_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 3, &out_len); + EXPECT_EQ(std::string(result, out_len), "X世x½5Ⅷ"); + // last 3 characters are ½ 5 Ⅷ + result = gdv_mask_last_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 3, &out_len); + EXPECT_EQ(std::string(result, out_len), "A世a½nⅧ"); + result = gdv_mask_show_first_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 3, &out_len); + EXPECT_EQ(std::string(result, out_len), "A世a½nⅧ"); + result = gdv_mask_show_last_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 3, &out_len); + EXPECT_EQ(std::string(result, out_len), "X世x½5Ⅷ"); + + // Nl in isolation, the specific character the *_N family used to mask. + data = "Ⅷ"; + data_len = static_cast(data.length()); + result = gdv_mask_first_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 1, &out_len); + EXPECT_EQ(std::string(result, out_len), data); + result = gdv_mask_last_n_utf8_int32(ctx_ptr, data.c_str(), data_len, 1, &out_len); + EXPECT_EQ(std::string(result, out_len), data); + + // Custom replacement characters apply to Lu/Ll/Nd only; pass-through categories + // are unaffected by them. + data = "A世a½5Ⅷ"; + data_len = static_cast(data.length()); + result = mask_utf8_utf8_utf8_utf8(ctx_ptr, data.c_str(), data_len, "U", 1, "l", 1, "#", + 1, &out_len); + EXPECT_EQ(std::string(result, out_len), "U世l½#Ⅷ"); + + EXPECT_TRUE(ctx.get_error().empty()) << ctx.get_error(); +} + TEST(TestGdvFnStubs, TestAesEncryptDecrypt16) { gandiva::ExecutionContext ctx; std::string key16 = "12345678abcdefgh"; diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 268cb55a6422..f391e7d01f5e 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -3366,8 +3366,10 @@ TEST_F(TestProjector, TestMaskAll) { auto array3 = MakeArrowArrayUtf8({"n", "#", "[0-9]"}, {true, true, true}); // expected output + // Row 2 contains 的 (U+4E16 family, category Lo). Lo has no case, so it is not + // masked as "lowercase" -- it passes through, per the Hive MASK specification. auto exp_mask = MakeArrowArrayUtf8( - {"XxXxx-nnn", "CAPlowCAPlowlow-###", "Ç-l-Ç-l-l--[0-9][0-9][0-9]"}, + {"XxXxx-nnn", "CAP的CAPlowlow-###", "Ç-l-Ç-l-l--[0-9][0-9][0-9]"}, {true, true, true}); // prepare input record batch @@ -3408,8 +3410,9 @@ TEST_F(TestProjector, TestMaskUpperLower) { auto array2 = MakeArrowArrayUtf8({"x", "low", "l-"}, {true, true, true}); // expected output + // 的 (category Lo) passes through unmasked; see TestMaskAll. auto exp_mask = MakeArrowArrayUtf8( - {"XxXxx-nnn", "CAPlowCAPlowlow-nnn", "Ç-l-Ç-l-l--nnn"}, {true, true, true}); + {"XxXxx-nnn", "CAP的CAPlowlow-nnn", "Ç-l-Ç-l-l--nnn"}, {true, true, true}); // prepare input record batch auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0, array1, array2}); @@ -3446,7 +3449,8 @@ TEST_F(TestProjector, TestMaskUpper) { auto array1 = MakeArrowArrayUtf8({"X", "CAP", "Ç-"}, {true, true, true}); // expected output - auto exp_mask = MakeArrowArrayUtf8({"XxXxx-nnn", "CAPxCAPxx-nnn", "Ç-xÇ-xx-nnn"}, + // 的 (category Lo) passes through unmasked; see TestMaskAll. + auto exp_mask = MakeArrowArrayUtf8({"XxXxx-nnn", "CAP的CAPxx-nnn", "Ç-xÇ-xx-nnn"}, {true, true, true}); // prepare input record batch @@ -3482,8 +3486,10 @@ TEST_F(TestProjector, TestMaskDefault) { MakeArrowArrayUtf8({"ABCcd-123", "A的Ççd-123", "abcd-Ⅷ"}, {true, true, true}); // expected output + // Only Lu / Ll / Nd are masked. 的 (Lo) and Ⅷ (U+2167, Nl) pass through: neither + // has a case or decimal-digit meaning to map onto x or n. auto exp_mask = - MakeArrowArrayUtf8({"XXXxx-nnn", "XxXxx-nnn", "xxxx-n"}, {true, true, true}); + MakeArrowArrayUtf8({"XXXxx-nnn", "X的Xxx-nnn", "xxxx-Ⅷ"}, {true, true, true}); // prepare input record batch auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0}); @@ -3497,6 +3503,145 @@ TEST_F(TestProjector, TestMaskDefault) { EXPECT_ARROW_ARRAY_EQUALS(exp_mask, outputs.at(0)); } +// mask_internal end to end, in the shape an engine emits it: a text column plus +// literal mode, char_count and replacements. This is what exercises the generated +// call, so it is the real check on the stub's argument mapping. +TEST_F(TestProjector, TestMaskInternal) { + auto f0 = field("f0", arrow::utf8()); + auto schema = arrow::schema({f0}); + auto text = TreeExprBuilder::MakeField(f0); + + auto make_mask = [&](const std::string& mode, int32_t char_count, + const std::string& upper, const std::string& lower, + const std::string& digit, const std::string& other, + const std::string& out_name) { + auto node = TreeExprBuilder::MakeFunction( + "mask_internal", + {text, TreeExprBuilder::MakeStringLiteral(mode), + TreeExprBuilder::MakeLiteral(char_count), + TreeExprBuilder::MakeStringLiteral(upper), + TreeExprBuilder::MakeStringLiteral(lower), + TreeExprBuilder::MakeStringLiteral(digit), + TreeExprBuilder::MakeStringLiteral(other)}, + arrow::utf8()); + return TreeExprBuilder::MakeExpression(node, field(out_name, arrow::utf8())); + }; + + // Defaults, Ranger's MASK_SHOW_LAST_4 shape, and an explicit otherChar. + auto expr_full = make_mask("FULL", -1, "X", "x", "n", "-1", "full"); + auto expr_show_last = make_mask("SHOW_LAST_N", 4, "x", "x", "x", "-1", "show_last"); + auto expr_other = make_mask("FULL", -1, "X", "x", "n", "*", "other"); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr_full, expr_show_last, expr_other}, + TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + int num_records = 4; + auto array0 = MakeArrowArrayUtf8({"Abc-123456", "王小明", "Abc-123456", ""}, + {true, true, false, true}); + + auto exp_full = + MakeArrowArrayUtf8({"Xxx-nnnnnn", "王小明", "", ""}, {true, true, false, true}); + auto exp_show_last = + MakeArrowArrayUtf8({"xxx-xx3456", "王小明", "", ""}, {true, true, false, true}); + auto exp_other = + MakeArrowArrayUtf8({"Xxx*nnnnnn", "***", "", ""}, {true, true, false, true}); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + EXPECT_ARROW_ARRAY_EQUALS(exp_full, outputs.at(0)); + EXPECT_ARROW_ARRAY_EQUALS(exp_show_last, outputs.at(1)); + EXPECT_ARROW_ARRAY_EQUALS(exp_other, outputs.at(2)); +} + +// The five-argument mask, which supplies the Hive otherChar. This is the only form +// that masks characters with no case and no decimal-digit meaning, so it is what a +// masking policy over CJK, Hebrew, Arabic or Thai data needs. +TEST_F(TestProjector, TestMaskOtherChar) { + auto f0 = field("f0", arrow::utf8()); + auto f1 = field("f1", arrow::utf8()); + auto f2 = field("f2", arrow::utf8()); + auto f3 = field("f3", arrow::utf8()); + auto f4 = field("f4", arrow::utf8()); + auto schema = arrow::schema({f0, f1, f2, f3, f4}); + + auto res_mask = field("output", arrow::utf8()); + auto expr_mask = + TreeExprBuilder::MakeExpression("mask", {f0, f1, f2, f3, f4}, res_mask); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr_mask}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + int num_records = 3; + auto array0 = MakeArrowArrayUtf8({"A的Ç-1", "王小明", "user@dom.com"}, {true, true, true}); + auto array1 = MakeArrowArrayUtf8({"X", "X", "X"}, {true, true, true}); + auto array2 = MakeArrowArrayUtf8({"x", "x", "x"}, {true, true, true}); + auto array3 = MakeArrowArrayUtf8({"n", "n", "n"}, {true, true, true}); + auto array4 = MakeArrowArrayUtf8({"*", "*", "*"}, {true, true, true}); + + // 的 (Lo) and '-' both take the otherChar; 王小明 is masked instead of passing through. + auto exp_mask = + MakeArrowArrayUtf8({"X*X*n", "***", "xxxx*xxx*xxx"}, {true, true, true}); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, + {array0, array1, array2, array3, array4}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + EXPECT_ARROW_ARRAY_EQUALS(exp_mask, outputs.at(0)); +} + +// All mask variants are registered kResultNullIfNull, so a null in any argument +// yields a null result and the stub is never invoked for that row. None of the +// other mask tests exercises a null, so this pins that contract. +TEST_F(TestProjector, TestMaskNullInput) { + auto f0 = field("f0", arrow::utf8()); + auto f1 = field("f1", arrow::int32()); + auto schema = arrow::schema({f0, f1}); + + auto res_mask = field("res_mask", arrow::utf8()); + auto res_first_n = field("res_first_n", arrow::utf8()); + + auto expr_mask = TreeExprBuilder::MakeExpression("mask", {f0}, res_mask); + auto expr_first_n = + TreeExprBuilder::MakeExpression("mask_first_n", {f0, f1}, res_first_n); + + std::shared_ptr projector; + auto status = + Projector::Make(schema, {expr_mask, expr_first_n}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + // Row 1: both valid. Row 2: null text. Row 3: valid text but null n. + // Row 4: empty string, which is a valid value and distinct from null. + int num_records = 4; + auto array0 = + MakeArrowArrayUtf8({"Abc-123", "Abc-123", "Xyz", ""}, {true, false, true, true}); + auto array1 = MakeArrowArrayInt32({3, 3, 3, 3}, {true, true, false, true}); + + auto exp_mask = + MakeArrowArrayUtf8({"Xxx-nnn", "", "Xxx", ""}, {true, false, true, true}); + // mask_first_n is null wherever either argument is null. + auto exp_first_n = + MakeArrowArrayUtf8({"Xxx-123", "", "", ""}, {true, false, false, true}); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0, array1}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + EXPECT_ARROW_ARRAY_EQUALS(exp_mask, outputs.at(0)); + EXPECT_ARROW_ARRAY_EQUALS(exp_first_n, outputs.at(1)); +} + TEST_F(TestProjector, TestSqrtInt32) { auto in_field = field("in", arrow::int32()); auto schema = arrow::schema({in_field});