Add isValidASCII function with tests

Author: rajatmohan22

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -1,4 +1,4 @@
-personal_ws-1.1 en 3286 
+personal_ws-1.1 en 3452 
 AArch
 ACLs
 ALTERs
@@ -495,25 +495,25 @@ JSONEachRow
 JSONEachRowWithProgress
 JSONExtract
 JSONExtractArrayRaw
-JSONExtractBool
-JSONExtractFloat
-JSONExtractInt
-JSONExtractKeys
-JSONExtractKeysAndValues
-JSONExtractKeysAndValuesRaw
-JSONExtractRaw
-JSONExtractString
-JSONExtractUInt
 JSONExtractArrayRawCaseInsensitive
+JSONExtractBool
 JSONExtractBoolCaseInsensitive
 JSONExtractCaseInsensitive
+JSONExtractFloat
 JSONExtractFloatCaseInsensitive
+JSONExtractInt
 JSONExtractIntCaseInsensitive
+JSONExtractKeys
+JSONExtractKeysAndValues
 JSONExtractKeysAndValuesCaseInsensitive
+JSONExtractKeysAndValuesRaw
 JSONExtractKeysAndValuesRawCaseInsensitive
 JSONExtractKeysCaseInsensitive
+JSONExtractRaw
 JSONExtractRawCaseInsensitive
+JSONExtractString
 JSONExtractStringCaseInsensitive
+JSONExtractUInt
 JSONExtractUIntCaseInsensitive
 JSONHas
 JSONLength
@@ -1258,6 +1258,7 @@ YYYYMMDDToDate
 YYYYMMDDhhmmssToDateTime
 Yandex
 Yasm
+YTsaurus
 ZCurve
 ZSTDQAT
 Zabbix
@@ -1953,6 +1954,7 @@ gRPC
 gaugehistogram
 gccMurmurHash
 gcem
+gdb's
 generateRandom
 generateRandomStructure
 generateSerialID
@@ -2166,6 +2168,7 @@ isNotDistinctFrom
 isNotNull
 isNull
 isNullable
+isValidASCII
 isValidJSON
 isValidUTF
 isZeroOrNull
@@ -2732,8 +2735,8 @@ quantiletdigest
 quantiletdigestweighted
 quantiletiming
 quantiletimingweighted
-quantized
 quantizations
+quantized
 quartile
 queryID
 queryString
@@ -3354,10 +3357,10 @@ updateNode
 uploader
 uploaders
 upperUTF
-uptime
-uptrace
 upsert
 upserts
+uptime
+uptrace
 uring
 url
 urlCluster
@@ -3441,6 +3444,7 @@ xz
 yaml
 yandex
 youtube
+ytsaurus
 zLib
 zLinux
 zabbix

src/Functions/isValidASCII.cpp

@@ -0,0 +1,144 @@
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/IFunction.h>
+#include <Functions/FunctionHelpers.h>
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnFixedString.h>
+#include <Columns/ColumnArray.h>
+#include <Columns/ColumnsNumber.h>
+#include <Common/Exception.h>
+#include <base/types.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+namespace
+{
+
+UInt8 isValidASCII(const UInt8 * data, UInt64 len)
+{
+    if (len == 0)
+        return 1;
+
+    for (UInt64 i = 0; i < len; ++i)
+    {
+        if (data[i] > 0x7F)
+            return 0;
+    }
+
+    return 1;
+}
+
+}
+
+class FunctionIsValidASCII : public IFunction
+{
+public:
+    static constexpr auto name = "isValidASCII";
+
+    static FunctionPtr create(ContextPtr)
+    {
+        return std::make_shared<DB::FunctionIsValidASCII>();
+    }
+
+    String getName() const override
+    {
+        return name;
+    }
+
+    size_t getNumberOfArguments() const override
+    {
+        return 1;
+    }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
+    {
+        return std::make_shared<DataTypeUInt8>();
+    }
+
+    bool useDefaultImplementationForConstants() const override
+    {
+        return true;
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    {
+        const auto & column = arguments[0];
+
+        if (const auto * col_str = checkAndGetColumn<ColumnString>(column.column.get()))
+        {
+            return executeString(col_str, input_rows_count);
+        }
+        else if (const auto * col_fixed_str = checkAndGetColumn<ColumnFixedString>(column.column.get()))
+        {
+            return executeFixedString(col_fixed_str, input_rows_count);
+        }
+        else
+        {
+            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "Cannot apply function {} to column {}, expected String or FixedString",
+                getName(), column.column->getName());
+        }
+    }
+
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override
+    {
+        return true;
+    }
+
+private:
+    ColumnPtr executeString(const ColumnString * col_str, size_t input_rows_count) const
+    {
+        auto result = ColumnUInt8::create(input_rows_count);
+        auto & result_data = result->getData();
+
+        const auto & chars = col_str->getChars();
+        const auto & offsets = col_str->getOffsets();
+
+        size_t prev_offset = 0;
+        for (size_t i = 0; i < input_rows_count; ++i)
+        {
+            size_t current_offset = offsets[i];
+            size_t string_size = current_offset - prev_offset;
+            result_data[i] = isValidASCII(chars.data() + prev_offset, string_size);
+            prev_offset = current_offset;
+        }
+
+        return result;
+    }
+
+    ColumnPtr executeFixedString(const ColumnFixedString * col_fixed_str, size_t input_rows_count) const
+    {
+        auto result = ColumnUInt8::create(input_rows_count);
+        auto & result_data = result->getData();
+
+        const auto & chars = col_fixed_str->getChars();
+        size_t string_size = col_fixed_str->getN();
+
+        for (size_t i = 0; i < input_rows_count; ++i)
+        {
+            result_data[i] = isValidASCII(chars.data() + i * string_size, string_size);
+        }
+
+        return result;
+    }
+};
+
+}
+
+REGISTER_FUNCTION(IsValidASCII)
+{
+    factory.registerFunction<DB::FunctionIsValidASCII>(DB::FunctionDocumentation{
+        .description = R"(Returns 1 if the input String or FixedString contains only ASCII bytes (0x00–0x7F), otherwise 0.)",
+        .examples = {{"isValidASCII", "SELECT isValidASCII('hello') AS is_ascii, isValidASCII('你好') AS is_not_ascii", ""}},
+        .introduced_in = {25, 8},
+        .category = DB::FunctionDocumentation::Category::Conditional,
+    });
+    factory.registerAlias("isASCII", "isValidASCII", DB::FunctionFactory::Case::Sensitive);
+}

tests/performance/isValidASCII.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <query>CREATE TABLE IF NOT EXISTS test_ascii_data (
+        id UInt32,
+        ascii_string String,
+        mixed_string String,
+        long_string String,
+        empty_string String
+    ) ENGINE = Memory</query>
+    
+    <query>INSERT INTO test_ascii_data SELECT 
+        number as id,
+        repeat('A', 100) as ascii_string,
+        repeat('Hello World!', 50) as mixed_string,
+        repeat('Pure ASCII 123 !@#', 30) as long_string,
+        '' as empty_string
+    FROM numbers(1000000)</query>
+    
+    <query>SELECT isValidASCII(ascii_string) FROM test_ascii_data FORMAT Null</query>
+    <query>SELECT isValidASCII(mixed_string) FROM test_ascii_data FORMAT Null</query>
+    <query>SELECT isValidASCII(long_string) FROM test_ascii_data FORMAT Null</query>
+    <query>SELECT isValidASCII(empty_string) FROM test_ascii_data FORMAT Null</query>
+    
+    <query>SELECT isValidASCII('Mixed\x80Content') FROM test_ascii_data LIMIT 1000000 FORMAT Null</query>
+    
+    <query>DROP TABLE IF EXISTS test_ascii_data</query>
+</clickhouse>

tests/queries/0_stateless/03594_is_valid_ascii.reference

@@ -0,0 +1,39 @@
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1

tests/queries/0_stateless/03594_is_valid_ascii.sql

@@ -0,0 +1,35 @@
+SELECT 1 = isValidASCII('') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('some text') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('\x00') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('\x66') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('\x7F') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('\x00\x7F') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('\x7F\x00') FROM system.numbers LIMIT 1;
+SELECT 0 = isValidASCII('какой-то текст') FROM system.numbers LIMIT 1;
+SELECT 0 = isValidASCII('\xC2\x80') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('hello world!') FROM system.numbers LIMIT 1;
+
+SELECT 1 = isASCII('') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('some text') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('\x00') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('\x66') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('\x7F') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('\x00\x7F') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('\x7F\x00') FROM system.numbers LIMIT 1;
+SELECT 0 = isASCII('какой-то текст') FROM system.numbers LIMIT 1;
+SELECT 0 = isASCII('\xC2\x80') FROM system.numbers LIMIT 1;
+SELECT 1 = isASCII('hello world!') FROM system.numbers LIMIT 1;
+
+SELECT isValidASCII(toString(number)) FROM system.numbers WHERE number < 10;
+
+SELECT 1 = isValidASCII('\x00') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('\x7F') FROM system.numbers LIMIT 1;
+SELECT 0 = isValidASCII('\x80') FROM system.numbers LIMIT 1;
+SELECT 0 = isValidASCII('\xFF') FROM system.numbers LIMIT 1;
+
+SELECT 0 = isValidASCII('Hello\x80World') FROM system.numbers LIMIT 1;
+SELECT 0 = isValidASCII('ASCII\xC2\x80Text') FROM system.numbers LIMIT 1;
+SELECT 1 = isValidASCII('Pure ASCII 123 !@#') FROM system.numbers LIMIT 1;
+
+SELECT 1 = isValidASCII(toFixedString('ASCII', 5)) FROM system.numbers LIMIT 1;
+SELECT 0 = isValidASCII(toFixedString('ASCII\x80', 6)) FROM system.numbers LIMIT 1; 

tests/queries/0_stateless/03594_is_valid_ascii_errors.reference

@@ -0,0 +1,4 @@
+SELECT isValidASCII([1, 2, 3]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT isValidASCII(toUUID('00000000-0000-0000-0000-000000000000')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT isValidASCII(toIPv6('::1')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT isValidASCII(toIPv4('127.0.0.1')); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }