[feature](timestamp_ns) Add end-to-end TIMESTAMP_NS support

be/src/agent/be_exec_version_manager.cpp

@@ -131,8 +131,10 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers
 // 13: start from master
 //   a. support strict ownership hash routing for external table sink writers.
 //   b. support Paimon default fixed-bucket routing in the external sink exchange.
+// 14: start from master
+//   a. support TIMESTAMP_NS in Thrift descriptors and PBlock exchange.
 
-const int BeExecVersionManager::max_be_exec_version = 13;
+const int BeExecVersionManager::max_be_exec_version = SUPPORT_TIMESTAMP_NS_VERSION;
 const int BeExecVersionManager::min_be_exec_version = 0;
 std::map<std::string, std::set<int>> BeExecVersionManager::_function_change_map {};
 std::set<std::string> BeExecVersionManager::_function_restrict_map;

be/src/agent/be_exec_version_manager.h

@@ -29,6 +29,7 @@ constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10;
 constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11;
 constexpr inline int SUPPORT_ICEBERG_VARIANT_VERSION = 12;
 constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 13;
+constexpr inline int SUPPORT_TIMESTAMP_NS_VERSION = 14;
 
 class BeExecVersionManager {
 public:

be/src/core/call_on_type_index.h

@@ -37,6 +37,7 @@ struct TypePair {
 class DataTypeDate;
 class DataTypeDateV2;
 class DataTypeDateTimeV2;
+class DataTypeTimeStampNs;
 class DataTypeDateTime;
 class DataTypeIPv4;
 class DataTypeIPv6;
@@ -83,6 +84,8 @@ bool call_on_index_and_data_type(PrimitiveType number, F&& f) {
         return f(TypePair<DataTypeDateV2, T>());
     case PrimitiveType::TYPE_DATETIMEV2:
         return f(TypePair<DataTypeDateTimeV2, T>());
+    case PrimitiveType::TYPE_TIMESTAMP_NS:
+        return f(TypePair<DataTypeTimeStampNs, T>());
     case PrimitiveType::TYPE_DATETIME:
         return f(TypePair<DataTypeDateTime, T>());
     case PrimitiveType::TYPE_TIMEV2:
@@ -236,6 +239,8 @@ bool dispatch_type_base(PrimitiveType number, F&& f) {
             return f(DispatchDataType<TYPE_DATEV2>());
         case PrimitiveType::TYPE_DATETIMEV2:
             return f(DispatchDataType<TYPE_DATETIMEV2>());
+        case PrimitiveType::TYPE_TIMESTAMP_NS:
+            return f(DispatchDataType<TYPE_TIMESTAMP_NS>());
         case PrimitiveType::TYPE_DATETIME:
             return f(DispatchDataType<TYPE_DATETIME>());
         case PrimitiveType::TYPE_TIMEV2:

be/src/core/column/column_array.cpp

@@ -874,6 +874,9 @@ ColumnArrayDataOffsets filter_return_new_dispatch(const Filter& filt, ssize_t re
         return filter_number_return_new<TYPE_DATETIME>(filt, result_size_hint, data, offsets);
     if (check_and_get_column<ColumnDateTimeV2>(data.get()))
         return filter_number_return_new<TYPE_DATETIMEV2>(filt, result_size_hint, data, offsets);
+    if (check_and_get_column<ColumnTimeStampNs>(data.get())) {
+        return filter_number_return_new<TYPE_TIMESTAMP_NS>(filt, result_size_hint, data, offsets);
+    }
     if (check_and_get_column<ColumnTimeStampTz>(data.get()))
         return filter_number_return_new<TYPE_TIMESTAMPTZ>(filt, result_size_hint, data, offsets);
     if (check_and_get_column<ColumnTimeV2>(data.get()))
@@ -940,6 +943,9 @@ size_t filter_inplace_dispatch(const Filter& filter, IColumn& src_data,
         return filter_number_inplace<TYPE_DATETIME>(filter, src_data, src_offsets);
     if (check_and_get_column<ColumnDateTimeV2>(&src_data))
         return filter_number_inplace<TYPE_DATETIMEV2>(filter, src_data, src_offsets);
+    if (check_and_get_column<ColumnTimeStampNs>(&src_data)) {
+        return filter_number_inplace<TYPE_TIMESTAMP_NS>(filter, src_data, src_offsets);
+    }
     if (check_and_get_column<ColumnTimeStampTz>(&src_data))
         return filter_number_inplace<TYPE_TIMESTAMPTZ>(filter, src_data, src_offsets);
     if (check_and_get_column<ColumnTimeV2>(&src_data))

be/src/core/column/column_array.h

@@ -46,7 +46,7 @@ class SipHash;
             ColumnFloat64, ColumnDecimal32, ColumnDecimal64, ColumnDecimal128V3,                 \
             ColumnDecimal128V2, ColumnDecimal256
 #define ALL_COLUMNS_TIME \
-    ColumnDate, ColumnDateTime, ColumnDateV2, ColumnDateTimeV2, ColumnTimeStampTz
+    ColumnDate, ColumnDateTime, ColumnDateV2, ColumnDateTimeV2, ColumnTimeStampNs, ColumnTimeStampTz
 #define ALL_COLUMNS_NUMERIC ALL_COLUMNS_NUMBER, ALL_COLUMNS_TIME
 #define ALL_COLUMNS_SIMPLE ALL_COLUMNS_NUMERIC, ColumnString, ColumnIPv4, ColumnIPv6
 

be/src/core/column/column_vector.cpp

@@ -563,6 +563,7 @@ template class ColumnVector<TYPE_DATE>;
 template class ColumnVector<TYPE_DATEV2>;
 template class ColumnVector<TYPE_DATETIME>;
 template class ColumnVector<TYPE_DATETIMEV2>;
+template class ColumnVector<TYPE_TIMESTAMP_NS>;
 template class ColumnVector<TYPE_TIMEV2>;
 template class ColumnVector<TYPE_TIMESTAMPTZ>;
 template class ColumnVector<TYPE_UINT32>;

be/src/core/column/column_vector.h

@@ -47,6 +47,7 @@
 #include "core/string_ref.h"
 #include "core/types.h"
 #include "core/uint24.h"
+#include "core/value/timestamp_ns_value.h"
 #include "core/value/vdatetime_value.h"
 #include "util/unaligned.h"
 
@@ -70,7 +71,7 @@ template <PrimitiveType T>
 class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
     static_assert(is_int_or_bool(T) || is_ip(T) || is_date_type(T) || is_float_or_double(T) ||
                   T == TYPE_TIMEV2 || T == TYPE_UINT32 || T == TYPE_UINT64 ||
-                  T == TYPE_TIMESTAMPTZ);
+                  T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T));
 
 private:
     using Self = ColumnVector;
@@ -120,7 +121,7 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
 
     void insert_range_of_integer(value_type begin, value_type end) {
         if constexpr (!is_float_or_double(T) && T != TYPE_TIMEV2 && T != TYPE_TIMESTAMPTZ &&
-                      !is_date_type(T)) {
+                      !is_date_type(T) && !is_timestamp_ns_type(T)) {
             auto old_size = data.size();
             auto new_size = old_size + static_cast<size_t>(end - begin);
             data.resize(new_size);
@@ -321,7 +322,7 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
     }
 
     Int64 get_int(size_t n) const override {
-        if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ) {
+        if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T)) {
             throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
                                    "Method get_int is not supported for " + get_name());
             return 0;
@@ -437,6 +438,7 @@ using ColumnDate = ColumnVector<TYPE_DATE>;
 using ColumnDateTime = ColumnVector<TYPE_DATETIME>;
 using ColumnDateV2 = ColumnVector<TYPE_DATEV2>;
 using ColumnDateTimeV2 = ColumnVector<TYPE_DATETIMEV2>;
+using ColumnTimeStampNs = ColumnVector<TYPE_TIMESTAMP_NS>;
 using ColumnFloat32 = ColumnVector<TYPE_FLOAT>;
 using ColumnFloat64 = ColumnVector<TYPE_DOUBLE>;
 using ColumnIPv4 = ColumnVector<TYPE_IPV4>;
@@ -461,6 +463,7 @@ extern template class ColumnVector<TYPE_DATE>;
 extern template class ColumnVector<TYPE_DATEV2>;
 extern template class ColumnVector<TYPE_DATETIME>;
 extern template class ColumnVector<TYPE_DATETIMEV2>;
+extern template class ColumnVector<TYPE_TIMESTAMP_NS>;
 extern template class ColumnVector<TYPE_TIMEV2>;
 extern template class ColumnVector<TYPE_TIMESTAMPTZ>;
 extern template class ColumnVector<TYPE_UINT32>;

be/src/core/column/columns_common.cpp

@@ -359,6 +359,8 @@ INSTANTIATE(DateV2Value<DateV2ValueType>, IColumn::Offset)
 INSTANTIATE(DateV2Value<DateV2ValueType>, ColumnArray::Offset64)
 INSTANTIATE(DateV2Value<DateTimeV2ValueType>, IColumn::Offset)
 INSTANTIATE(DateV2Value<DateTimeV2ValueType>, ColumnArray::Offset64)
+INSTANTIATE(TimeStampNsValue, IColumn::Offset)
+INSTANTIATE(TimeStampNsValue, ColumnArray::Offset64)
 INSTANTIATE(TimestampTzValue, IColumn::Offset)
 INSTANTIATE(TimestampTzValue, ColumnArray::Offset64)
 INSTANTIATE(UInt32, IColumn::Offset)

be/src/core/column/variant_v2/column_variant_v2_typed_column.cpp

@@ -37,6 +37,7 @@ bool is_supported_variant_typed_identity(PrimitiveType type) {
     case TYPE_DATEV2:
     case TYPE_DATETIME:
     case TYPE_DATETIMEV2:
+    case TYPE_TIMESTAMP_NS:
     case TYPE_TIMESTAMPTZ:
     case TYPE_CHAR:
     case TYPE_VARCHAR:

be/src/core/column/variant_v2/column_variant_v2_typed_column.h

@@ -117,6 +117,8 @@ void with_variant_typed_scalar(const Column& column, size_t row, uint8_t scale,
     } else if constexpr (Type == TYPE_DATETIMEV2) {
         const int64_t value = variant_timestamp_micros(column.get_data()[row], row, "DATETIMEV2");
         callback(VariantScalarRef::timestamp_micros(value, false));
+    } else if constexpr (Type == TYPE_TIMESTAMP_NS) {
+        callback(VariantScalarRef::timestamp_nanos(column.get_data()[row].epoch_nanos(), false));
     } else if constexpr (Type == TYPE_TIMESTAMPTZ) {
         const int64_t value = variant_timestamp_micros(column.get_data()[row], row, "TIMESTAMPTZ");
         callback(VariantScalarRef::timestamp_micros(value, true));

be/src/core/data_type/convert_field_to_type.cpp

@@ -58,6 +58,9 @@ void dispatch(F&& f, const Field& field) {
     case PrimitiveType::TYPE_DATETIMEV2:
         f(field.template get<TYPE_DATETIMEV2>());
         return;
+    case PrimitiveType::TYPE_TIMESTAMP_NS:
+        f(field.template get<TYPE_TIMESTAMP_NS>());
+        return;
     case PrimitiveType::TYPE_TIMESTAMPTZ:
         f(field.template get<TYPE_TIMESTAMPTZ>());
         return;
@@ -180,6 +183,9 @@ class FieldVisitorToJsonb : public StaticVisitor<void> {
     void operator()(const DateV2Value<DateTimeV2ValueType>& x, JsonbWriter* writer) const {
         writer->writeInt64(*(UInt64*)&x);
     }
+    void operator()(const TimeStampNsValue& x, JsonbWriter* writer) const {
+        writer->writeInt64(x.epoch_nanos());
+    }
     void operator()(const TimestampTzValue& x, JsonbWriter* writer) const {
         writer->writeInt64(*(UInt64*)&x);
     }

be/src/core/data_type/data_type.cpp

@@ -126,6 +126,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
         return PGenericType::VARIANT;
     case PrimitiveType::TYPE_DATETIMEV2:
         return PGenericType::DATETIMEV2;
+    case PrimitiveType::TYPE_TIMESTAMP_NS:
+        return PGenericType::TIMESTAMP_NS;
     case PrimitiveType::TYPE_TIMESTAMPTZ:
         return PGenericType::TIMESTAMPTZ;
     case PrimitiveType::TYPE_BITMAP:

be/src/core/data_type/data_type_factory.cpp

@@ -59,6 +59,7 @@
 #include "core/data_type/data_type_string.h"
 #include "core/data_type/data_type_struct.h"
 #include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
 #include "core/data_type/data_type_timestamptz.h"
 #include "core/data_type/data_type_varbinary.h"
 #include "core/data_type/data_type_variant.h"
@@ -157,6 +158,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
     case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
         result = create_datetimev2(scale);
         break;
+    case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+        result = std::make_shared<DataTypeTimeStampNs>();
+        break;
     case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
         result = std::make_shared<DataTypeTimeStampTz>(scale);
         break;
@@ -263,6 +267,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
     case PGenericType::DATETIMEV2:
         nested = std::make_shared<DataTypeDateTimeV2>(pcolumn.decimal_param().scale());
         break;
+    case PGenericType::TIMESTAMP_NS:
+        nested = std::make_shared<DataTypeTimeStampNs>();
+        break;
     case PGenericType::DATETIME:
         nested = std::make_shared<DataTypeDateTime>();
         break;
@@ -441,6 +448,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type
     case TYPE_DATETIMEV2:
         nested = create_datetimev2(scale);
         break;
+    case TYPE_TIMESTAMP_NS:
+        nested = std::make_shared<DataTypeTimeStampNs>();
+        break;
     case TYPE_DATETIME:
         nested = std::make_shared<DataTypeDateTime>();
         break;

be/src/core/data_type/data_type_number_base.cpp

@@ -63,7 +63,7 @@ std::string DataTypeNumberBase<T>::to_string(
         return std::to_string(value);
     } else if constexpr (T == TYPE_DATETIME || T == TYPE_DATE) {
         return std::to_string(binary_cast<doris::VecDateTimeValue, Int64>(value));
-    } else if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ) {
+    } else if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T)) {
         return std::to_string(value.to_date_int_val());
     } else if constexpr (std::numeric_limits<typename PrimitiveTypeTraits<T>::CppType>::is_iec559) {
         return CastToString::from_number(value);
@@ -187,8 +187,12 @@ FieldWithDataType DataTypeNumberBase<T>::get_field_with_data_type(const IColumn&
             assert_cast<const ColumnVector<T>&, TypeCheckOnRelease::DISABLE>(column);
     Field field;
     column_data.get(row_num, field);
-    return FieldWithDataType {.field = std::move(field),
-                              .base_scalar_type_id = get_primitive_type()};
+    auto field_with_data_type = FieldWithDataType {.field = std::move(field),
+                                                   .base_scalar_type_id = get_primitive_type()};
+    if constexpr (T == TYPE_TIMESTAMP_NS) {
+        field_with_data_type.scale = static_cast<int>(get_scale());
+    }
+    return field_with_data_type;
 }
 
 /// Explicit template instantiations - to avoid code bloat in headers.
@@ -204,6 +208,7 @@ template class DataTypeNumberBase<TYPE_DATE>;
 template class DataTypeNumberBase<TYPE_DATEV2>;
 template class DataTypeNumberBase<TYPE_DATETIME>;
 template class DataTypeNumberBase<TYPE_DATETIMEV2>;
+template class DataTypeNumberBase<TYPE_TIMESTAMP_NS>;
 template class DataTypeNumberBase<TYPE_IPV4>;
 template class DataTypeNumberBase<TYPE_IPV6>;
 template class DataTypeNumberBase<TYPE_TIMEV2>;

be/src/core/data_type/data_type_number_base.h

@@ -48,7 +48,7 @@ class IColumn;
 template <PrimitiveType T>
 class DataTypeNumberBase : public IDataType {
     static_assert(is_int_or_bool(T) || is_ip(T) || is_date_type(T) || is_float_or_double(T) ||
-                  T == TYPE_TIMEV2 || T == TYPE_TIMESTAMPTZ);
+                  T == TYPE_TIMEV2 || T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T));
 
 public:
     static constexpr bool is_parametric = false;
@@ -108,6 +108,7 @@ extern template class DataTypeNumberBase<TYPE_DATE>;
 extern template class DataTypeNumberBase<TYPE_DATEV2>;
 extern template class DataTypeNumberBase<TYPE_DATETIME>;
 extern template class DataTypeNumberBase<TYPE_DATETIMEV2>;
+extern template class DataTypeNumberBase<TYPE_TIMESTAMP_NS>;
 extern template class DataTypeNumberBase<TYPE_IPV4>;
 extern template class DataTypeNumberBase<TYPE_IPV6>;
 extern template class DataTypeNumberBase<TYPE_TIMEV2>;

be/src/core/data_type/data_type_time.h

@@ -40,8 +40,10 @@ class IColumn;
 
 class DataTypeTimeV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_TIMEV2> {
 public:
+    static constexpr UInt32 MAX_SCALE = 6;
+
     DataTypeTimeV2(int scale = 0) : _scale(scale) {
-        if (UNLIKELY(scale > 6)) {
+        if (UNLIKELY(scale > static_cast<int>(MAX_SCALE))) {
             throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Scale {} is out of bounds", scale);
         }
         if (scale == -1) {

be/src/core/data_type/data_type_timestamp_ns.cpp

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/data_type/data_type_timestamp_ns.h"
+
+#include <typeinfo>
+
+#include "common/exception.h"
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
+#include "core/string_ref.h"
+
+namespace doris {
+
+DataTypeSerDeSPtr DataTypeTimeStampNs::get_serde(int nesting_level) const {
+    return std::make_shared<DataTypeTimeStampNsSerDe>(nesting_level);
+}
+
+Field DataTypeTimeStampNs::get_field(const TExprNode& node) const {
+    TimeStampNsValue value;
+    const StringRef string_value(node.date_literal.value.data(), node.date_literal.value.size());
+    const auto status = parse_timestamp_ns(string_value, &value);
+    if (!status.ok()) {
+        throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
+                               "Invalid value: {} for type TimeStampNs: {}",
+                               node.date_literal.value, status.to_string());
+    }
+    return Field::create_field<TYPE_TIMESTAMP_NS>(value);
+}
+
+bool DataTypeTimeStampNs::equals(const IDataType& rhs) const {
+    return typeid(rhs) == typeid(*this);
+}
+
+} // namespace doris

be/src/core/data_type/data_type_timestamp_ns.h

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "core/data_type/data_type_number_base.h"
+#include "core/value/timestamp_ns_value.h"
+
+namespace doris {
+
+class DataTypeTimeStampNs final : public DataTypeNumberBase<PrimitiveType::TYPE_TIMESTAMP_NS> {
+public:
+    const std::string get_family_name() const override { return "TimeStampNs"; }
+
+    bool equals(const IDataType& rhs) const override;
+
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override;
+
+    Field get_field(const TExprNode& node) const override;
+    UInt32 get_scale() const override { return TimeStampNsValue::FRACTIONAL_DIGITS; }
+};
+
+} // namespace doris

be/src/core/data_type/define_primitive_type.h

@@ -73,7 +73,8 @@ enum PrimitiveType : PrimitiveNative {
     TYPE_UINT64,                         /* 39, used as offset */
     TYPE_FIXED_LENGTH_OBJECT,            /* 40, represent fixed-length object on BE */
     TYPE_VARBINARY,                      /* 41, varbinary */
-    TYPE_TIMESTAMPTZ                     /* 42, timestamptz */
+    TYPE_TIMESTAMPTZ,                    /* 42, timestamptz */
+    TYPE_TIMESTAMP_NS                    /* 43, signed Int64 epoch nanoseconds */
 };
 
 } // namespace doris