diff --git a/e2e_test/batch/catalog/pg_cast.slt.part b/e2e_test/batch/catalog/pg_cast.slt.part index 00bbfbe1c0aaf..c56368863c937 100644 --- a/e2e_test/batch/catalog/pg_cast.slt.part +++ b/e2e_test/batch/catalog/pg_cast.slt.part @@ -46,46 +46,55 @@ SELECT * FROM pg_catalog.pg_cast; 42 1700 1043 a 43 1082 1043 a 44 1082 1114 i -45 1082 1184 i -46 1043 16 e -47 1043 21 e -48 1043 23 e -49 1043 20 e -50 1043 700 e -51 1043 701 e -52 1043 1700 e -53 1043 1082 e -54 1043 1083 e -55 1043 1114 e -56 1043 1184 e -57 1043 1186 e -58 1043 17 e -59 1043 3802 e -60 1043 1301 e -61 1083 1043 a -62 1083 1186 i -63 1114 1082 a -64 1114 1043 a -65 1114 1083 a -66 1114 1184 i -67 1184 1082 a -68 1184 1043 a -69 1184 1083 a -70 1184 1114 a -71 1186 1043 a -72 1186 1083 a -73 17 1043 a -74 3802 16 e -75 3802 21 e -76 3802 23 e -77 3802 20 e -78 3802 700 e -79 3802 701 e -80 3802 1700 e -81 3802 1043 a -82 20 20 e -83 1301 701 e -84 1301 1043 a +45 1082 1305 i +46 1082 1184 i +47 1043 16 e +48 1043 21 e +49 1043 23 e +50 1043 20 e +51 1043 700 e +52 1043 701 e +53 1043 1700 e +54 1043 1082 e +55 1043 1083 e +56 1043 1114 e +57 1043 1305 e +58 1043 1184 e +59 1043 1186 e +60 1043 17 e +61 1043 3802 e +62 1043 1301 e +63 1083 1043 a +64 1083 1186 i +65 1114 1082 a +66 1114 1043 a +67 1114 1083 a +68 1114 1305 i +69 1114 1184 i +70 1305 1082 a +71 1305 1043 a +72 1305 1083 a +73 1305 1114 a +74 1305 1184 i +75 1184 1082 a +76 1184 1043 a +77 1184 1083 a +78 1184 1114 a +79 1184 1305 a +80 1186 1043 a +81 1186 1083 a +82 17 1043 a +83 3802 16 e +84 3802 21 e +85 3802 23 e +86 3802 20 e +87 3802 700 e +88 3802 701 e +89 3802 1700 e +90 3802 1043 a +91 20 20 e +92 1301 701 e +93 1301 1043 a query TT rowsort SELECT s.typname, t.typname diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part index 6eb727f678380..85f5e5b5ee47a 100644 --- a/e2e_test/batch/types/timestamp_ns.slt.part +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -2,7 +2,7 @@ statement ok SET RW_IMPLICIT_FLUSH TO true; statement ok -create table t1(v1 int, v2 timestamp); +create table t1(v1 int, v2 timestamp_ns); statement ok insert into t1 values(1,'2013-01-01 01:01:01.123456789'),(2,'2012-01-01 01:01:01.123456'),(3,'0000-01-01 01:01:01.123456789'),(4,'2213-01-01 01:01:01.123456789'),(5,null),(6,'2013-01-01 01:01:01.123456789'); @@ -62,7 +62,7 @@ select * from t1 where v2 >= '2012-01-01 01:01:01.123456'; 6 2013-01-01 01:01:01.123456789 query T rowsort -select v1, cast(v2 as date) as date_v2, cast(v2 as timestamp with time zone) as timestamptz_v2 from t1; +select v1, cast(v2 as date) as date_v2, cast(v2 as timestamptz) as timestamptz_v2 from t1; ---- 1 2013-01-01 2013-01-01 01:01:01.123456+00:00 2 2012-01-01 2012-01-01 01:01:01.123456+00:00 @@ -102,7 +102,7 @@ select v1, to_char(v2, 'YYYY-MM-DD HH24:MI:SS.NS') as formatted_v2 from t1; 6 2013-01-01 01:01:01.123456789 query T rowsort -select generate_series('2013-01-01 01:01:01.123456789'::timestamp,'2013-01-01 01:01:05.123456790'::timestamp, '1 s'); +select generate_series('2013-01-01 01:01:01.123456789'::timestamp_ns,'2013-01-01 01:01:05.123456790'::timestamp_ns, '1 s'); ---- 2013-01-01 01:01:01.123456789 2013-01-01 01:01:02.123456789 @@ -194,7 +194,7 @@ select v1, extract(nanosecond from v2) from t1; 6 1123456789 query T rowsort -select make_timestamp(2013, 01, 01, 01, 01, 1.123456789); +select make_timestamp_ns(2013, 01, 01, 01, 01, 1.123456789); ---- 2013-01-01 01:01:01.123456789 diff --git a/e2e_test/iceberg/test_case/iceberg_engine.slt b/e2e_test/iceberg/test_case/iceberg_engine.slt index 1856a39b26f59..8cda03475ddfc 100644 --- a/e2e_test/iceberg/test_case/iceberg_engine.slt +++ b/e2e_test/iceberg/test_case/iceberg_engine.slt @@ -58,6 +58,7 @@ v_bool boolean, v_date date, v_timestamp timestamptz, v_ts_ntz timestamp, +v_timestamp_ns timestamp_ns, v_decimal decimal, v_map map(int, int), v_array int[], @@ -69,10 +70,10 @@ v_nested_type struct[] statement ok INSERT INTO full_type_t VALUES -(1, 1, 1, 1000, 1.1, 1.11, '1-1', true, '2022-03-11', '2022-03-11 01:00:00Z'::timestamptz, '2022-03-11 01:00:00',1.11, map {1:100,2:200}, array[1,2,3], row(1,2), '{"a":"foo", "b":"bar"}', row(1, 1, 1, 1000, 1.1, 1.11, '1-1', true, '2022-03-11', '2022-03-11 01:00:00Z'::timestamptz, '2022-03-11 01:00:00',1.11, '{"a":"foo", "b":"bar"}'), array[row(1,map {1:100,2:200}), row(2,map {3:500,4:200})]), -(2, 2, 2, 2000, 2.2, 2.22, '2-2', false, '2022-03-12', '2022-03-12 02:00:00Z'::timestamptz, '2022-03-12 02:00:00',2.22, map {3:300}, array[1,null,3], row(3,null), '{"k2":[2,true,4]}', row(2, 2, 2, 2000, 2.2, 2.22, '2-2', false, '2022-03-12', '2022-03-12 02:00:00Z'::timestamptz, '2022-03-12 02:00:00',2.22, '{"k2":[2,true,4]}'), array[]::struct[]), -(3, 3, 3, 3000, 3.3, 3.33, '3-3', true, '2022-03-13', '2022-03-13 03:00:00Z'::timestamptz, '2022-03-13 03:00:00','inf', null, null, null, '1', row(3, 3, 3, 3000, 3.3, 3.33, '3-3', true, '2022-03-13', '2022-03-13 03:00:00Z'::timestamptz, '2022-03-13 03:00:00','inf', '1'), array[row(1,map {1:100,2:200})]), -(4, 4, 4, 4000, 4.4, 4.44, '4-4', false, '2022-03-14', '2022-03-14 04:00:00Z'::timestamptz, '2022-03-14 04:00:00','-inf', null, null, null, 'true', row(4, 4, 4, 4000, 4.4, 4.44, '4-4', false, '2022-03-14', '2022-03-14 04:00:00Z'::timestamptz, '2022-03-14 04:00:00','-inf', 'true'), null); +(1, 1, 1, 1000, 1.1, 1.11, '1-1', true, '2022-03-11', '2022-03-11 01:00:00Z'::timestamptz, '2022-03-11 01:00:00','2022-03-11 01:00:00.123456789',1.11, map {1:100,2:200}, array[1,2,3], row(1,2), '{"a":"foo", "b":"bar"}', row(1, 1, 1, 1000, 1.1, 1.11, '1-1', true, '2022-03-11', '2022-03-11 01:00:00Z'::timestamptz, '2022-03-11 01:00:00',1.11, '{"a":"foo", "b":"bar"}'), array[row(1,map {1:100,2:200}), row(2,map {3:500,4:200})]), +(2, 2, 2, 2000, 2.2, 2.22, '2-2', false, '2022-03-12', '2022-03-12 02:00:00Z'::timestamptz, '2022-03-12 02:00:00','2022-03-11 01:00:00.123456789',2.22, map {3:300}, array[1,null,3], row(3,null), '{"k2":[2,true,4]}', row(2, 2, 2, 2000, 2.2, 2.22, '2-2', false, '2022-03-12', '2022-03-12 02:00:00Z'::timestamptz, '2022-03-12 02:00:00',2.22, '{"k2":[2,true,4]}'), array[]::struct[]), +(3, 3, 3, 3000, 3.3, 3.33, '3-3', true, '2022-03-13', '2022-03-13 03:00:00Z'::timestamptz, '2022-03-13 03:00:00','2022-03-11 01:00:00.123456789','inf', null, null, null, '1', row(3, 3, 3, 3000, 3.3, 3.33, '3-3', true, '2022-03-13', '2022-03-13 03:00:00Z'::timestamptz, '2022-03-13 03:00:00','inf', '1'), array[row(1,map {1:100,2:200})]), +(4, 4, 4, 4000, 4.4, 4.44, '4-4', false, '2022-03-14', '2022-03-14 04:00:00Z'::timestamptz, '2022-03-14 04:00:00','2022-03-11 01:00:00.123456789','-inf', null, null, null, 'true', row(4, 4, 4, 4000, 4.4, 4.44, '4-4', false, '2022-03-14', '2022-03-14 04:00:00Z'::timestamptz, '2022-03-14 04:00:00','-inf', 'true'), null); statement ok FLUSH; @@ -80,10 +81,10 @@ FLUSH; query ????????????????? rowsort select * from full_type_t ---- -1 1 1 1000 1.1 1.11 1-1 t 2022-03-11 2022-03-11 01:00:00+00:00 2022-03-11 01:00:00 1.1100000000 {1:100,2:200} {1,2,3} (1,2) {"a": "foo", "b": "bar"} (1,1,1,1000,1.1,1.11,1-1,t,2022-03-11,"2022-03-11 01:00:00+00:00","2022-03-11 01:00:00",1.1100000000,"{""a"": ""foo"", ""b"": ""bar""}") {"(1,\"{1:100,2:200}\")","(2,\"{3:500,4:200}\")"} -2 2 2 2000 2.2 2.22 2-2 f 2022-03-12 2022-03-12 02:00:00+00:00 2022-03-12 02:00:00 2.2200000000 {3:300} {1,NULL,3} (3,) {"k2": [2, true, 4]} (2,2,2,2000,2.2,2.22,2-2,f,2022-03-12,"2022-03-12 02:00:00+00:00","2022-03-12 02:00:00",2.2200000000,"{""k2"": [2, true, 4]}") {} -3 3 3 3000 3.3 3.33 3-3 t 2022-03-13 2022-03-13 03:00:00+00:00 2022-03-13 03:00:00 999999999999999999.9999999999 NULL NULL NULL 1 (3,3,3,3000,3.3,3.33,3-3,t,2022-03-13,"2022-03-13 03:00:00+00:00","2022-03-13 03:00:00",999999999999999999.9999999999,1) {"(1,\"{1:100,2:200}\")"} -4 4 4 4000 4.4 4.44 4-4 f 2022-03-14 2022-03-14 04:00:00+00:00 2022-03-14 04:00:00 -999999999999999999.9999999999 NULL NULL NULL true (4,4,4,4000,4.4,4.44,4-4,f,2022-03-14,"2022-03-14 04:00:00+00:00","2022-03-14 04:00:00",-999999999999999999.9999999999,true) NULL +1 1 1 1000 1.1 1.11 1-1 t 2022-03-11 2022-03-11 01:00:00+00:00 2022-03-11 01:00:00 2022-03-11 01:00:00.123456789 1.1100000000 {1:100,2:200} {1,2,3} (1,2) {"a": "foo", "b": "bar"} (1,1,1,1000,1.1,1.11,1-1,t,2022-03-11,"2022-03-11 01:00:00+00:00","2022-03-11 01:00:00",1.1100000000,"{""a"": ""foo"", ""b"": ""bar""}") {"(1,\"{1:100,2:200}\")","(2,\"{3:500,4:200}\")"} +2 2 2 2000 2.2 2.22 2-2 f 2022-03-12 2022-03-12 02:00:00+00:00 2022-03-12 02:00:00 2022-03-11 01:00:00.123456789 2.2200000000 {3:300} {1,NULL,3} (3,) {"k2": [2, true, 4]} (2,2,2,2000,2.2,2.22,2-2,f,2022-03-12,"2022-03-12 02:00:00+00:00","2022-03-12 02:00:00",2.2200000000,"{""k2"": [2, true, 4]}") {} +3 3 3 3000 3.3 3.33 3-3 t 2022-03-13 2022-03-13 03:00:00+00:00 2022-03-13 03:00:00 2022-03-11 01:00:00.123456789 999999999999999999.9999999999 NULL NULL NULL 1 (3,3,3,3000,3.3,3.33,3-3,t,2022-03-13,"2022-03-13 03:00:00+00:00","2022-03-13 03:00:00",999999999999999999.9999999999,1) {"(1,\"{1:100,2:200}\")"} +4 4 4 4000 4.4 4.44 4-4 f 2022-03-14 2022-03-14 04:00:00+00:00 2022-03-14 04:00:00 2022-03-11 01:00:00.123456789 -999999999999999999.9999999999 NULL NULL NULL true (4,4,4,4000,4.4,4.44,4-4,f,2022-03-14,"2022-03-14 04:00:00+00:00","2022-03-14 04:00:00",-999999999999999999.9999999999,true) NULL statement ok DROP TABLE full_type_t; diff --git a/proto/data.proto b/proto/data.proto index 9bb15ebcc8d62..70859b4b74a07 100644 --- a/proto/data.proto +++ b/proto/data.proto @@ -53,6 +53,7 @@ message DataType { SERIAL = 19; INT256 = 20; MAP = 21; + TIMESTAMP_NS = 22; } TypeName type_name = 1; // Data length for char. @@ -104,6 +105,7 @@ enum ArrayType { SERIAL = 17; INT256 = 18; MAP = 20; + TIMESTAMP_NS = 21; } message Array { diff --git a/proto/expr.proto b/proto/expr.proto index 57f49a1b3d7fe..880aa8b13c994 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -56,6 +56,7 @@ message ExprNode { MAKE_DATE = 113; MAKE_TIME = 114; MAKE_TIMESTAMP = 115; + MAKE_TIMESTAMP_NS = 116; // From f64 to timestamp. // e.g. `select to_timestamp(1672044740.0)` SEC_TO_TIMESTAMPTZ = 104; diff --git a/src/common/src/array/arrow/arrow_iceberg.rs b/src/common/src/array/arrow/arrow_iceberg.rs index 6882ec6f63224..172172bd97208 100644 --- a/src/common/src/array/arrow/arrow_iceberg.rs +++ b/src/common/src/array/arrow/arrow_iceberg.rs @@ -97,6 +97,7 @@ impl ToArrow for IcebergArrowConvert { DataType::Date => self.date_type_to_arrow(), DataType::Time => self.time_type_to_arrow(), DataType::Timestamp => self.timestamp_type_to_arrow(), + DataType::TimestampNs => self.timestamp_ns_type_to_arrow(), DataType::Timestamptz => self.timestamptz_type_to_arrow(), DataType::Interval => self.interval_type_to_arrow(), DataType::Varchar => self.varchar_type_to_arrow(), @@ -246,6 +247,7 @@ impl ToArrow for IcebergCreateTableArrowConvert { DataType::Date => self.date_type_to_arrow(), DataType::Time => self.time_type_to_arrow(), DataType::Timestamp => self.timestamp_type_to_arrow(), + DataType::TimestampNs => self.timestamp_ns_type_to_arrow(), DataType::Timestamptz => self.timestamptz_type_to_arrow(), DataType::Interval => self.interval_type_to_arrow(), DataType::Varchar => self.varchar_type_to_arrow(), diff --git a/src/common/src/array/arrow/arrow_impl.rs b/src/common/src/array/arrow/arrow_impl.rs index edf83f0a6f54c..ba36dea80dd47 100644 --- a/src/common/src/array/arrow/arrow_impl.rs +++ b/src/common/src/array/arrow/arrow_impl.rs @@ -107,6 +107,7 @@ pub trait ToArrow { ArrayImpl::Date(array) => self.date_to_arrow(array), ArrayImpl::Time(array) => self.time_to_arrow(array), ArrayImpl::Timestamp(array) => self.timestamp_to_arrow(array), + ArrayImpl::TimestampNs(array) => self.timestamp_ns_to_arrow(array), ArrayImpl::Timestamptz(array) => self.timestamptz_to_arrow(array), ArrayImpl::Interval(array) => self.interval_to_arrow(array), ArrayImpl::Utf8(array) => self.utf8_to_arrow(array), @@ -180,6 +181,14 @@ pub trait ToArrow { ))) } + #[inline] + fn timestamp_ns_to_arrow( + &self, + array: &TimestampNsArray, + ) -> Result { + Ok(Arc::new(arrow_array::TimestampNanosecondArray::from(array))) + } + #[inline] fn timestamptz_to_arrow( &self, @@ -319,6 +328,7 @@ pub trait ToArrow { DataType::Date => self.date_type_to_arrow(), DataType::Time => self.time_type_to_arrow(), DataType::Timestamp => self.timestamp_type_to_arrow(), + DataType::TimestampNs => self.timestamp_ns_type_to_arrow(), DataType::Timestamptz => self.timestamptz_type_to_arrow(), DataType::Interval => self.interval_type_to_arrow(), DataType::Varchar => self.varchar_type_to_arrow(), @@ -382,6 +392,10 @@ pub trait ToArrow { fn timestamp_type_to_arrow(&self) -> arrow_schema::DataType { arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, None) } + #[inline] + fn timestamp_ns_type_to_arrow(&self) -> arrow_schema::DataType { + arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, None) + } #[inline] fn timestamptz_type_to_arrow(&self) -> arrow_schema::DataType { @@ -523,7 +537,7 @@ pub trait FromArrow { Timestamp(Second, Some(_)) => DataType::Timestamptz, Timestamp(Millisecond, None) => DataType::Timestamp, Timestamp(Millisecond, Some(_)) => DataType::Timestamptz, - Timestamp(Nanosecond, None) => DataType::Timestamp, + Timestamp(Nanosecond, None) => DataType::TimestampNs, Timestamp(Nanosecond, Some(_)) => DataType::Timestamptz, Interval(MonthDayNano) => DataType::Interval, Utf8 => DataType::Varchar, @@ -622,10 +636,10 @@ pub trait FromArrow { self.from_timestampus_some_array(array.as_any().downcast_ref().unwrap()) } Timestamp(Nanosecond, None) => { - self.from_timestampns_array(array.as_any().downcast_ref().unwrap()) + self.from_timestamp_ns_array(array.as_any().downcast_ref().unwrap()) } Timestamp(Nanosecond, Some(_)) => { - self.from_timestampns_some_array(array.as_any().downcast_ref().unwrap()) + self.from_timestamp_ns_some_array(array.as_any().downcast_ref().unwrap()) } Interval(MonthDayNano) => { self.from_interval_array(array.as_any().downcast_ref().unwrap()) @@ -797,14 +811,14 @@ pub trait FromArrow { Ok(ArrayImpl::Timestamptz(array.into())) } - fn from_timestampns_array( + fn from_timestamp_ns_array( &self, array: &arrow_array::TimestampNanosecondArray, ) -> Result { - Ok(ArrayImpl::Timestamp(array.into())) + Ok(ArrayImpl::TimestampNs(array.into())) } - fn from_timestampns_some_array( + fn from_timestamp_ns_some_array( &self, array: &arrow_array::TimestampNanosecondArray, ) -> Result { @@ -1046,6 +1060,11 @@ converts_with_timeunit!(TimestamptzArray, arrow_array::TimestampMillisecondArray converts_with_timeunit!(TimestamptzArray, arrow_array::TimestampMicrosecondArray, TimeUnit::Microsecond, @map); converts_with_timeunit!(TimestamptzArray, arrow_array::TimestampNanosecondArray, TimeUnit::Nanosecond, @map); +converts_with_timeunit!(TimestampNsArray, arrow_array::TimestampSecondArray, TimeUnit::Second, @map); +converts_with_timeunit!(TimestampNsArray, arrow_array::TimestampMillisecondArray, TimeUnit::Millisecond, @map); +converts_with_timeunit!(TimestampNsArray, arrow_array::TimestampMicrosecondArray, TimeUnit::Microsecond, @map); +converts_with_timeunit!(TimestampNsArray, arrow_array::TimestampNanosecondArray, TimeUnit::Nanosecond, @map); + /// Converts RisingWave value from and into Arrow value. trait FromIntoArrow { /// The corresponding element type in the Arrow array. @@ -1162,6 +1181,35 @@ impl FromIntoArrowWithUnit for Timestamp { } } +impl FromIntoArrowWithUnit for TimestampNs { + type ArrowType = i64; + type TimestampType = TimeUnit; + + fn from_arrow_with_unit(value: Self::ArrowType, time_unit: Self::TimestampType) -> Self { + match time_unit { + TimeUnit::Second => { + TimestampNs(DateTime::from_timestamp(value as _, 0).unwrap().naive_utc()) + } + TimeUnit::Millisecond => { + TimestampNs(DateTime::from_timestamp_millis(value).unwrap().naive_utc()) + } + TimeUnit::Microsecond => { + TimestampNs(DateTime::from_timestamp_micros(value).unwrap().naive_utc()) + } + TimeUnit::Nanosecond => TimestampNs(DateTime::from_timestamp_nanos(value).naive_utc()), + } + } + + fn into_arrow_with_unit(self, time_unit: Self::TimestampType) -> Self::ArrowType { + match time_unit { + TimeUnit::Second => self.0.and_utc().timestamp(), + TimeUnit::Millisecond => self.0.and_utc().timestamp_millis(), + TimeUnit::Microsecond => self.0.and_utc().timestamp_micros(), + TimeUnit::Nanosecond => self.0.and_utc().timestamp_nanos_opt().unwrap(), + } + } +} + impl FromIntoArrowWithUnit for Timestamptz { type ArrowType = i64; type TimestampType = TimeUnit; diff --git a/src/common/src/array/chrono_array.rs b/src/common/src/array/chrono_array.rs index 0e6c9f523d9f7..91399fb62d15c 100644 --- a/src/common/src/array/chrono_array.rs +++ b/src/common/src/array/chrono_array.rs @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{PrimitiveArray, PrimitiveArrayBuilder}; +use super::{PrimitiveArray, PrimitiveArrayBuilder, TimestampNs}; use crate::types::{Date, Time, Timestamp, Timestamptz}; pub type DateArray = PrimitiveArray; pub type TimeArray = PrimitiveArray