Skip to content

Commit 0aad80b

Browse files
TCeasonTCeason
committed
feat(query): enhance datetime functions
1 parent d87b65c commit 0aad80b

File tree

15 files changed

+237
-13
lines changed

15 files changed

+237
-13
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/query/ast/src/parser/expr.rs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1214,6 +1214,13 @@ pub fn expr_element(i: Input) -> IResult<WithSpan<ExprElement>> {
12141214
|(_, _, unit, _, date, _)| ExprElement::DateTrunc { unit, date },
12151215
);
12161216

1217+
let trunc = map(
1218+
rule! {
1219+
TRUNC ~ "(" ~ #subexpr(0) ~ "," ~ #interval_kind ~ ")"
1220+
},
1221+
|(_, _, date, _, unit, _)| ExprElement::DateTrunc { unit, date },
1222+
);
1223+
12171224
let last_day = map(
12181225
rule! {
12191226
LAST_DAY ~ "(" ~ #subexpr(0) ~ ("," ~ #interval_kind)? ~ ")"
@@ -1326,7 +1333,8 @@ pub fn expr_element(i: Input) -> IResult<WithSpan<ExprElement>> {
13261333
| #date_diff : "`DATE_DIFF(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`"
13271334
| #date_sub : "`DATE_SUB(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`"
13281335
| #date_between : "`DATE_BETWEEN((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW), ..., ...,)`"
1329-
| #date_trunc : "`DATE_TRUNC((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND), ...)`"
1336+
| #date_trunc : "`DATE_TRUNC((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK), ...)`"
1337+
| #trunc : "`TRUNC(..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK))`"
13301338
| #last_day : "`LAST_DAY(..., (YEAR | QUARTER | MONTH | WEEK)))`"
13311339
| #previous_day : "`PREVIOUS_DAY(..., (Sunday | Monday | Tuesday | Wednesday | Thursday | Friday | Saturday))`"
13321340
| #next_day : "`NEXT_DAY(..., (Sunday | Monday | Tuesday | Wednesday | Thursday | Friday | Saturday))`"

‎src/query/ast/src/parser/token.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -517,6 +517,8 @@ pub enum TokenKind {
517517
DATESUB,
518518
#[token("DATE_TRUNC", ignore(ascii_case))]
519519
DATE_TRUNC,
520+
#[token("TRUNC", ignore(ascii_case))]
521+
TRUNC,
520522
#[token("DATETIME", ignore(ascii_case))]
521523
DATETIME,
522524
#[token("DAY", ignore(ascii_case))]
@@ -1672,6 +1674,7 @@ impl TokenKind {
16721674
| TokenKind::DATE_SUB
16731675
| TokenKind::DATE_BETWEEN
16741676
| TokenKind::DATE_TRUNC
1677+
| TokenKind::TRUNC
16751678
| TokenKind::LAST_DAY
16761679
| TokenKind::PREVIOUS_DAY
16771680
| TokenKind::NEXT_DAY

‎src/query/ast/tests/it/parser.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,6 +1264,8 @@ fn test_expr() {
12641264
r#"extract(year from d)"#,
12651265
r#"date_part(year, d)"#,
12661266
r#"datepart(year, d)"#,
1267+
r#"date_trunc(week, to_timestamp(1630812366))"#,
1268+
r#"trunc(to_timestamp(1630812366), week)"#,
12671269
r#"DATEDIFF(SECOND, to_timestamp('2024-01-01 21:01:35.423179'), to_timestamp('2023-12-31 09:38:18.165575'))"#,
12681270
r#"last_day(to_date('2024-10-22'), week)"#,
12691271
r#"last_day(to_date('2024-10-22'))"#,

‎src/query/ast/tests/it/testdata/expr-error.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ error:
5252
--> SQL:1:10
5353
|
5454
1 | CAST(col1)
55-
| ---- ^ unexpected `)`, expecting `AS`, `,`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, or 41 more ...
55+
| ---- ^ unexpected `)`, expecting `AS`, `,`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, or 42 more ...
5656
| |
5757
| while parsing `CAST(... AS ...)`
5858
| while parsing expression
@@ -81,7 +81,7 @@ error:
8181
1 | $ abc + 3
8282
| ^
8383
| |
84-
| unexpected `$`, expecting `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATEDIFF`, `DATESUB`, or 39 more ...
84+
| unexpected `$`, expecting `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATEDIFF`, `DATESUB`, or 40 more ...
8585
| while parsing expression
8686

8787

‎src/query/ast/tests/it/testdata/expr.txt

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1842,6 +1842,92 @@ DatePart {
18421842
}
18431843

18441844

1845+
---------- Input ----------
1846+
date_trunc(week, to_timestamp(1630812366))
1847+
---------- Output ---------
1848+
DATE_TRUNC(WEEK, to_timestamp(1630812366))
1849+
---------- AST ------------
1850+
DateTrunc {
1851+
span: Some(
1852+
0..42,
1853+
),
1854+
unit: Week,
1855+
date: FunctionCall {
1856+
span: Some(
1857+
17..41,
1858+
),
1859+
func: FunctionCall {
1860+
distinct: false,
1861+
name: Identifier {
1862+
span: Some(
1863+
17..29,
1864+
),
1865+
name: "to_timestamp",
1866+
quote: None,
1867+
ident_type: None,
1868+
},
1869+
args: [
1870+
Literal {
1871+
span: Some(
1872+
30..40,
1873+
),
1874+
value: UInt64(
1875+
1630812366,
1876+
),
1877+
},
1878+
],
1879+
params: [],
1880+
order_by: [],
1881+
window: None,
1882+
lambda: None,
1883+
},
1884+
},
1885+
}
1886+
1887+
1888+
---------- Input ----------
1889+
trunc(to_timestamp(1630812366), week)
1890+
---------- Output ---------
1891+
DATE_TRUNC(WEEK, to_timestamp(1630812366))
1892+
---------- AST ------------
1893+
DateTrunc {
1894+
span: Some(
1895+
0..37,
1896+
),
1897+
unit: Week,
1898+
date: FunctionCall {
1899+
span: Some(
1900+
6..30,
1901+
),
1902+
func: FunctionCall {
1903+
distinct: false,
1904+
name: Identifier {
1905+
span: Some(
1906+
6..18,
1907+
),
1908+
name: "to_timestamp",
1909+
quote: None,
1910+
ident_type: None,
1911+
},
1912+
args: [
1913+
Literal {
1914+
span: Some(
1915+
19..29,
1916+
),
1917+
value: UInt64(
1918+
1630812366,
1919+
),
1920+
},
1921+
],
1922+
params: [],
1923+
order_by: [],
1924+
window: None,
1925+
lambda: None,
1926+
},
1927+
},
1928+
}
1929+
1930+
18451931
---------- Input ----------
18461932
DATEDIFF(SECOND, to_timestamp('2024-01-01 21:01:35.423179'), to_timestamp('2023-12-31 09:38:18.165575'))
18471933
---------- Output ---------

‎src/query/expression/src/function.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,8 @@ pub struct FunctionContext {
172172
pub parse_datetime_ignore_remainder: bool,
173173
pub enable_strict_datetime_parser: bool,
174174
pub random_function_seed: bool,
175+
pub week_start: u8,
176+
pub date_format_style: String,
175177
}
176178

177179
impl Default for FunctionContext {
@@ -192,6 +194,8 @@ impl Default for FunctionContext {
192194
parse_datetime_ignore_remainder: false,
193195
enable_strict_datetime_parser: true,
194196
random_function_seed: false,
197+
week_start: 0,
198+
date_format_style: "mysql".to_string(),
195199
}
196200
}
197201
}

‎src/query/expression/src/utils/date_helper.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1324,9 +1324,9 @@ impl PGDateTimeFormatter {
13241324

13251325
format_map.push(("YYYY", |dt| dt.strftime("%Y").to_string()));
13261326
format_map.push(("YY", |dt| dt.strftime("%y").to_string()));
1327-
format_map.push(("MM", |dt| dt.strftime("%m").to_string()));
1328-
format_map.push(("MON", |dt| dt.strftime("%b").to_string()));
13291327
format_map.push(("MMMM", |dt| dt.strftime("%B").to_string()));
1328+
format_map.push(("MON", |dt| dt.strftime("%b").to_string()));
1329+
format_map.push(("MM", |dt| dt.strftime("%m").to_string()));
13301330
format_map.push(("DD", |dt| dt.strftime("%d").to_string()));
13311331
format_map.push(("DY", |dt| dt.strftime("%a").to_string()));
13321332
format_map.push(("HH24", |dt| dt.strftime("%H").to_string()));

‎src/query/functions/src/scalars/timestamp/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ databend-common-expression = { workspace = true }
1111
dtparse = { workspace = true }
1212
jiff = { workspace = true }
1313
num-traits = { workspace = true }
14+
regex = { workspace = true }

‎src/query/functions/src/scalars/timestamp/src/datetime.rs

Lines changed: 68 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -351,11 +351,17 @@ fn register_string_to_timestamp(registry: &mut FunctionRegistry) {
351351
"to_date",
352352
|_, _, _| FunctionDomain::MayThrow,
353353
vectorize_with_builder_2_arg::<StringType, StringType, NullableType<DateType>>(
354-
|date, format, output, ctx| {
354+
|date_string, format, output, ctx| {
355355
if format.is_empty() {
356356
output.push_null();
357357
} else {
358-
match NaiveDate::parse_from_str(date, format) {
358+
let format = if ctx.func_ctx.date_format_style == *"mysql" {
359+
format.to_string()
360+
} else {
361+
pg_format_to_strftime(format)
362+
};
363+
println!("format is {}", format.clone());
364+
match NaiveDate::parse_from_str(date_string, &format) {
359365
Ok(res) => {
360366
output.push(res.num_days_from_ce() - EPOCH_DAYS_FROM_CE);
361367
}
@@ -372,11 +378,16 @@ fn register_string_to_timestamp(registry: &mut FunctionRegistry) {
372378
"try_to_date",
373379
|_, _, _| FunctionDomain::MayThrow,
374380
vectorize_with_builder_2_arg::<StringType, StringType, NullableType<DateType>>(
375-
|date, format, output, _| {
381+
|date, format, output, ctx| {
376382
if format.is_empty() {
377383
output.push_null();
378384
} else {
379-
match NaiveDate::parse_from_str(date, format) {
385+
let format = if ctx.func_ctx.date_format_style == *"mysql" {
386+
format.to_string()
387+
} else {
388+
pg_format_to_strftime(format)
389+
};
390+
match NaiveDate::parse_from_str(date, &format) {
380391
Ok(res) => {
381392
output.push(res.num_days_from_ce() - EPOCH_DAYS_FROM_CE);
382393
}
@@ -400,7 +411,13 @@ fn string_to_format_datetime(
400411
return Ok((0, true));
401412
}
402413

403-
let (mut tm, offset) = BrokenDownTime::parse_prefix(format, timestamp)
414+
let format = if ctx.func_ctx.date_format_style == *"mysql" {
415+
format.to_string()
416+
} else {
417+
pg_format_to_strftime(format)
418+
};
419+
420+
let (mut tm, offset) = BrokenDownTime::parse_prefix(&format, timestamp)
404421
.map_err(|err| Box::new(ErrorCode::BadArguments(format!("{err}"))))?;
405422

406423
if !ctx.func_ctx.parse_datetime_ignore_remainder && offset != timestamp.len() {
@@ -705,7 +722,12 @@ fn register_to_string(registry: &mut FunctionRegistry) {
705722
vectorize_with_builder_2_arg::<TimestampType, StringType, NullableType<StringType>>(
706723
|micros, format, output, ctx| {
707724
let ts = micros.to_timestamp(ctx.func_ctx.tz.clone());
708-
let format = replace_time_format(format);
725+
let format = if ctx.func_ctx.date_format_style == *"mysql" {
726+
format.to_string()
727+
} else {
728+
pg_format_to_strftime(format)
729+
};
730+
let format = replace_time_format(&format);
709731
let mut buf = String::new();
710732
let mut formatter = fmt::Formatter::new(&mut buf, FormattingOptions::new());
711733
if Display::fmt(&ts.strftime(format.as_ref()), &mut formatter).is_err() {
@@ -2387,3 +2409,43 @@ where T: ToNumber<i32> {
23872409
}),
23882410
);
23892411
}
2412+
2413+
#[inline]
2414+
pub fn pg_format_to_strftime(pg_format_string: &str) -> String {
2415+
let mut result = pg_format_string.to_string();
2416+
2417+
let mut mappings = vec![
2418+
("YYYY", "%Y"),
2419+
("YY", "%y"),
2420+
("MMMM", "%B"),
2421+
("MON", "%b"),
2422+
("MM", "%m"),
2423+
("DD", "%d"),
2424+
("DY", "%a"),
2425+
("HH24", "%H"),
2426+
("HH12", "%I"),
2427+
("AM", "%p"),
2428+
("PM", "%p"), // AM/PM both map to %p
2429+
("MI", "%M"),
2430+
("SS", "%S"),
2431+
("FF", "%f"),
2432+
("UUUU", "%G"),
2433+
("TZH", "%z"),
2434+
("TZM", "%z"),
2435+
];
2436+
mappings.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
2437+
2438+
for (pg_key, strftime_code) in mappings {
2439+
let pattern = if pg_key == "MON" {
2440+
// should keep "month". Only "MON" as a single string escape it.
2441+
format!(r"(?i)\b{}\b", regex::escape(pg_key))
2442+
} else {
2443+
format!(r"(?i){}", regex::escape(pg_key))
2444+
};
2445+
let reg = regex::Regex::new(&pattern).expect("Failed to compile regex for format key");
2446+
2447+
// Use replace_all to substitute all occurrences of the PG key with the strftime code.
2448+
result = reg.replace_all(&result, strftime_code).to_string();
2449+
}
2450+
result
2451+
}

‎src/query/service/src/sessions/query_ctx.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,8 @@ impl TableContext for QueryContext {
966966
let geometry_output_format = settings.get_geometry_output_format()?;
967967
let parse_datetime_ignore_remainder = settings.get_parse_datetime_ignore_remainder()?;
968968
let enable_strict_datetime_parser = settings.get_enable_strict_datetime_parser()?;
969+
let week_start = settings.get_week_start()? as u8;
970+
let date_format_style = settings.get_date_format_style()?;
969971
let query_config = &GlobalConfig::instance().query;
970972
let random_function_seed = settings.get_random_function_seed()?;
971973

@@ -986,6 +988,8 @@ impl TableContext for QueryContext {
986988
parse_datetime_ignore_remainder,
987989
enable_strict_datetime_parser,
988990
random_function_seed,
991+
week_start,
992+
date_format_style,
989993
})
990994
}
991995

‎src/query/settings/src/settings_default.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,13 @@ impl DefaultSettings {
150150
scope: SettingScope::Both,
151151
range: Some(SettingRange::Numeric(1..=u64::MAX)),
152152
}),
153+
("week_start", DefaultSettingValue {
154+
value: UserSettingValue::UInt64(1),
155+
desc: "Specifies the first day of the week.",
156+
mode: SettingMode::Both,
157+
scope: SettingScope::Both,
158+
range: Some(SettingRange::Numeric(0..=1)),
159+
}),
153160
("parquet_max_block_size", DefaultSettingValue {
154161
value: UserSettingValue::UInt64(8192),
155162
desc: "Max block size for parquet reader",
@@ -324,6 +331,13 @@ impl DefaultSettings {
324331
scope: SettingScope::Both,
325332
range: Some(SettingRange::String(vec!["PostgreSQL".into(), "MySQL".into(), "Experimental".into(), "Hive".into(), "Prql".into()])),
326333
}),
334+
("date_format_style", DefaultSettingValue {
335+
value: UserSettingValue::String("MySQL".to_owned()),
336+
desc: "Sets the date format style. Available values include \"MySQL\", \"Oracle\".",
337+
mode: SettingMode::Both,
338+
scope: SettingScope::Both,
339+
range: Some(SettingRange::String(vec!["Oracle".into(), "MySQL".into()])),
340+
}),
327341
("query_tag", DefaultSettingValue {
328342
value: UserSettingValue::String("".to_owned()),
329343
desc: "Sets the query tag for this session.",

‎src/query/settings/src/settings_getter_setter.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,10 @@ impl Settings {
403403
}
404404
}
405405

406+
pub fn get_date_format_style(&self) -> Result<String> {
407+
Ok(self.try_get_string("date_format_style")?.to_lowercase())
408+
}
409+
406410
pub fn get_collation(&self) -> Result<&str> {
407411
match self.try_get_string("collation")?.to_lowercase().as_str() {
408412
"utf8" => Ok("utf8"),
@@ -762,6 +766,10 @@ impl Settings {
762766
self.try_get_u64("cost_factor_aggregate_per_row")
763767
}
764768

769+
pub fn get_week_start(&self) -> Result<u64> {
770+
self.try_get_u64("week_start")
771+
}
772+
765773
pub fn get_cost_factor_network_per_row(&self) -> Result<u64> {
766774
self.try_get_u64("cost_factor_network_per_row")
767775
}

0 commit comments

Comments
 (0)