diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 1b09399c20121..80e9187d68140 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -332,8 +332,9 @@ export class BaseQuery { this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner'); this.canUseNativeSqlPlannerPreAggregation = false; if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) { - const hasMultiStageMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }).multiStageMembers.length > 0; - this.canUseNativeSqlPlannerPreAggregation = hasMultiStageMeasures; + const fullAggregateMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }); + + this.canUseNativeSqlPlannerPreAggregation = fullAggregateMeasures.multiStageMembers.length > 0 || fullAggregateMeasures.cumulativeMeasures.length > 0; } this.queryLevelJoinHints = this.options.joinHints ?? []; this.prebuildJoin(); @@ -775,6 +776,13 @@ export class BaseQuery { ); } + driverTools(external) { + if (external && !this.options.disableExternalPreAggregations && this.externalQueryClass) { + return this.externalQuery(); + } + return this; + } + buildSqlAndParamsRust(exportAnnotatedSql) { const order = this.options.order && R.pipe( R.map((hash) => ((!hash || !hash.id) ? null : hash)), diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index 5717969612fd7..7ecc8c1aad79f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -334,4 +334,13 @@ export class CubeStoreQuery extends BaseQuery { } ); } + + public sqlTemplates() { + const templates = super.sqlTemplates(); + templates.statements.time_series_select = '{% for time_item in seria %}' + + 'select to_timestamp(\'{{ time_item[0] }}\') date_from, to_timestamp(\'{{ time_item[1] }}\') date_to \n' + + '{% if not loop.last %} UNION ALL\n{% endif %}' + + '{% endfor %}'; + return templates; + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs index 6633f487858e4..1df0a7958a6e7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs @@ -1,4 +1,5 @@ use super::base_query_options::FilterItem; +use super::driver_tools::{DriverTools, NativeDriverTools}; use super::filter_group::{FilterGroup, NativeFilterGroup}; use super::filter_params::{FilterParams, NativeFilterParams}; use super::pre_aggregation_obj::{NativePreAggregationObj, PreAggregationObj}; @@ -16,12 +17,7 @@ use std::rc::Rc; #[nativebridge::native_bridge] pub trait BaseTools { - fn convert_tz(&self, field: String) -> Result; - fn time_grouped_column( - &self, - granularity: String, - dimension: String, - ) -> Result; + fn driver_tools(&self, external: bool) -> Result, CubeError>; fn sql_templates(&self) -> Result, CubeError>; fn security_context_for_rust(&self) -> Result, CubeError>; fn sql_utils_for_rust(&self) -> Result, CubeError>; @@ -33,10 +29,6 @@ pub trait BaseTools { &self, used_filters: Option>, ) -> Result, CubeError>; - fn timestamp_precision(&self) -> Result; - fn time_stamp_cast(&self, field: String) -> Result; //TODO move to templates - fn date_time_cast(&self, field: String) -> Result; //TODO move to templates - fn in_db_time_zone(&self, date: String) -> Result; fn generate_time_series( &self, granularity: String, @@ -49,23 +41,8 @@ pub trait BaseTools { origin: String, ) -> Result>, CubeError>; fn get_allocated_params(&self) -> Result, CubeError>; - fn subtract_interval(&self, date: String, interval: String) -> Result; - fn add_interval(&self, date: String, interval: String) -> Result; - fn add_timestamp_interval(&self, date: String, interval: String) -> Result; fn all_cube_members(&self, path: String) -> Result, CubeError>; fn interval_and_minimal_time_unit(&self, interval: String) -> Result, CubeError>; - //===== TODO Move to templates - fn hll_init(&self, sql: String) -> Result; - fn hll_merge(&self, sql: String) -> Result; - fn hll_cardinality_merge(&self, sql: String) -> Result; - fn count_distinct_approx(&self, sql: String) -> Result; - fn date_bin( - &self, - interval: String, - source: String, - origin: String, - ) -> Result; - fn get_pre_aggregation_by_name( &self, cube_name: String, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/driver_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/driver_tools.rs new file mode 100644 index 0000000000000..31efcb3e94ee4 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/driver_tools.rs @@ -0,0 +1,39 @@ +use super::sql_templates_render::{NativeSqlTemplatesRender, SqlTemplatesRender}; +use cubenativeutils::wrappers::serializer::{ + NativeDeserialize, NativeDeserializer, NativeSerialize, +}; +use cubenativeutils::wrappers::NativeContextHolder; +use cubenativeutils::wrappers::NativeObjectHandle; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +#[nativebridge::native_bridge] +pub trait DriverTools { + fn convert_tz(&self, field: String) -> Result; + fn time_grouped_column( + &self, + granularity: String, + dimension: String, + ) -> Result; + fn sql_templates(&self) -> Result, CubeError>; + fn timestamp_precision(&self) -> Result; + fn time_stamp_cast(&self, field: String) -> Result; //TODO move to templates + fn date_time_cast(&self, field: String) -> Result; //TODO move to templates + fn in_db_time_zone(&self, date: String) -> Result; + fn get_allocated_params(&self) -> Result, CubeError>; + fn subtract_interval(&self, date: String, interval: String) -> Result; + fn add_interval(&self, date: String, interval: String) -> Result; + fn add_timestamp_interval(&self, date: String, interval: String) -> Result; + fn interval_and_minimal_time_unit(&self, interval: String) -> Result, CubeError>; + fn hll_init(&self, sql: String) -> Result; + fn hll_merge(&self, sql: String) -> Result; + fn hll_cardinality_merge(&self, sql: String) -> Result; + fn count_distinct_approx(&self, sql: String) -> Result; + fn date_bin( + &self, + interval: String, + source: String, + origin: String, + ) -> Result; +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs index 995438cb995b1..1262ca05575c5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs @@ -6,6 +6,7 @@ pub mod case_item; pub mod case_label; pub mod cube_definition; pub mod dimension_definition; +pub mod driver_tools; pub mod evaluator; pub mod filter_group; pub mod filter_params; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs index cf1e80b170918..1fff70496060f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs @@ -1,5 +1,4 @@ use super::*; -use crate::cube_bridge::pre_aggregation_obj::PreAggregationObj; use crate::logical_plan::*; use crate::plan::FilterItem; use crate::planner::query_tools::QueryTools; @@ -29,7 +28,7 @@ impl MatchState { pub struct PreAggregationOptimizer { query_tools: Rc, - used_pre_aggregations: HashMap<(String, String), Rc>, + used_pre_aggregations: HashMap<(String, String), Rc>, } impl PreAggregationOptimizer { @@ -71,7 +70,7 @@ impl PreAggregationOptimizer { Ok(None) } - pub fn get_used_pre_aggregations(&self) -> Vec> { + pub fn get_used_pre_aggregations(&self) -> Vec> { self.used_pre_aggregations.values().cloned().collect() } @@ -445,15 +444,14 @@ impl PreAggregationOptimizer { granularity: pre_aggregation.granularity.clone(), table_name: table_name.clone(), cube_name: pre_aggregation.cube_name.clone(), + pre_aggregation_obj, }; + let result = Rc::new(pre_aggregation); self.used_pre_aggregations.insert( - ( - pre_aggregation.cube_name.clone(), - pre_aggregation.name.clone(), - ), - pre_aggregation_obj.clone(), + (result.cube_name.clone(), result.name.clone()), + result.clone(), ); - Ok(Rc::new(pre_aggregation)) + Ok(result) } else { Err(CubeError::internal(format!( "Cannot find pre aggregation object for cube {} and name {}", diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs index 7ed2add6d21f2..85ec32be2fbc4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs @@ -1,4 +1,5 @@ use super::*; +use crate::cube_bridge::pre_aggregation_obj::PreAggregationObj; use crate::planner::sql_evaluator::MemberSymbol; use itertools::Itertools; use std::rc::Rc; @@ -13,6 +14,7 @@ pub struct PreAggregation { pub granularity: Option, pub table_name: String, pub cube_name: String, + pub pre_aggregation_obj: Rc, } impl PrettyPrint for PreAggregation { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index aa98fb7cd4b83..e1eb80557b057 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -41,15 +41,14 @@ impl PhysicalPlanBuilderContext { pub struct PhysicalPlanBuilder { query_tools: Rc, - _plan_sql_templates: PlanSqlTemplates, + plan_sql_templates: PlanSqlTemplates, } impl PhysicalPlanBuilder { - pub fn new(query_tools: Rc) -> Self { - let plan_sql_templates = query_tools.plan_sql_templates(); + pub fn new(query_tools: Rc, plan_sql_templates: PlanSqlTemplates) -> Self { Self { query_tools, - _plan_sql_templates: plan_sql_templates, + plan_sql_templates, } } @@ -102,6 +101,7 @@ impl PhysicalPlanBuilder { ) -> Result, CubeError> { let mut render_references = HashMap::new(); let mut measure_references = HashMap::new(); + let mut dimensions_references = HashMap::new(); let mut context_factory = context.make_sql_nodes_factory(); let from = match &logical_plan.source { SimpleQuerySource::LogicalJoin(join) => self.process_logical_join( @@ -114,8 +114,8 @@ impl PhysicalPlanBuilder { let res = self.process_pre_aggregation( pre_aggregation, context, - &mut render_references, &mut measure_references, + &mut dimensions_references, )?; for member in logical_plan.schema.time_dimensions.iter() { context_factory.add_dimensions_with_ignored_timezone(member.full_name()); @@ -128,6 +128,7 @@ impl PhysicalPlanBuilder { let mut select_builder = SelectBuilder::new(from); context_factory.set_ungrouped(logical_plan.ungrouped); context_factory.set_pre_aggregation_measures_references(measure_references); + context_factory.set_pre_aggregation_dimensions_references(dimensions_references); let mut group_by = Vec::new(); for member in logical_plan.schema.dimensions.iter() { @@ -185,8 +186,8 @@ impl PhysicalPlanBuilder { &self, pre_aggregation: &Rc, _context: &PhysicalPlanBuilderContext, - render_references: &mut HashMap, measure_references: &mut HashMap, + dimensions_references: &mut HashMap, ) -> Result, CubeError> { let mut pre_aggregation_schema = Schema::empty(); let pre_aggregation_alias = PlanSqlTemplates::memeber_alias_name( @@ -201,7 +202,7 @@ impl PhysicalPlanBuilder { &dim.alias_suffix(), self.query_tools.clone(), )?; - render_references.insert( + dimensions_references.insert( dim.full_name(), QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), ); @@ -214,16 +215,10 @@ impl PhysicalPlanBuilder { granularity, self.query_tools.clone(), )?; - render_references.insert( + dimensions_references.insert( dim.full_name(), QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), ); - if let Some(granularity) = &granularity { - render_references.insert( - format!("{}_{}", dim.full_name(), granularity), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), - ); - } pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(dim.full_name()))); } for meas in pre_aggregation.measures.iter() { @@ -970,9 +965,7 @@ impl PhysicalPlanBuilder { )); }; - let templates = self.query_tools.plan_sql_templates(); - - let ts_date_range = if templates.supports_generated_time_series() + let ts_date_range = if self.plan_sql_templates.supports_generated_time_series() && granularity_obj.is_predefined_granularity() { if let Some(date_range) = time_dimension_symbol diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index 5998be84e3917..d7d7b896f191c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -52,9 +52,7 @@ impl RegularRollingWindowJoinCondition { }; let trailing_start = if let Some(trailing_interval) = &self.trailing_interval { - templates - .base_tools() - .subtract_interval(start_date, trailing_interval.clone())? + templates.subtract_interval(start_date, trailing_interval.clone())? } else { start_date }; @@ -72,9 +70,7 @@ impl RegularRollingWindowJoinCondition { }; let leading_end = if let Some(leading_interval) = &self.leading_interval { - templates - .base_tools() - .add_interval(end_date, leading_interval.clone())? + templates.add_interval(end_date, leading_interval.clone())? } else { end_date }; @@ -121,7 +117,7 @@ pub struct ToDateRollingWindowJoinCondition { time_series_source: String, granularity: String, time_dimension: Expr, - query_tools: Rc, + _query_tools: Rc, } impl ToDateRollingWindowJoinCondition { @@ -135,7 +131,7 @@ impl ToDateRollingWindowJoinCondition { time_series_source, granularity, time_dimension, - query_tools, + _query_tools: query_tools, } } @@ -150,10 +146,7 @@ impl ToDateRollingWindowJoinCondition { templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?; let date_to = templates.column_reference(&Some(self.time_series_source.clone()), "date_from")?; - let grouped_from = self - .query_tools - .base_tools() - .time_grouped_column(self.granularity.clone(), date_from)?; + let grouped_from = templates.time_grouped_column(self.granularity.clone(), date_from)?; let result = format!("{date_column} >= {grouped_from} and {date_column} <= {date_to}"); Ok(result) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs index 5661affe94f59..cdad6a3c9e20b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/time_series.rs @@ -44,7 +44,6 @@ impl TimeSeries { && self.granularity.is_predefined_granularity() { let interval_description = templates - .base_tools() .interval_and_minimal_time_unit(self.granularity.granularity_interval().clone())?; if interval_description.len() != 2 { return Err(CubeError::internal( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index 14651d0a4febf..681be99b46773 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -2,8 +2,9 @@ use super::planners::QueryPlanner; use super::query_tools::QueryTools; use super::QueryProperties; use crate::cube_bridge::base_query_options::BaseQueryOptions; -use crate::cube_bridge::pre_aggregation_obj::{NativePreAggregationObj, PreAggregationObj}; +use crate::cube_bridge::pre_aggregation_obj::NativePreAggregationObj; use crate::logical_plan::optimizers::*; +use crate::logical_plan::PreAggregation; use crate::logical_plan::Query; use crate::physical_plan_builder::PhysicalPlanBuilder; use cubenativeutils::wrappers::inner_types::InnerTypes; @@ -78,19 +79,30 @@ impl BaseQuery { } fn build_sql_and_params_impl(&self) -> Result, CubeError> { - let templates = self.query_tools.plan_sql_templates(); let query_planner = QueryPlanner::new(self.request.clone(), self.query_tools.clone()); let logical_plan = query_planner.plan()?; let (optimized_plan, used_pre_aggregations) = self.try_pre_aggregations(logical_plan.clone())?; - let physical_plan_builder = PhysicalPlanBuilder::new(self.query_tools.clone()); + let is_external = if !used_pre_aggregations.is_empty() { + used_pre_aggregations + .iter() + .all(|pre_aggregation| pre_aggregation.external) + } else { + false + }; + + let templates = self.query_tools.plan_sql_templates(is_external)?; + + let physical_plan_builder = + PhysicalPlanBuilder::new(self.query_tools.clone(), templates.clone()); let original_sql_pre_aggregations = if !self.request.is_pre_aggregation_query() { OriginalSqlCollector::new(self.query_tools.clone()).collect(&optimized_plan)? } else { HashMap::new() }; + let physical_plan = physical_plan_builder.build( optimized_plan, original_sql_pre_aggregations, @@ -98,7 +110,9 @@ impl BaseQuery { )?; let sql = physical_plan.to_sql(&templates)?; - let (result_sql, params) = self.query_tools.build_sql_and_params(&sql, true)?; + let (result_sql, params) = self + .query_tools + .build_sql_and_params(&sql, true, &templates)?; let res = self.context.empty_array()?; res.set(0, result_sql.to_native(self.context.clone())?)?; @@ -107,6 +121,7 @@ impl BaseQuery { res.set( 2, used_pre_aggregations + .pre_aggregation_obj .clone() .as_any() .downcast::>() @@ -122,7 +137,7 @@ impl BaseQuery { fn try_pre_aggregations( &self, plan: Rc, - ) -> Result<(Rc, Vec>), CubeError> { + ) -> Result<(Rc, Vec>), CubeError> { let result = if !self.request.is_pre_aggregation_query() { let mut pre_aggregation_optimizer = PreAggregationOptimizer::new(self.query_tools.clone()); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index fe3fa953d4bd8..e6d616c5c950c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -1,8 +1,8 @@ use super::filter_operator::FilterOperator; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::MemberSymbol; -use crate::planner::sql_templates::filter::FilterTemplates; use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_templates::TemplateProjectionColumn; use crate::planner::QueryDateTimeHelper; use crate::planner::{evaluate_with_context, FiltersContext, VisitorContext}; use cubenativeutils::CubeError; @@ -26,7 +26,6 @@ pub struct BaseFilter { filter_operator: FilterOperator, values: Vec>, use_raw_values: bool, - templates: FilterTemplates, } impl PartialEq for BaseFilter { @@ -45,7 +44,6 @@ impl BaseFilter { filter_operator: FilterOperator, values: Option>>, ) -> Result, CubeError> { - let templates = FilterTemplates::new(query_tools.templates_render()); let values = if let Some(values) = values { values } else { @@ -57,7 +55,6 @@ impl BaseFilter { filter_type, filter_operator, values, - templates, use_raw_values: false, })) } @@ -74,7 +71,6 @@ impl BaseFilter { filter_type: self.filter_type.clone(), filter_operator, values, - templates: self.templates.clone(), use_raw_values, }) } @@ -134,47 +130,83 @@ impl BaseFilter { let filters_context = context.filters_context(); let res = match self.filter_operator { - FilterOperator::Equal => self.equals_where(&member_sql, filters_context)?, - FilterOperator::NotEqual => self.not_equals_where(&member_sql, filters_context)?, - FilterOperator::InDateRange => self.in_date_range(&member_sql, filters_context)?, - FilterOperator::BeforeDate => self.before_date(&member_sql, filters_context)?, + FilterOperator::Equal => { + self.equals_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::NotEqual => { + self.not_equals_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::InDateRange => { + self.in_date_range(&member_sql, plan_templates, filters_context)? + } + FilterOperator::BeforeDate => { + self.before_date(&member_sql, plan_templates, filters_context)? + } FilterOperator::BeforeOrOnDate => { - self.before_or_on_date(&member_sql, filters_context)? + self.before_or_on_date(&member_sql, plan_templates, filters_context)? + } + FilterOperator::AfterDate => { + self.after_date(&member_sql, plan_templates, filters_context)? } - FilterOperator::AfterDate => self.after_date(&member_sql, filters_context)?, FilterOperator::AfterOrOnDate => { - self.after_or_on_date(&member_sql, filters_context)? + self.after_or_on_date(&member_sql, plan_templates, filters_context)? } FilterOperator::NotInDateRange => { - self.not_in_date_range(&member_sql, filters_context)? + self.not_in_date_range(&member_sql, plan_templates, filters_context)? + } + FilterOperator::RegularRollingWindowDateRange => self + .regular_rolling_window_date_range( + &member_sql, + plan_templates, + filters_context, + )?, + FilterOperator::ToDateRollingWindowDateRange => self + .to_date_rolling_window_date_range( + &member_sql, + plan_templates, + filters_context, + )?, + FilterOperator::In => { + self.in_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::NotIn => { + self.not_in_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::Set => { + self.set_where(&member_sql, plan_templates, filters_context)? } - FilterOperator::RegularRollingWindowDateRange => { - self.regular_rolling_window_date_range(&member_sql, filters_context)? + FilterOperator::NotSet => { + self.not_set_where(&member_sql, plan_templates, filters_context)? } - FilterOperator::ToDateRollingWindowDateRange => { - self.to_date_rolling_window_date_range(&member_sql, filters_context)? + FilterOperator::Gt => { + self.gt_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::Gte => { + self.gte_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::Lt => { + self.lt_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::Lte => { + self.lte_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::Contains => { + self.contains_where(&member_sql, plan_templates, filters_context)? } - FilterOperator::In => self.in_where(&member_sql, filters_context)?, - FilterOperator::NotIn => self.not_in_where(&member_sql, filters_context)?, - FilterOperator::Set => self.set_where(&member_sql, filters_context)?, - FilterOperator::NotSet => self.not_set_where(&member_sql, filters_context)?, - FilterOperator::Gt => self.gt_where(&member_sql, filters_context)?, - FilterOperator::Gte => self.gte_where(&member_sql, filters_context)?, - FilterOperator::Lt => self.lt_where(&member_sql, filters_context)?, - FilterOperator::Lte => self.lte_where(&member_sql, filters_context)?, - FilterOperator::Contains => self.contains_where(&member_sql, filters_context)?, FilterOperator::NotContains => { - self.not_contains_where(&member_sql, filters_context)? + self.not_contains_where(&member_sql, plan_templates, filters_context)? } FilterOperator::StartsWith => { - self.starts_with_where(&member_sql, filters_context)? + self.starts_with_where(&member_sql, plan_templates, filters_context)? } FilterOperator::NotStartsWith => { - self.not_starts_with_where(&member_sql, filters_context)? + self.not_starts_with_where(&member_sql, plan_templates, filters_context)? + } + FilterOperator::EndsWith => { + self.ends_with_where(&member_sql, plan_templates, filters_context)? } - FilterOperator::EndsWith => self.ends_with_where(&member_sql, filters_context)?, FilterOperator::NotEndsWith => { - self.not_ends_with_where(&member_sql, filters_context)? + self.not_ends_with_where(&member_sql, plan_templates, filters_context)? } FilterOperator::MeasureFilter => { return Err(CubeError::internal(format!( @@ -196,7 +228,7 @@ impl BaseFilter { if measure_symbol.measure_filters().is_empty() && measure_symbol.measure_drill_filters().is_empty() { - self.templates.always_true()? + plan_templates.always_true()? } else { let visitor = context.make_visitor(self.query_tools.clone()); let node_processor = context.node_processor(); @@ -220,7 +252,7 @@ impl BaseFilter { .join(" AND ") } } - _ => self.templates.always_true()?, + _ => plan_templates.always_true()?, }; Ok(res) } @@ -228,107 +260,111 @@ impl BaseFilter { fn equals_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { let need_null_check = self.is_need_null_chek(false); if self.is_array_value() { - self.templates.in_where( + plan_templates.in_where( member_sql.to_string(), self.filter_and_allocate_values(), need_null_check, ) } else if self.is_values_contains_null() { - self.templates.not_set_where(member_sql.to_string()) + plan_templates.not_set_where(member_sql.to_string()) } else { - self.templates - .equals(member_sql.to_string(), self.first_param()?, need_null_check) + plan_templates.equals(member_sql.to_string(), self.first_param()?, need_null_check) } } fn not_equals_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { let need_null_check = self.is_need_null_chek(true); if self.is_array_value() { - self.templates.not_in_where( + plan_templates.not_in_where( member_sql.to_string(), self.filter_and_allocate_values(), need_null_check, ) } else if self.is_values_contains_null() { - self.templates.set_where(member_sql.to_string()) + plan_templates.set_where(member_sql.to_string()) } else { - self.templates - .not_equals(member_sql.to_string(), self.first_param()?, need_null_check) + plan_templates.not_equals(member_sql.to_string(), self.first_param()?, need_null_check) } } fn in_date_range( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, filters_context: &FiltersContext, ) -> Result { let use_db_time_zone = !filters_context.use_local_tz; - let (from, to) = self.allocate_date_params(use_db_time_zone, false)?; - self.templates - .time_range_filter(member_sql.to_string(), from, to) + let (from, to) = self.allocate_date_params(use_db_time_zone, false, plan_templates)?; + plan_templates.time_range_filter(member_sql.to_string(), from, to) } fn not_in_date_range( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, filters_context: &FiltersContext, ) -> Result { let use_db_time_zone = !filters_context.use_local_tz; - let (from, to) = self.allocate_date_params(use_db_time_zone, false)?; - self.templates - .time_not_in_range_filter(member_sql.to_string(), from, to) + let (from, to) = self.allocate_date_params(use_db_time_zone, false, plan_templates)?; + plan_templates.time_not_in_range_filter(member_sql.to_string(), from, to) } fn before_date( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, filters_context: &FiltersContext, ) -> Result { let use_db_time_zone = !filters_context.use_local_tz; - let value = self.first_timestamp_param(use_db_time_zone, false)?; + let value = self.first_timestamp_param(use_db_time_zone, false, plan_templates)?; - self.templates.lt(member_sql.to_string(), value) + plan_templates.lt(member_sql.to_string(), value) } fn before_or_on_date( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, filters_context: &FiltersContext, ) -> Result { let use_db_time_zone = !filters_context.use_local_tz; - let value = self.first_timestamp_param(use_db_time_zone, false)?; + let value = self.first_timestamp_param(use_db_time_zone, false, plan_templates)?; - self.templates.lte(member_sql.to_string(), value) + plan_templates.lte(member_sql.to_string(), value) } fn after_date( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, filters_context: &FiltersContext, ) -> Result { let use_db_time_zone = !filters_context.use_local_tz; - let value = self.first_timestamp_param(use_db_time_zone, false)?; + let value = self.first_timestamp_param(use_db_time_zone, false, plan_templates)?; - self.templates.gt(member_sql.to_string(), value) + plan_templates.gt(member_sql.to_string(), value) } fn after_or_on_date( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, filters_context: &FiltersContext, ) -> Result { let use_db_time_zone = !filters_context.use_local_tz; - let value = self.first_timestamp_param(use_db_time_zone, false)?; + let value = self.first_timestamp_param(use_db_time_zone, false, plan_templates)?; - self.templates.gte(member_sql.to_string(), value) + plan_templates.gte(member_sql.to_string(), value) } fn extend_date_range_bound( @@ -336,21 +372,16 @@ impl BaseFilter { date: String, interval: &Option, is_sub: bool, + plan_templates: &PlanSqlTemplates, ) -> Result, CubeError> { if let Some(interval) = interval { if interval != "unbounded" { if is_sub { Ok(Some( - self.query_tools - .base_tools() - .subtract_interval(date, interval.clone())?, + plan_templates.subtract_interval(date, interval.clone())?, )) } else { - Ok(Some( - self.query_tools - .base_tools() - .add_interval(date, interval.clone())?, - )) + Ok(Some(plan_templates.add_interval(date, interval.clone())?)) } } else { Ok(None) @@ -360,53 +391,96 @@ impl BaseFilter { } } + fn date_range_from_time_series( + &self, + plan_templates: &PlanSqlTemplates, + ) -> Result<(String, String), CubeError> { + let from_expr = format!("min(date_from)"); + let to_expr = format!("max(date_to)"); + let alias = format!("value"); + let time_series_cte_name = format!("time_series"); // FIXME May be should be passed as parameter + + let from_column = TemplateProjectionColumn { + expr: from_expr.clone(), + alias: alias.clone(), + aliased: plan_templates.column_aliased(&from_expr, &alias)?, + }; + + let to_column = TemplateProjectionColumn { + expr: to_expr.clone(), + alias: alias.clone(), + aliased: plan_templates.column_aliased(&to_expr, &alias)?, + }; + let from = plan_templates.select( + vec![], + &time_series_cte_name, + vec![from_column], + None, + vec![], + None, + vec![], + None, + None, + false, + )?; + let to = plan_templates.select( + vec![], + &time_series_cte_name, + vec![to_column], + None, + vec![], + None, + vec![], + None, + None, + false, + )?; + Ok((format!("({})", from), format!("({})", to))) + } + fn regular_rolling_window_date_range( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - let (from, to) = self.allocate_date_params(false, true)?; + let (from, to) = self.date_range_from_time_series(plan_templates)?; let from = if self.values.len() >= 3 { - self.extend_date_range_bound(from, &self.values[2], true)? + self.extend_date_range_bound(from, &self.values[2], true, plan_templates)? } else { Some(from) }; let to = if self.values.len() >= 4 { - self.extend_date_range_bound(to, &self.values[3], false)? + self.extend_date_range_bound(to, &self.values[3], false, plan_templates)? } else { Some(to) }; - let date_field = self - .query_tools - .base_tools() - .convert_tz(member_sql.to_string())?; + let date_field = plan_templates.convert_tz(member_sql.to_string())?; if let (Some(from), Some(to)) = (&from, &to) { - self.templates - .time_range_filter(date_field, from.clone(), to.clone()) + plan_templates.time_range_filter(date_field, from.clone(), to.clone()) } else if let Some(from) = &from { - self.templates.gte(date_field, from.clone()) + plan_templates.gte(date_field, from.clone()) } else if let Some(to) = &to { - self.templates.lte(date_field, to.clone()) + plan_templates.lte(date_field, to.clone()) } else { - self.templates.always_true() + plan_templates.always_true() } } fn to_date_rolling_window_date_range( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - let (from, to) = self.allocate_date_params(false, true)?; + let (from, to) = self.date_range_from_time_series(plan_templates)?; let from = if self.values.len() >= 3 { if let Some(granularity) = &self.values[2] { - self.query_tools - .base_tools() - .time_grouped_column(granularity.clone(), from)? + plan_templates.time_grouped_column(granularity.clone(), from)? } else { return Err(CubeError::user(format!( "Granularity required for to_date rolling window" @@ -418,20 +492,18 @@ impl BaseFilter { ))); }; - let date_field = self - .query_tools - .base_tools() - .convert_tz(member_sql.to_string())?; - self.templates.time_range_filter(date_field, from, to) + let date_field = plan_templates.convert_tz(member_sql.to_string())?; + plan_templates.time_range_filter(date_field, from, to) } fn in_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { let need_null_check = self.is_need_null_chek(false); - self.templates.in_where( + plan_templates.in_where( member_sql.to_string(), self.filter_and_allocate_values(), need_null_check, @@ -441,10 +513,11 @@ impl BaseFilter { fn not_in_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { let need_null_check = self.is_need_null_chek(true); - self.templates.not_in_where( + plan_templates.not_in_where( member_sql.to_string(), self.filter_and_allocate_values(), need_null_check, @@ -454,101 +527,109 @@ impl BaseFilter { fn set_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.templates.set_where(member_sql.to_string()) + plan_templates.set_where(member_sql.to_string()) } fn not_set_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.templates.not_set_where(member_sql.to_string()) + plan_templates.not_set_where(member_sql.to_string()) } fn gt_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.templates - .gt(member_sql.to_string(), self.first_param()?) + plan_templates.gt(member_sql.to_string(), self.first_param()?) } fn gte_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.templates - .gte(member_sql.to_string(), self.first_param()?) + plan_templates.gte(member_sql.to_string(), self.first_param()?) } fn lt_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.templates - .lt(member_sql.to_string(), self.first_param()?) + plan_templates.lt(member_sql.to_string(), self.first_param()?) } fn lte_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.templates - .lte(member_sql.to_string(), self.first_param()?) + plan_templates.lte(member_sql.to_string(), self.first_param()?) } fn contains_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.like_or_where(member_sql, false, true, true) + self.like_or_where(member_sql, false, true, true, plan_templates) } fn not_contains_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.like_or_where(member_sql, true, true, true) + self.like_or_where(member_sql, true, true, true, plan_templates) } fn starts_with_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.like_or_where(member_sql, false, false, true) + self.like_or_where(member_sql, false, false, true, plan_templates) } fn not_starts_with_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.like_or_where(member_sql, true, false, true) + self.like_or_where(member_sql, true, false, true, plan_templates) } fn ends_with_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.like_or_where(member_sql, false, true, false) + self.like_or_where(member_sql, false, true, false, plan_templates) } fn not_ends_with_where( &self, member_sql: &str, + plan_templates: &PlanSqlTemplates, _filters_context: &FiltersContext, ) -> Result { - self.like_or_where(member_sql, true, true, false) + self.like_or_where(member_sql, true, true, false, plan_templates) } fn like_or_where( @@ -557,18 +638,16 @@ impl BaseFilter { not: bool, start_wild: bool, end_wild: bool, + plan_templates: &PlanSqlTemplates, ) -> Result { let values = self.filter_and_allocate_values(); let like_parts = values .into_iter() - .map(|v| { - self.templates - .ilike(member_sql, &v, start_wild, end_wild, not) - }) + .map(|v| plan_templates.ilike(member_sql, &v, start_wild, end_wild, not)) .collect::, _>>()?; let logical_symbol = if not { " AND " } else { " OR " }; let null_check = if self.is_need_null_chek(not) { - self.templates.or_is_null_check(member_sql.to_string())? + plan_templates.or_is_null_check(member_sql.to_string())? } else { "".to_string() }; @@ -583,14 +662,15 @@ impl BaseFilter { &self, value: &String, use_db_time_zone: bool, + plan_templates: &PlanSqlTemplates, ) -> Result { if self.use_raw_values { return Ok(value.clone()); } - let from = self.format_from_date(value)?; + let from = self.format_from_date(value, plan_templates)?; let res = if use_db_time_zone && from != FROM_PARTITION_RANGE { - self.query_tools.base_tools().in_db_time_zone(from)? + plan_templates.in_db_time_zone(from)? } else { from }; @@ -601,14 +681,15 @@ impl BaseFilter { &self, value: &String, use_db_time_zone: bool, + plan_templates: &PlanSqlTemplates, ) -> Result { if self.use_raw_values { return Ok(value.clone()); } - let from = self.format_to_date(value)?; + let from = self.format_to_date(value, plan_templates)?; let res = if use_db_time_zone && from != TO_PARTITION_RANGE { - self.query_tools.base_tools().in_db_time_zone(from)? + plan_templates.in_db_time_zone(from)? } else { from }; @@ -619,10 +700,11 @@ impl BaseFilter { &self, use_db_time_zone: bool, as_date_time: bool, + plan_templates: &PlanSqlTemplates, ) -> Result<(String, String), CubeError> { if self.values.len() >= 2 { let from = if let Some(from_str) = &self.values[0] { - self.from_date_in_db_time_zone(from_str, use_db_time_zone)? + self.from_date_in_db_time_zone(from_str, use_db_time_zone, plan_templates)? } else { return Err(CubeError::user(format!( "Arguments for date range is not valid" @@ -630,14 +712,14 @@ impl BaseFilter { }; let to = if let Some(to_str) = &self.values[1] { - self.to_date_in_db_time_zone(to_str, use_db_time_zone)? + self.to_date_in_db_time_zone(to_str, use_db_time_zone, plan_templates)? } else { return Err(CubeError::user(format!( "Arguments for date range is not valid" ))); }; - let from = self.allocate_timestamp_param(&from, as_date_time)?; - let to = self.allocate_timestamp_param(&to, as_date_time)?; + let from = self.allocate_timestamp_param(&from, as_date_time, plan_templates)?; + let to = self.allocate_timestamp_param(&to, as_date_time, plan_templates)?; Ok((from, to)) } else { Err(CubeError::user(format!( @@ -647,12 +729,20 @@ impl BaseFilter { } } - fn format_from_date(&self, date: &str) -> Result { - QueryDateTimeHelper::format_from_date(date, self.query_tools.clone()) + fn format_from_date( + &self, + date: &str, + plan_templates: &PlanSqlTemplates, + ) -> Result { + QueryDateTimeHelper::format_from_date(date, plan_templates.timestamp_precision()?) } - fn format_to_date(&self, date: &str) -> Result { - QueryDateTimeHelper::format_to_date(date, self.query_tools.clone()) + fn format_to_date( + &self, + date: &str, + plan_templates: &PlanSqlTemplates, + ) -> Result { + QueryDateTimeHelper::format_to_date(date, plan_templates.timestamp_precision()?) } fn allocate_param(&self, param: &str) -> String { @@ -663,15 +753,16 @@ impl BaseFilter { &self, param: &str, as_date_time: bool, + plan_templates: &PlanSqlTemplates, ) -> Result { if self.use_raw_values { return Ok(param.to_string()); } let placeholder = self.query_tools.allocate_param(param); if as_date_time { - self.query_tools.base_tools().date_time_cast(placeholder) + plan_templates.date_time_cast(placeholder) } else { - self.query_tools.base_tools().time_stamp_cast(placeholder) + plan_templates.time_stamp_cast(placeholder) } } @@ -693,6 +784,7 @@ impl BaseFilter { &self, use_db_time_zone: bool, as_date_time: bool, + plan_templates: &PlanSqlTemplates, ) -> Result { if self.values.is_empty() { Err(CubeError::user(format!( @@ -701,8 +793,9 @@ impl BaseFilter { } else { if let Some(value) = &self.values[0] { self.allocate_timestamp_param( - &self.from_date_in_db_time_zone(value, use_db_time_zone)?, + &self.from_date_in_db_time_zone(value, use_db_time_zone, plan_templates)?, as_date_time, + plan_templates, ) } else { Err(CubeError::user(format!( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs index b9fb6a2b6e5a3..10bce7430a05f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs @@ -9,15 +9,13 @@ lazy_static! { static ref PARAMS_MATCH_RE: Regex = Regex::new(r"\$_(\d+)_\$").unwrap(); } pub struct ParamsAllocator { - sql_templates: PlanSqlTemplates, params: Vec, export_annotated_sql: bool, } impl ParamsAllocator { - pub fn new(sql_templates: PlanSqlTemplates, export_annotated_sql: bool) -> ParamsAllocator { + pub fn new(export_annotated_sql: bool) -> ParamsAllocator { ParamsAllocator { - sql_templates, params: Vec::new(), export_annotated_sql, } @@ -41,6 +39,7 @@ impl ParamsAllocator { sql: &str, native_allocated_params: Vec, should_reuse_params: bool, + templates: &PlanSqlTemplates, ) -> Result<(String, Vec), CubeError> { let (sql, params) = self.add_native_allocated_params(sql, &native_allocated_params)?; let mut params_in_sql_order = Vec::new(); @@ -61,7 +60,7 @@ impl ParamsAllocator { if self.export_annotated_sql { format!("${}$", new_index) } else { - match self.sql_templates.param(new_index) { + match templates.param(new_index) { Ok(res) => res, Err(e) => { if error.is_none() { @@ -79,7 +78,7 @@ impl ParamsAllocator { let ind: usize = caps[1].to_string().parse().unwrap(); let index = params_in_sql_order.len(); params_in_sql_order.push(params[ind].clone()); - match self.sql_templates.param(index) { + match templates.param(index) { Ok(res) => res, Err(e) => { if error.is_none() { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs index 17159420954b4..a67c040d005de 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/rolling_window_planner.rs @@ -6,7 +6,6 @@ use super::{ use crate::cube_bridge::measure_definition::RollingWindow; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::MemberSymbol; -use crate::planner::sql_templates::TemplateProjectionColumn; use crate::planner::BaseMeasure; use crate::planner::{BaseMember, BaseTimeDimension, GranularityHelper, QueryProperties}; use cubenativeutils::CubeError; @@ -269,53 +268,6 @@ impl RollingWindowPlanner { } } - fn make_time_seires_from_to_dates_suqueries_conditions( - &self, - time_series_cte_name: &str, - ) -> Result<(String, String), CubeError> { - let templates = self.query_tools.plan_sql_templates(); - let from_expr = format!("min(date_from)"); - let to_expr = format!("max(date_to)"); - let alias = format!("value"); - - let from_column = TemplateProjectionColumn { - expr: from_expr.clone(), - alias: alias.clone(), - aliased: templates.column_aliased(&from_expr, &alias)?, - }; - - let to_column = TemplateProjectionColumn { - expr: to_expr.clone(), - alias: alias.clone(), - aliased: templates.column_aliased(&to_expr, &alias)?, - }; - let from = templates.select( - vec![], - &time_series_cte_name, - vec![from_column], - None, - vec![], - None, - vec![], - None, - None, - false, - )?; - let to = templates.select( - vec![], - &time_series_cte_name, - vec![to_column], - None, - vec![], - None, - vec![], - None, - None, - false, - )?; - Ok((format!("({})", from), format!("({})", to))) - } - fn make_rolling_base_state( &self, time_dimension: Rc, @@ -335,36 +287,6 @@ impl RollingWindowPlanner { &time_dimension.resolve_granularity()?, )?; - let templates = self.query_tools.plan_sql_templates(); - - if templates.supports_generated_time_series() { - let (from, to) = - self.make_time_seires_from_to_dates_suqueries_conditions("time_series")?; - new_state.replace_range_to_subquery_in_date_filter(&time_dimension_base_name, from, to); - } else if time_dimension.get_date_range().is_some() && result_granularity.is_some() { - let granularity = time_dimension.get_granularity_obj().clone().unwrap(); - let date_range = time_dimension.get_date_range().unwrap(); - let series = if granularity.is_predefined_granularity() { - self.query_tools - .base_tools() - .generate_time_series(granularity.granularity().clone(), date_range.clone())? - } else { - self.query_tools.base_tools().generate_custom_time_series( - granularity.granularity_interval().clone(), - date_range.clone(), - granularity.origin_local_formatted(), - )? - }; - if !series.is_empty() { - let new_from_date = series.first().unwrap()[0].clone(); - let new_to_date = series.last().unwrap()[1].clone(); - new_state.replace_range_in_date_filter( - &time_dimension_base_name, - new_from_date, - new_to_date, - ); - } - } let new_time_dimension = time_dimension.change_granularity(result_granularity.clone())?; //We keep only one time_dimension in the leaf query because, even if time_dimension values have different granularity, in the leaf query we need to group by the lowest granularity. new_state.set_time_dimensions(vec![new_time_dimension.clone()]); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index 2ab26de23d7d5..2e1e7f1a2b128 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -144,16 +144,12 @@ impl QueryTools { cube_evaluator.clone(), timezone.clone(), ))); - let sql_templates = PlanSqlTemplates::new(templates_render.clone(), base_tools.clone()); Ok(Rc::new(Self { cube_evaluator, base_tools, join_graph, templates_render, - params_allocator: Rc::new(RefCell::new(ParamsAllocator::new( - sql_templates, - export_annotated_sql, - ))), + params_allocator: Rc::new(RefCell::new(ParamsAllocator::new(export_annotated_sql))), evaluator_compiler, cached_data: RefCell::new(QueryToolsCachedData::new()), timezone, @@ -164,8 +160,9 @@ impl QueryTools { &self.cube_evaluator } - pub fn plan_sql_templates(&self) -> PlanSqlTemplates { - PlanSqlTemplates::new(self.templates_render.clone(), self.base_tools.clone()) + pub fn plan_sql_templates(&self, external: bool) -> Result { + let driver_tools = self.base_tools.driver_tools(external)?; + PlanSqlTemplates::try_new(driver_tools) } pub fn base_tools(&self) -> &Rc { @@ -240,12 +237,14 @@ impl QueryTools { &self, sql: &str, should_reuse_params: bool, + templates: &PlanSqlTemplates, ) -> Result<(String, Vec), CubeError> { let native_allocated_params = self.base_tools.get_allocated_params()?; self.params_allocator.borrow().build_sql_and_params( sql, native_allocated_params, should_reuse_params, + templates, ) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index c080f5d74e011..6f49f3b471fb5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -17,6 +17,7 @@ pub struct SqlNodesFactory { ungrouped_measure: bool, count_approx_as_state: bool, render_references: HashMap, + pre_aggregation_dimensions_references: HashMap, pre_aggregation_measures_references: HashMap, rendered_as_multiplied_measures: HashSet, ungrouped_measure_references: HashMap, @@ -37,6 +38,7 @@ impl SqlNodesFactory { ungrouped_measure: false, count_approx_as_state: false, render_references: HashMap::new(), + pre_aggregation_dimensions_references: HashMap::new(), pre_aggregation_measures_references: HashMap::new(), ungrouped_measure_references: HashMap::new(), cube_name_references: HashMap::new(), @@ -82,6 +84,13 @@ impl SqlNodesFactory { self.rendered_as_multiplied_measures = value; } + pub fn set_pre_aggregation_dimensions_references( + &mut self, + value: HashMap, + ) { + self.pre_aggregation_dimensions_references = value; + } + pub fn set_original_sql_pre_aggregations(&mut self, value: HashMap) { self.original_sql_pre_aggregations = value; } @@ -240,10 +249,15 @@ impl SqlNodesFactory { } fn dimension_processor(&self, input: Rc) -> Rc { + let input = if !self.pre_aggregation_dimensions_references.is_empty() { + RenderReferencesSqlNode::new(input, self.pre_aggregation_dimensions_references.clone()) + } else { + let input: Rc = GeoDimensionSqlNode::new(input); + let input: Rc = CaseDimensionSqlNode::new(input); + input + }; let input: Rc = TimeDimensionNode::new(self.dimensions_with_ignored_timezone.clone(), input); - let input: Rc = GeoDimensionSqlNode::new(input); - let input: Rc = CaseDimensionSqlNode::new(input); let input: Rc = AutoPrefixSqlNode::new(input, self.cube_name_references.clone()); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs index db903803f729b..3e5323ac968ae 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs @@ -64,9 +64,9 @@ impl SqlNode for FinalMeasureSqlNode { input } else if ev.measure_type() == "countDistinctApprox" { if self.count_approx_as_state { - query_tools.base_tools().hll_init(input)? + templates.hll_init(input)? } else { - query_tools.base_tools().count_distinct_approx(input)? + templates.count_distinct_approx(input)? } } else if self.is_count_distinct(ev) { templates.count_distinct(&input)? diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_pre_aggregation_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_pre_aggregation_measure.rs index f1bf73c76115b..aa0660911bb45 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_pre_aggregation_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_pre_aggregation_measure.rs @@ -52,9 +52,7 @@ impl SqlNode for FinalPreAggregationMeasureSqlNode { if ev.measure_type() == "count" || ev.measure_type() == "sum" { format!("sum({})", pre_aggregation_measure) } else if ev.measure_type() == "countDistinctApprox" { - query_tools - .base_tools() - .count_distinct_approx(pre_aggregation_measure)? + templates.count_distinct_approx(pre_aggregation_measure)? } else if ev.measure_type() == "min" || ev.measure_type() == "max" { format!("{}({})", ev.measure_type(), pre_aggregation_measure) } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs index ea1821540635f..c8546ecd85d9f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/rolling_window.rs @@ -44,7 +44,7 @@ impl SqlNode for RollingWindowNode { templates, )?; if m.measure_type() == "countDistinctApprox" { - query_tools.base_tools().hll_cardinality_merge(input)? + templates.hll_cardinality_merge(input)? } else { if m.measure_type() == "sum" || m.measure_type() == "count" diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs index 56413bd0f4a99..004d74bb65563 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_dimension.rs @@ -50,29 +50,26 @@ impl SqlNode for TimeDimensionNode { { input_sql } else { - query_tools.base_tools().convert_tz(input_sql)? + templates.convert_tz(input_sql)? }; let res = if granularity_obj.is_natural_aligned() { if let Some(granularity_offset) = granularity_obj.granularity_offset() { - let dt = query_tools - .base_tools() + let dt = templates .subtract_interval(converted_tz, granularity_offset.clone())?; - let dt = query_tools.base_tools().time_grouped_column( + let dt = templates.time_grouped_column( granularity_obj.granularity_from_interval()?, dt, )?; - query_tools - .base_tools() - .add_interval(dt, granularity_offset.clone())? + templates.add_interval(dt, granularity_offset.clone())? } else { - query_tools.base_tools().time_grouped_column( + templates.time_grouped_column( granularity_obj.granularity().clone(), converted_tz, )? } } else { - query_tools.base_tools().date_bin( + templates.date_bin( granularity_obj.granularity_interval().clone(), converted_tz, granularity_obj.origin_local_formatted(), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs index 5b3f81fc3825f..4cc97efb89791 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/time_shift.rs @@ -47,9 +47,7 @@ impl SqlNode for TimeShiftSqlNode { MemberSymbol::Dimension(ev) => { if let Some(shift) = self.shifts.get(&ev.full_name()) { let shift = shift.interval.to_sql(); - let res = templates - .base_tools() - .add_timestamp_interval(input, shift)?; + let res = templates.add_timestamp_interval(input, shift)?; format!("({})", res) } else { input diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs index 407877faf89dc..69eba92875e85 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs @@ -134,10 +134,8 @@ impl TimeDimensionSymbol { ) -> Result, CubeError> { if let Some(date_range) = &self.date_range { let tz = query_tools.timezone(); - let from_date_str = - QueryDateTimeHelper::format_from_date(&date_range.0, query_tools.clone())?; - let to_date_str = - QueryDateTimeHelper::format_to_date(&date_range.1, query_tools.clone())?; + let from_date_str = QueryDateTimeHelper::format_from_date(&date_range.0, 3)?; + let to_date_str = QueryDateTimeHelper::format_to_date(&date_range.1, 3)?; let start = QueryDateTime::from_date_str(tz, &from_date_str)?; let end = QueryDateTime::from_date_str(tz, &to_date_str)?; let end = end.add_duration(Duration::milliseconds(1))?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs index 564d1a57f3cac..a5931f5f708f4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs @@ -1,5 +1,5 @@ use super::{TemplateGroupByColumn, TemplateOrderByColumn, TemplateProjectionColumn}; -use crate::cube_bridge::base_tools::BaseTools; +use crate::cube_bridge::driver_tools::DriverTools; use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use crate::plan::join::JoinType; use convert_case::{Boundary, Case, Casing}; @@ -10,7 +10,7 @@ use std::rc::Rc; #[derive(Clone)] pub struct PlanSqlTemplates { render: Rc, - base_tools: Rc, + driver_tools: Rc, } pub const UNDERSCORE_UPPER_BOUND: Boundary = Boundary { name: "UnderscoreUpper", @@ -41,10 +41,90 @@ pub const UPPER_UPPER_BOUND: Boundary = Boundary { }; impl PlanSqlTemplates { - pub fn new(render: Rc, base_tools: Rc) -> Self { - Self { render, base_tools } + pub fn try_new(driver_tools: Rc) -> Result { + let render = driver_tools.sql_templates()?; + Ok(Self { + render, + driver_tools, + }) } + pub fn convert_tz(&self, field: String) -> Result { + self.driver_tools.convert_tz(field) + } + + pub fn time_grouped_column( + &self, + granularity: String, + dimension: String, + ) -> Result { + self.driver_tools + .time_grouped_column(granularity, dimension) + } + + pub fn timestamp_precision(&self) -> Result { + self.driver_tools.timestamp_precision() + } + + pub fn time_stamp_cast(&self, field: String) -> Result { + self.driver_tools.time_stamp_cast(field) + } + + pub fn date_time_cast(&self, field: String) -> Result { + self.driver_tools.date_time_cast(field) + } + + pub fn in_db_time_zone(&self, date: String) -> Result { + self.driver_tools.in_db_time_zone(date) + } + + pub fn subtract_interval(&self, date: String, interval: String) -> Result { + self.driver_tools.subtract_interval(date, interval) + } + + pub fn add_interval(&self, date: String, interval: String) -> Result { + self.driver_tools.add_interval(date, interval) + } + + pub fn add_timestamp_interval( + &self, + date: String, + interval: String, + ) -> Result { + self.driver_tools.add_timestamp_interval(date, interval) + } + + pub fn interval_and_minimal_time_unit( + &self, + interval: String, + ) -> Result, CubeError> { + self.driver_tools.interval_and_minimal_time_unit(interval) + } + + pub fn hll_init(&self, sql: String) -> Result { + self.driver_tools.hll_init(sql) + } + + pub fn hll_merge(&self, sql: String) -> Result { + self.driver_tools.hll_merge(sql) + } + + pub fn hll_cardinality_merge(&self, sql: String) -> Result { + self.driver_tools.hll_cardinality_merge(sql) + } + + pub fn count_distinct_approx(&self, sql: String) -> Result { + self.driver_tools.count_distinct_approx(sql) + } + + pub fn date_bin( + &self, + interval: String, + source: String, + origin: String, + ) -> Result { + self.driver_tools.date_bin(interval, source, origin) + } pub fn alias_name(name: &str) -> String { let res = name .with_boundaries(&[ @@ -59,8 +139,8 @@ impl PlanSqlTemplates { res } - pub fn base_tools(&self) -> &Rc { - &self.base_tools + pub fn driver_tools(&self) -> &Rc { + &self.driver_tools } pub fn memeber_alias_name(cube_name: &str, name: &str, suffix: &Option) -> String { @@ -124,10 +204,6 @@ impl PlanSqlTemplates { ) } - pub fn always_true(&self) -> Result { - Ok(self.render.get_template("filters/always_true")?.clone()) - } - pub fn query_aliased(&self, query: &str, alias: &str) -> Result { let quoted_alias = self.quote_identifier(alias)?; self.render.render_template( @@ -423,4 +499,207 @@ impl PlanSqlTemplates { }, ) } + + pub fn equals( + &self, + column: String, + value: String, + is_null_check: bool, + ) -> Result { + self.render.render_template( + &"filters/equals", + context! { + value => value, + is_null_check => self.additional_null_check(is_null_check, &column)?, + column => column, + }, + ) + } + + pub fn not_equals( + &self, + column: String, + value: String, + is_null_check: bool, + ) -> Result { + self.render.render_template( + &"filters/not_equals", + context! { + value => value, + is_null_check => self.additional_null_check(is_null_check, &column)?, + column => column, + }, + ) + } + + pub fn time_range_filter( + &self, + column: String, + from_timestamp: String, + to_timestamp: String, + ) -> Result { + self.render.render_template( + &"filters/time_range_filter", + context! { + column => column, + from_timestamp => from_timestamp, + to_timestamp => to_timestamp, + }, + ) + } + + pub fn time_not_in_range_filter( + &self, + column: String, + from_timestamp: String, + to_timestamp: String, + ) -> Result { + self.render.render_template( + &"filters/time_not_in_range_filter", + context! { + column => column, + from_timestamp => from_timestamp, + to_timestamp => to_timestamp, + }, + ) + } + + pub fn in_where( + &self, + column: String, + values: Vec, + is_null_check: bool, + ) -> Result { + let values_concat = values.join(", "); + self.render.render_template( + &"filters/in", + context! { + is_null_check => self.additional_null_check(is_null_check, &column)?, + values_concat => values_concat, + column => column, + }, + ) + } + + pub fn not_in_where( + &self, + column: String, + values: Vec, + is_null_check: bool, + ) -> Result { + let values_concat = values.join(", "); + self.render.render_template( + &"filters/not_in", + context! { + is_null_check => self.additional_null_check(is_null_check, &column)?, + values_concat => values_concat, + column => column, + }, + ) + } + + pub fn or_is_null_check(&self, column: String) -> Result { + self.render.render_template( + &"filters/or_is_null_check", + context! { + column => column, + }, + ) + } + + pub fn set_where(&self, column: String) -> Result { + self.render.render_template( + &"filters/set_where", + context! { + column => column, + }, + ) + } + + pub fn not_set_where(&self, column: String) -> Result { + self.render.render_template( + &"filters/not_set_where", + context! { + column => column, + }, + ) + } + + pub fn gt(&self, column: String, param: String) -> Result { + self.render.render_template( + &"filters/gt", + context! { + column => column, + param => param + }, + ) + } + + pub fn always_true(&self) -> Result { + Ok(self.render.get_template("filters/always_true")?.clone()) + } + + pub fn gte(&self, column: String, param: String) -> Result { + self.render.render_template( + &"filters/gte", + context! { + column => column, + param => param + }, + ) + } + + pub fn lt(&self, column: String, param: String) -> Result { + self.render.render_template( + &"filters/lt", + context! { + column => column, + param => param + }, + ) + } + + pub fn lte(&self, column: String, param: String) -> Result { + self.render.render_template( + &"filters/lte", + context! { + column => column, + param => param + }, + ) + } + + pub fn additional_null_check(&self, need: bool, column: &String) -> Result { + if need { + self.or_is_null_check(column.clone()) + } else { + Ok(String::default()) + } + } + + pub fn ilike( + &self, + column: &str, + value: &str, + start_wild: bool, + end_wild: bool, + not: bool, + ) -> Result { + let pattern = self.render.render_template( + &"filters/like_pattern", + context! { + start_wild => start_wild, + value => value, + end_wild => end_wild + }, + )?; + self.render.render_template( + &"tesseract/ilike", + context! { + expr => column, + negated => not, + pattern => pattern + }, + ) + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs index 5d281a0871107..4d20e5b414dc4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/time_dimension/date_time_helper.rs @@ -1,6 +1,3 @@ -use std::rc::Rc; - -use crate::planner::query_tools::QueryTools; use chrono::{DateTime, Duration, LocalResult, NaiveDate, NaiveDateTime, TimeZone}; use chrono_tz::Tz; use cubenativeutils::CubeError; @@ -92,8 +89,7 @@ impl QueryDateTimeHelper { } } - pub fn format_from_date(date: &str, query_tools: Rc) -> Result { - let precision = query_tools.base_tools().timestamp_precision()?; + pub fn format_from_date(date: &str, precision: u32) -> Result { if precision == 3 { if DATE_TIME_LOCAL_MS_RE.is_match(date) { return Ok(date.to_string()); @@ -121,8 +117,7 @@ impl QueryDateTimeHelper { Ok(date.to_string()) } - pub fn format_to_date(date: &str, query_tools: Rc) -> Result { - let precision = query_tools.base_tools().timestamp_precision()?; + pub fn format_to_date(date: &str, precision: u32) -> Result { if precision == 3 { if DATE_TIME_LOCAL_MS_RE.is_match(date) { return Ok(date.to_string());