in work

Author: waralexrom

packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts

@@ -28,7 +28,6 @@ export class BaseDimension {
       const dimensionPath = dimension as string | null;
       if (dimensionPath !== null) {
         const { path, joinHint } = CubeSymbols.joinHintFromPath(dimensionPath);
-        console.log("!!! dimension", dimensionPath, path, joinHint);
         this.dimension = path;
         this.joinHint = joinHint;
       }

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -289,6 +289,12 @@ export class BaseQuery {
     }).filter(R.identity).map(this.newTimeDimension.bind(this));
     this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);
     this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner');
+    if (this.useNativeSqlPlanner) {
+      const hasMultiStageMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }).multiStageMembers.length > 0;
+      this.canUseNativeSqlPlannerPreAggregation = hasMultiStageMeasures;
+    } else {
+      this.useNativePreAggregations = false;
+    }
     this.prebuildJoin();
 
     this.cubeAliasPrefix = this.options.cubeAliasPrefix;
@@ -471,6 +477,17 @@ export class BaseQuery {
   }
 
   newDimension(dimensionPath) {
+    const memberArr = dimensionPath.split('.');
+    if (memberArr.length > 3 &&
+          memberArr[memberArr.length - 2] === 'granularities' &&
+          this.cubeEvaluator.isDimension(memberArr.slice(0, -2))) {
+      return this.newTimeDimension(
+        {
+          dimension: this.cubeEvaluator.pathFromArray(memberArr.slice(0, -2)),
+          granularity: memberArr[memberArr.length - 1]
+        }
+      );
+    }
     return new BaseDimension(this, dimensionPath);
   }
 
@@ -637,33 +654,34 @@ export class BaseQuery {
    */
   buildSqlAndParams(exportAnnotatedSql) {
     if (this.useNativeSqlPlanner) {
-      const isRelatedToPreAggregation = false;
-      /*       if (this.options.preAggregationQuery) {
-        isRelatedToPreAggregation = true;
-      } else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) {
-        if (this.externalPreAggregationQuery()) {
+      let isRelatedToPreAggregation = false;
+
+      if (!this.canUseNativeSqlPlannerPreAggregation) {
+        if (this.options.preAggregationQuery) {
           isRelatedToPreAggregation = true;
+        } else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) {
+          if (this.externalPreAggregationQuery()) {
+            isRelatedToPreAggregation = true;
+          }
+        } else {
+          let preAggForQuery =
+            this.preAggregations.findPreAggregationForQuery();
+          if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) {
+            preAggForQuery = undefined;
+          }
+          if (preAggForQuery) {
+            isRelatedToPreAggregation = true;
+          }
         }
-      } else {
-        let preAggForQuery =
-          this.preAggregations.findPreAggregationForQuery();
-        if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) {
-          preAggForQuery = undefined;
-        }
-        if (preAggForQuery) {
-          isRelatedToPreAggregation = true;
+
+        if (isRelatedToPreAggregation) {
+          return this.newQueryWithoutNative().buildSqlAndParams(exportAnnotatedSql);
         }
       }
 
-      if (isRelatedToPreAggregation) {
-        return this.newQueryWithoutNative().buildSqlAndParams(exportAnnotatedSql);
-      } */
-
-      console.log("!!!!!! EEEEEEEEEE");
       return this.buildSqlAndParamsRust(exportAnnotatedSql);
     }
 
-      console.log("!!!!!! RRRRRRRRR");
     if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
       if (this.externalPreAggregationQuery()) { // TODO performance
         return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
@@ -1333,7 +1351,6 @@ export class BaseQuery {
 
   childrenMultiStageContext(memberPath, queryContext, wouldNodeApplyFilters) {
     let member;
-    console.log("!!!! children ", memberPath);
     if (this.cubeEvaluator.isMeasure(memberPath)) {
       member = this.newMeasure(memberPath);
     } else if (this.cubeEvaluator.isDimension(memberPath)) {

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -21,7 +21,6 @@ export class PreAggregations {
    * @return {unknown[]}
    */
   preAggregationsDescription() {
-    return [];
     const preAggregations = [this.preAggregationsDescriptionLocal()].concat(
       this.query.subQueryDimensions.map(d => this.query.subQueryDescription(d).subQuery)
         .map(q => q.preAggregations.preAggregationsDescription())
@@ -312,8 +311,6 @@ export class PreAggregations {
     const dimensionsList = query.dimensions.map(dim => dim.expressionPath());
     const segmentsList = query.segments.map(s => s.expressionPath());
     const ownedDimensions = PreAggregations.ownedMembers(query, flattenDimensionMembers);
-    console.log('!!! dimensions', dimensionsList);
-    console.log('!!! ownedDimensions', ownedDimensions);
     const ownedTimeDimensions = query.timeDimensions.map(td => {
       const owned = PreAggregations.ownedMembers(query, [td]);
       let { dimension } = td;
@@ -785,7 +782,7 @@ export class PreAggregations {
    */
   findPreAggregationForQuery() {
     if (!this.preAggregationForQuery) {
-      if (this.query.useNativeSqlPlanner) {
+      if (this.query.useNativeSqlPlanner && this.query.canUseNativeSqlPlannerPreAggregation) {
         this.query.findPreAggregationForQueryRust();
       } else {
         this.preAggregationForQuery =
@@ -834,7 +831,6 @@ export class PreAggregations {
    * @returns {Array<Object>}
    */
   rollupMatchResults() {
-    return [];
     const { query } = this;
 
     const canUsePreAggregation = this.canUsePreAggregationFn(query);

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -670,7 +670,6 @@ export class CubeEvaluator extends CubeSymbols {
 
     const cubeAndName = Array.isArray(path) ? path : path.split('.');
     if (!this.evaluatedCubes[cubeAndName[0]]) {
-      console.trace("!!!!");
       throw new UserError(`Cube '${cubeAndName[0]}' not found for path '${path}'`);
     }
 

packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-multi-stage.test.ts

@@ -86,13 +86,6 @@ describe('PreAggregationsMultiStage', () => {
       createdAt: {
         type: 'time',
         sql: 'created_at',
-        granularities: {
-            three_days: {
-              interval: '1 days',
-              title: '1 days',
-              origin: '2017-01-01'
-            }
-        }
       },
       checkinsCount: {
         type: 'number',
@@ -107,7 +100,7 @@ describe('PreAggregationsMultiStage', () => {
 
       createdAtDay: {
         type: 'time',
-        sql: \`\${createdAt.three_days}\`,
+        sql: \`\${createdAt.day}\`,
       },
 
 
@@ -129,9 +122,9 @@ describe('PreAggregationsMultiStage', () => {
             granularity: 'day',
             partitionGranularity: 'month',
         },
-        revenueAndTimeRollup: {
+        revenueAndTimeAndCountRollup: {
             type: 'rollup',
-            measureReferences: [revenue],
+            measureReferences: [revenue, count],
             dimensionReferences: [source],
             timeDimensionReference: createdAt,
             granularity: 'day',
@@ -200,7 +193,6 @@ describe('PreAggregationsMultiStage', () => {
 
       const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
       const sqlAndParams = query.buildSqlAndParams();
-      console.log('!!!! sqlAndParamsl', sqlAndParams);
       expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_per_id_rollup');
       expect(sqlAndParams[0]).toContain('vis_revenue_per_id_rollup');
 
@@ -228,9 +220,8 @@ describe('PreAggregationsMultiStage', () => {
         );
       });
     }));
-  }
 
-    it('simple multi stage with add_group_by and time proxy dimension 11', () => compiler.compile().then(() => {
+    it('simple multi stage with add_group_by and time proxy dimension', () => compiler.compile().then(() => {
       const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
         measures: [
           'visitors.revenueAndTime'
@@ -245,12 +236,10 @@ describe('PreAggregationsMultiStage', () => {
 
       const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
       const sqlAndParams = query.buildSqlAndParams();
-      console.log('!!!! sqlAndParamsl', sqlAndParams);
-      expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_rollup');
-      expect(sqlAndParams[0]).toContain('vis_revenue_and_time_rollup');
+      expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_and_count_rollup');
+      expect(sqlAndParams[0]).toContain('vis_revenue_and_time_and_count_rollup');
 
       return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
-        console.log('!!!! res', res);
         expect(res).toEqual(
           [
             { vis__source: 'google', vis__revenue_and_time: '25' },
@@ -262,36 +251,40 @@ describe('PreAggregationsMultiStage', () => {
       });
     }));
 
-    it('simple multi stage with add_group_by and time proxy dimension tttmp', () => compiler.compile().then(() => {
+    it('multi stage with add_group_by and time proxy dimension and regular measure', () => compiler.compile().then(() => {
       const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
         measures: [
-          'visitors.testMeas'
+          'visitors.revenueAndTime',
+          'visitors.count'
         ],
-        dimensions: ['visitor_checkins.source'],
+        dimensions: ['visitors.source'],
         timezone: 'America/Los_Angeles',
         order: [{
-          id: 'visitor_checkins.source'
+          id: 'visitors.source'
         }],
         preAggregationsSchema: ''
       });
 
       const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
       const sqlAndParams = query.buildSqlAndParams();
-     console.log('!!!! sqlAndParamsl', sqlAndParams);
-/*       console.log('!!!! sqlAndParamsl', sqlAndParams);
-      expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_rollup');
-      expect(sqlAndParams[0]).toContain('vis_revenue_and_time_rollup'); */
+      expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_and_count_rollup');
+      expect(sqlAndParams[0]).toContain('vis_revenue_and_time_and_count_rollup');
+      expect(sqlAndParams[0]).not.toContain('select * from visitors');
 
       return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
-        console.log('!!!! res', res);
         expect(res).toEqual(
           [
-            { vis__source: 'google', vis__revenue_and_time: '25' },
-            { vis__source: 'some', vis__revenue_and_time: '50' },
-            { vis__source: null, vis__revenue_and_time: '50' }
+            { vis__source: 'google', vis__count: '1', vis__revenue_and_time: '25' },
+            { vis__source: 'some', vis__count: '2', vis__revenue_and_time: '50' },
+            { vis__source: null, vis__count: '3', vis__revenue_and_time: '50' }
           ]
 
         );
       });
     }));
+  } else {
+    it.skip('multi stage pre-aggregations', () => {
+      // Skipping because it works only in Tesseract
+    });
+  }
 });

packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts

@@ -1769,7 +1769,7 @@ describe('PreAggregations', () => {
     });
   }));
 
-  it('partitioned rolling 11', () => compiler.compile().then(() => {
+  it('partitioned rolling', () => compiler.compile().then(() => {
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
       measures: [
         'visitors.checkinsRollingTotal',
@@ -1792,7 +1792,6 @@ describe('PreAggregations', () => {
     });
 
     const queryAndParams = query.buildSqlAndParams();
-    console.log('!!! ---', queryAndParams);
     const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
     console.log(preAggregationsDescription);
     expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true);
@@ -1955,7 +1954,7 @@ describe('PreAggregations', () => {
     });
   }));
 
-  it('segment 111', () => compiler.compile().then(() => {
+  it('segment', () => compiler.compile().then(() => {
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
       measures: [
         'visitors.checkinsTotal'
@@ -1998,7 +1997,7 @@ describe('PreAggregations', () => {
     });
   }));
 
-  it('rollup join 11', () => compiler.compile().then(() => {
+  it('rollup join', () => compiler.compile().then(() => {
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
       measures: [
         'visitor_checkins.count',
@@ -2214,7 +2213,7 @@ describe('PreAggregations', () => {
     });
   }));
 
-  it('simple view 11', () => compiler.compile().then(() => {
+  it('simple view', () => compiler.compile().then(() => {
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
       measures: [
         'visitors_view.checkinsTotal'

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -1,10 +1,9 @@
 use super::base_query_options::FilterItem;
 use super::filter_group::{FilterGroup, NativeFilterGroup};
 use super::filter_params::{FilterParams, NativeFilterParams};
-use cubenativeutils::wrappers::NativeArray;
+use super::pre_aggregation_obj::{NativePreAggregationObj, PreAggregationObj};
 use super::security_context::{NativeSecurityContext, SecurityContext};
 use super::sql_templates_render::{NativeSqlTemplatesRender, SqlTemplatesRender};
-use super::pre_aggregation_obj::{NativePreAggregationObj, PreAggregationObj};
 use super::sql_utils::{NativeSqlUtils, SqlUtils};
 use cubenativeutils::wrappers::serializer::{
     NativeDeserialize, NativeDeserializer, NativeSerialize,
@@ -61,6 +60,14 @@ pub trait BaseTools {
         origin: String,
     ) -> Result<String, CubeError>;
 
-    fn get_pre_aggregation_by_name(&self, cube_name: String, name: String) -> Result<Rc<dyn PreAggregationObj>, CubeError>;
-    fn pre_aggregation_table_name(&self, cube_name: String, name: String) -> Result<String, CubeError>; //TODO move to rust
+    fn get_pre_aggregation_by_name(
+        &self,
+        cube_name: String,
+        name: String,
+    ) -> Result<Rc<dyn PreAggregationObj>, CubeError>;
+    fn pre_aggregation_table_name(
+        &self,
+        cube_name: String,
+        name: String,
+    ) -> Result<String, CubeError>; //TODO move to rust
 }

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs

@@ -22,9 +22,9 @@ pub mod member_order_by;
 pub mod member_sql;
 pub mod options_member;
 pub mod pre_aggregation_description;
+pub mod pre_aggregation_obj;
 pub mod security_context;
 pub mod segment_definition;
 pub mod sql_templates_render;
 pub mod sql_utils;
 pub mod struct_with_sql_member;
-pub mod pre_aggregation_obj;

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs

@@ -2,7 +2,6 @@ use super::member_sql::{MemberSql, NativeMemberSql};
 use cubenativeutils::wrappers::serializer::{
     NativeDeserialize, NativeDeserializer, NativeSerialize,
 };
-use cubenativeutils::wrappers::NativeArray;
 use cubenativeutils::wrappers::NativeContextHolder;
 use cubenativeutils::wrappers::NativeObjectHandle;
 use cubenativeutils::CubeError;

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_obj.rs

@@ -1,8 +1,4 @@
-use super::member_sql::{MemberSql, NativeMemberSql};
-use cubenativeutils::wrappers::serializer::{
-    NativeDeserialize, NativeDeserializer, NativeSerialize,
-};
-use cubenativeutils::wrappers::NativeArray;
+use cubenativeutils::wrappers::serializer::{NativeDeserialize, NativeSerialize};
 use cubenativeutils::wrappers::NativeContextHolder;
 use cubenativeutils::wrappers::NativeObjectHandle;
 use cubenativeutils::CubeError;
@@ -22,5 +18,4 @@ pub struct PreAggregationObjStatic {
 }
 
 #[nativebridge::native_bridge(PreAggregationObjStatic)]
-pub trait PreAggregationObj {
-}
+pub trait PreAggregationObj {}

rust/cubesqlplanner/cubesqlplanner/src/logical_plan/aggregate_multiplied_subquery.rs

@@ -1,6 +1,5 @@
 use super::pretty_print::*;
 use super::*;
-use crate::planner::BaseCube;
 use std::rc::Rc;
 
 pub enum AggregateMultipliedSubquerySouce {
@@ -35,7 +34,8 @@ impl PrettyPrint for AggregateMultipliedSubquery {
         match self.source.as_ref() {
             AggregateMultipliedSubquerySouce::Cube => {
                 result.println("Cube:", &details_state);
-                self.pk_cube.pretty_print(result, &details_state.new_level());
+                self.pk_cube
+                    .pretty_print(result, &details_state.new_level());
             }
             AggregateMultipliedSubquerySouce::MeasureSubquery(measure_subquery) => {
                 result.println(