Skip to content

Commit dcabac1

Browse files
waralexromwaralexrom
committed
in work
1 parent 09309d0 commit dcabac1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+758
-697
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ export class BaseDimension {
2828
const dimensionPath = dimension as string | null;
2929
if (dimensionPath !== null) {
3030
const { path, joinHint } = CubeSymbols.joinHintFromPath(dimensionPath);
31-
console.log("!!! dimension", dimensionPath, path, joinHint);
3231
this.dimension = path;
3332
this.joinHint = joinHint;
3433
}

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,12 @@ export class BaseQuery {
289289
}).filter(R.identity).map(this.newTimeDimension.bind(this));
290290
this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);
291291
this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner');
292+
if (this.useNativeSqlPlanner) {
293+
const hasMultiStageMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }).multiStageMembers.length > 0;
294+
this.canUseNativeSqlPlannerPreAggregation = hasMultiStageMeasures;
295+
} else {
296+
this.useNativePreAggregations = false;
297+
}
292298
this.prebuildJoin();
293299

294300
this.cubeAliasPrefix = this.options.cubeAliasPrefix;
@@ -471,6 +477,17 @@ export class BaseQuery {
471477
}
472478

473479
newDimension(dimensionPath) {
480+
const memberArr = dimensionPath.split('.');
481+
if (memberArr.length > 3 &&
482+
memberArr[memberArr.length - 2] === 'granularities' &&
483+
this.cubeEvaluator.isDimension(memberArr.slice(0, -2))) {
484+
return this.newTimeDimension(
485+
{
486+
dimension: this.cubeEvaluator.pathFromArray(memberArr.slice(0, -2)),
487+
granularity: memberArr[memberArr.length - 1]
488+
}
489+
);
490+
}
474491
return new BaseDimension(this, dimensionPath);
475492
}
476493

@@ -637,33 +654,34 @@ export class BaseQuery {
637654
*/
638655
buildSqlAndParams(exportAnnotatedSql) {
639656
if (this.useNativeSqlPlanner) {
640-
const isRelatedToPreAggregation = false;
641-
/* if (this.options.preAggregationQuery) {
642-
isRelatedToPreAggregation = true;
643-
} else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) {
644-
if (this.externalPreAggregationQuery()) {
657+
let isRelatedToPreAggregation = false;
658+
659+
if (!this.canUseNativeSqlPlannerPreAggregation) {
660+
if (this.options.preAggregationQuery) {
645661
isRelatedToPreAggregation = true;
662+
} else if (!this.options.disableExternalPreAggregations && this.externalQueryClass) {
663+
if (this.externalPreAggregationQuery()) {
664+
isRelatedToPreAggregation = true;
665+
}
666+
} else {
667+
let preAggForQuery =
668+
this.preAggregations.findPreAggregationForQuery();
669+
if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) {
670+
preAggForQuery = undefined;
671+
}
672+
if (preAggForQuery) {
673+
isRelatedToPreAggregation = true;
674+
}
646675
}
647-
} else {
648-
let preAggForQuery =
649-
this.preAggregations.findPreAggregationForQuery();
650-
if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) {
651-
preAggForQuery = undefined;
652-
}
653-
if (preAggForQuery) {
654-
isRelatedToPreAggregation = true;
676+
677+
if (isRelatedToPreAggregation) {
678+
return this.newQueryWithoutNative().buildSqlAndParams(exportAnnotatedSql);
655679
}
656680
}
657681

658-
if (isRelatedToPreAggregation) {
659-
return this.newQueryWithoutNative().buildSqlAndParams(exportAnnotatedSql);
660-
} */
661-
662-
console.log("!!!!!! EEEEEEEEEE");
663682
return this.buildSqlAndParamsRust(exportAnnotatedSql);
664683
}
665684

666-
console.log("!!!!!! RRRRRRRRR");
667685
if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
668686
if (this.externalPreAggregationQuery()) { // TODO performance
669687
return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
@@ -1333,7 +1351,6 @@ export class BaseQuery {
13331351

13341352
childrenMultiStageContext(memberPath, queryContext, wouldNodeApplyFilters) {
13351353
let member;
1336-
console.log("!!!! children ", memberPath);
13371354
if (this.cubeEvaluator.isMeasure(memberPath)) {
13381355
member = this.newMeasure(memberPath);
13391356
} else if (this.cubeEvaluator.isDimension(memberPath)) {

‎packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ export class PreAggregations {
2121
* @return {unknown[]}
2222
*/
2323
preAggregationsDescription() {
24-
return [];
2524
const preAggregations = [this.preAggregationsDescriptionLocal()].concat(
2625
this.query.subQueryDimensions.map(d => this.query.subQueryDescription(d).subQuery)
2726
.map(q => q.preAggregations.preAggregationsDescription())
@@ -312,8 +311,6 @@ export class PreAggregations {
312311
const dimensionsList = query.dimensions.map(dim => dim.expressionPath());
313312
const segmentsList = query.segments.map(s => s.expressionPath());
314313
const ownedDimensions = PreAggregations.ownedMembers(query, flattenDimensionMembers);
315-
console.log('!!! dimensions', dimensionsList);
316-
console.log('!!! ownedDimensions', ownedDimensions);
317314
const ownedTimeDimensions = query.timeDimensions.map(td => {
318315
const owned = PreAggregations.ownedMembers(query, [td]);
319316
let { dimension } = td;
@@ -785,7 +782,7 @@ export class PreAggregations {
785782
*/
786783
findPreAggregationForQuery() {
787784
if (!this.preAggregationForQuery) {
788-
if (this.query.useNativeSqlPlanner) {
785+
if (this.query.useNativeSqlPlanner && this.query.canUseNativeSqlPlannerPreAggregation) {
789786
this.query.findPreAggregationForQueryRust();
790787
} else {
791788
this.preAggregationForQuery =
@@ -834,7 +831,6 @@ export class PreAggregations {
834831
* @returns {Array<Object>}
835832
*/
836833
rollupMatchResults() {
837-
return [];
838834
const { query } = this;
839835

840836
const canUsePreAggregation = this.canUsePreAggregationFn(query);

‎packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,6 @@ export class CubeEvaluator extends CubeSymbols {
670670

671671
const cubeAndName = Array.isArray(path) ? path : path.split('.');
672672
if (!this.evaluatedCubes[cubeAndName[0]]) {
673-
console.trace("!!!!");
674673
throw new UserError(`Cube '${cubeAndName[0]}' not found for path '${path}'`);
675674
}
676675

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-multi-stage.test.ts

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,6 @@ describe('PreAggregationsMultiStage', () => {
8686
createdAt: {
8787
type: 'time',
8888
sql: 'created_at',
89-
granularities: {
90-
three_days: {
91-
interval: '1 days',
92-
title: '1 days',
93-
origin: '2017-01-01'
94-
}
95-
}
9689
},
9790
checkinsCount: {
9891
type: 'number',
@@ -107,7 +100,7 @@ describe('PreAggregationsMultiStage', () => {
107100
108101
createdAtDay: {
109102
type: 'time',
110-
sql: \`\${createdAt.three_days}\`,
103+
sql: \`\${createdAt.day}\`,
111104
},
112105
113106
@@ -129,9 +122,9 @@ describe('PreAggregationsMultiStage', () => {
129122
granularity: 'day',
130123
partitionGranularity: 'month',
131124
},
132-
revenueAndTimeRollup: {
125+
revenueAndTimeAndCountRollup: {
133126
type: 'rollup',
134-
measureReferences: [revenue],
127+
measureReferences: [revenue, count],
135128
dimensionReferences: [source],
136129
timeDimensionReference: createdAt,
137130
granularity: 'day',
@@ -200,7 +193,6 @@ describe('PreAggregationsMultiStage', () => {
200193

201194
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
202195
const sqlAndParams = query.buildSqlAndParams();
203-
console.log('!!!! sqlAndParamsl', sqlAndParams);
204196
expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_per_id_rollup');
205197
expect(sqlAndParams[0]).toContain('vis_revenue_per_id_rollup');
206198

@@ -228,9 +220,8 @@ describe('PreAggregationsMultiStage', () => {
228220
);
229221
});
230222
}));
231-
}
232223

233-
it('simple multi stage with add_group_by and time proxy dimension 11', () => compiler.compile().then(() => {
224+
it('simple multi stage with add_group_by and time proxy dimension', () => compiler.compile().then(() => {
234225
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
235226
measures: [
236227
'visitors.revenueAndTime'
@@ -245,12 +236,10 @@ describe('PreAggregationsMultiStage', () => {
245236

246237
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
247238
const sqlAndParams = query.buildSqlAndParams();
248-
console.log('!!!! sqlAndParamsl', sqlAndParams);
249-
expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_rollup');
250-
expect(sqlAndParams[0]).toContain('vis_revenue_and_time_rollup');
239+
expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_and_count_rollup');
240+
expect(sqlAndParams[0]).toContain('vis_revenue_and_time_and_count_rollup');
251241

252242
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
253-
console.log('!!!! res', res);
254243
expect(res).toEqual(
255244
[
256245
{ vis__source: 'google', vis__revenue_and_time: '25' },
@@ -262,36 +251,40 @@ describe('PreAggregationsMultiStage', () => {
262251
});
263252
}));
264253

265-
it('simple multi stage with add_group_by and time proxy dimension tttmp', () => compiler.compile().then(() => {
254+
it('multi stage with add_group_by and time proxy dimension and regular measure', () => compiler.compile().then(() => {
266255
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
267256
measures: [
268-
'visitors.testMeas'
257+
'visitors.revenueAndTime',
258+
'visitors.count'
269259
],
270-
dimensions: ['visitor_checkins.source'],
260+
dimensions: ['visitors.source'],
271261
timezone: 'America/Los_Angeles',
272262
order: [{
273-
id: 'visitor_checkins.source'
263+
id: 'visitors.source'
274264
}],
275265
preAggregationsSchema: ''
276266
});
277267

278268
const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();
279269
const sqlAndParams = query.buildSqlAndParams();
280-
console.log('!!!! sqlAndParamsl', sqlAndParams);
281-
/* console.log('!!!! sqlAndParamsl', sqlAndParams);
282-
expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_rollup');
283-
expect(sqlAndParams[0]).toContain('vis_revenue_and_time_rollup'); */
270+
expect(preAggregationsDescription[0].tableName).toEqual('vis_revenue_and_time_and_count_rollup');
271+
expect(sqlAndParams[0]).toContain('vis_revenue_and_time_and_count_rollup');
272+
expect(sqlAndParams[0]).not.toContain('select * from visitors');
284273

285274
return dbRunner.evaluateQueryWithPreAggregations(query).then(res => {
286-
console.log('!!!! res', res);
287275
expect(res).toEqual(
288276
[
289-
{ vis__source: 'google', vis__revenue_and_time: '25' },
290-
{ vis__source: 'some', vis__revenue_and_time: '50' },
291-
{ vis__source: null, vis__revenue_and_time: '50' }
277+
{ vis__source: 'google', vis__count: '1', vis__revenue_and_time: '25' },
278+
{ vis__source: 'some', vis__count: '2', vis__revenue_and_time: '50' },
279+
{ vis__source: null, vis__count: '3', vis__revenue_and_time: '50' }
292280
]
293281

294282
);
295283
});
296284
}));
285+
} else {
286+
it.skip('multi stage pre-aggregations', () => {
287+
// Skipping because it works only in Tesseract
288+
});
289+
}
297290
});

‎packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1769,7 +1769,7 @@ describe('PreAggregations', () => {
17691769
});
17701770
}));
17711771

1772-
it('partitioned rolling 11', () => compiler.compile().then(() => {
1772+
it('partitioned rolling', () => compiler.compile().then(() => {
17731773
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
17741774
measures: [
17751775
'visitors.checkinsRollingTotal',
@@ -1792,7 +1792,6 @@ describe('PreAggregations', () => {
17921792
});
17931793

17941794
const queryAndParams = query.buildSqlAndParams();
1795-
console.log('!!! ---', queryAndParams);
17961795
const preAggregationsDescription = query.preAggregations?.preAggregationsDescription();
17971796
console.log(preAggregationsDescription);
17981797
expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true);
@@ -1955,7 +1954,7 @@ describe('PreAggregations', () => {
19551954
});
19561955
}));
19571956

1958-
it('segment 111', () => compiler.compile().then(() => {
1957+
it('segment', () => compiler.compile().then(() => {
19591958
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
19601959
measures: [
19611960
'visitors.checkinsTotal'
@@ -1998,7 +1997,7 @@ describe('PreAggregations', () => {
19981997
});
19991998
}));
20001999

2001-
it('rollup join 11', () => compiler.compile().then(() => {
2000+
it('rollup join', () => compiler.compile().then(() => {
20022001
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
20032002
measures: [
20042003
'visitor_checkins.count',
@@ -2214,7 +2213,7 @@ describe('PreAggregations', () => {
22142213
});
22152214
}));
22162215

2217-
it('simple view 11', () => compiler.compile().then(() => {
2216+
it('simple view', () => compiler.compile().then(() => {
22182217
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
22192218
measures: [
22202219
'visitors_view.checkinsTotal'

‎rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
use super::base_query_options::FilterItem;
22
use super::filter_group::{FilterGroup, NativeFilterGroup};
33
use super::filter_params::{FilterParams, NativeFilterParams};
4-
use cubenativeutils::wrappers::NativeArray;
4+
use super::pre_aggregation_obj::{NativePreAggregationObj, PreAggregationObj};
55
use super::security_context::{NativeSecurityContext, SecurityContext};
66
use super::sql_templates_render::{NativeSqlTemplatesRender, SqlTemplatesRender};
7-
use super::pre_aggregation_obj::{NativePreAggregationObj, PreAggregationObj};
87
use super::sql_utils::{NativeSqlUtils, SqlUtils};
98
use cubenativeutils::wrappers::serializer::{
109
NativeDeserialize, NativeDeserializer, NativeSerialize,
@@ -61,6 +60,14 @@ pub trait BaseTools {
6160
origin: String,
6261
) -> Result<String, CubeError>;
6362

64-
fn get_pre_aggregation_by_name(&self, cube_name: String, name: String) -> Result<Rc<dyn PreAggregationObj>, CubeError>;
65-
fn pre_aggregation_table_name(&self, cube_name: String, name: String) -> Result<String, CubeError>; //TODO move to rust
63+
fn get_pre_aggregation_by_name(
64+
&self,
65+
cube_name: String,
66+
name: String,
67+
) -> Result<Rc<dyn PreAggregationObj>, CubeError>;
68+
fn pre_aggregation_table_name(
69+
&self,
70+
cube_name: String,
71+
name: String,
72+
) -> Result<String, CubeError>; //TODO move to rust
6673
}

‎rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ pub mod member_order_by;
2222
pub mod member_sql;
2323
pub mod options_member;
2424
pub mod pre_aggregation_description;
25+
pub mod pre_aggregation_obj;
2526
pub mod security_context;
2627
pub mod segment_definition;
2728
pub mod sql_templates_render;
2829
pub mod sql_utils;
2930
pub mod struct_with_sql_member;
30-
pub mod pre_aggregation_obj;

‎rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ use super::member_sql::{MemberSql, NativeMemberSql};
22
use cubenativeutils::wrappers::serializer::{
33
NativeDeserialize, NativeDeserializer, NativeSerialize,
44
};
5-
use cubenativeutils::wrappers::NativeArray;
65
use cubenativeutils::wrappers::NativeContextHolder;
76
use cubenativeutils::wrappers::NativeObjectHandle;
87
use cubenativeutils::CubeError;

‎rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_obj.rs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
use super::member_sql::{MemberSql, NativeMemberSql};
2-
use cubenativeutils::wrappers::serializer::{
3-
NativeDeserialize, NativeDeserializer, NativeSerialize,
4-
};
5-
use cubenativeutils::wrappers::NativeArray;
1+
use cubenativeutils::wrappers::serializer::{NativeDeserialize, NativeSerialize};
62
use cubenativeutils::wrappers::NativeContextHolder;
73
use cubenativeutils::wrappers::NativeObjectHandle;
84
use cubenativeutils::CubeError;
@@ -22,5 +18,4 @@ pub struct PreAggregationObjStatic {
2218
}
2319

2420
#[nativebridge::native_bridge(PreAggregationObjStatic)]
25-
pub trait PreAggregationObj {
26-
}
21+
pub trait PreAggregationObj {}

‎rust/cubesqlplanner/cubesqlplanner/src/logical_plan/aggregate_multiplied_subquery.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
use super::pretty_print::*;
22
use super::*;
3-
use crate::planner::BaseCube;
43
use std::rc::Rc;
54

65
pub enum AggregateMultipliedSubquerySouce {
@@ -35,7 +34,8 @@ impl PrettyPrint for AggregateMultipliedSubquery {
3534
match self.source.as_ref() {
3635
AggregateMultipliedSubquerySouce::Cube => {
3736
result.println("Cube:", &details_state);
38-
self.pk_cube.pretty_print(result, &details_state.new_level());
37+
self.pk_cube
38+
.pretty_print(result, &details_state.new_level());
3939
}
4040
AggregateMultipliedSubquerySouce::MeasureSubquery(measure_subquery) => {
4141
result.println(

0 commit comments

Comments
 (0)