Skip to content

fix(schema-compiler): Fix incorrect truncated time dimensions over time series queries for BigQuery #9615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

fix(schema-compiler): Fix incorrect truncated time dimensions over time series queries for BigQuery #9615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
84cd46b
fix(schema-compiler): Fix incorrect truncated time dimensions over ti…
KSDaemon May 23, 2025
13420fc
neat yarn cmd
KSDaemon May 26, 2025
5e6fbbb
add drivers tests
KSDaemon May 26, 2025
823790d
update excluded tests
KSDaemon May 26, 2025
a8bd91e
update snapshots
KSDaemon May 26, 2025
5e14c3f
update snapshot
KSDaemon May 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 0 additions & 24 deletions packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,6 @@ export class BigqueryQuery extends BaseQuery {
return new BigqueryFilter(this, filter);
}

public dateSeriesSql(timeDimension: BaseTimeDimension) {
return `${timeDimension.dateSeriesAliasName()} AS (${this.seriesSql(timeDimension)})`;
}

public seriesSql(timeDimension: BaseTimeDimension) {
const values = timeDimension.timeSeries().map(
([from, to]) => `select '${from}' f, '${to}' t`
Expand All @@ -161,26 +157,6 @@ export class BigqueryQuery extends BaseQuery {
return 6;
}

public overTimeSeriesSelect(cumulativeMeasures, dateSeriesSql, baseQuery, dateJoinConditionSql, baseQueryAlias) {
const forSelect = this.overTimeSeriesForSelect(cumulativeMeasures);
const outerSeriesAlias = this.cubeAlias('outer_series');
const outerBase = this.cubeAlias('outer_base');
const timeDimensionAlias = this.timeDimensions.map(d => d.aliasName()).filter(d => !!d)[0];
const aliasesForSelect = this.timeDimensions.map(d => d.dateSeriesSelectColumn(outerSeriesAlias)).concat(
this.dimensions.concat(cumulativeMeasures).map(s => s.aliasName())
).filter(c => !!c).join(', ');
const dateSeriesAlias = this.timeDimensions.map(d => `${d.dateSeriesAliasName()}`).filter(c => !!c)[0];
return `
WITH ${dateSeriesSql} SELECT ${aliasesForSelect} FROM
${dateSeriesAlias} ${outerSeriesAlias}
LEFT JOIN (
SELECT ${forSelect} FROM ${dateSeriesAlias}
INNER JOIN (${baseQuery}) AS ${baseQueryAlias} ON ${dateJoinConditionSql}
${this.groupByClause()}
) AS ${outerBase} ON ${outerSeriesAlias}.${this.escapeColumnName('date_from')} = ${outerBase}.${timeDimensionAlias}
`;
}

public subtractInterval(date, interval) {
return `DATETIME_SUB(${date}, INTERVAL ${this.formatInterval(interval)[0]})`;
}
Expand Down
3 changes: 1 addition & 2 deletions packages/cubejs-testing-drivers/fixtures/athena.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,10 +150,9 @@
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"--------------------",
"week granularity is not supported for intervals",
"--------------------",
"querying BigECommerce: rolling window by 2 week",
Expand Down
2 changes: 1 addition & 1 deletion packages/cubejs-testing-drivers/fixtures/bigquery.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"---------------------------------------",
"SKIPPED SQL API (Need work)",
"---------------------------------------",
"SQL API: reuse params",
Expand Down
18 changes: 14 additions & 4 deletions packages/cubejs-testing-drivers/fixtures/clickhouse.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,20 +179,25 @@
"querying BigECommerce: partitioned pre-agg",
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean",

"---------------------------------------",
"Unsupported JOIN ON conditions. Unexpected 'big_e_commerce__order_date_month > subtractWeeks(date_to, 2)'",
"--------------------",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day",
"querying BigECommerce: rolling window by 2 week",
"querying BigECommerce: rolling window by 2 month",
"querying BigECommerce: rolling window YTD (month)",
"querying BigECommerce: rolling window YTD (month + week)",
"querying BigECommerce: rolling window YTD (month + week + no gran)",
"querying BigECommerce: rolling window YTD (month + week + day)",
"querying BigECommerce: rolling window YTD (month + week + day + no gran)",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"---------------------------------------",
"Custom Granularities ",
Expand All @@ -210,6 +215,11 @@
"SQL API: Rollup over exprs",
"SQL API: Rollup with aliases",
"SQL API: Simple Rollup",
"SQL API: SQL push down push to cube quoted alias"
"SQL API: SQL push down push to cube quoted alias",

"SKIPPED SQL API (due to inconsistency)",
"---------------------------------------",
"Below doesn't work probably due to strict type comparison in ClickHouse, but tests run across all DBs",
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
]
}
2 changes: 0 additions & 2 deletions packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,7 @@
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"---------------------------------------",
"Custom Granularities ",
Expand Down
11 changes: 8 additions & 3 deletions packages/cubejs-testing-drivers/fixtures/mssql.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,9 +141,8 @@
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",
"--------------------",

"---------------------------------------",
"SKIPPED SQL API (Need work)",
"---------------------------------------",
Expand All @@ -161,6 +160,12 @@
"SQL API: Extended nested Rollup over asterisk",
"SQL API: ungrouped pre-agg",
"SQL API: NULLS FIRST/LAST SQL push down",
"SQL API: SQL push down push to cube quoted alias"
"SQL API: SQL push down push to cube quoted alias",

"---------------------------------------",
"Error during rewrite: Can't detect Cube query and it may be not supported yet.",
"---------------------------------------",
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
]
}
10 changes: 8 additions & 2 deletions packages/cubejs-testing-drivers/fixtures/mysql.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,12 @@
"querying BigECommerce: partitioned pre-agg",
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",

"---------------------------------------",
Expand All @@ -157,6 +157,12 @@
"SQL API: Nested Rollup with aliases",
"SQL API: Nested Rollup over asterisk",
"SQL API: Extended nested Rollup over asterisk",
"SQL API: SQL push down push to cube quoted alias"
"SQL API: SQL push down push to cube quoted alias",

"---------------------------------------",
"Error during rewrite: Can't detect Cube query and it may be not supported yet.",
"---------------------------------------",
"SQL API: Rolling Window YTD (year + month + day + date_trunc equal)",
"SQL API: Rolling Window YTD (year + month + day + date_trunc IN)"
]
}
1 change: 1 addition & 0 deletions packages/cubejs-testing-drivers/fixtures/postgres.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@
"querying Products: dimensions -- doesn't work wo ordering",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
Expand Down
4 changes: 2 additions & 2 deletions packages/cubejs-testing-drivers/fixtures/redshift.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,14 @@
"querying Products: dimensions -- doesn't work wo ordering",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range",

"---------------------------------------",
"SKIPPED SQL API (Need work) ",
"---------------------------------------",
Expand Down
2 changes: 1 addition & 1 deletion packages/cubejs-testing-drivers/fixtures/snowflake.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,12 @@
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
"querying BigECommerce: null sum",
"querying BigECommerce: null boolean",

"---------------------------------------",
"Requires Tesseract. ",
"---------------------------------------",
"querying BigECommerce: rolling window by 2 day without date range",
"querying BigECommerce: rolling window by 2 month without date range",
"querying BigECommerce: rolling window YTD",
"querying BigECommerce: rolling window YTD without date range"
]
}
1 change: 1 addition & 0 deletions packages/cubejs-testing-drivers/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"redshift-core": "yarn test-driver -i dist/test/redshift-core.test.js",
"redshift-full": "yarn test-driver -i dist/test/redshift-full.test.js",
"redshift-export-bucket-s3-full": "yarn test-driver -i dist/test/redshift-export-bucket-s3-full.test.js",
"update-all-snapshots-local": "yarn run athena-export-bucket-s3-full --mode=local -u; yarn run bigquery-export-bucket-gcs-full --mode=local -u; yarn run clickhouse-full --mode=local -u; yarn run clickhouse-export-bucket-s3-full --mode=local -u; yarn run clickhouse-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-full --mode=local -u; yarn run mssql-full --mode=local -u; yarn run mysql-full --mode=local -u; yarn run postgres-full --mode=local -u; yarn run redshift-export-bucket-s3-full --mode=local -u; yarn run redshift-full --mode=local -u; yarn run snowflake-encrypted-pk-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-via-storage-integration-full --mode=local -u; yarn run snowflake-export-bucket-gcs-full --mode=local -u; yarn run snowflake-export-bucket-gcs-prefix-full --mode=local -u; yarn run snowflake-export-bucket-s3-full --mode=local -u; yarn run snowflake-full --mode=local -u",
"tst": "clear && yarn tsc && yarn bigquery-core"
},
"files": [
Expand Down
148 changes: 147 additions & 1 deletion packages/cubejs-testing-drivers/src/tests/testQueries.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1579,7 +1579,7 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
expect(response.rawData()).toMatchSnapshot();
});

execute('querying BigECommerce: rolling window YTD', async () => {
execute('querying BigECommerce: rolling window YTD (month)', async () => {
const response = await client.load({
measures: [
'BigECommerce.rollingCountYTD',
Expand All @@ -1589,6 +1589,107 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
granularity: 'month',
dateRange: ['2020-01-01', '2020-12-31'],
}],
order: [
['BigECommerce.orderDate', 'asc'],
],
});
expect(response.rawData()).toMatchSnapshot();
});

execute('querying BigECommerce: rolling window YTD (month + week)', async () => {
const response = await client.load({
measures: [
'BigECommerce.rollingCountYTD',
],
timeDimensions: [{
dimension: 'BigECommerce.orderDate',
granularity: 'month',
dateRange: ['2020-01-01', '2020-12-31'],
}, {
dimension: 'BigECommerce.orderDate',
granularity: 'week',
dateRange: ['2020-01-01', '2020-12-31'],
}],
order: [
['BigECommerce.orderDate', 'asc'],
],
});
expect(response.rawData()).toMatchSnapshot();
});

execute('querying BigECommerce: rolling window YTD (month + week + no gran)', async () => {
const response = await client.load({
measures: [
'BigECommerce.rollingCountYTD',
],
timeDimensions: [{
dimension: 'BigECommerce.orderDate',
granularity: 'month',
dateRange: ['2020-01-01', '2020-12-31'],
}, {
dimension: 'BigECommerce.orderDate',
granularity: 'week',
dateRange: ['2020-01-01', '2020-12-31'],
}, {
dimension: 'BigECommerce.orderDate',
dateRange: ['2020-01-01', '2020-12-31'],
}],
order: [
['BigECommerce.orderDate', 'asc'],
],
});
expect(response.rawData()).toMatchSnapshot();
});

execute('querying BigECommerce: rolling window YTD (month + week + day)', async () => {
const response = await client.load({
measures: [
'BigECommerce.rollingCountYTD',
],
timeDimensions: [{
dimension: 'BigECommerce.orderDate',
granularity: 'month',
dateRange: ['2020-01-01', '2020-03-01'],
}, {
dimension: 'BigECommerce.orderDate',
granularity: 'week',
dateRange: ['2020-01-01', '2020-03-01'],
}, {
dimension: 'BigECommerce.orderDate',
granularity: 'day',
dateRange: ['2020-01-01', '2020-03-01'],
}],
order: [
['BigECommerce.orderDate', 'asc'],
],
});
expect(response.rawData()).toMatchSnapshot();
});

execute('querying BigECommerce: rolling window YTD (month + week + day + no gran)', async () => {
const response = await client.load({
measures: [
'BigECommerce.rollingCountYTD',
],
timeDimensions: [{
dimension: 'BigECommerce.orderDate',
granularity: 'month',
dateRange: ['2020-01-01', '2020-03-01'],
}, {
dimension: 'BigECommerce.orderDate',
granularity: 'week',
dateRange: ['2020-01-01', '2020-03-01'],
}, {
dimension: 'BigECommerce.orderDate',
granularity: 'day',
dateRange: ['2020-01-01', '2020-03-01'],
}, {
dimension: 'BigECommerce.orderDate',
dateRange: ['2020-01-01', '2020-03-01'],
}],
order: [
['BigECommerce.orderDate', 'asc'],
],
});
expect(response.rawData()).toMatchSnapshot();
});
Expand All @@ -1606,6 +1707,19 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
expect(response.rawData()).toMatchSnapshot();
});

execute('querying BigECommerce: rolling window YTD without granularity', async () => {
const response = await client.load({
measures: [
'BigECommerce.rollingCountYTD',
],
timeDimensions: [{
dimension: 'BigECommerce.orderDate',
dateRange: ['2020-01-01', '2020-03-01'],
}],
});
expect(response.rawData()).toMatchSnapshot();
});

if (includeHLLSuite) {
execute('querying BigECommerce: rolling count_distinct_approx window by 2 day', async () => {
const response = await client.load({
Expand Down Expand Up @@ -2093,6 +2207,38 @@ from
expect(res.rows).toMatchSnapshot();
});

executePg('SQL API: Rolling Window YTD (year + month + day + date_trunc equal)', async (connection) => {
// It's important to use day granularity - it tests for ambiguous names
const res = await connection.query(`
SELECT
DATE_TRUNC('year', orderDate) AS "orderDateY",
DATE_TRUNC('month', orderDate) AS "orderDateM",
DATE_TRUNC('day', orderDate) AS "orderDateD",
MEASURE(rollingCountYTD) AS "rollingCountYTD"
FROM "BigECommerce"
WHERE DATE_TRUNC('year', orderDate) = CAST('2020-01-01' AS DATE)
GROUP BY 1, 2, 3
ORDER BY 3 ASC NULLS FIRST;
`);
expect(res.rows).toMatchSnapshot();
});

executePg('SQL API: Rolling Window YTD (year + month + day + date_trunc IN)', async (connection) => {
// It's important to use day granularity - it tests for ambiguous names
const res = await connection.query(`
SELECT
DATE_TRUNC('year', orderDate) AS "orderDateY",
DATE_TRUNC('month', orderDate) AS "orderDateM",
DATE_TRUNC('day', orderDate) AS "orderDateD",
MEASURE(rollingCountYTD) AS "rollingCountYTD"
FROM "BigECommerce"
WHERE DATE_TRUNC('year', orderDate) IN (CAST('2020-01-01' AS DATE))
GROUP BY 1, 2, 3
ORDER BY 3 ASC NULLS FIRST;
`);
expect(res.rows).toMatchSnapshot();
});

executePg('SQL API: SQL push down push to cube quoted alias', async (connection) => {
const res = await connection.query(`
SELECT
Expand Down
Loading
Loading