Skip to content

Commit 89e1eb5

Browse files
SergeyZ06SergeyZ06
authored
Merge pull request #25 from epoch8/development
Multi sources now available
2 parents aa99be8 + 93a0c7d commit 89e1eb5

File tree

79 files changed

+1222
-1094
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+1222
-1094
lines changed

‎dbt_project.yml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,13 @@ clean-targets: # directories to be removed by `dbt clean`
2525
- "dbt_packages"
2626

2727
# vars:
28-
# DBT_PACKAGE_GA4__PROJECT: "holybricks"
29-
# DBT_PACKAGE_GA4__DATASET: "analytics_202294062"
28+
# DBT_PACKAGE_GA4__PROJECT: "commonwealth-356813"
29+
# DBT_PACKAGE_GA4__DATASET: "analytics_375681484"
30+
# DBT_PACKAGE_GA4__DATASETS: "analytics_375681484,analytics_247024976"
3031
# DBT_PACKAGE_GA4__INTERVAL: 14
3132
# DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL: 4
32-
# DBT_PACKAGE_GA4__STREAM_ID: ['2059071580', '2771133919']
33-
# DBT_PACKAGE_GA4__TIME_ZONE: 'America/Los_Angeles'
33+
# DBT_PACKAGE_GA4__STREAM_ID: "['2059071580', '5200866205', '5200058214']"
34+
# DBT_PACKAGE_GA4__TIME_ZONE: "America/Los_Angeles"
3435

3536

3637
# Configuring models
@@ -43,5 +44,11 @@ clean-targets: # directories to be removed by `dbt clean`
4344
models:
4445
dbt_package_ga4:
4546

47+
integration:
48+
+dataset: integration
49+
4650
anchor:
4751
+dataset: anchor
52+
53+
bi:
54+
+dataset: bi

‎models/anchor/ga4_event/anchor__ga4_event.sql‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{
1+
{{-
22
config(
33
enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
44
tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
1111
},
1212
cluster_by = 'ga4_event_id'
1313
)
14-
}}
14+
-}}
1515

1616

1717
WITH t1 AS (
1818
SELECT DISTINCT
19-
PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
19+
PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
2020
TO_HEX(
2121
MD5(
2222
CONCAT(
@@ -27,16 +27,17 @@ WITH t1 AS (
2727
)
2828
) AS ga4_event_id
2929
FROM
30-
{{ source('dbt_package_ga4', 'events') }} AS events
30+
{{ ref('src_ga4__events') }} AS events
3131
WHERE
32-
_TABLE_SUFFIX NOT LIKE '%intraday%'
33-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
34-
AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
32+
TABLE_SUFFIX NOT LIKE '%intraday%'
33+
{%- if not is_incremental() %}
34+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
35+
{%- endif %}
3536

36-
{% if is_incremental() %}
37-
{% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
38-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
39-
{% endif %}
37+
{%- if is_incremental() %}
38+
{%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
39+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
40+
{%- endif %}
4041
),
4142

4243
t2 AS (
@@ -60,7 +61,7 @@ final AS (
6061

6162
SELECT * FROM final
6263

63-
{% if is_incremental() %}
64+
{%- if is_incremental() %}
6465
WHERE
6566
final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
66-
{% endif %}
67+
{%- endif %}

‎models/anchor/ga4_event/attr__ga4_event__ga4_event_bundle_sequence_id.sql‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{
1+
{{-
22
config(
33
enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
44
tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
1111
},
1212
cluster_by = 'ga4_event_id'
1313
)
14-
}}
14+
-}}
1515

1616

1717
WITH t1 AS (
1818
SELECT DISTINCT
19-
PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
19+
PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
2020
TO_HEX(
2121
MD5(
2222
CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
2828
) AS ga4_event_id,
2929
events.event_bundle_sequence_id AS ga4_event_bundle_sequence_id
3030
FROM
31-
{{ source('dbt_package_ga4', 'events') }} AS events
31+
{{ ref('src_ga4__events') }} AS events
3232
WHERE
33-
_TABLE_SUFFIX NOT LIKE '%intraday%'
34-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
35-
AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
36-
37-
{% if is_incremental() %}
38-
{% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
39-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
40-
{% endif %}
33+
TABLE_SUFFIX NOT LIKE '%intraday%'
34+
{%- if not is_incremental() %}
35+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
36+
{%- endif %}
37+
38+
{%- if is_incremental() %}
39+
{%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
40+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
41+
{%- endif %}
4142
),
4243

4344
t2 AS (
@@ -76,7 +77,7 @@ final AS (
7677

7778
SELECT * FROM final
7879

79-
{% if is_incremental() %}
80+
{%- if is_incremental() %}
8081
WHERE
8182
final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
82-
{% endif %}
83+
{%- endif %}

‎models/anchor/ga4_event/attr__ga4_event__ga4_event_date.sql‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{
1+
{{-
22
config(
33
enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
44
tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
1111
},
1212
cluster_by = 'ga4_event_id'
1313
)
14-
}}
14+
-}}
1515

1616

1717
WITH t1 AS (
1818
SELECT DISTINCT
19-
PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
19+
PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
2020
TO_HEX(
2121
MD5(
2222
CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
2828
) AS ga4_event_id,
2929
SAFE_CAST(events.event_date AS DATE FORMAT 'YYYYMMDD') AS ga4_event_date
3030
FROM
31-
{{ source('dbt_package_ga4', 'events') }} AS events
31+
{{ ref('src_ga4__events') }} AS events
3232
WHERE
33-
_TABLE_SUFFIX NOT LIKE '%intraday%'
34-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
35-
AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
36-
37-
{% if is_incremental() %}
38-
{% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
39-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
40-
{% endif %}
33+
TABLE_SUFFIX NOT LIKE '%intraday%'
34+
{%- if not is_incremental() %}
35+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
36+
{%- endif %}
37+
38+
{%- if is_incremental() %}
39+
{%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
40+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
41+
{%- endif %}
4142
),
4243

4344
t2 AS (
@@ -64,7 +65,7 @@ final AS (
6465

6566
SELECT * FROM final
6667

67-
{% if is_incremental() %}
68+
{%- if is_incremental() %}
6869
WHERE
6970
final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
70-
{% endif %}
71+
{%- endif %}

‎models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_purchase_revenue.sql‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{
1+
{{-
22
config(
33
enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
44
tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
1111
},
1212
cluster_by = 'ga4_event_id'
1313
)
14-
}}
14+
-}}
1515

1616

1717
WITH t1 AS (
1818
SELECT DISTINCT
19-
PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
19+
PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
2020
TO_HEX(
2121
MD5(
2222
CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
2828
) AS ga4_event_id,
2929
events.ecommerce.purchase_revenue AS ga4_event_ecommerce_purchase_revenue
3030
FROM
31-
{{ source('dbt_package_ga4', 'events') }} AS events
31+
{{ ref('src_ga4__events') }} AS events
3232
WHERE
33-
_TABLE_SUFFIX NOT LIKE '%intraday%'
34-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
35-
AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
36-
37-
{% if is_incremental() %}
38-
{% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
39-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
40-
{% endif %}
33+
TABLE_SUFFIX NOT LIKE '%intraday%'
34+
{%- if not is_incremental() %}
35+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
36+
{%- endif %}
37+
38+
{%- if is_incremental() %}
39+
{%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
40+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
41+
{%- endif %}
4142
),
4243

4344
t2 AS (
@@ -64,7 +65,7 @@ final AS (
6465

6566
SELECT * FROM final
6667

67-
{% if is_incremental() %}
68+
{%- if is_incremental() %}
6869
WHERE
6970
final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
70-
{% endif %}
71+
{%- endif %}

‎models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_purchase_revenue_in_usd.sql‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{
1+
{{-
22
config(
33
enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
44
tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
1111
},
1212
cluster_by = 'ga4_event_id'
1313
)
14-
}}
14+
-}}
1515

1616

1717
WITH t1 AS (
1818
SELECT DISTINCT
19-
PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
19+
PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
2020
TO_HEX(
2121
MD5(
2222
CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
2828
) AS ga4_event_id,
2929
events.ecommerce.purchase_revenue_in_usd AS ga4_event_ecommerce_purchase_revenue_in_usd
3030
FROM
31-
{{ source('dbt_package_ga4', 'events') }} AS events
31+
{{ ref('src_ga4__events') }} AS events
3232
WHERE
33-
_TABLE_SUFFIX NOT LIKE '%intraday%'
34-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
35-
AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
36-
37-
{% if is_incremental() %}
38-
{% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
39-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
40-
{% endif %}
33+
TABLE_SUFFIX NOT LIKE '%intraday%'
34+
{%- if not is_incremental() %}
35+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
36+
{%- endif %}
37+
38+
{%- if is_incremental() %}
39+
{%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
40+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
41+
{%- endif %}
4142
),
4243

4344
t2 AS (
@@ -64,7 +65,7 @@ final AS (
6465

6566
SELECT * FROM final
6667

67-
{% if is_incremental() %}
68+
{%- if is_incremental() %}
6869
WHERE
6970
final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
70-
{% endif %}
71+
{%- endif %}

‎models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_refund_value.sql‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{
1+
{{-
22
config(
33
enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
44
tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
1111
},
1212
cluster_by = 'ga4_event_id'
1313
)
14-
}}
14+
-}}
1515

1616

1717
WITH t1 AS (
1818
SELECT DISTINCT
19-
PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
19+
PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
2020
TO_HEX(
2121
MD5(
2222
CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
2828
) AS ga4_event_id,
2929
events.ecommerce.refund_value AS ga4_event_ecommerce_refund_value
3030
FROM
31-
{{ source('dbt_package_ga4', 'events') }} AS events
31+
{{ ref('src_ga4__events') }} AS events
3232
WHERE
33-
_TABLE_SUFFIX NOT LIKE '%intraday%'
34-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
35-
AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
36-
37-
{% if is_incremental() %}
38-
{% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
39-
AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
40-
{% endif %}
33+
TABLE_SUFFIX NOT LIKE '%intraday%'
34+
{%- if not is_incremental() %}
35+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
36+
{%- endif %}
37+
38+
{%- if is_incremental() %}
39+
{%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
40+
AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
41+
{%- endif %}
4142
),
4243

4344
t2 AS (
@@ -64,7 +65,7 @@ final AS (
6465

6566
SELECT * FROM final
6667

67-
{% if is_incremental() %}
68+
{%- if is_incremental() %}
6869
WHERE
6970
final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
70-
{% endif %}
71+
{%- endif %}

0 commit comments

Comments
 (0)