Skip to content

Commit dd46da5

Browse files
ataftataft
committed
Support dbt 1.4
* metrics to 1.4.1 * support for Azure and sqlserver adapter, when it upgrades to 1.4 * updated data to 2021 through 2022
1 parent ee0d329 commit dd46da5

File tree

5 files changed

+1841
-1557
lines changed

5 files changed

+1841
-1557
lines changed

‎README.md‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,10 @@ To run this project (assuming you have dbt installed):
3232
1. Clone this repo
3333
2. [Set up a profile](https://docs.getdbt.com/reference/profiles.yml) to connect to your database
3434
3. Run `dbt deps`
35-
4. Run `dbt seed`
36-
5. Run `dbt run`
37-
6. Run `dbt snapshot`
38-
7. Run `dbt test` (tests and add constraints)
39-
8. Run `dbt source freshness`
40-
9. Run `dbt docs generate`
41-
10. Run `dbt docs serve` (if you want to run a local docs server)
35+
4. Run `dbt build`, which does the following: seed, test, run, snapshot
36+
5. Run `dbt source freshness`
37+
6. Run `dbt docs generate`
38+
7. Run `dbt docs serve` (if you want to run a local docs server)
4239

4340
### dbt Cloud
4441

@@ -68,23 +65,19 @@ To run this project (assuming you have dbt installed):
6865

6966
7. In the command line at the bottom, run `dbt deps`
7067

71-
8. Run `dbt seed`
68+
8. Run `dbt build`
7269

73-
9. Run `dbt run`
70+
9. Run `dbt source freshness`
7471

75-
10. Run `dbt snapshot`
76-
77-
11. Run `dbt source freshness`
78-
79-
12. If all runs OK, then [Set up an Environment](https://docs.getdbt.com/docs/dbt-cloud/cloud-quickstart#create-an-environment)
72+
10. If all runs OK, then [Set up an Environment](https://docs.getdbt.com/docs/dbt-cloud/cloud-quickstart#create-an-environment)
8073

8174
> Make sure it is a **deployment** environment
8275
83-
13. [Create a new Job](https://docs.getdbt.com/docs/dbt-cloud/cloud-quickstart#create-a-new-job)
76+
11. [Create a new Job](https://docs.getdbt.com/docs/dbt-cloud/cloud-quickstart#create-a-new-job)
8477

8578
> In *Execution Settings*, check **Generate Docs** and **Run Source Freshness**
8679
87-
14. Run the new job
80+
12. Run the new job
8881

8982

9083
## Connect Business Intelligence tool

‎dbt_project.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# and underscores. A good package name should reflect your organization's
44
# name or the intended use of these models
55
name: 'dbt_business_intelligence'
6-
version: '1.3.0'
6+
version: '1.4.0'
77
config-version: 2
88

99
require-dbt-version: ">=1.3.0"
@@ -55,3 +55,9 @@ models:
5555
seeds:
5656
dbt_business_intelligence:
5757
schema: staging # all seeds in this project will use the mapping schema by default
58+
59+
dispatch:
60+
- macro_namespace: dbt_utils
61+
search_order: ['tsql_utils', 'dbt_utils']
62+
- macro_namespace: dbt_date
63+
search_order: ['tsql_utils', 'dbt_date']

‎macros/sqlserver__create_constraints.sql‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
{# SQL Server specific implementation to create a primary key #}
2+
{%- macro sqlserver__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false) -%}
3+
{%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%}
4+
5+
{%- if constraint_name|length > 63 %}
6+
{%- set constraint_name_query %}
7+
select 'PK_' || md5( '{{ constraint_name }}' )::varchar as "constraint_name"
8+
{%- endset -%}
9+
{%- set results = run_query(constraint_name_query) -%}
10+
{%- set constraint_name = results.columns[0].values()[0] -%}
11+
{% endif %}
12+
13+
{%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%}
14+
15+
{#- Check that the table does not already have this PK/UK -#}
16+
{%- if not dbt_constraints.unique_constraint_exists(table_relation, column_names) -%}
17+
18+
{%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions) -%}
19+
20+
{%- do log("Creating primary key: " ~ constraint_name, info=true) -%}
21+
{%- call statement('add_pk', fetch_result=False, auto_begin=True) -%}
22+
ALTER TABLE {{table_relation}} ADD CONSTRAINT {{constraint_name}} PRIMARY KEY ( {{columns_csv}} )
23+
{%- endcall -%}
24+
{{ adapter.commit() }}
25+
26+
{%- else -%}
27+
{%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ table_relation, info=false) -%}
28+
{%- endif -%}
29+
30+
{%- else -%}
31+
{%- do log("Skipping " ~ constraint_name ~ " because PK/UK already exists: " ~ table_relation ~ " " ~ column_names, info=false) -%}
32+
{%- endif -%}
33+
34+
{%- endmacro -%}
35+
36+
37+
38+
{# SQL Server specific implementation to create a unique key #}
39+
{%- macro sqlserver__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false) -%}
40+
{%- set constraint_name = (table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%}
41+
42+
{%- if constraint_name|length > 63 %}
43+
{%- set constraint_name_query %}
44+
select 'UK_' || md5( '{{ constraint_name }}' )::varchar as "constraint_name"
45+
{%- endset -%}
46+
{%- set results = run_query(constraint_name_query) -%}
47+
{%- set constraint_name = results.columns[0].values()[0] -%}
48+
{% endif %}
49+
50+
{%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%}
51+
52+
{#- Check that the table does not already have this PK/UK -#}
53+
{%- if not dbt_constraints.unique_constraint_exists(table_relation, column_names) -%}
54+
55+
{%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions) -%}
56+
57+
{%- do log("Creating unique key: " ~ constraint_name, info=true) -%}
58+
{%- call statement('add_uk', fetch_result=False, auto_begin=True) -%}
59+
ALTER TABLE {{table_relation}} ADD CONSTRAINT {{constraint_name}} UNIQUE ( {{columns_csv}} )
60+
{%- endcall -%}
61+
{{ adapter.commit() }}
62+
63+
{%- else -%}
64+
{%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ table_relation, info=false) -%}
65+
{%- endif -%}
66+
67+
{%- else -%}
68+
{%- do log("Skipping " ~ constraint_name ~ " because PK/UK already exists: " ~ table_relation ~ " " ~ column_names, info=false) -%}
69+
{%- endif -%}
70+
71+
{%- endmacro -%}
72+
73+
{# SQL Server specific implementation to create a not null constraint #}
74+
{%- macro sqlserver__create_not_null(table_relation, column_names, verify_permissions, quote_columns=false) -%}
75+
{%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%}
76+
77+
{%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions) -%}
78+
79+
{%- set modify_statements= [] -%}
80+
{%- for column in columns_list -%}
81+
{%- set modify_statements = modify_statements.append( "ALTER COLUMN " ~ column ~ " SET NOT NULL" ) -%}
82+
{%- endfor -%}
83+
{%- set modify_statement_csv = modify_statements | join(", ") -%}
84+
{%- do log("Creating not null constraint for: " ~ columns_list | join(", ") ~ " in " ~ table_relation, info=true) -%}
85+
{%- call statement('add_nn', fetch_result=False, auto_begin=True) -%}
86+
ALTER TABLE {{table_relation}} {{ modify_statement_csv }};
87+
{%- endcall -%}
88+
{{ adapter.commit() }}
89+
90+
{%- else -%}
91+
{%- do log("Skipping not null constraint for " ~ columns_list | join(", ") ~ " in " ~ table_relation ~ " because of insufficient privileges: " ~ table_relation, info=true) -%}
92+
{%- endif -%}
93+
{%- endmacro -%}
94+
95+
{# SQL Server specific implementation to create a foreign key #}
96+
{%- macro sqlserver__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true) -%}
97+
{%- set constraint_name = (fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%}
98+
99+
{%- if constraint_name|length > 63 %}
100+
{%- set constraint_name_query %}
101+
select 'FK_' || md5( '{{ constraint_name }}' )::varchar as "constraint_name"
102+
{%- endset -%}
103+
{%- set results = run_query(constraint_name_query) -%}
104+
{%- set constraint_name = results.columns[0].values()[0] -%}
105+
{% endif %}
106+
107+
{%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%}
108+
{%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%}
109+
{#- Check that the PK table has a PK or UK -#}
110+
{%- if dbt_constraints.unique_constraint_exists(pk_table_relation, pk_column_names) -%}
111+
{#- Check if the table already has this foreign key -#}
112+
{%- if not dbt_constraints.foreign_key_exists(fk_table_relation, fk_column_names) -%}
113+
114+
{%- if dbt_constraints.have_ownership_priv(fk_table_relation, verify_permissions) and dbt_constraints.have_references_priv(pk_table_relation, verify_permissions) -%}
115+
116+
{%- do log("Creating foreign key: " ~ constraint_name ~ " referencing " ~ pk_table_relation.identifier ~ " " ~ pk_column_names, info=true) -%}
117+
{%- call statement('add_fk', fetch_result=False, auto_begin=True) -%}
118+
ALTER TABLE {{fk_table_relation}} ADD CONSTRAINT {{constraint_name}} FOREIGN KEY ( {{fk_columns_csv}} ) REFERENCES {{pk_table_relation}} ( {{pk_columns_csv}} ) ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED
119+
{%- endcall -%}
120+
{{ adapter.commit() }}
121+
122+
{%- else -%}
123+
{%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ fk_table_relation ~ " referencing " ~ pk_table_relation, info=true) -%}
124+
{%- endif -%}
125+
126+
{%- else -%}
127+
{%- do log("Skipping " ~ constraint_name ~ " because FK already exists: " ~ fk_table_relation ~ " " ~ fk_column_names, info=false) -%}
128+
{%- endif -%}
129+
{%- else -%}
130+
{%- do log("Skipping " ~ constraint_name ~ " because a PK/UK was not found on the PK table: " ~ pk_table_relation ~ " " ~ pk_column_names, info=true) -%}
131+
{%- endif -%}
132+
133+
{%- endmacro -%}
134+
135+
136+
137+
{#- This macro is used in create macros to avoid duplicate PK/UK constraints
138+
and to skip FK where no PK/UK constraint exists on the parent table -#}
139+
{%- macro sqlserver__unique_constraint_exists(table_relation, column_names) -%}
140+
{%- set lookup_query -%}
141+
select c.oid as constraint_name
142+
, upper(col.attname) as column_name
143+
from pg_constraint c
144+
cross join lateral unnest(c.conkey) as con(conkey)
145+
join pg_class tbl on tbl.oid = c.conrelid
146+
join pg_namespace ns on ns.oid = tbl.relnamespace
147+
join pg_attribute col on (col.attrelid = tbl.oid
148+
and col.attnum = con.conkey)
149+
where c.contype in ('p', 'u')
150+
and ns.nspname ilike '{{table_relation.schema}}'
151+
and tbl.relname ilike '{{table_relation.identifier}}'
152+
order by constraint_name
153+
{%- endset -%}
154+
{%- do log("Lookup: " ~ lookup_query, info=false) -%}
155+
{%- set constraint_list = run_query(lookup_query) -%}
156+
{%- if constraint_list.columns["column_name"].values() | count > 0 -%}
157+
{%- for constraint in constraint_list.group_by("constraint_name") -%}
158+
{%- if dbt_constraints.column_list_matches(constraint.columns["column_name"].values(), column_names ) -%}
159+
{%- do log("Found PK/UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%}
160+
{{ return(true) }}
161+
{%- endif -%}
162+
{% endfor %}
163+
{%- endif -%}#}
164+
165+
{#- If we get this far then the table does not have either constraint -#}
166+
{%- do log("No PK/UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%}
167+
{{ return(false) }}
168+
{%- endmacro -%}
169+
170+
171+
172+
{#- This macro is used in create macros to avoid duplicate FK constraints -#}
173+
{%- macro sqlserver__foreign_key_exists(table_relation, column_names) -%}
174+
{%- set lookup_query -%}
175+
select c.oid as fk_name
176+
, upper(col.attname) as fk_column_name
177+
from pg_constraint c
178+
cross join lateral unnest(c.conkey) as con(conkey)
179+
join pg_class tbl on tbl.oid = c.conrelid
180+
join pg_namespace ns on ns.oid = tbl.relnamespace
181+
join pg_attribute col on (col.attrelid = tbl.oid
182+
and col.attnum = con.conkey)
183+
where c.contype in ('f')
184+
and ns.nspname ilike '{{table_relation.schema}}'
185+
and tbl.relname ilike '{{table_relation.identifier}}'
186+
order by fk_name
187+
{%- endset -%}
188+
{%- do log("Lookup: " ~ lookup_query, info=false) -%}
189+
{%- set constraint_list = run_query(lookup_query) -%}
190+
{%- if constraint_list.columns["fk_column_name"].values() | count > 0 -%}
191+
{%- for constraint in constraint_list.group_by("fk_name") -%}
192+
{%- if dbt_constraints.column_list_matches(constraint.columns["fk_column_name"].values(), column_names ) -%}
193+
{%- do log("Found FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%}
194+
{{ return(true) }}
195+
{%- endif -%}
196+
{% endfor %}
197+
{%- endif -%}
198+
199+
{#- If we get this far then the table does not have this constraint -#}
200+
{%- do log("No FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%}
201+
{{ return(false) }}
202+
{%- endmacro -%}
203+
204+
205+
{%- macro sqlserver__have_references_priv(table_relation, verify_permissions) -%}
206+
{%- if verify_permissions is sameas true -%}
207+
208+
{%- set lookup_query -%}
209+
select case when count(*) > 0 then 'y' else 'n' end as "have_references"
210+
from information_schema.table_privileges t
211+
join information_schema.enabled_roles er on t.grantee = er.role_name
212+
where upper(t.table_schema) = upper('{{table_relation.schema}}')
213+
and upper(t.table_name) = upper('{{table_relation.identifier}}')
214+
{%- endset -%}
215+
{%- do log("Lookup: " ~ lookup_query, info=false) -%}
216+
{%- set results = run_query(lookup_query) -%}
217+
{%- if "y" in( results.columns["have_references"].values() ) -%}
218+
{{ return(true) }}
219+
{%- endif -%}
220+
221+
{{ return(false) }}
222+
{%- else -%}
223+
{{ return(true) }}
224+
{%- endif -%}
225+
{%- endmacro -%}
226+
227+
228+
{%- macro sqlserver__have_ownership_priv(table_relation, verify_permissions) -%}
229+
{%- if verify_permissions is sameas true -%}
230+
231+
{%- set lookup_query -%}
232+
select case when count(*) > 0 then 'y' else 'n' end as "have_ownership"
233+
from pg_catalog.pg_tables t
234+
join information_schema.enabled_roles er on t.tableowner = er.role_name
235+
where upper(t.schemaname) = upper('{{table_relation.schema}}')
236+
and upper(t.tablename) = upper('{{table_relation.identifier}}')
237+
{%- endset -%}
238+
{%- do log("Lookup: " ~ lookup_query, info=false) -%}
239+
{%- set results = run_query(lookup_query) -%}
240+
{%- if "y" in( results.columns["have_ownership"].values() ) -%}
241+
{{ return(true) }}
242+
{%- endif -%}
243+
244+
{{ return(false) }}
245+
{%- else -%}
246+
{{ return(true) }}
247+
{%- endif -%}
248+
{%- endmacro -%}
249+
250+
251+
{% macro sqlserver__drop_referential_constraints(relation) -%}
252+
{%- set lookup_query -%}
253+
select constraint_name
254+
from information_schema.table_constraints
255+
where table_schema = '{{relation.schema}}'
256+
and table_name='{{relation.identifier}}'
257+
and constraint_type in ('FOREIGN KEY', 'PRIMARY KEY', 'UNIQUE')
258+
{%- endset -%}
259+
{%- set constraint_list = run_query(lookup_query) -%}
260+
261+
{%- for constraint_name in constraint_list.columns["constraint_name"].values() -%}
262+
{%- do log("Dropping constraint: " ~ constraint_name ~ " from table " ~ relation, info=false) -%}
263+
{%- call statement('drop_constraint_cascade', fetch_result=False, auto_begin=True) -%}
264+
ALTER TABLE {{relation}} DROP CONSTRAINT IF EXISTS "{{constraint_name}}" CASCADE
265+
{%- endcall -%}
266+
{{ adapter.commit() }}
267+
{% endfor %}
268+
269+
{% endmacro %}
270+
271+
{#- SQL Server will error if you try to truncate tables with FK constraints or tables with PK/UK constraints
272+
referenced by FK so we will drop all constraints before truncating tables -#}
273+
{% macro sqlserver__truncate_relation(relation) -%}
274+
{{ sqlserver__drop_referential_constraints(relation) }}
275+
{{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}
276+
{% endmacro %}
277+
278+
{#- SQL Server will get deadlocks if you try to drop tables with FK constraints or tables with PK/UK constraints
279+
referenced by FK so we will drop all constraints before dropping tables -#}
280+
{% macro sqlserver__drop_relation(relation) -%}
281+
{{ sqlserver__drop_referential_constraints(relation) }}
282+
{{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}
283+
{% endmacro %}

‎packages.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ packages:
22
- package: calogica/dbt_date
33
version: 0.6.2
44
- package: dbt-labs/metrics
5-
version: 1.3.0
6-
- git: "https://github.com/flexanalytics/dbt_constraints.git"
7-
revision: main
5+
version: 1.4.1
6+
- package: Snowflake-Labs/dbt_constraints
7+
version: 0.6.0
8+
- package: dbt-msft/tsql_utils
9+
version: 0.9.0

0 commit comments

Comments
 (0)