Skip to content

Commit d46ee6c

Browse files
korawicakorawica
committed
🧪 tests: add templates on tests.
1 parent f472640 commit d46ee6c

File tree

7 files changed

+132
-103
lines changed

7 files changed

+132
-103
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ from datetime import datetime
3737
from sqlplate import SQLPlate
3838

3939
statement: str = (
40-
SQLPlate.system('databricks')
40+
SQLPlate.format('databricks')
4141
.template('etl.delta')
4242
.option('catalog', 'catalog-name')
4343
.option('schema', 'schema-name')

‎src/sqlplate/sqlplate.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,15 @@ def __init__(self, name: str, path: Path) -> None:
2929
raise FileNotFoundError(f"Path {path} does not exists.")
3030

3131
self.path: Path = path
32+
33+
# NOTE: Make default arguments.
3234
self._template_name: str | None = None
3335
self._template_type: str | None = None
3436
self._template: Template | None = None
3537
self._option: dict[str, Any] = {}
3638

3739
@classmethod
38-
def system(cls, name: str, path: Path | None = None) -> 'SQLPlate':
40+
def format(cls, name: str, path: Path | None = None) -> 'SQLPlate':
3941
"""Construction this class from a system value name.
4042
4143
Args:

‎tests/conftest.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from pathlib import Path
2+
from typing import Callable
23

34
import pytest
45

@@ -11,3 +12,15 @@ def test_path() -> Path:
1112
@pytest.fixture(scope="session")
1213
def template_path(test_path) -> Path:
1314
return test_path.parent / 'templates'
15+
16+
17+
@pytest.fixture(scope='session')
18+
def respect_sql(test_path: Path) -> Callable[[str, str], str]:
19+
from tests.utils import load_respect_file
20+
21+
def wrap_respect(template: str, fmt: str) -> str:
22+
return load_respect_file(
23+
test_path / f'templates/{fmt}.{template}.sql'
24+
)
25+
26+
return wrap_respect

‎tests/templates/databricks.etl.delta.query.sql‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
DELETE FROM catalog-name.schema-name.table-name
2+
WHERE
3+
load_src = 'SOURCE_FOO'
4+
AND load_date = 20250201
5+
;
6+
MERGE INTO catalog-name.schema-name.table-name AS target
7+
USING (
8+
WITH change_query AS (
9+
SELECT
10+
src.*,
11+
CASE WHEN tgt.pk_col IS NULL THEN 99
12+
WHEN hash(src.col01, src.col02) <> hash(tgt.col01, tgt.col02) THEN 1
13+
ELSE 0 END AS data_change
14+
FROM ( SELECT * FROM catalog-name.schema-name.source-name ) AS src
15+
LEFT JOIN catalog-name.schema-name.table-name AS tgt
16+
ON tgt.col01 = src.col01
17+
AND tgt.col02 = src.col02
18+
)
19+
SELECT * EXCEPT( data_change ) FROM change_query WHERE data_change IN (99, 1)
20+
) AS source
21+
ON target.pk_col = source.pk_col
22+
WHEN MATCHED THEN UPDATE
23+
SET target.col01= source.col01
24+
,target.col02= source.col02
25+
, target.updt_load_src = 'SOURCE_FOO'
26+
, target.updt_load_id = 1
27+
, target.updt_load_date = to_timestamp('20250201', 'yyyyMMdd')
28+
WHEN NOT MATCHED THEN INSERT
29+
(
30+
col01, col02, pk_col, load_src, load_id, load_date, updt_load_src, updt_load_id, updt_load_date
31+
)
32+
VALUES (
33+
source.col01,
34+
source.col02,
35+
source.pk_col,
36+
'SOURCE_FOO',
37+
1,
38+
20250201,
39+
'SOURCE_FOO',
40+
1,
41+
to_timestamp('20250201', 'yyyyMMdd')
42+
)
43+
;

‎tests/templates/databricks.etl.delta.sql‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
DELETE FROM catalog-name.schema-name.table-name
2+
WHERE
3+
load_src = 'SOURCE_FOO'
4+
AND load_date = 20250201
5+
;
6+
MERGE INTO catalog-name.schema-name.table-name AS target
7+
USING (
8+
WITH change_query AS (
9+
SELECT
10+
src.*,
11+
CASE WHEN tgt.pk_col01 IS NULL THEN 99
12+
WHEN hash(src.col01, src.col02) <> hash(tgt.col01, tgt.col02) THEN 1
13+
ELSE 0 END AS data_change
14+
FROM catalog-name.schema-name.source-name AS src
15+
LEFT JOIN catalog-name.schema-name.table-name AS tgt
16+
ON tgt.col01 = src.col01
17+
AND tgt.col02 = src.col02
18+
)
19+
SELECT * EXCEPT( data_change ) FROM change_query WHERE data_change IN (99, 1)
20+
) AS source
21+
ON target.pk_col01 = source.pk_col01
22+
AND target.pk_col02 = source.pk_col02
23+
WHEN MATCHED THEN UPDATE
24+
SET target.col01= source.col01
25+
,target.col02= source.col02
26+
, target.updt_load_src = 'SOURCE_FOO'
27+
, target.updt_load_id = 1
28+
, target.updt_load_date = to_timestamp('20250201', 'yyyyMMdd')
29+
WHEN NOT MATCHED THEN INSERT
30+
(
31+
col01, col02, pk_col01, pk_col02, load_src, load_id, load_date, updt_load_src, updt_load_id, updt_load_date
32+
)
33+
VALUES (
34+
source.col01,
35+
source.col02,
36+
source.pk_col01,
37+
source.pk_col02,
38+
'SOURCE_FOO',
39+
1,
40+
20250201,
41+
'SOURCE_FOO',
42+
1,
43+
to_timestamp('20250201', 'yyyyMMdd')
44+
)
45+
;

‎tests/test_databricks.py‎

Lines changed: 18 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
from datetime import datetime
22
from textwrap import dedent
3+
from typing import Callable
4+
from functools import partial
35

46
import pytest
57
from jinja2.exceptions import UndefinedError
68
from src.sqlplate import SQLPlate
79

8-
from .utils import prepare_statement
10+
from tests.utils import prepare_statement
11+
12+
13+
@pytest.fixture(scope='module')
14+
def respect_databricks(respect_sql: Callable[[str, str], str]) -> Callable[[str], str]:
15+
return partial(respect_sql, fmt='databricks')
916

1017

1118
def test_sql_select(template_path):
1219
select_sql: SQLPlate = (
13-
SQLPlate.system('databricks', path=template_path)
20+
SQLPlate.format('databricks', path=template_path)
1421
.template('select')
1522
.option('schema', 'schema-name')
1623
.option('table', 'table-name')
@@ -49,9 +56,9 @@ def test_sql_select(template_path):
4956
)
5057

5158

52-
def test_sql_delta(template_path):
59+
def test_sql_delta(template_path, respect_databricks):
5360
select_sql: SQLPlate = (
54-
SQLPlate.system('databricks', path=template_path)
61+
SQLPlate.format('databricks', path=template_path)
5562
.template('etl.delta')
5663
.option('catalog', 'catalog-name')
5764
.option('schema', 'schema-name')
@@ -71,110 +78,20 @@ def test_sql_delta(template_path):
7178
.option('query', 'SELECT * FROM catalog-name.schema-name.source-name')
7279
.load()
7380
)
74-
assert prepare_statement(statement) == dedent("""
75-
DELETE FROM catalog-name.schema-name.table-name
76-
WHERE
77-
load_src = 'SOURCE_FOO'
78-
AND load_date = 20250201
79-
;
80-
MERGE INTO catalog-name.schema-name.table-name AS target
81-
USING (
82-
WITH change_query AS (
83-
SELECT
84-
src.*,
85-
CASE WHEN tgt.pk_col IS NULL THEN 99
86-
WHEN hash(src.col01, src.col02) <> hash(tgt.col01, tgt.col02) THEN 1
87-
ELSE 0 END AS data_change
88-
FROM ( SELECT * FROM catalog-name.schema-name.source-name ) AS src
89-
LEFT JOIN catalog-name.schema-name.table-name AS tgt
90-
ON tgt.col01 = src.col01
91-
AND tgt.col02 = src.col02
92-
)
93-
SELECT * EXCEPT( data_change ) FROM change_query WHERE data_change IN (99, 1)
94-
) AS source
95-
ON target.pk_col = source.pk_col
96-
WHEN MATCHED THEN UPDATE
97-
SET target.col01= source.col01
98-
,target.col02= source.col02
99-
, target.updt_load_src = 'SOURCE_FOO'
100-
, target.updt_load_id = 1
101-
, target.updt_load_date = to_timestamp('20250201', 'yyyyMMdd')
102-
WHEN NOT MATCHED THEN INSERT
103-
(
104-
col01, col02, pk_col, load_src, load_id, load_date, updt_load_src, updt_load_id, updt_load_date
105-
)
106-
VALUES (
107-
source.col01,
108-
source.col02,
109-
source.pk_col,
110-
'SOURCE_FOO',
111-
1,
112-
20250201,
113-
'SOURCE_FOO',
114-
1,
115-
to_timestamp('20250201', 'yyyyMMdd')
116-
)
117-
;
118-
""").strip('\n')
81+
assert prepare_statement(statement) == respect_databricks('etl.delta.query')
11982

12083
statement: str = (
12184
select_sql
12285
.option('pk', ['pk_col01', 'pk_col02'])
12386
.option('source', 'catalog-name.schema-name.source-name')
12487
.load()
12588
)
126-
assert prepare_statement(statement) == dedent("""
127-
DELETE FROM catalog-name.schema-name.table-name
128-
WHERE
129-
load_src = 'SOURCE_FOO'
130-
AND load_date = 20250201
131-
;
132-
MERGE INTO catalog-name.schema-name.table-name AS target
133-
USING (
134-
WITH change_query AS (
135-
SELECT
136-
src.*,
137-
CASE WHEN tgt.pk_col01 IS NULL THEN 99
138-
WHEN hash(src.col01, src.col02) <> hash(tgt.col01, tgt.col02) THEN 1
139-
ELSE 0 END AS data_change
140-
FROM catalog-name.schema-name.source-name AS src
141-
LEFT JOIN catalog-name.schema-name.table-name AS tgt
142-
ON tgt.col01 = src.col01
143-
AND tgt.col02 = src.col02
144-
)
145-
SELECT * EXCEPT( data_change ) FROM change_query WHERE data_change IN (99, 1)
146-
) AS source
147-
ON target.pk_col01 = source.pk_col01
148-
AND target.pk_col02 = source.pk_col02
149-
WHEN MATCHED THEN UPDATE
150-
SET target.col01= source.col01
151-
,target.col02= source.col02
152-
, target.updt_load_src = 'SOURCE_FOO'
153-
, target.updt_load_id = 1
154-
, target.updt_load_date = to_timestamp('20250201', 'yyyyMMdd')
155-
WHEN NOT MATCHED THEN INSERT
156-
(
157-
col01, col02, pk_col01, pk_col02, load_src, load_id, load_date, updt_load_src, updt_load_id, updt_load_date
158-
)
159-
VALUES (
160-
source.col01,
161-
source.col02,
162-
source.pk_col01,
163-
source.pk_col02,
164-
'SOURCE_FOO',
165-
1,
166-
20250201,
167-
'SOURCE_FOO',
168-
1,
169-
to_timestamp('20250201', 'yyyyMMdd')
170-
)
171-
;
172-
""").strip('\n')
89+
assert prepare_statement(statement) == respect_databricks('etl.delta')
17390

17491

17592
def test_sql_scd1_soft_delete(template_path):
17693
select_sql: SQLPlate = (
177-
SQLPlate.system('databricks', path=template_path)
94+
SQLPlate.format('databricks', path=template_path)
17895
.template('etl.scd1-soft-delete')
17996
.option('catalog', 'catalog-name')
18097
.option('schema', 'schema-name')
@@ -330,7 +247,7 @@ def test_sql_scd1_soft_delete(template_path):
330247

331248
def test_sql_scd2(template_path):
332249
select_sql: SQLPlate = (
333-
SQLPlate.system('databricks', path=template_path)
250+
SQLPlate.format('databricks', path=template_path)
334251
.template('etl.scd2')
335252
.option('catalog', 'catalog-name')
336253
.option('schema', 'schema-name')
@@ -415,7 +332,7 @@ def test_sql_scd2(template_path):
415332

416333
def test_sql_scd2_delete_src(template_path):
417334
select_sql: SQLPlate = (
418-
SQLPlate.system('databricks', path=template_path)
335+
SQLPlate.format('databricks', path=template_path)
419336
.template('etl.scd2-delete-src')
420337
.option('catalog', 'catalog-name')
421338
.option('schema', 'schema-name')
@@ -509,7 +426,7 @@ def test_sql_scd2_delete_src(template_path):
509426

510427
def test_sql_transaction(template_path):
511428
select_sql: SQLPlate = (
512-
SQLPlate.system('databricks', path=template_path)
429+
SQLPlate.format('databricks', path=template_path)
513430
.template('etl.transaction')
514431
.option('catalog', 'catalog-name')
515432
.option('schema', 'schema-name')
@@ -547,7 +464,7 @@ def test_sql_transaction(template_path):
547464

548465
def test_sql_full_dump(template_path):
549466
select_sql: SQLPlate = (
550-
SQLPlate.system('databricks', path=template_path)
467+
SQLPlate.format('databricks', path=template_path)
551468
.template('etl.fulldump')
552469
.option('catalog', 'catalog-name')
553470
.option('schema', 'schema-name')

‎tests/utils.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,11 @@
1+
from pathlib import Path
2+
from textwrap import dedent
3+
4+
15
def prepare_statement(statement: str) -> str:
26
return statement.replace('\t', '').strip().strip('\n')
7+
8+
9+
def load_respect_file(template: Path) -> str:
10+
with template.open(mode='rt', encoding='utf-8') as f:
11+
return dedent(f.read()).strip('\n')

0 commit comments

Comments
 (0)