Skip to content

Commit a4b4a59

Browse files
committed
⚙️ fixed: change way to get dq.
1 parent 9e1111e commit a4b4a59

File tree

5 files changed

+50
-37
lines changed

5 files changed

+50
-37
lines changed

README.md

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ print(statement.strip().strip('\n'))
6969

7070
The result SQL statement:
7171

72-
```text
72+
```sql
7373
MERGE INTO catalog-name.schema-name.table-name AS target
7474
USING (
7575
WITH change_query AS (
@@ -117,20 +117,26 @@ WHEN NOT MATCHED THEN INSERT
117117
118118
```python
119119
from sqlplate import SQLPlate
120-
from sqlplate.rules import Unique, NotNull, Count
121120

122-
report: str = (
121+
statement: str = (
123122
SQLPlate.format('databricks')
124-
.quality(mode="pushdown")
123+
.template('quality.check')
125124
.option('catalog', 'catalog-name')
126125
.option('schema', 'schema-name')
127126
.option('table', 'table-name')
128-
.check('unique', Unique(cols=['pk_col']))
129-
.check('not-null', NotNull(cols=['col01', 'col02']))
130-
.check('row-count', Count())
131-
.validate(output='html')
127+
.option('filter', "load_date >= to_timestamp('20250201', 'yyyyMMdd')")
128+
.option('unique', ['pk_col'])
129+
.option('notnull', ['col01', 'col02'])
130+
.option('row_count', True)
131+
.load()
132132
)
133-
print(report.strip().strip('\n'))
133+
print(statement.strip().strip('\n'))
134+
```
135+
136+
The result SQL statement:
137+
138+
```sql
139+
134140
```
135141

136142
## :chains: Support Systems

src/sqlplate/rules.py

Lines changed: 0 additions & 10 deletions
This file was deleted.

src/sqlplate/sqlplate.py

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -147,21 +147,3 @@ def stream(
147147
)
148148
if trim(s) != ''
149149
)
150-
151-
def check(
152-
self,
153-
name: str,
154-
rule: Any,
155-
) -> 'SQLPlate':
156-
return self
157-
158-
def validate(
159-
self,
160-
output: Literal["json", "html"],
161-
):
162-
return self
163-
164-
def filter(
165-
self,
166-
):
167-
return self
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{% extends "base.jinja" %}
2+
3+
{% block statement %}
4+
WITH source AS (
5+
SELECT
6+
*
7+
FROM {{ catalog }}.{{ schema }}.{{ table }}
8+
{%+ if filter %}WHERE {{ filter }}{% endif +%}
9+
)
10+
SELECT
11+
*
12+
{%+ if row_count %}, (SELECT COUNT(1) FROM source) AS table_records{% endif +%}
13+
{%+ if unique -%}
14+
{%- for col in unique -%}
15+
, (SELECT COUNT {{ col }} FROM (SELECT DISTINCT {{ col}} FROM source)) AS unique_{{ col }}
16+
{%- endfor -%}
17+
{%- endif +%}
18+
FROM source
19+
{% endblock statement %}

tests/test_databricks.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,3 +277,19 @@ def test_sql_full_dump(template_path):
277277
FROM ( SELECT * FROM catalog-name.schema-name.source-name ) AS sub_query
278278
;
279279
""").strip('\n')
280+
281+
282+
def test_quality_check(template_path):
283+
statement: SQLPlate = (
284+
SQLPlate.format('databricks', path=template_path)
285+
.template('quality.check')
286+
.option('catalog', 'catalog-name')
287+
.option('schema', 'schema-name')
288+
.option('table', 'table-name')
289+
.option('filter', "load_date >= to_timestamp('20250201', 'yyyyMMdd')")
290+
.option('unique', ['pk_col'])
291+
.option('notnull', ['col01', 'col02'])
292+
.option("row_count", True)
293+
.load()
294+
)
295+
print(statement)

0 commit comments

Comments
 (0)