From 8516ed267263d0aaba274810d42db70c4d3e0694 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 16 Apr 2025 13:45:23 +0200 Subject: [PATCH 1/3] feat: support TTL as a column-level configuration --- CHANGELOG.md | 29 +++++++++++++++++ README.md | 23 ++++++++++++- dbt/adapters/clickhouse/impl.py | 6 +++- .../constraints/fixtures_contraints.py | 32 +++++++++++++++++++ .../adapter/constraints/test_constraints.py | 13 ++++++++ 5 files changed, 101 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8062cf83..9615a2e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +### Unreleased + +#### Improvements + +* Added support for [TTL (time-to-live)](https://clickhouse.com/docs/guides/developer/ttl) as a column configuration for `table` and `ephemeral` materializations. This feature is implemented as a [custom constraint](https://docs.getdbt.com/reference/resource-properties/constraints#custom-constraints), which requires model contracts to be enforced. For example: + + ```sql + -- test_ttl.sql + {{ config(order_by='(ts)', engine='MergeTree()', materialized='table') }} + + SELECT now() AS ts, + 'Some value that should expire!' AS col_ttl + ``` + + ```yaml + models: + - name: test_ttl + description: 'Testing column-level TTL' + config: + contract: + enforced: true + columns: + - name: ts + data_type: timestamp + - name: col_ttl + data_type: String + ttl: ts + INTERVAL 1 DAY + ``` + ### Release [1.8.9], 2025-02-16 #### Improvements diff --git a/README.md b/README.md index 232f5ee5..f491c7bd 100644 --- a/README.md +++ b/README.md @@ -123,9 +123,30 @@ your_profile_name: ## Column Configuration +> **_NOTE:_** The column configuration options below require [model contracts](https://docs.getdbt.com/docs/collaborate/govern/model-contracts) to be enforced. + | Option | Description | Default if any | |--------|------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------| -| codec | A string consisting of arguments passed to `CODEC()` in the column's DDL. For example: `codec: "Delta, ZSTD"` will be interpreted as `CODEC(Delta, ZSTD)`. | | +| codec | A string consisting of arguments passed to `CODEC()` in the column's DDL. For example: `codec: "Delta, ZSTD"` will be compiled as `CODEC(Delta, ZSTD)`. | +| ttl | A string consisting of a [TTL (time-to-live) expression](https://clickhouse.com/docs/guides/developer/ttl) that defines a TTL rule in the column's DDL. For example: `ttl: ts + INTERVAL 1 DAY` will be compiled as `TTL ts + INTERVAL 1 DAY`. | + +### Example + +```yaml +models: + - name: table_column_configs + description: 'Testing column-level configurations' + config: + contract: + enforced: true + columns: + - name: ts + data_type: timestamp + codec: ZSTD + - name: x + data_type: UInt8 + ttl: ts + INTERVAL 1 DAY +``` ## ClickHouse Cluster diff --git a/dbt/adapters/clickhouse/impl.py b/dbt/adapters/clickhouse/impl.py index 91238ddf..7acb6051 100644 --- a/dbt/adapters/clickhouse/impl.py +++ b/dbt/adapters/clickhouse/impl.py @@ -545,8 +545,12 @@ def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) rendered_columns = [] for v in raw_columns.values(): codec = f"CODEC({_codec})" if (_codec := v.get('codec')) else "" + ttl = f"TTL {ttl}" if (ttl := v.get('ttl')) else "" + # Codec and TTL are optional clauses. The adapter should support scenarios where one + # or both are omitted. If specified together, the codec clause should appear first. + clauses = " ".join(filter(None, [codec, ttl])) rendered_columns.append( - f"{quote_identifier(v['name'])} {v['data_type']} {codec}".rstrip() + f"{quote_identifier(v['name'])} {v['data_type']} {clauses}".rstrip() ) if v.get("constraints"): warn_or_error(ConstraintNotSupported(constraint='column', adapter='clickhouse')) diff --git a/tests/integration/adapter/constraints/fixtures_contraints.py b/tests/integration/adapter/constraints/fixtures_contraints.py index 508b25b1..a6483895 100644 --- a/tests/integration/adapter/constraints/fixtures_contraints.py +++ b/tests/integration/adapter/constraints/fixtures_contraints.py @@ -256,3 +256,35 @@ 1::Int32 as id, toDate('2019-01-01') as date_day """ + +custom_constraint_model_schema_yml = """ +version: 2 +models: + - name: custom_column_constraint_model + materialized: table + config: + contract: + enforced: true + columns: + - name: id + data_type: Int32 + codec: ZSTD + - name: ts + data_type: timestamp + - name: col_ttl + data_type: String + ttl: ts + INTERVAL 1 DAY +""" + +check_custom_constraints_model_sql = """ +{{ + config( + materialized = "table", + ) +}} + +select + 101::Int32 as id, + timestamp('2025-04-16') as ts, + 'blue' as col_ttl +""" diff --git a/tests/integration/adapter/constraints/test_constraints.py b/tests/integration/adapter/constraints/test_constraints.py index f18a7ca9..bc3846cb 100644 --- a/tests/integration/adapter/constraints/test_constraints.py +++ b/tests/integration/adapter/constraints/test_constraints.py @@ -5,8 +5,10 @@ bad_foreign_key_model_sql, check_constraints_model_fail_sql, check_constraints_model_sql, + check_custom_constraints_model_sql, constraint_model_schema_yml, contract_model_schema_yml, + custom_constraint_model_schema_yml, model_data_type_schema_yml, my_model_data_type_sql, my_model_incremental_wrong_name_sql, @@ -188,3 +190,14 @@ def test_model_constraints_fail_ddl(self, project): ["run", "-s", "check_constraints_model"], expect_pass=False ) assert 'violated' in log_output.lower() + +class TestModelCustomConstraints: + @pytest.fixture(scope="class") + def models(self): + return { + "check_custom_constraints_model.sql": check_custom_constraints_model_sql, + "constraints_schema.yml": custom_constraint_model_schema_yml, + } + + def test_model_constraints_ddl(self, project): + run_dbt(["run", "-s", "check_custom_constraints_model"]) From 8bf3b514465a5d8ec9fdcd283c581443c497c872 Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 16 Apr 2025 15:44:20 +0200 Subject: [PATCH 2/3] Fix linting --- dbt/adapters/clickhouse/impl.py | 2 +- tests/integration/adapter/constraints/test_constraints.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/clickhouse/impl.py b/dbt/adapters/clickhouse/impl.py index 7acb6051..7902e058 100644 --- a/dbt/adapters/clickhouse/impl.py +++ b/dbt/adapters/clickhouse/impl.py @@ -546,7 +546,7 @@ def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) for v in raw_columns.values(): codec = f"CODEC({_codec})" if (_codec := v.get('codec')) else "" ttl = f"TTL {ttl}" if (ttl := v.get('ttl')) else "" - # Codec and TTL are optional clauses. The adapter should support scenarios where one + # Codec and TTL are optional clauses. The adapter should support scenarios where one # or both are omitted. If specified together, the codec clause should appear first. clauses = " ".join(filter(None, [codec, ttl])) rendered_columns.append( diff --git a/tests/integration/adapter/constraints/test_constraints.py b/tests/integration/adapter/constraints/test_constraints.py index bc3846cb..e27ca625 100644 --- a/tests/integration/adapter/constraints/test_constraints.py +++ b/tests/integration/adapter/constraints/test_constraints.py @@ -191,6 +191,7 @@ def test_model_constraints_fail_ddl(self, project): ) assert 'violated' in log_output.lower() + class TestModelCustomConstraints: @pytest.fixture(scope="class") def models(self): From 800d2a4edde544f991af46d40c8787d8eaa1607c Mon Sep 17 00:00:00 2001 From: morsapaes Date: Wed, 16 Apr 2025 15:48:03 +0200 Subject: [PATCH 3/3] Add Copilot suggestion to fix typo --- .../{fixtures_contraints.py => fixtures_constraints.py} | 0 tests/integration/adapter/constraints/test_constraints.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename tests/integration/adapter/constraints/{fixtures_contraints.py => fixtures_constraints.py} (100%) diff --git a/tests/integration/adapter/constraints/fixtures_contraints.py b/tests/integration/adapter/constraints/fixtures_constraints.py similarity index 100% rename from tests/integration/adapter/constraints/fixtures_contraints.py rename to tests/integration/adapter/constraints/fixtures_constraints.py diff --git a/tests/integration/adapter/constraints/test_constraints.py b/tests/integration/adapter/constraints/test_constraints.py index e27ca625..f3724fe8 100644 --- a/tests/integration/adapter/constraints/test_constraints.py +++ b/tests/integration/adapter/constraints/test_constraints.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file -from fixtures_contraints import ( + +from tests.integration.adapter.constraints.fixtures_constraints import ( bad_column_constraint_model_sql, bad_foreign_key_model_sql, check_constraints_model_fail_sql,