Dbt 1 3 (#109)

* Clean up some datatypes, add datatypes tests

* Update array macros, add array and concurrency tests

* Minor tweaks

* Update test actions

* Force case sensitive relation searches

Author: genzgd

.github/workflows/lint.yml

@@ -7,6 +7,7 @@ on:  # yamllint disable-line rule:truthy
     branches-ignore:
       - '*_test'
       - '*_dev'
+      - '*_cloud'
 
 jobs:
   lint:

.github/workflows/test_cloud.yml

@@ -22,7 +22,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup Python 3.10
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
 

.github/workflows/test_matrix.yml

@@ -46,7 +46,7 @@ jobs:
              clickhouse/clickhouse-server:${{ matrix.clickhouse-version }}
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

CHANGELOG.md

@@ -1,3 +1,16 @@
+### Release [1.3.0], 2022-10-30
+#### Improvement
+- Support dbt [1.3.0]  https://github.com/ClickHouse/dbt-clickhouse/issues/105
+  - Adds additional dbt 1.3.0 core tests
+  - Adds array utility macros ported from dbt-utils
+  - Does NOT add support for Python models (not implemented)
+  - Does NOT utilize default/standard incremental materialization macros (standard strategies do not work in ClickHouse)
+
+#### Bug Fix
+- Require exact match for relations.  ClickHouse databases and tables are all case sensitive, so all searches are now case sensitive.  Closes https://github.com/ClickHouse/dbt-clickhouse/issues/100 and https://github.com/ClickHouse/dbt-clickhouse/issues/110
+
+</br></br>
+
 ### Release [1.2.1], 2022-09-19
 #### Improvements
 - Support dbt 1.2.1  https://github.com/ClickHouse/dbt-clickhouse/issues/79

dbt/adapters/clickhouse/__version__.py

@@ -1 +1 @@
-version = '1.2.2'
+version = '1.3.0'

dbt/adapters/clickhouse/column.py

@@ -13,8 +13,8 @@ class ClickHouseColumn(Column):
     TYPE_LABELS = {
         'STRING': 'String',
         'TIMESTAMP': 'DateTime',
-        'FLOAT': 'Float64',
-        'INTEGER': 'Int64',
+        'FLOAT': 'Float32',
+        'INTEGER': 'Int32',
     }
     is_nullable: bool = False
     _brackets_regex = re.compile(r'^(Nullable|LowCardinality)\((.*)\)$')
@@ -109,7 +109,7 @@ def string_type(cls, size: int) -> str:
 
     @classmethod
     def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str:
-        return dtype
+        return f'Decimal({precision}, {scale})'
 
     def literal(self, value):
         return f'to{self.dtype}({value})'

dbt/adapters/clickhouse/impl.py

@@ -2,7 +2,7 @@
 import io
 from concurrent.futures import Future
 from dataclasses import dataclass
-from typing import Callable, List, Optional, Set, Union
+from typing import Callable, Dict, List, Optional, Set, Union
 
 import agate
 import dbt.exceptions
@@ -14,7 +14,7 @@
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.relation import RelationType
 from dbt.events import AdapterLogger
-from dbt.utils import executor
+from dbt.utils import executor, filter_null_values
 
 from dbt.adapters.clickhouse.column import ClickHouseColumn
 from dbt.adapters.clickhouse.connections import ClickHouseConnectionManager
@@ -49,6 +49,7 @@ def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
     @classmethod
     def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
         decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
+        # We match these type to the Column.TYPE_LABELS for consistency
         return 'Float32' if decimals else 'Int32'
 
     @classmethod
@@ -112,6 +113,14 @@ def drop_schema(self, relation: BaseRelation) -> None:
         if conn:
             conn.handle.database_dropped(relation.schema)
 
+    def _make_match_kwargs(self, database: str, schema: str, identifier: str) -> Dict[str, str]:
+        return filter_null_values(
+            {
+                "database": database,
+                "identifier": identifier,
+            }
+        )
+
     def list_relations_without_caching(
         self, schema_relation: ClickHouseRelation
     ) -> List[ClickHouseRelation]:
@@ -170,7 +179,6 @@ def parse_clickhouse_columns(
 
     def get_columns_in_relation(self, relation: ClickHouseRelation) -> List[ClickHouseColumn]:
         rows: List[agate.Row] = super().get_columns_in_relation(relation)
-
         return self.parse_clickhouse_columns(relation, rows)
 
     def get_catalog(self, manifest):
@@ -220,6 +228,7 @@ def get_rows_different_sql(
         relation_a: ClickHouseRelation,
         relation_b: ClickHouseRelation,
         column_names: Optional[List[str]] = None,
+        except_operator: str = None,
     ) -> str:
         names: List[str]
         if column_names is None:

dbt/adapters/clickhouse/relation.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Optional
 
 import dbt.exceptions
 from dbt.adapters.base.relation import BaseRelation, Policy
@@ -38,3 +39,15 @@ def render(self):
                 'include, but only one can be set'
             )
         return super().render()
+
+    def matches(
+        self,
+        database: Optional[str] = None,
+        schema: Optional[str] = None,
+        identifier: Optional[str] = None,
+    ):
+        if schema:
+            raise dbt.exceptions.RuntimeException(
+                f'Passed unexpected schema value {schema} to Relation.matches'
+            )
+        return self.database == database and self.identifier == identifier

dbt/include/clickhouse/macros/adapters.sql

@@ -1,110 +1,3 @@
-{% macro engine_clause(label) %}
-  {%- set engine = config.get('engine', validator=validation.any[basestring]) -%}
-  {%- if engine is not none %}
-    {{ label }} = {{ engine }}
-  {%- else %}
-    {{ label }} = MergeTree()
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro partition_cols(label) %}
-  {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}
-  {%- if cols is not none %}
-    {%- if cols is string -%}
-      {%- set cols = [cols] -%}
-    {%- endif -%}
-    {{ label }} (
-    {%- for item in cols -%}
-      {{ item }}
-      {%- if not loop.last -%},{%- endif -%}
-    {%- endfor -%}
-    )
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro primary_key_clause(label) %}
-  {%- set primary_key = config.get('primary_key', validator=validation.any[basestring]) -%}
-
-  {%- if primary_key is not none %}
-    {{ label }} {{ primary_key }}
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro order_cols(label) %}
-  {%- set cols = config.get('order_by', validator=validation.any[list, basestring]) -%}
-  {%- set engine = config.get('engine', validator=validation.any[basestring]) -%}
-  {%- set supported = [
-    'HDFS',
-    'MaterializedPostgreSQL',
-    'S3',
-    'EmbeddedRocksDB',
-    'Hive'
-  ] -%}
-
-  {%- if engine is none or 'MergeTree' in engine or engine in supported %}
-    {%- if cols is not none %}
-      {%- if cols is string -%}
-        {%- set cols = [cols] -%}
-      {%- endif -%}
-      {{ label }} (
-      {%- for item in cols -%}
-        {{ item }}
-        {%- if not loop.last -%},{%- endif -%}
-      {%- endfor -%}
-      )
-    {%- else %}
-      {{ label }} (tuple())
-    {%- endif %}
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro on_cluster_clause(label) %}
-  {% set on_cluster = adapter.get_clickhouse_cluster_name() %}
-  {%- if on_cluster is not none %}
-    {{ label }} {{ on_cluster }}
-  {%- endif %}
-{%- endmacro -%}
-
-{% macro clickhouse__create_table_as(temporary, relation, sql) -%}
-    {% set create_table = create_table_or_empty(temporary, relation, sql) %}
-    {% if adapter.is_before_version('22.7.1') -%}
-        {{ create_table }}
-    {%- else %}
-        {% call statement('create_table_empty') %}
-            {{ create_table }}
-        {% endcall %}
-        {{ clickhouse__insert_into(relation.include(database=False), sql) }}
-    {%- endif %}
-{%- endmacro %}
-
-{% macro create_table_or_empty(temporary, relation, sql) -%}
-    {%- set sql_header = config.get('sql_header', none) -%}
-
-    {{ sql_header if sql_header is not none }}
-
-    {% if temporary -%}
-        create temporary table {{ relation.name }}
-        engine Memory
-        {{ order_cols(label="order by") }}
-        {{ partition_cols(label="partition by") }}
-        {{ adapter.get_model_settings(model) }}
-    {%- else %}
-        create table {{ relation.include(database=False) }}
-        {{ on_cluster_clause(label="on cluster") }}
-        {{ engine_clause(label="engine") }}
-        {{ order_cols(label="order by") }}
-        {{ primary_key_clause(label="primary key") }}
-        {{ partition_cols(label="partition by") }}
-        {{ adapter.get_model_settings(model) }}
-        {% if not adapter.is_before_version('22.7.1') -%}
-            empty
-        {%- endif %}
-    {%- endif %}
-    as (
-        {{ sql }}
-    )
-{%- endmacro %}
-
 {% macro clickhouse__create_view_as(relation, sql) -%}
   {%- set sql_header = config.get('sql_header', none) -%}
 
@@ -221,16 +114,9 @@
   {% endcall %}
 {% endmacro %}
 
-{% macro clickhouse__insert_into(target_relation, sql) %}
-  {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
-  {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
-
-  insert into {{ target_relation }} ({{ dest_cols_csv }})
-  {{ sql }}
-{%- endmacro %}
-
 {% macro exchange_tables_atomic(old_relation, target_relation) %}
   {%- call statement('exchange_tables_atomic') -%}
     EXCHANGE TABLES {{ old_relation }} AND {{ target_relation }}
   {% endcall %}
 {% endmacro %}
+

dbt/include/clickhouse/macros/materializations/incremental.sql

@@ -49,6 +49,7 @@
 
     -- First create a temporary table with all of the new data
     {% set new_data_relation = existing_relation.incorporate(path={"identifier": model['name'] + '__dbt_new_data'}) %}
+    {{ drop_relation_if_exists(new_data_relation) }}
     {% call statement('create_new_data_temp') %}
         {{ get_create_table_as_sql(False, new_data_relation, sql) }}
     {% endcall %}