add spark types (#27)

* add spark types
* use dbt relation name

Co-authored-by: oliverlaslett <[email protected]>

Author: chaimt

CHANGELOG.md

@@ -3,8 +3,12 @@
 Recent and upcoming changes to dbt2looker
 
 ## 0.8.3
+### Added
+- Support for spark adapter (@chaimt)
+
 ### Changed
 - Updated with support for dbt2looker (@chaimt)
+- Lookml views now populate their "sql_table_name" using the dbt relation name
 
 ## 0.8.2
 ### Changed

dbt2looker/generator.py

@@ -1,4 +1,5 @@
 import logging
+import re
 
 import lkml
 
@@ -154,6 +155,21 @@
         'TIME WITHOUT TIME ZONE': 'string',
         # TIMETZ not supported
         # TIME WITH TIME ZONE not supported
+    },
+    'spark': {
+        'byte':        'number',
+        'short':       'number',
+        'integer':     'number',
+        'long':        'number',
+        'float':       'number',
+        'double':      'number',
+        'decimal':     'number',
+        'string':      'string',
+        'varchar':     'string',
+        'char':        'string',
+        'boolean':     'yesno',
+        'timestamp':   'timestamp',
+        'date':        'datetime',
     }
 }
 
@@ -172,8 +188,13 @@
 ]
 
 
+def normalise_spark_types(column_type: str) -> str:
+    return re.match(r'^[^\(]*', column_type)
+
+
 def map_adapter_type_to_looker(adapter_type: models.SupportedDbtAdapters, column_type: str):
-    looker_type = LOOKER_DTYPE_MAP[adapter_type].get(column_type)
+    normalised_column_type = normalise_spark_types(column_type) if adapter_type == models.SupportedDbtAdapters.spark.value else column_type
+    looker_type = LOOKER_DTYPE_MAP[adapter_type].get(normalised_column_type)
     if (column_type is not None) and (looker_type is None):
         logging.warning(f'Column type {column_type} not supported for conversion from {adapter_type} to looker. No dimension will be created.')
     return looker_type
@@ -282,7 +303,7 @@ def lookml_view_from_dbt_model(model: models.DbtModel, adapter_type: models.Supp
     lookml = {
         'view': {
             'name': model.name,
-            'sql_table_name': f'{model.database}.{model.db_schema}.{model.name}',
+            'sql_table_name': model.relation_name,
             'dimension_groups': lookml_dimension_groups_from_model(model, adapter_type),
             'dimensions': lookml_dimensions_from_model(model, adapter_type),
             'measures': lookml_measures_from_model(model),

dbt2looker/models.py

@@ -18,6 +18,7 @@ class SupportedDbtAdapters(str, Enum):
     postgres = 'postgres'
     redshift = 'redshift'
     snowflake = 'snowflake'
+    spark = 'spark'
 
 
 # Lookml types
@@ -152,7 +153,7 @@ class DbtModelMeta(Dbt2LookerModelMeta):
 
 class DbtModel(DbtNode):
     resource_type: Literal['model']
-    database: str
+    relation_name: str
     db_schema: str = Field(..., alias='schema')
     name: str
     description: str
@@ -187,7 +188,6 @@ class DbtManifest(BaseModel):
 
 class DbtCatalogNodeMetadata(BaseModel):
     type: str
-    database: str
     db_schema: str = Field(..., alias='schema')
     name: str
     comment: Optional[str]

dbt2looker/parser.py

@@ -87,7 +87,7 @@ def parse_typed_models(raw_manifest: dict, raw_catalog: dict, tag: Optional[str]
         if model.unique_id not in catalog_nodes:
             logging.warning(
                 f'Model {model.unique_id} not found in catalog. No looker view will be generated. '
-                f'Check if model has materialized in {adapter_type} at {model.database}.{model.db_schema}.{model.name}')
+                f'Check if model has materialized in {adapter_type} at {model.relation_name}')
 
     # Update dbt models with data types from catalog
     dbt_typed_models = [