warn on models not materialized

Author: owlas

dbt2looker/cli.py

@@ -28,7 +28,7 @@ def get_manifest(prefix: str):
     with open(path, 'r') as f:
         raw_manifest = json.load(f)
     parser.validate_manifest(raw_manifest)    # FIX
-    logging.info(f'Detected valid manifest at {path}')
+    logging.debug(f'Detected valid manifest at {path}')
     return raw_manifest
 
 
@@ -42,7 +42,7 @@ def get_catalog(prefix: str):
     with open(path, 'r') as f:
         raw_catalog = json.load(f)
     parser.validate_catalog(raw_catalog)
-    logging.info(f'Detected valid catalog at {path}')
+    logging.debug(f'Detected valid catalog at {path}')
     return raw_catalog
 
 
@@ -55,7 +55,7 @@ def get_dbt_project_config(prefix: str):
         raise SystemExit('Failed') from e
     with open(path, 'r') as f:
         project_config = yaml.load(f, Loader=Loader)
-    logging.info(f'Detected valid dbt config at {path}')
+    logging.debug(f'Detected valid dbt config at {path}')
     return project_config
 
 

dbt2looker/generator.py

@@ -110,8 +110,8 @@
 
 def map_adapter_type_to_looker(adapter_type: models.SupportedDbtAdapters, column_type: str):
     looker_type = LOOKER_DTYPE_MAP[adapter_type].get(column_type)
-    if looker_type is None:
-        logging.warn(f'Column type {column_type} not supported for conversion from {adapter_type} to looker. No dimension will be created.')
+    if (column_type is not None) and (looker_type is None):
+        logging.warning(f'Column type {column_type} not supported for conversion from {adapter_type} to looker. No dimension will be created.')
     return looker_type
 
 

dbt2looker/parser.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Dict, Optional
 
 from . import models
@@ -44,6 +45,14 @@ def parse_models(raw_manifest: dict, tag=None):
 def parse_typed_models(raw_manifest: dict, raw_catalog: dict, tag: Optional[str] = None):
     catalog_nodes = parse_catalog_nodes(raw_catalog)
     dbt_models = parse_models(raw_manifest, tag=tag)
+    adapter_type = parse_adapter_type(raw_manifest)
+
+    # Check catalog for models
+    for model in dbt_models:
+        if model.unique_id not in catalog_nodes:
+            logging.warning(
+                f'Model {model.unique_id} not found in catalog. No looker view will be generated. '
+                f'Check if model has materialized in {adapter_type} at {model.database}.{model.db_schema}.{model.name}')
 
     # Update dbt models with data types from catalog
     dbt_typed_models = [
@@ -54,6 +63,7 @@ def parse_typed_models(raw_manifest: dict, raw_catalog: dict, tag: Optional[str]
             for column in model.columns.values()
         }})
         for model in dbt_models
+        if model.unique_id in catalog_nodes
     ]
     return dbt_typed_models
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbt2looker"
-version = "0.6.1"
+version = "0.6.2"
 description = "Generate lookml view files from dbt models"
 authors = ["oliverlaslett <[email protected]>"]
 license = "MIT"