Support dbt-core version 1.4 with dbt log_version 3 (#29)

Author: robastel

.github/workflows/test.yml

@@ -31,12 +31,16 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ['3.7', '3.10']
-        dbt-version: [0.18.x, 0.19.x, 1.x]
+        dbt-version: [0.18.x, 0.19.x, 1.1.x, 1.4.x]
         exclude:
           - python-version: '3.10'
             dbt-version: 0.18.x
           - python-version: '3.10'
             dbt-version: 0.19.x
+          - python-version: '3.7'
+            dbt-version: 1.1.x
+          - python-version: '3.7'
+            dbt-version: 1.4.x
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

README.md

@@ -216,7 +216,6 @@ dbt-invoke properties.delete <options>
   - This may be partially remedied by increasing the value of the `--threads` 
     option in `dbt-invoke properties.update`.
 - dbt-invoke is tested against:
-  - Python 3.7 and 3.10
-  - dbt 0.18, 0.19, and 1.1
+  - dbt 0.18, 0.19, 1.1, and 1.4
   - macos-latest, windows-latest, ubuntu-latest
 - dbt-invoke has not been tested across different types of data warehouses.

dbt_invoke/properties.py

@@ -780,19 +780,49 @@ def _get_columns(ctx, resource_location, resource_dict, **kwargs):
             ctx, _MACRO_NAME, hide=True, logger=_LOGGER, sql=sql, **kwargs
         )
 
-    # from dbt-core v1.0.0 onwards, run_operations INFO logs have code M011 and messages have key 'msg'
-    # https://github.com/dbt-labs/dbt-core/blob/22b1a09aa218e8152b0c2dd261abe2503ea15ddb/core/dbt/events/types.py#L401
     relevant_lines = list(
-        filter(lambda x: x.get('code') == 'M011', result_lines)
+        filter(
+            lambda x: x.get(
+                # dbt-core>=1.0,<1.4
+                #   run-operation logs contain structure
+                #   {
+                #       'code': 'M011',
+                #       'msg': ['column1', 'column2', ...]
+                #   }
+                'code',
+                # dbt-core>=1.4
+                #   run-operation logs contain structure
+                #   {
+                #       'info': {
+                #           'code': 'M011',
+                #           'msg': "['column1', 'column2', ...]" # string value
+                #       }
+                #   }
+                x.get('info', dict()).get('code'),
+            )
+            == 'M011',
+            result_lines,
+        )
     )
     if len(relevant_lines) >= 1:
-        columns = relevant_lines[-1].get('msg')
+        relevant_line = relevant_lines[-1]
+        columns = relevant_line.get(
+            'msg',
+            relevant_line.get('info', dict()).get('msg'),
+        )
     else:
         # for older dbt-core versions, we need to cross fingers a little harder
         relevant_lines = result_lines[1:]
         # also, the message key is different
         columns = relevant_lines[-1].get('message')
-        # columns are not passed as valid json but as a string representation of a list
+    # In some version of dbt columns are not passed as valid json but as
+    # a string representation of a list
+    is_string_list = (
+        isinstance(columns, str)
+        and columns.startswith('[')
+        and columns.endswith(']')
+    )
+    if is_string_list:
         columns = ast.literal_eval(columns)
     return columns
 

requirements/requirements_dbt_1.x.txt requirements/requirements_dbt_1.1.x.txtrequirements/requirements_dbt_1.x.txt renamed to requirements/requirements_dbt_1.1.x.txt

@@ -0,0 +1,7 @@
+agate>=1.6,<1.7.1
+dbt-core~=1.4.0
+dbt-sqlite~=1.4.0
+invoke>=1.4.1
+PyYAML>=5.1
+ruamel.yaml>=0.17.12
+-e .