Merge pull request #35 from fivetran/feature/databricks-compatibility

Feature: Databricks compatibility

Author: fivetran-avinash

.buildkite/pipeline.yml

@@ -57,3 +57,17 @@ steps:
             - "CI_REDSHIFT_DBT_USER"
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,3 +1,7 @@
+# dbt_jira_source v0.6.1
+## 🎉 Feature Updates 🎉
+- Databricks compatibility 🧱 ([#35](https://github.com/fivetran/dbt_jira_source/pull/35))
+
 # dbt_jira_source v0.6.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -23,7 +23,15 @@
 ## Step 1: Prerequisites
 To use this dbt package, you must have the following:
 - At least one Fivetran Jira connector syncing data into your destination. 
-- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+- A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
+
+### Databricks Dispatch Configuration
+If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+```yml
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
+```
 
 ## Step 2: Install the package
 Include the following jira_source package version in your `packages.yml` file.
@@ -92,6 +100,9 @@ packages:
 
     - package: dbt-labs/dbt_utils
       version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'jira_source'
-version: '0.6.0'
+version: '0.6.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

docs/catalog.json

@@ -1,10 +1,11 @@
 name: 'jira_source_integration_tests'
-version: '0.6.0'
+version: '0.6.1'
 config-version: 2
 profile: 'integration_tests'
 
 
 vars:
+  jira_schema: jira_source_integrations_tests
   jira_source:
     jira_comment_identifier: "comment"
     jira_component_identifier: "component"
@@ -75,4 +76,8 @@ seeds:
         start_date: timestamp
     field: 
       +column_types:
-        id: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        id: "{{ 'string' if target.name in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']

docs/manifest.json

@@ -1,3 +1,6 @@
 packages:
 - package: fivetran/fivetran_utils
   version: [">=0.4.0", "<0.5.0"]
+
+- package: dbt-labs/spark_utils
+  version: [">=0.3.0", "<0.4.0"]