Feature/add titan (#251) (#252)

* Feature/add titan (#251)

* Titan role updates
  * Add role grants.
  * update roles for covid and snowflake dbs
  * grant transformer access to additional warehouses and dbs
  * update roles and grant permissions macro
  * make quoting consistent
  * fix role name in dbt macro

* fix macro and script to get artifacts to not fail on first run and add stage permissions
* change workflow 20 to only run on PR to main
* include seeds when running dynamic models for the first time
* account for not having catalog.json on the first run

* update model for country geo due to source schema change
* fix linting issue

---------

Co-authored-by: Noel Gomez
Co-authored-by: Sebastian Sassi

Author: noel

.github/workflows/10_feature_dbt_checks.yml

@@ -99,6 +99,7 @@ jobs:
         run: "../automate/dbt/get_artifacts.sh"
 
       - name: Clone incremental models that are directly or indirectly affected by the change
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
         run: |
           dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs
           dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs
@@ -110,9 +111,15 @@ jobs:
       - name: Run Observe Model
         run: "dbt build --fail-fast -s L1_inlets.observe"
 
-      # There is an issue with --empty and dynamic tables so need to exclude them
+      # There is an issue with --empty and dynamic tables so need to run them by themselves
       - name: Governance run of dynamic tables
-        run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures --defer --state logs"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }}
+        run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures resource_type:seed"
+
+      # There is an issue with --empty and dynamic tables so need to run them by themselves
+      - name: Governance run of dynamic tables
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
+        run: "dbt build --fail-fast -s config.materialized:dynamic_table stg_test_failures --defer --state logs"
 
       # There is an issue with --empty and dynamic tables so need to exclude them
       - name: Governance run of dbt with EMPTY models using slim mode
@@ -125,8 +132,13 @@ jobs:
         run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Generate Docs Combining Prod and branch catalog.json
+        if: ${{ steps.prod_manifest.outputs.catalog_found == 'true' }}
         run: "dbt-coves generate docs --merge-deferred --state logs"
 
+      - name: Generate dbt Docs
+        if: ${{ steps.prod_manifest.outputs.catalog_found == 'false' }}
+        run: "dbt docs generate"
+
       - name: Run governance checks
         run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"
 

.github/workflows/20_release_dbt_checks.yml

@@ -3,8 +3,8 @@ name: "20: 🏎️ dbt Release Branch Validations"
 on:  # yamllint disable-line rule:truthy
   pull_request:
     types: [opened, synchronize, labeled, unlabeled]
-    # No branch restrictions for validate-branch job to run on all PRs
-    # dbt job will be conditionally run based on target branch
+    branches:
+      - main
     paths:
       - transform/**/*
 
@@ -112,7 +112,7 @@ jobs:
       run: automate/git/branch_validator.py
 
   dbt:
-    if: github.base_ref == 'main' && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh')
+    if: github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh')
     name: Pull Request dbt Tests
     runs-on: ubuntu-latest
     needs: [validate-branch]

automate/dbt/get_artifacts.sh

@@ -8,22 +8,57 @@ cd $DATACOVES__DBT_HOME
 mkdir -p logs
 
 dbt run-operation get_last_artifacts
-LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)"
+
+# Check if manifest.json exists before trying to grep it
+if [ -f "logs/manifest.json" ]; then
+    LINES_IN_MANIFEST="$(grep -c '^' logs/manifest.json)"
+else
+    LINES_IN_MANIFEST=0
+fi
 
 if [ $LINES_IN_MANIFEST -eq 0 ]
 then
-    echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake."
+    echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest.json to snowflake."
     # This is used by github actions
-    echo "manifest_found=false" >> $GITHUB_OUTPUT
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "manifest_found=false" >> $GITHUB_OUTPUT
+    fi
 
     # This is used by Jenkins
     # echo "false" > temp_MANIFEST_FOUND.txt
 else
     echo "Updated manifest from production"
 
     # This is used by github actions
-    echo "manifest_found=true" >> $GITHUB_OUTPUT
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "manifest_found=true" >> $GITHUB_OUTPUT
+    fi
 
     # This is used by Jenkins
     # echo "true" > temp_MANIFEST_FOUND.txt
 fi
+
+
+# Check if catalog.json exists before trying to grep it
+if [ -f "logs/catalog.json" ]; then
+    LINES_IN_CATALOG="$(grep -c '^' logs/catalog.json)"
+else
+    LINES_IN_CATALOG=0
+fi
+
+
+if [ $LINES_IN_CATALOG -eq 0 ]
+then
+    echo "Catalog not found in Snowflake stage, contact the Snowflake administrator to load a updated catalog.json to snowflake."
+    # This is used by github actions
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "catalog_found=false" >> $GITHUB_OUTPUT
+    fi
+else
+    echo "Updated catalog from production"
+
+    # This is used by github actions
+    if [ -n "$GITHUB_OUTPUT" ]; then
+        echo "catalog_found=true" >> $GITHUB_OUTPUT
+    fi
+fi