Feature/handle full refresh (#245) (#247)

noel · web-flow · commit 33b6181b7d4a · 2025-08-26T17:27:49.000-07:00
* fix issue with earthquake staging model
* update workflow to handle incremental models
* update get artifacts script to use newer github actions syntax
* update workflow to run for any branch name
diff --git a/.github/workflows/10_feature_airflow_checks.yml b/.github/workflows/10_feature_airflow_checks.yml
@@ -37,6 +37,7 @@ jobs:
     name: Pull Request Airflow Tests
     runs-on: ubuntu-latest
     container: datacoves/ci-airflow-dbt-snowflake:3.4
+    needs: [validate-branch]
 
     env:
       AIRFLOW__CORE__DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/orchestrate/dags
@@ -83,17 +84,19 @@ jobs:
 
   airflow-job-status:
     runs-on: ubuntu-latest
-    needs: [airflow]
+    needs: [validate-branch, airflow]
     if: always()
     steps:
       - name: Comment PR with Airflow status
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
             ## 🎯 Airflow Workflow Status
-            - **Job**: Pull Request Airflow Tests → **${{ needs.airflow.result }}**
+            - **Branch Validation**: ${{ needs.validate-branch.result }}
+            - **Airflow Validations**: ${{ needs.airflow.result }}
 
-            ${{ needs.airflow.result == 'skipped' && '⏭️ _Skipped because no orchestrate/ files were changed_' || '' }}
+            ${{ needs.validate-branch.result == 'failure' && '❌ _Branch validation failed_' || '' }}
+            ${{ needs.airflow.result == 'skipped' && '⏭️ _Airflow validations skipped_' || '' }}
             ${{ needs.airflow.result == 'success' && '✅ _All Airflow validations passed_' || '' }}
             ${{ needs.airflow.result == 'failure' && '❌ _Airflow validations failed_' || '' }}
           comment_tag: airflow-workflow-status
diff --git a/.github/workflows/10_feature_dbt_checks.yml b/.github/workflows/10_feature_dbt_checks.yml
@@ -36,6 +36,7 @@ jobs:
   dbt:
     name: Pull Request dbt Tests
     runs-on: ubuntu-latest
+    needs: [validate-branch]
 
     # Set environment variables in
     # https://github.com//<your org>/<your repo>/settings/variables/actions
@@ -71,6 +72,9 @@ jobs:
       # cannot be applied when using the Datacoves permifrost security model.
       DATACOVES__DROP_DB_ON_FAIL:  ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}
 
+      # Full refresh control variables
+      FULL_REFRESH_FLAG: ${{ contains(github.event.pull_request.labels.*.name, 'full-refresh') && '--full-refresh' || '' }}
+
     steps:
       - name: Checkout branch
         uses: actions/checkout@v3.5.0
@@ -88,12 +92,17 @@ jobs:
         run: "dbt deps"
 
       - name: Create PR database
-        run: "dbt --no-write-json run-operation create_database"
+        run: dbt --no-write-json run-operation create_database
 
       - name: Get prod manifest
         id: prod_manifest
         run: "../automate/dbt/get_artifacts.sh"
 
+      - name: Clone incremental models that are directly or indirectly affected by the change
+        run: |
+          dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs
+          dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs
+
       ##### Governance Checks
       # this first runs dbt but creates enpty tables, this is enough to then run the hooks and fail fast
 
@@ -107,13 +116,13 @@ jobs:
 
       # There is an issue with --empty and dynamic tables so need to exclude them
       - name: Governance run of dbt with EMPTY models using slim mode
-        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }}
-        run: "dbt build --fail-fast --defer --state logs --select state:modified+ --empty --exclude config.materialized:dynamic_table"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
+        run: "dbt build --fail-fast --defer --state logs --select state:modified+ --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}"
 
       # There is an issue with --empty and dynamic tables so need to exclude
       - name: Governance run of dbt with EMPTY models using full run
-        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }}
-        run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }}
+        run: "dbt build --fail-fast --empty --exclude config.materialized:dynamic_table ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Generate Docs Combining Prod and branch catalog.json
         run: "dbt-coves generate docs --merge-deferred --state logs"
@@ -123,12 +132,12 @@ jobs:
 
       ##### Real dbt run given that we passed governance checks
       - name: Run dbt build slim mode
-        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }}
-        run: "dbt build --fail-fast --defer --state logs --select state:modified+"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
+        run: "dbt build --fail-fast --defer --state logs --select state:modified+ ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Run dbt build full run
-        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }}
-        run: "dbt build --fail-fast"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }}
+        run: "dbt build --fail-fast ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Grant access to PR database
         id: grant-access-to-database
@@ -142,17 +151,19 @@ jobs:
 
   dbt-job-status:
     runs-on: ubuntu-latest
-    needs: [dbt]
+    needs: [validate-branch, dbt]
     if: always()
     steps:
       - name: Comment PR with dbt status
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
             ## 🧪 dbt Workflow Status
-            - **Job**: Pull Request dbt Tests → **${{ needs.dbt.result }}**
+            - **Branch Validation**: ${{ needs.validate-branch.result }}
+            - **dbt Validations**: ${{ needs.dbt.result }}
 
-            ${{ needs.dbt.result == 'skipped' && '⏭️ _Skipped because no transform/ files were changed_' || '' }}
+            ${{ needs.validate-branch.result == 'failure' && '❌ _Branch validation failed_' || '' }}
+            ${{ needs.dbt.result == 'skipped' && '⏭️ _dbt validations skipped_' || '' }}
             ${{ needs.dbt.result == 'success' && '✅ _All dbt validations passed_' || '' }}
             ${{ needs.dbt.result == 'failure' && '❌ _dbt validations failed_' || '' }}
           comment_tag: dbt-workflow-status
diff --git a/.github/workflows/20_release_dbt_checks.yml b/.github/workflows/20_release_dbt_checks.yml
@@ -2,8 +2,9 @@ name: "20: 🏎️ dbt Release Branch Validations"
 
 on:  # yamllint disable-line rule:truthy
   pull_request:
-    branches:
-      - main
+    types: [opened, synchronize, labeled, unlabeled]
+    # No branch restrictions for validate-branch job to run on all PRs
+    # dbt job will be conditionally run based on target branch
     paths:
       - transform/**/*
 
@@ -16,6 +17,83 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  add-deploy-marker:
+    if: github.event.action == 'labeled' && github.event.label.name == 'full-refresh'
+    runs-on: ubuntu-latest
+    name: Add Deploy Marker to PR Description
+
+    steps:
+      - name: Add deploy marker to PR description
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { data: pullRequest } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+
+            let body = pullRequest.body || '';
+
+            // Check if marker already exists
+            if (!body.includes('[deploy:full-refresh]')) {
+              // Add marker to the end of the description
+              const newBody = body + '\n\n[deploy:full-refresh]';
+
+              await github.rest.pulls.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.issue.number,
+                body: newBody
+              });
+
+              // Also add a comment explaining what happened
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: '🚀 **Full-refresh deployment marker added to PR description**\n\nThe `[deploy:full-refresh]` marker has been automatically added to this PR description. When merged, this will trigger a full-refresh deployment.'
+              });
+            }
+
+  remove-deploy-marker:
+    if: github.event.action == 'unlabeled' && github.event.label.name == 'full-refresh'
+    runs-on: ubuntu-latest
+    name: Remove Deploy Marker from PR Description
+
+    steps:
+      - name: Remove deploy marker from PR description
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { data: pullRequest } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+
+            let body = pullRequest.body || '';
+
+            // Remove the marker if it exists
+            if (body.includes('[deploy:full-refresh]')) {
+              const newBody = body.replace(/\n*\[deploy:full-refresh\]\n*/g, '').trim();
+
+              await github.rest.pulls.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.issue.number,
+                body: newBody
+              });
+
+              // Add a comment explaining what happened
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: '✅ **Full-refresh deployment marker removed**\n\nThe `[deploy:full-refresh]` marker has been removed from this PR description. Normal deployment will occur on merge.'
+              });
+            }
+
   validate-branch:
     runs-on: ubuntu-latest
     name: Validate Branch Names and Merge Rules and assure branch has changes from main
@@ -34,8 +112,10 @@ jobs:
       run: automate/git/branch_validator.py
 
   dbt:
+    if: github.base_ref == 'main' && github.event.action != 'unlabeled' && (github.event.action != 'labeled' || github.event.label.name == 'full-refresh')
     name: Pull Request dbt Tests
     runs-on: ubuntu-latest
+    needs: [validate-branch]
 
     # Set environment variables in
     # https://github.com//<your org>/<your repo>/settings/variables/actions
@@ -44,7 +124,7 @@ jobs:
     # https://github.com/<org>/<repo>/settings/environments
     # environment: PR_ENV
 
-    # most people should use this one
+    # Most people should use this docker image
     container: datacoves/ci-basic-dbt-snowflake:3.4
 
     defaults:
@@ -74,6 +154,9 @@ jobs:
       # cannot be applied when using the Datacoves permifrost security model.
       DATACOVES__DROP_DB_ON_FAIL:  ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}
 
+      # Full refresh control variables
+      FULL_REFRESH_FLAG: ${{ contains(github.event.pull_request.labels.*.name, 'full-refresh') && '--full-refresh' || '' }}
+
     steps:
       - name: Checkout branch
         uses: actions/checkout@v3.5.0
@@ -123,12 +206,12 @@ jobs:
 
       ##### Real dbt run given that we passed governance checks
       - name: Run dbt build slim mode
-        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }}
-        run: "dbt build --fail-fast --defer --state logs --select state:modified+"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' }}
+        run: "dbt build --fail-fast --defer --state logs --select state:modified+ ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Run dbt build full run
-        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' || contains(github.event.pull_request.labels.*.name, 'full-refresh') }}
-        run: "dbt build --fail-fast"
+        if: ${{ steps.prod_manifest.outputs.manifest_found == 'false' }}
+        run: "dbt build --fail-fast ${{ env.FULL_REFRESH_FLAG }}"
 
       - name: Grant access to QA_TEMP database
         if: steps.check_qa_created_today.outputs.qa_created_today != 'true'
@@ -141,6 +224,7 @@ jobs:
           dbt --no-write-json run-operation swap_database --args '{db1: ${{ env.DATACOVES__MAIN__DATABASE }}, db2: ${{ env.DATACOVES__MAIN__DATABASE_QA }}, create_missing_db: true}'
           dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{ env.DATACOVES__MAIN__DATABASE }}, recreate: False}'
 
+
       # # We drop the database when there is a failure to grant access to the db because
       # # most likely the schema was not set properly in dbt_project.yml so models built to default schema
       # - name: Drop PR database on Failure to grant security access
@@ -149,17 +233,21 @@ jobs:
 
   dbt-job-status:
     runs-on: ubuntu-latest
-    needs: [dbt]
+    needs: [add-deploy-marker, remove-deploy-marker, validate-branch, dbt]
     if: always()
     steps:
-      - name: Comment PR with dbt status
+      - name: Comment PR with workflow status
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
             ## 🧪 dbt Workflow Status
-            - **Job**: Pull Request dbt Tests → **${{ needs.dbt.result }}**
+            - **Branch Validation**: ${{ needs.validate-branch.result }}
+            - **Deploy Marker Management**: ${{ (needs.add-deploy-marker.result != 'skipped' && needs.add-deploy-marker.result) || (needs.remove-deploy-marker.result != 'skipped' && needs.remove-deploy-marker.result) || 'skipped' }}
+            - **dbt Validations**: ${{ needs.dbt.result }}
 
-            ${{ needs.dbt.result == 'skipped' && '⏭️ _Skipped because no transform/ files were changed_' || '' }}
+            ${{ needs.validate-branch.result == 'failure' && '❌ _Branch validation failed_' || '' }}
+            ${{ needs.dbt.result == 'skipped' && '⏭️ _dbt validations skipped_' || '' }}
             ${{ needs.dbt.result == 'success' && '✅ _All dbt validations passed_' || '' }}
             ${{ needs.dbt.result == 'failure' && '❌ _dbt validations failed_' || '' }}
+            ${{ (needs.add-deploy-marker.result == 'success' || needs.remove-deploy-marker.result == 'success') && '📝 _Deploy marker updated in PR description_' || '' }}
           comment_tag: dbt-workflow-status
diff --git a/.github/workflows/30_deploy_changes_to_production.yml b/.github/workflows/30_deploy_changes_to_production.yml
@@ -73,15 +73,27 @@ jobs:
       - name: Refresh dynamic tables before blue-green
         run: "dbt run -s materialization config.materialized:dynamic_table --full-refresh"
 
+      - name: Check for full-refresh marker in merge commit message
+        id: check-full-refresh
+        run: |
+          # Check the latest commit message for the deployment marker
+          if git log -1 --pretty=%B | grep -q "\[deploy:full-refresh\]"; then
+            echo "full_refresh=true" >> $GITHUB_OUTPUT
+            echo "Found '[deploy:full-refresh]' marker in merge commit message, will run with --full-refresh"
+          else
+            echo "full_refresh=false" >> $GITHUB_OUTPUT
+            echo "No full-refresh deployment marker found in merge commit message, running normal deployment"
+          fi
+
       - name: Run blue / green deployment
-        if: contains(github.event.pull_request.labels.*.name, 'full-refresh') != true
+        if: steps.check-full-refresh.outputs.full_refresh != 'true'
         id: run-blue-green
         env:
           MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }}
         run: "dbt-coves blue-green --defer"
 
       - name: Run blue / green deployment with full-refresh
-        if: contains(github.event.pull_request.labels.*.name, 'full-refresh')
+        if: steps.check-full-refresh.outputs.full_refresh == 'true'
         id: run-blue-green-full-refresh
         env:
           MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }}
diff --git a/automate/dbt/get_artifacts.sh b/automate/dbt/get_artifacts.sh
@@ -14,15 +14,15 @@ if [ $LINES_IN_MANIFEST -eq 0 ]
 then
     echo "Manifest not found in Snowflake stage, contact the Snowflake administrator to load a updated manifest to snowflake."
     # This is used by github actions
-    echo "::set-output name=manifest_found::false"
+    echo "manifest_found=false" >> $GITHUB_OUTPUT
 
     # This is used by Jenkins
     # echo "false" > temp_MANIFEST_FOUND.txt
 else
     echo "Updated manifest from production"
 
     # This is used by github actions
-    echo "::set-output name=manifest_found::true"
+    echo "manifest_found=true" >> $GITHUB_OUTPUT
 
     # This is used by Jenkins
     # echo "true" > temp_MANIFEST_FOUND.txt
diff --git a/automate/git/branch_validator.py b/automate/git/branch_validator.py
@@ -37,6 +37,7 @@ def get_commit_count(source_branch, github_token, repository):
             print(f"✅ The {source_branch} is up to date with the main branch.")
         else:
             print(f"❌ There are {commits_behind} commit(s) in the main branch that are not in Source branch: {source_branch}. Pull main into the {source_branch}")
+            sys.exit(1)
 
     except requests.exceptions.RequestException as e:
         raise GitCommandError(f"❌ Failed to compare branches via GitHub API: {e}")
diff --git a/transform/models/L1_inlets/country_geo/stg_country_polygons.sql b/transform/models/L1_inlets/country_geo/stg_country_polygons.sql
@@ -8,12 +8,12 @@ with raw_source as (
 final as (
 
     select
-        features:properties:ADMIN::STRING as country_name,
-        features:properties:ISO_A2::STRING as country_code_2,
-        features:properties:ISO_A3::STRING as country_code_3,
-        features:type::STRING as feature_type,
-        features:geometry:type::STRING as geometry_type,
-        TRY_TO_GEOGRAPHY(features:geometry) as geography,
+        name as country_name,
+        iso3166_1_alpha_2 as country_code_2,
+        iso3166_1_alpha_3 as country_code_3,
+        type as feature_type,
+        geometry_type,
+        try_to_geography(geometry) as geography,
         features as raw_geojson
     from raw_source
 
@@ -26,8 +26,8 @@ select
     feature_type,
     geometry_type,
     geography,
-    case when geography is not NULL then ST_AREA(geography) end as area_m2,
-    case when geography is not NULL then ST_PERIMETER(geography) end as perimeter_m,
-    case when geography is not NULL then ST_CENTROID(geography) end as centroid,
+    case when geography is not null then st_area(geography) end as area_m2,
+    case when geography is not null then st_perimeter(geography) end as perimeter_m,
+    case when geography is not null then st_centroid(geography) end as centroid,
     raw_geojson
 from final
diff --git a/transform/models/L3_coves/earthquake_analytics/earthquakes_around_the_world.sql b/transform/models/L3_coves/earthquake_analytics/earthquakes_around_the_world.sql
diff --git a/transform/models/L3_coves/earthquake_analytics/earthquakes_around_the_world.yml b/transform/models/L3_coves/earthquake_analytics/earthquakes_around_the_world.yml
