move us_population to its own schema (#228)

* move us_population to its own schema
* run observe model for post hook
* Add S3 upload to deploy script

Author: noel

.github/workflows/10_integrate_dbt_changes.yml

@@ -79,6 +79,10 @@ jobs:
       ##### Governance Checks
       # this first runs dbt but creates enpty tables, this is enough to then run the hooks and fail fast
 
+      # We need to run observe model so that post hook works
+      - name: Run Observe Model
+        run: "dbt build --fail-fast -s L1_inlets.observe"
+
       # There is an issue with --empty and dynamic tables so need to exclude them
       - name: Governance run of dynamic tables
         run: "dbt build --fail-fast -s config.materialized:dynamic_table -s test_failures --defer --state logs"

.github/workflows/30_deploy_changes_to_production.yml

@@ -104,6 +104,25 @@ jobs:
       - name: Upload dbt artifacts
         run: "dbt run-operation upload_artifacts"
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Upload artifacts to S3 for Datahub
+        run: |
+          # Check and upload each file individually
+          for file in target/sources.json target/manifest.json target/catalog.json target/run_results.json; do
+            if [ -f "$file" ]; then
+              echo "Uploading $file to S3..."
+              aws s3 cp "$file" "s3://${DBT_ARTIFACTS_BUCKET}/dbt_artifacts/$(basename $file)"
+            else
+              echo "File $file does not exist, skipping..."
+            fi
+          done
+
       - uses: fregante/setup-git-user@v2
       - name: Bump dbt project and git project version
         run: "../automate/dbt/bump_dbt_project.sh"

secure/databases.yml

@@ -9,11 +9,11 @@
     schemas:
       - L1_ACCOUNT_USAGE
       - L1_COUNTRY_DATA
-      - L1_GOOGLE_ANALYTICS_4
-      - L1_DBT_ARTIFACTS
       - L1_COVID19_EPIDEMIOLOGICAL_DATA
+      - L1_GOOGLE_ANALYTICS_4
       - L1_LOANS
       - L1_OBSERVE
+      - L1_US_POPULATION
 
       - L2_COUNTRY_DEMOGRAPHICS
       - L2_COVID_OBSERVATIONS

secure/roles.yml

@@ -16,16 +16,15 @@
 
       - z_schema_seeds
 
-      - z_schema_l1_dbt_artifacts
+      - z_schema_dbt_test__audit
+
       - z_schema_l1_account_usage
       - z_schema_l1_country_data
-      - z_schema_l1_google_analytics_4
-      - z_schema_l1_dbt_artifacts
       - z_schema_l1_covid19_epidemiological_data
+      - z_schema_l1_google_analytics_4
       - z_schema_l1_loans
       - z_schema_l1_observe
-
-      - z_schema_dbt_test__audit
+      - z_schema_l1_us_population
 
       - z_schema_l2_country_demographics
       - z_schema_l2_covid_observations
@@ -355,66 +354,66 @@
         read:
           - raw.snapshots
 
-# INLETS
-- z_schema_l1_account_usage:
+- z_schema_resources:
     privileges:
       schemas:
         read:
-          - balboa.l1_account_usage
+          - balboa_apps.resources
 
-- z_schema_l1_country_data:
+- z_schema_seeds:
     privileges:
       schemas:
         read:
-          - balboa.l1_country_data
+          - balboa.seeds
 
-- z_schema_l1_google_analytics_4:
+- z_schema_dbt_test__audit:
     privileges:
       schemas:
         read:
-          - balboa.l1_google_analytics_4
+          - balboa.dbt_test__audit
 
-- z_schema_l1_covid19_epidemiological_data:
+# INLETS
+- z_schema_l1_account_usage:
     privileges:
       schemas:
         read:
-          - balboa.l1_covid19_epidemiological_data
+          - balboa.l1_account_usage
 
-- z_schema_l1_dbt_artifacts:
+- z_schema_l1_country_data:
     privileges:
       schemas:
         read:
-          - balboa.l1_dbt_artifacts
+          - balboa.l1_country_data
 
-- z_schema_l1_loans:
+- z_schema_l1_covid19_epidemiological_data:
     privileges:
       schemas:
         read:
-          - balboa.l1_loans
+          - balboa.l1_covid19_epidemiological_data
 
-- z_schema_l1_observe:
+- z_schema_l1_google_analytics_4:
     privileges:
       schemas:
         read:
-          - balboa.l1_observe
+          - balboa.l1_google_analytics_4
 
-- z_schema_resources:
+- z_schema_l1_loans:
     privileges:
       schemas:
         read:
-          - balboa_apps.resources
+          - balboa.l1_loans
 
-- z_schema_seeds:
+- z_schema_l1_observe:
     privileges:
       schemas:
         read:
-          - balboa.seeds
+          - balboa.l1_observe
 
-- z_schema_dbt_test__audit:
+- z_schema_l1_us_population:
     privileges:
       schemas:
         read:
-          - balboa.dbt_test__audit
+          - balboa.l1_us_population
 
 # BAYS
 - z_schema_l2_country_demographics:

transform/dbt_project.yml

@@ -77,7 +77,7 @@ models:
             observe:
                 +schema: L1_OBSERVE
             us_population:
-                +schema: L1_COUNTRY_DATA
+                +schema: L1_US_POPULATION
 
         L2_bays:
             +group: marketing