Source data migration

.github/workflows/pr-compare.yml

@@ -0,0 +1,25 @@
+name: PR Compare
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: ['main']
+    paths:
+      - models/**
+      - seeds/**
+      - tests/**
+
+jobs:
+  piperider-compare:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: PipeRider Compare
+      uses: InfuseAI/piperider-compare-action@v1
+      with:
+        cloud_api_token: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }}
+        cloud_project: ${{ secrets.PIPERIDER_API_PROJECT }}
+        upload: true
+        share: true      

.github/workflows/pr-comparison-job-slim-ci.yml

@@ -0,0 +1,81 @@
+name: PR Transformation and Comparison (Slim CI)
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+      # Trigger when PR is created and/or head of branch is updated.
+    branches: [ "main" ]
+      # Only PR target is `main`
+    paths:
+      - models/**
+      - seeds/**
+      - tests/**
+
+env:
+  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+  SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+  SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+  SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+  SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+  SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+  SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }}
+  DBT_CLOUD_API_TOKEN: ${{ secrets.DBT_CLOUD_API_TOKEN }}
+  DBT_CLOUD_ACCOUNT_ID: ${{ secrets.DBT_CLOUD_ACCOUNT_ID }}
+  DBT_CLOUD_JOB_ID: 173664
+  PIPERIDER_CLOUD_TOKEN_ID: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }}
+  PIPERIDER_CLOUD_PROJECT_ID: ${{ secrets.PIPERIDER_CLOUD_PROJECT_ID }}
+  PIPERIDER_API_PROJECT: ${{ secrets.PIPERIDER_API_PROJECT }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Setup Piperider
+      run: |
+        pip install dbt-cloud-cli
+        pip install dbt-snowflake
+        pip install 'piperider-nightly[snowflake]'
+
+    - name: Trigger DBT Cloud Job (PR Job)
+      run: |
+        dbt-cloud job run \
+          --cause "GitHub PR Job" \
+          --git-branch ${{ github.head_ref }} \
+          --schema-override ${SNOWFLAKE_SCHEMA} \
+          --wait \
+          --file response.json
+        echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV
+
+    - name: Piperider Cloud Login
+      run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --no-interaction
+
+    - name: Profile on PR environment
+      run: |
+        mkdir -p state/pr
+        dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path manifest.json -f state/pr/manifest.json
+        dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path run_results.json -f state/pr/run_results.json
+        piperider run --datasource jaffle_shop_pr --dbt-state state/pr --dbt-run-results -o /tmp/pipe-target
+        piperider cloud upload-report --run .piperider/outputs/latest/run.json
+
+    - name: Generate Comparison Summary
+      run: |
+        piperider cloud compare-reports \
+          --base datasource:jaffle_shop \
+          --target datasource:jaffle_shop_pr \
+          --summary-file summary.md \
+          --tables-from target-only
+
+    - name: Create PR Comment
+      uses: peter-evans/create-or-update-comment@v2.1.0
+      with:
+        issue-number: ${{ github.event.number }}
+        body-file: summary.md
+

.github/workflows/pr-comparison-job.yml

@@ -0,0 +1,74 @@
+name: PR Transformation and Comparison
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: ['main']
+    paths:
+      - models/**
+      - seeds/**
+      - tests/**
+
+env:
+  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+  SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+  SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+  SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+  SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+  SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+  SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Setup DBT + Piperider
+      run: |
+        pip install dbt-snowflake
+        pip install piperider['snowflake']
+
+    - name: Run DBT on PR environment
+      run: dbt build --target jaffle_shop_pr
+
+    - name: Profile on PR environment
+      run: piperider run --datasource jaffle_shop_pr --dbt-state target -o /tmp/pr-report/
+
+    - name: Profile on PROD environment
+      run: piperider run --datasource jaffle_shop -o /tmp/prod-report/
+
+    - name: Compare and Prepare
+      run: |
+        piperider compare-reports \
+          --debug \
+          --base /tmp/prod-report/run.json \
+          --target /tmp/pr-report/run.json \
+          --output comparison-report/ \
+          --tables-from target-only
+        zip -r comparison-report.zip comparison-report/
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: comparison-report-artifact
+        path: comparison-report.zip
+
+    - name: Prepare Comparison Summary
+      run: |
+        echo "# :bar_chart: Piperider Comparison Summary" > summary.md
+        cat comparison-report/summary.md >> summary.md
+        echo -e "\n" >> summary.md
+        echo "## :paperclip: Generated Comparison Report ZIP" >> summary.md
+        echo "Find it in the [Github Action Runs Page](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md
+
+    - name: Create PR Comment
+      uses: peter-evans/create-or-update-comment@v2.1.0
+      with:
+        issue-number: ${{ github.event.number }}
+        body-file: summary.md
+

.github/workflows/production-daily-job-slim-ci.yml

@@ -0,0 +1,69 @@
+name: Daily Production Transformation (Slim CI)
+on:
+  schedule:
+    - cron: '0 18 * * *' # run at 2 AM (UTC + 8) every day
+  workflow_dispatch:
+  pull_request:
+    types:
+      - closed
+    branches: ['main']
+    paths:
+      - models/**
+      - seeds/**
+      - tests/**
+
+env:
+  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+  SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+  SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+  SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+  SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+  SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+  DBT_CLOUD_API_TOKEN: ${{ secrets.DBT_CLOUD_API_TOKEN }}
+  DBT_CLOUD_ACCOUNT_ID: ${{ secrets.DBT_CLOUD_ACCOUNT_ID }}
+  DBT_CLOUD_JOB_ID: 173663
+  PIPERIDER_CLOUD_TOKEN_ID: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }}
+  PIPERIDER_API_PROJECT: ${{ secrets. PIPERIDER_API_PROJECT }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Setup DBT + Piperider
+      run: |
+        pip install dbt-cloud-cli
+        pip install dbt-snowflake
+        pip install piperider['snowflake']
+
+    - name: Run DBT on PROD environment
+      run: |
+        dbt-cloud job run --wait --file response.json
+        echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV
+
+    - name: Piperider Cloud Login
+      run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --no-interaction
+
+    - name: Profile on PROD environment
+      run: |
+        mkdir -p state/prod
+        dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path manifest.json -f state/prod/manifest.json
+        dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path run_results.json -f state/prod/run_results.json
+        piperider run --datasource jaffle_shop --dbt-state state/prod -o report/
+        piperider cloud upload-report --run .piperider/outputs/latest/run.json
+
+    - name: Create run artifact
+      run: zip -r run-report.zip report/
+
+    - name: Upload profiling result
+      uses: actions/upload-artifact@v3
+      with:
+        name: run-report-artifact
+        path: run-report.zip

.github/workflows/production-daily-job.yml

@@ -0,0 +1,54 @@
+name: Daily Production Transformation
+on:
+  schedule:
+    - cron: '0 18 * * *' # run at 2 AM (UTC + 8) every day
+  workflow_dispatch:
+  pull_request:
+    types:
+      - closed
+    branches: ['main']
+    paths:
+      - models/**
+      - seeds/**
+      - tests/**
+
+env:
+  SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+  SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+  SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+  SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+  SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+  SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Setup DBT + Piperider
+      run: |
+        pip install dbt-snowflake
+        pip install piperider['snowflake']
+
+    - name: Run DBT on PROD environment
+      run: dbt build --target jaffle_shop
+
+    - name: Profile on PROD environment
+      run: piperider run --datasource jaffle_shop --dbt-state target -o report/
+
+    - name: Create run artifact
+      run: zip -r run-report.zip report/
+
+    - name: Upload profiling result
+      uses: actions/upload-artifact@v3
+      with:
+        name: run-report-artifact
+        path: run-report.zip
+

.gitignore

@@ -2,4 +2,7 @@
 target/
 dbt_modules/
 logs/
+venv/
 **/.DS_Store
+*.duckdb
+

.piperider/.gitignore

@@ -0,0 +1,8 @@
+**.log
+__pycache__/
+logs/
+outputs/
+reports/
+comparisons/
+credentials.yml
+.unsend_events.json

.piperider/compare/default.yml

@@ -0,0 +1,15 @@
+base:
+  branch: main
+  dbt:
+    commands:
+    - dbt deps
+    - dbt build
+  piperider:
+    command: piperider run
+target:
+  dbt:
+    commands:
+    - dbt deps
+    - dbt build
+  piperider:
+    command: piperider run

.piperider/config.yml

@@ -0,0 +1,6 @@
+dataSources: []
+dbt:
+  projectDir: .
+
+telemetry:
+  id: 2e470b721f5a4114aa7cf9b73eca760d

dbt_project.yml

@@ -22,5 +22,6 @@ require-dbt-version: [">=1.0.0", "<2.0.0"]
 models:
   jaffle_shop:
       materialized: table
-      staging:
+      +tags: piperider
+      staging:      
         materialized: view

models/customers.sql → models/int_customer_order_history_joined.sql

@@ -34,7 +34,7 @@ customer_payments as (
 
     select
         orders.customer_id,
-        sum(amount) as total_amount
+        sum(amount)::bigint as total_amount
 
     from payments
 

models/orders.sql → models/int_order_payments_pivoted.sql

@@ -18,10 +18,10 @@ order_payments as (
         order_id,
 
         {% for payment_method in payment_methods -%}
-        sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,
+        sum(case when payment_method = '{{ payment_method }}' then amount else 0 end)::bigint as {{ payment_method }}_amount,
         {% endfor -%}
 
-        sum(amount) as total_amount
+        sum(amount)::bigint as total_amount
 
     from payments