From d7d8d2eb68830bd01e8d8e5875fe995fbb5ad397 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Mon, 5 Dec 2022 15:08:53 +0800 Subject: [PATCH 01/22] Add dbt profiles setting and piperider config setting Signed-off-by: Wei-Chun, Chang --- .piperider/.gitignore | 8 ++++++++ .piperider/config.yml | 35 +++++++++++++++++++++++++++++++++++ profiles.yml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 .piperider/.gitignore create mode 100644 .piperider/config.yml create mode 100644 profiles.yml diff --git a/.piperider/.gitignore b/.piperider/.gitignore new file mode 100644 index 000000000..e015b8d38 --- /dev/null +++ b/.piperider/.gitignore @@ -0,0 +1,8 @@ +**.log +__pycache__/ +logs/ +outputs/ +reports/ +comparisons/ +credentials.yml +.unsend_events.json \ No newline at end of file diff --git a/.piperider/config.yml b/.piperider/config.yml new file mode 100644 index 000000000..bbccdd71e --- /dev/null +++ b/.piperider/config.yml @@ -0,0 +1,35 @@ +dataSources: +- name: jaffle_shop_pr + type: snowflake + dbt: + profile: jaffle_shop + target: pr + projectDir: . +- name: jaffle_shop + type: snowflake + dbt: + profile: jaffle_shop + target: prod + projectDir: . + +profiler: +# table: +# # the maximum row count to profile. (Default unlimited) +# limit: 1000000 +# duplicateRows: false + +# The tables to include/exclude +# includes: [] +# excludes: [] + +# tables: +# my-table-name: +# # description of the table +# description: "this is a table description" +# columns: +# my-col-name: +# # description of the column +# description: "this is a column description" + +telemetry: + id: 2e470b721f5a4114aa7cf9b73eca760d diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 000000000..f2b2f9cec --- /dev/null +++ b/profiles.yml @@ -0,0 +1,30 @@ +jaffle_shop: + target: prod + outputs: + pr: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" + + # User/password auth + user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" + + role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" + threads: 4 + prod: + type: snowflak + account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" + + # User/password auth + user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" + + role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" + schema: PUBLIC + threads: 4 + From 6a1d673fee084367fbf531b62c856ffbe5ff38e5 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Fri, 9 Dec 2022 15:06:25 +0800 Subject: [PATCH 02/22] Add production and pr workflows for ci Signed-off-by: Wei-Chun, Chang --- .github/workflows/pr-comparison-job.yml | 60 ++++++++++++++++++++++ .github/workflows/production-daily-job.yml | 54 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 .github/workflows/pr-comparison-job.yml create mode 100644 .github/workflows/production-daily-job.yml diff --git a/.github/workflows/pr-comparison-job.yml b/.github/workflows/pr-comparison-job.yml new file mode 100644 index 000000000..ff701c705 --- /dev/null +++ b/.github/workflows/pr-comparison-job.yml @@ -0,0 +1,60 @@ +name: PR Transformation and Comparison +on: + pull_request: + types: [opened, synchronize, reopened] + branches: ['main'] + paths: + - models/** + - seeds/** + - tests/** + +env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Setup DBT + Piperider + run: | + pip install dbt-snowflake + pip install piperider['snowflake'] + + - name: Run DBT on PR environment + run: dbt build --target pr + + - name: Profile on PR environment + run: piperider run --datasource jaffle_shop_pr --dbt-state target -o /tmp/pr-report/ + + - name: Profile on PROD environment + run: piperider run --datasource jaffle_shop -o /tmp/prod-report/ + + - name: Compare and Prepare + run: | + piperider compare-reports \ + --debug \ + --base /tmp/prod-report/run.json \ + --target /tmp/pr-report/run.json \ + --output comparison-report/ \ + --tables-from target-only + zip -r comparison-report.zip comparison-report/ + + - uses: actions/upload-artifact@v3 + with: + name: comparison-report-artifact + path: comparison-report.zip + diff --git a/.github/workflows/production-daily-job.yml b/.github/workflows/production-daily-job.yml new file mode 100644 index 000000000..a5e3cd7e5 --- /dev/null +++ b/.github/workflows/production-daily-job.yml @@ -0,0 +1,54 @@ +name: Daily Production Transformation +on: + schedule: + - cron: '0 18 * * *' # run at 2 AM (UTC + 8) every day + workflow_dispatch: + pull_request: + types: + - closed + branches: ['main'] + paths: + - models/** + - seeds/** + - tests/** + +env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Setup DBT + Piperider + run: | + pip install dbt-snowflake + pip install piperider['snowflake'] + + - name: Run DBT on PROD environment + run: dbt build --target prod + + - name: Profile on PROD environment + run: piperider run --datasource jaffle_shop --dbt-state target -o report/ + + - name: Create run artifact + run: zip -r run-report.zip report/ + + - name: Upload profiling result + uses: actions/upload-artifact@v3 + with: + name: run-report-artifact + path: run-report.zip + From d17535ff3e3399741ea7262c227aa3c68b0f2cff Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Fri, 9 Dec 2022 16:42:45 +0800 Subject: [PATCH 03/22] Update dbt profile type Signed-off-by: Wei-Chun, Chang --- profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles.yml b/profiles.yml index f2b2f9cec..b76ed4294 100644 --- a/profiles.yml +++ b/profiles.yml @@ -15,7 +15,7 @@ jaffle_shop: schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" threads: 4 prod: - type: snowflak + type: snowflake account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" # User/password auth From 22789fb15eb42c0ae9f7fe89c38071142923a0d5 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Sat, 10 Dec 2022 21:39:53 +0800 Subject: [PATCH 04/22] Add production and pr workflows for slim ci Signed-off-by: Wei-Chun, Chang --- .../workflows/pr-comparison-job-slim-ci.yml | 92 +++++++++++++++++++ .../production-daily-job-slim-ci.yml | 67 ++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 .github/workflows/pr-comparison-job-slim-ci.yml create mode 100644 .github/workflows/production-daily-job-slim-ci.yml diff --git a/.github/workflows/pr-comparison-job-slim-ci.yml b/.github/workflows/pr-comparison-job-slim-ci.yml new file mode 100644 index 000000000..55d2888e7 --- /dev/null +++ b/.github/workflows/pr-comparison-job-slim-ci.yml @@ -0,0 +1,92 @@ +name: PR Transformation and Comparison (Slim CI) + +on: + pull_request: + types: [opened, synchronize, reopened] + # Trigger when PR is created and/or head of branch is updated. + branches: [ "main" ] + # Only PR target is `main` + paths: + - models/** + - seeds/** + - tests/** + +env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }} + DBT_CLOUD_API_TOKEN: ${{ secrets.DBT_CLOUD_API_TOKEN }} + DBT_CLOUD_ACCOUNT_ID: ${{ secrets.DBT_CLOUD_ACCOUNT_ID }} + DBT_CLOUD_JOB_ID: 173664 + PIPERIDER_CLOUD_TOKEN_ID: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }} + PIPERIDER_CLOUD_PROJECT_ID: ${{ secrets.PIPERIDER_CLOUD_PROJECT_ID }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Setup Piperider + run: | + pip install dbt-cloud-cli + pip install dbt-snowflake + pip install piperider['snowflake'] + + - name: Trigger DBT Cloud Job (PR Job) + run: | + dbt-cloud job run \ + --cause "GitHub PR Job" \ + --git-branch ${{ github.head_ref }} \ + --schema-override ${SNOWFLAKE_SCHEMA} \ + --wait \ + --file response.json + echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV + + - name: Piperider Cloud Login + run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --disable-auto-upload + + - name: Profile on PR environment + run: | + mkdir -p state/pr + dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path manifest.json -f state/pr/manifest.json + dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path run_results.json -f state/pr/run_results.json + piperider run --datasource jaffle_shop_pr --dbt-state state/pr -o /tmp/pipe-target + piperider cloud upload-report --run .piperider/outputs/latest/run.json + + - name: Fetch Base Run + run: | + curl -X GET \ + -H "Authorization: Bearer $PIPERIDER_CLOUD_TOKEN_ID" \ + "https://cloud.piperider.io/api/projects/$PIPERIDER_CLOUD_PROJECT_ID/reports?datasource=jaffle_shop" > response.json + base_report_id=$(cat response.json | jq -r '.[-1].id') + mkdir -p /tmp/pipe-base + curl -X GET \ + -H "Authorization: Bearer $PIPERIDER_CLOUD_TOKEN_ID" \ + "https://cloud.piperider.io/api/projects/$PIPERIDER_CLOUD_PROJECT_ID/reports/$base_report_id" | jq -r '.report_content' > /tmp/pipe-base/run.json + + - name: Generate Comparison Artifact + Data Summary (local) + run: | + piperider compare-reports \ + --debug \ + --base /tmp/pipe-base/run.json \ + --target /tmp/pipe-target/run.json \ + --tables-from target-only \ + -o comparison-report/ + zip -r comparison-report.zip comparison-report/ + + - uses: actions/upload-artifact@v3 + with: + name: comparison-report-artifact + path: comparison-report.zip + diff --git a/.github/workflows/production-daily-job-slim-ci.yml b/.github/workflows/production-daily-job-slim-ci.yml new file mode 100644 index 000000000..f7c4904f8 --- /dev/null +++ b/.github/workflows/production-daily-job-slim-ci.yml @@ -0,0 +1,67 @@ +name: Daily Production Transformation (Slim CI) +on: + schedule: + - cron: '0 18 * * *' # run at 2 AM (UTC + 8) every day + workflow_dispatch: + pull_request: + types: + - closed + branches: ['main'] + paths: + - models/** + - seeds/** + - tests/** + +env: + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + DBT_CLOUD_API_TOKEN: ${{ secrets.DBT_CLOUD_API_TOKEN }} + DBT_CLOUD_ACCOUNT_ID: ${{ secrets.DBT_CLOUD_ACCOUNT_ID }} + DBT_CLOUD_JOB_ID: 173663 + PIPERIDER_CLOUD_TOKEN_ID: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Setup DBT + Piperider + run: | + pip install dbt-snowflake + pip install piperider['snowflake'] + + - name: Run DBT on PROD environment + run: | + dbt-cloud job run --wait --file response.json + echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV + + - name: Piperider Cloud Login + run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --disable-auto-upload + + - name: Profile on PROD environment + run: | + mkdir -p state/prod + dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path manifest.json -f state/prod/manifest.json + dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path run_results.json -f state/prod/run_results.json + piperider run --datasource infusetude --dbt-state state/prod -o report/ + piperider cloud upload-report --run .piperider/outputs/latest/run.json + + - name: Create run artifact + run: zip -r run-report.zip report/ + + - name: Upload profiling result + uses: actions/upload-artifact@v3 + with: + name: run-report-artifact + path: run-report.zip From 41f8ddf60091e7a88e3152266903f80a721fbf62 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Sun, 11 Dec 2022 00:27:36 +0800 Subject: [PATCH 05/22] Update slim ci production job Signed-off-by: Wei-Chun, Chang --- .github/workflows/production-daily-job-slim-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/production-daily-job-slim-ci.yml b/.github/workflows/production-daily-job-slim-ci.yml index f7c4904f8..e1441d2d4 100644 --- a/.github/workflows/production-daily-job-slim-ci.yml +++ b/.github/workflows/production-daily-job-slim-ci.yml @@ -38,6 +38,7 @@ jobs: - name: Setup DBT + Piperider run: | + pip install dbt-cloud-cli pip install dbt-snowflake pip install piperider['snowflake'] @@ -54,7 +55,7 @@ jobs: mkdir -p state/prod dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path manifest.json -f state/prod/manifest.json dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path run_results.json -f state/prod/run_results.json - piperider run --datasource infusetude --dbt-state state/prod -o report/ + piperider run --datasource jaffle_shop --dbt-state state/prod -o report/ piperider cloud upload-report --run .piperider/outputs/latest/run.json - name: Create run artifact From dec661ea60a2af5860a88761762fd96ac0651975 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Tue, 20 Dec 2022 19:03:34 +0800 Subject: [PATCH 06/22] Post comparison summary to the PR comment Signed-off-by: Wei-Chun, Chang --- .github/workflows/pr-comparison-job-slim-ci.yml | 14 ++++++++++++++ .github/workflows/pr-comparison-job.yml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/.github/workflows/pr-comparison-job-slim-ci.yml b/.github/workflows/pr-comparison-job-slim-ci.yml index 55d2888e7..7433ef287 100644 --- a/.github/workflows/pr-comparison-job-slim-ci.yml +++ b/.github/workflows/pr-comparison-job-slim-ci.yml @@ -90,3 +90,17 @@ jobs: name: comparison-report-artifact path: comparison-report.zip + - name: Prepare Comparison Summary + run: | + echo "# :bar_chart: Piperider Comparison Summary" > summary.md + cat comparison-report/summary.md >> summary.md + echo -e "\n" >> summary.md + echo "## :paperclip: Generated Comparison Report ZIP" >> summary.md + echo "Find it in the [Github Action Runs Page](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md + + - name: Create PR Comment + uses: peter-evans/create-or-update-comment@v2.1.0 + with: + issue-number: ${{ github.event.number }} + body-file: summary.md + diff --git a/.github/workflows/pr-comparison-job.yml b/.github/workflows/pr-comparison-job.yml index ff701c705..760087231 100644 --- a/.github/workflows/pr-comparison-job.yml +++ b/.github/workflows/pr-comparison-job.yml @@ -58,3 +58,17 @@ jobs: name: comparison-report-artifact path: comparison-report.zip + - name: Prepare Comparison Summary + run: | + echo "# :bar_chart: Piperider Comparison Summary" > summary.md + cat comparison-report/summary.md >> summary.md + echo -e "\n" >> summary.md + echo "## :paperclip: Generated Comparison Report ZIP" >> summary.md + echo "Find it in the [Github Action Runs Page](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md + + - name: Create PR Comment + uses: peter-evans/create-or-update-comment@v2.1.0 + with: + issue-number: ${{ github.event.number }} + body-file: summary.md + From 925ef921188f3060c9e0f2f57cd6a3e4e6fc6080 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Fri, 23 Dec 2022 15:24:32 +0800 Subject: [PATCH 07/22] Update the slim CI workflow with piperider cloud command Signed-off-by: Wei-Chun, Chang --- .../workflows/pr-comparison-job-slim-ci.yml | 38 +++---------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/.github/workflows/pr-comparison-job-slim-ci.yml b/.github/workflows/pr-comparison-job-slim-ci.yml index 7433ef287..1eaa54d43 100644 --- a/.github/workflows/pr-comparison-job-slim-ci.yml +++ b/.github/workflows/pr-comparison-job-slim-ci.yml @@ -64,39 +64,13 @@ jobs: piperider run --datasource jaffle_shop_pr --dbt-state state/pr -o /tmp/pipe-target piperider cloud upload-report --run .piperider/outputs/latest/run.json - - name: Fetch Base Run + - name: Generate Comparison Summary run: | - curl -X GET \ - -H "Authorization: Bearer $PIPERIDER_CLOUD_TOKEN_ID" \ - "https://cloud.piperider.io/api/projects/$PIPERIDER_CLOUD_PROJECT_ID/reports?datasource=jaffle_shop" > response.json - base_report_id=$(cat response.json | jq -r '.[-1].id') - mkdir -p /tmp/pipe-base - curl -X GET \ - -H "Authorization: Bearer $PIPERIDER_CLOUD_TOKEN_ID" \ - "https://cloud.piperider.io/api/projects/$PIPERIDER_CLOUD_PROJECT_ID/reports/$base_report_id" | jq -r '.report_content' > /tmp/pipe-base/run.json - - - name: Generate Comparison Artifact + Data Summary (local) - run: | - piperider compare-reports \ - --debug \ - --base /tmp/pipe-base/run.json \ - --target /tmp/pipe-target/run.json \ - --tables-from target-only \ - -o comparison-report/ - zip -r comparison-report.zip comparison-report/ - - - uses: actions/upload-artifact@v3 - with: - name: comparison-report-artifact - path: comparison-report.zip - - - name: Prepare Comparison Summary - run: | - echo "# :bar_chart: Piperider Comparison Summary" > summary.md - cat comparison-report/summary.md >> summary.md - echo -e "\n" >> summary.md - echo "## :paperclip: Generated Comparison Report ZIP" >> summary.md - echo "Find it in the [Github Action Runs Page](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md + piperider cloud compare-reports \ + --base datasource:jaffle_shop \ + --target datasource:jaffle_shop_pr \ + --summary-file summary.md \ + --tables-from target-only - name: Create PR Comment uses: peter-evans/create-or-update-comment@v2.1.0 From 470428c89863639afd371dc695e56962ab4a40a4 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Fri, 23 Dec 2022 15:47:32 +0800 Subject: [PATCH 08/22] Update piperider installation Signed-off-by: Wei-Chun, Chang --- .github/workflows/pr-comparison-job-slim-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-comparison-job-slim-ci.yml b/.github/workflows/pr-comparison-job-slim-ci.yml index 1eaa54d43..a2072e88a 100644 --- a/.github/workflows/pr-comparison-job-slim-ci.yml +++ b/.github/workflows/pr-comparison-job-slim-ci.yml @@ -41,7 +41,7 @@ jobs: run: | pip install dbt-cloud-cli pip install dbt-snowflake - pip install piperider['snowflake'] + pip install 'piperider-nightly[snowflake]' - name: Trigger DBT Cloud Job (PR Job) run: | From af65fe8815748cac2fa3ccffd2084094decf54ba Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Thu, 5 Jan 2023 23:33:03 +0800 Subject: [PATCH 09/22] Add business metrics Signed-off-by: Wei-Chun, Chang --- ... => int_customer_order_history_joined.sql} | 0 ...ers.sql => int_order_payments_pivoted.sql} | 0 models/marts/average_order_amount.yml | 16 ++++ models/{ => marts}/docs.md | 0 models/marts/expenses.yml | 21 ++++++ models/marts/orders.sql | 23 ++++++ models/marts/orders.yml | 73 +++++++++++++++++++ models/marts/profit.yml | 15 ++++ models/marts/revenue.yml | 21 ++++++ models/schema.yml | 6 +- 10 files changed, 172 insertions(+), 3 deletions(-) rename models/{customers.sql => int_customer_order_history_joined.sql} (100%) rename models/{orders.sql => int_order_payments_pivoted.sql} (100%) create mode 100644 models/marts/average_order_amount.yml rename models/{ => marts}/docs.md (100%) create mode 100644 models/marts/expenses.yml create mode 100644 models/marts/orders.sql create mode 100644 models/marts/orders.yml create mode 100644 models/marts/profit.yml create mode 100644 models/marts/revenue.yml diff --git a/models/customers.sql b/models/int_customer_order_history_joined.sql similarity index 100% rename from models/customers.sql rename to models/int_customer_order_history_joined.sql diff --git a/models/orders.sql b/models/int_order_payments_pivoted.sql similarity index 100% rename from models/orders.sql rename to models/int_order_payments_pivoted.sql diff --git a/models/marts/average_order_amount.yml b/models/marts/average_order_amount.yml new file mode 100644 index 000000000..d7179d3c2 --- /dev/null +++ b/models/marts/average_order_amount.yml @@ -0,0 +1,16 @@ +version: 2 + +metrics: + - name: average_order_amount + label: Average Order Amount + model: ref('orders') + description: "The average size of a jaffle order" + + calculation_method: average + expression: amount + + timestamp: order_date + time_grains: [day, week, month] + + tags: ['piperider'] + diff --git a/models/docs.md b/models/marts/docs.md similarity index 100% rename from models/docs.md rename to models/marts/docs.md diff --git a/models/marts/expenses.yml b/models/marts/expenses.yml new file mode 100644 index 000000000..584e309ce --- /dev/null +++ b/models/marts/expenses.yml @@ -0,0 +1,21 @@ +version: 2 + +metrics: + - name: expenses + label: Expenses + model: ref('orders') + description: "The total expenses of our jaffle business" + + calculation_method: sum + expression: amount / 4 + + timestamp: order_date + time_grains: [day, week, month, year] + + filters: + - field: status + operator: '=' + value: "'completed'" + + tags: ['piperider'] + diff --git a/models/marts/orders.sql b/models/marts/orders.sql new file mode 100644 index 000000000..0b5ae3888 --- /dev/null +++ b/models/marts/orders.sql @@ -0,0 +1,23 @@ +with orders as ( + + select * from {{ ref('int_order_payments_pivoted') }} + +) +, +customers as ( + + select * from {{ ref('int_customer_order_history_joined') }} + +) +, +final as ( + + select + * + from orders + left join customers using (customer_id) + +) + +select * from final + diff --git a/models/marts/orders.yml b/models/marts/orders.yml new file mode 100644 index 000000000..055a491c7 --- /dev/null +++ b/models/marts/orders.yml @@ -0,0 +1,73 @@ +version: 2 + +models: + - name: orders + description: This table has basic information about orders, as well as some derived facts based on payments + + columns: + - name: order_id + tests: + - unique + - not_null + description: This is a unique identifier for an order + + - name: customer_id + description: Foreign key to the customers table + tests: + - not_null + + - name: order_date + description: Date (UTC) that the order was placed + + - name: status + description: '{{ doc("orders_status") }}' + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: amount + description: Total amount (AUD) of the order + tests: + - not_null + + - name: credit_card_amount + description: Amount of the order (AUD) paid for by credit card + tests: + - not_null + + - name: coupon_amount + description: Amount of the order (AUD) paid for by coupon + tests: + - not_null + + - name: bank_transfer_amount + description: Amount of the order (AUD) paid for by bank transfer + tests: + - not_null + + - name: gift_card_amount + description: Amount of the order (AUD) paid for by gift card + tests: + - not_null + + - name: customer_id + description: This is a unique identifier for a customer + + - name: first_name + description: Customer's first name. PII. + + - name: last_name + description: Customer's last name. PII. + + - name: first_order + description: Date (UTC) of a customer's first order + + - name: most_recent_order + description: Date (UTC) of a customer's most recent order + + - name: number_of_orders + description: Count of the number of orders a customer has placed + + - name: total_order_amount + description: Total value (AUD) of a customer's orders + diff --git a/models/marts/profit.yml b/models/marts/profit.yml new file mode 100644 index 000000000..b6103e3f6 --- /dev/null +++ b/models/marts/profit.yml @@ -0,0 +1,15 @@ +version: 2 + +metrics: + - name: profit + label: Profit + description: "The total money we get to take home from our jaffle business" + + calculation_method: derived + expression: "{{metric('revenue')}} - {{metric('expenses')}}" + + timestamp: order_date + time_grains: [hour, day, week, month, year] + + tags: ['piperider'] + diff --git a/models/marts/revenue.yml b/models/marts/revenue.yml new file mode 100644 index 000000000..be4be6c9b --- /dev/null +++ b/models/marts/revenue.yml @@ -0,0 +1,21 @@ +version: 2 + +metrics: + - name: revenue + label: Revenue + model: ref('orders') + description: "The total revenue of our jaffle business" + + calculation_method: sum + expression: amount + + timestamp: order_date + time_grains: [day, week, month, year] + + filters: + - field: status + operator: '=' + value: "'completed'" + + tags: ['piperider'] + diff --git a/models/schema.yml b/models/schema.yml index 381349cfd..f7dcdad0c 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: customers + - name: int_customer_order_history_joined description: This table has basic information about a customer, as well as some derived facts based on a customer's orders columns: @@ -29,7 +29,7 @@ models: - name: total_order_amount description: Total value (AUD) of a customer's orders - - name: orders + - name: int_order_payments_pivoted description: This table has basic information about orders, as well as some derived facts based on payments columns: @@ -44,7 +44,7 @@ models: tests: - not_null - relationships: - to: ref('customers') + to: ref('int_customer_order_history_joined') field: customer_id - name: order_date From da24bbbf4c146c1aedffe53c1cb79d3305d3c0e1 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 8 Mar 2023 09:54:06 +0800 Subject: [PATCH 10/22] Update CI workflow with piperider cli v0.20.0 command Signed-off-by: Wei-Chun, Chang --- .github/workflows/pr-comparison-job-slim-ci.yml | 2 +- .github/workflows/production-daily-job-slim-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-comparison-job-slim-ci.yml b/.github/workflows/pr-comparison-job-slim-ci.yml index a2072e88a..359cd52eb 100644 --- a/.github/workflows/pr-comparison-job-slim-ci.yml +++ b/.github/workflows/pr-comparison-job-slim-ci.yml @@ -54,7 +54,7 @@ jobs: echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV - name: Piperider Cloud Login - run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --disable-auto-upload + run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID - name: Profile on PR environment run: | diff --git a/.github/workflows/production-daily-job-slim-ci.yml b/.github/workflows/production-daily-job-slim-ci.yml index e1441d2d4..6d3ed4abf 100644 --- a/.github/workflows/production-daily-job-slim-ci.yml +++ b/.github/workflows/production-daily-job-slim-ci.yml @@ -48,7 +48,7 @@ jobs: echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV - name: Piperider Cloud Login - run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --disable-auto-upload + run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID - name: Profile on PROD environment run: | From 64757d7be9c43104fb7051498570f8d15564ec66 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Fri, 31 Mar 2023 10:19:56 +0800 Subject: [PATCH 11/22] Update login option and run option for slim CI Signed-off-by: Wei-Chun, Chang --- .github/workflows/pr-comparison-job-slim-ci.yml | 5 +++-- .github/workflows/production-daily-job-slim-ci.yml | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-comparison-job-slim-ci.yml b/.github/workflows/pr-comparison-job-slim-ci.yml index 359cd52eb..e9c3cb1a0 100644 --- a/.github/workflows/pr-comparison-job-slim-ci.yml +++ b/.github/workflows/pr-comparison-job-slim-ci.yml @@ -24,6 +24,7 @@ env: DBT_CLOUD_JOB_ID: 173664 PIPERIDER_CLOUD_TOKEN_ID: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }} PIPERIDER_CLOUD_PROJECT_ID: ${{ secrets.PIPERIDER_CLOUD_PROJECT_ID }} + PIPERIDER_API_PROJECT: ${{ secrets.PIPERIDER_API_PROJECT }} jobs: build: @@ -54,14 +55,14 @@ jobs: echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV - name: Piperider Cloud Login - run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID + run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --no-interaction - name: Profile on PR environment run: | mkdir -p state/pr dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path manifest.json -f state/pr/manifest.json dbt-cloud run get-artifact --run-id ${{ env.run_id }} --path run_results.json -f state/pr/run_results.json - piperider run --datasource jaffle_shop_pr --dbt-state state/pr -o /tmp/pipe-target + piperider run --datasource jaffle_shop_pr --dbt-state state/pr --dbt-run-results -o /tmp/pipe-target piperider cloud upload-report --run .piperider/outputs/latest/run.json - name: Generate Comparison Summary diff --git a/.github/workflows/production-daily-job-slim-ci.yml b/.github/workflows/production-daily-job-slim-ci.yml index 6d3ed4abf..66feab19c 100644 --- a/.github/workflows/production-daily-job-slim-ci.yml +++ b/.github/workflows/production-daily-job-slim-ci.yml @@ -23,6 +23,7 @@ env: DBT_CLOUD_ACCOUNT_ID: ${{ secrets.DBT_CLOUD_ACCOUNT_ID }} DBT_CLOUD_JOB_ID: 173663 PIPERIDER_CLOUD_TOKEN_ID: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }} + PIPERIDER_API_PROJECT: ${{ secrets. PIPERIDER_API_PROJECT }} jobs: build: @@ -48,7 +49,7 @@ jobs: echo "run_id=$(cat response.json | jq -r '.data.id')" >> $GITHUB_ENV - name: Piperider Cloud Login - run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID + run: piperider cloud login --token $PIPERIDER_CLOUD_TOKEN_ID --no-interaction - name: Profile on PROD environment run: | From 6632c587134dba346684ad1db9880d16f22c0c92 Mon Sep 17 00:00:00 2001 From: popcorny Date: Fri, 31 Mar 2023 15:20:42 +0800 Subject: [PATCH 12/22] Change the piperider config Signed-off-by: popcorny --- .gitignore | 1 + .piperider/compare/default.yml | 16 ++++++++++++++++ .piperider/config.yml | 30 ++++-------------------------- profiles.yml | 19 ++++++++++++++++--- 4 files changed, 37 insertions(+), 29 deletions(-) create mode 100644 .piperider/compare/default.yml diff --git a/.gitignore b/.gitignore index 716442207..f5f3499cf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ target/ dbt_modules/ logs/ +venv/ **/.DS_Store diff --git a/.piperider/compare/default.yml b/.piperider/compare/default.yml new file mode 100644 index 000000000..dbb006e3a --- /dev/null +++ b/.piperider/compare/default.yml @@ -0,0 +1,16 @@ +base: + branch: main + dbt: + commands: + - dbt deps + - dbt build + piperider: + command: piperider run +target: + branch: feature/add-metrics + dbt: + commands: + - dbt deps + - dbt build + piperider: + command: piperider run diff --git a/.piperider/config.yml b/.piperider/config.yml index bbccdd71e..6e8f7e94b 100644 --- a/.piperider/config.yml +++ b/.piperider/config.yml @@ -1,16 +1,7 @@ -dataSources: -- name: jaffle_shop_pr - type: snowflake - dbt: - profile: jaffle_shop - target: pr - projectDir: . -- name: jaffle_shop - type: snowflake - dbt: - profile: jaffle_shop - target: prod - projectDir: . +dataSources: [] +dbt: + projectDir: . + # tag: 'piperider' profiler: # table: @@ -18,18 +9,5 @@ profiler: # limit: 1000000 # duplicateRows: false -# The tables to include/exclude -# includes: [] -# excludes: [] - -# tables: -# my-table-name: -# # description of the table -# description: "this is a table description" -# columns: -# my-col-name: -# # description of the column -# description: "this is a column description" - telemetry: id: 2e470b721f5a4114aa7cf9b73eca760d diff --git a/profiles.yml b/profiles.yml index b76ed4294..851bcdbf9 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,7 +1,20 @@ jaffle_shop: - target: prod + target: dev outputs: - pr: + dev: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" + + # User/password auth + user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" + + role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" + schema: PUBLIC + threads: 4 + jaffle_shop_pr: type: snowflake account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" @@ -14,7 +27,7 @@ jaffle_shop: warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" threads: 4 - prod: + jaffle_shop: type: snowflake account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" From ccd6e67599b1315f359e35676b28ce3609c48e8c Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 5 Apr 2023 22:02:24 +0800 Subject: [PATCH 13/22] Update workflows dbt profile target name Signed-off-by: Wei-Chun, Chang --- .github/workflows/pr-comparison-job.yml | 2 +- .github/workflows/production-daily-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-comparison-job.yml b/.github/workflows/pr-comparison-job.yml index 760087231..4a01248e7 100644 --- a/.github/workflows/pr-comparison-job.yml +++ b/.github/workflows/pr-comparison-job.yml @@ -35,7 +35,7 @@ jobs: pip install piperider['snowflake'] - name: Run DBT on PR environment - run: dbt build --target pr + run: dbt build --target jaffle_shop_pr - name: Profile on PR environment run: piperider run --datasource jaffle_shop_pr --dbt-state target -o /tmp/pr-report/ diff --git a/.github/workflows/production-daily-job.yml b/.github/workflows/production-daily-job.yml index a5e3cd7e5..1e37c1364 100644 --- a/.github/workflows/production-daily-job.yml +++ b/.github/workflows/production-daily-job.yml @@ -38,7 +38,7 @@ jobs: pip install piperider['snowflake'] - name: Run DBT on PROD environment - run: dbt build --target prod + run: dbt build --target jaffle_shop - name: Profile on PROD environment run: piperider run --datasource jaffle_shop --dbt-state target -o report/ From 456c7047aec398210c1d045855abcf8868e84028 Mon Sep 17 00:00:00 2001 From: popcorny Date: Mon, 17 Apr 2023 15:52:48 +0800 Subject: [PATCH 14/22] Use duckdb by default Signed-off-by: popcorny Co-authored-by: wcchang --- .gitignore | 2 + .piperider/compare/default.yml | 1 - .piperider/config.yml | 2 +- dbt_project.yml | 5 ++- models/int_customer_order_history_joined.sql | 2 +- models/int_order_payments_pivoted.sql | 4 +- models/staging/stg_orders.sql | 14 ++++++- profiles.yml | 41 +------------------ profiles.yml.snowflake | 43 ++++++++++++++++++++ requirements.txt | 2 + 10 files changed, 70 insertions(+), 46 deletions(-) create mode 100644 profiles.yml.snowflake create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index f5f3499cf..94bdc7656 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dbt_modules/ logs/ venv/ **/.DS_Store +*.duckdb + diff --git a/.piperider/compare/default.yml b/.piperider/compare/default.yml index dbb006e3a..c798e0419 100644 --- a/.piperider/compare/default.yml +++ b/.piperider/compare/default.yml @@ -7,7 +7,6 @@ base: piperider: command: piperider run target: - branch: feature/add-metrics dbt: commands: - dbt deps diff --git a/.piperider/config.yml b/.piperider/config.yml index 6e8f7e94b..ad6356366 100644 --- a/.piperider/config.yml +++ b/.piperider/config.yml @@ -1,7 +1,7 @@ dataSources: [] dbt: projectDir: . - # tag: 'piperider' + tag: 'piperider' profiler: # table: diff --git a/dbt_project.yml b/dbt_project.yml index acdce4c57..7e02c9c35 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -22,5 +22,8 @@ require-dbt-version: [">=1.0.0", "<2.0.0"] models: jaffle_shop: materialized: table - staging: + staging: materialized: view + +tags: piperider + marts: + +tags: piperider diff --git a/models/int_customer_order_history_joined.sql b/models/int_customer_order_history_joined.sql index 016a004fe..9aedd70a2 100644 --- a/models/int_customer_order_history_joined.sql +++ b/models/int_customer_order_history_joined.sql @@ -34,7 +34,7 @@ customer_payments as ( select orders.customer_id, - sum(amount) as total_amount + sum(amount)::bigint as total_amount from payments diff --git a/models/int_order_payments_pivoted.sql b/models/int_order_payments_pivoted.sql index cbb293491..681a7729e 100644 --- a/models/int_order_payments_pivoted.sql +++ b/models/int_order_payments_pivoted.sql @@ -18,10 +18,10 @@ order_payments as ( order_id, {% for payment_method in payment_methods -%} - sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, + sum(case when payment_method = '{{ payment_method }}' then amount else 0 end)::bigint as {{ payment_method }}_amount, {% endfor -%} - sum(amount) as total_amount + sum(amount)::bigint as total_amount from payments diff --git a/models/staging/stg_orders.sql b/models/staging/stg_orders.sql index a654dcb94..46cbb2d98 100644 --- a/models/staging/stg_orders.sql +++ b/models/staging/stg_orders.sql @@ -18,6 +18,18 @@ renamed as ( from source +), + +-- Shift the order_date by the number of days since 2018-04-09 (the max order_date in the raw data) +shift_date as ( + + select + order_id, + customer_id, + (order_date + datediff('day', date '2018-04-09', CURRENT_DATE)::int) as order_date, + status + + from renamed ) -select * from renamed +select * from shift_date diff --git a/profiles.yml b/profiles.yml index 851bcdbf9..5fa75b6a0 100644 --- a/profiles.yml +++ b/profiles.yml @@ -2,42 +2,5 @@ jaffle_shop: target: dev outputs: dev: - type: snowflake - account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" - - # User/password auth - user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - - role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" - database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" - warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" - schema: PUBLIC - threads: 4 - jaffle_shop_pr: - type: snowflake - account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" - - # User/password auth - user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - - role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" - database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" - warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" - schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" - threads: 4 - jaffle_shop: - type: snowflake - account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" - - # User/password auth - user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" - - role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" - database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" - warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" - schema: PUBLIC - threads: 4 - + type: duckdb + path: jaffle_shop.duckdb diff --git a/profiles.yml.snowflake b/profiles.yml.snowflake new file mode 100644 index 000000000..851bcdbf9 --- /dev/null +++ b/profiles.yml.snowflake @@ -0,0 +1,43 @@ +jaffle_shop: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" + + # User/password auth + user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" + + role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" + schema: PUBLIC + threads: 4 + jaffle_shop_pr: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" + + # User/password auth + user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" + + role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" + threads: 4 + jaffle_shop: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') | as_text }}" + + # User/password auth + user: "{{ env_var('SNOWFLAKE_USER') | as_text }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') | as_text }}" + + role: "{{ env_var('SNOWFLAKE_ROLE') | as_text }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') | as_text }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') | as_text }}" + schema: PUBLIC + threads: 4 + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..ad3da7e71 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +dbt-duckdb +piperider[duckdb] From fc5880351e9cb922a4c708158dc7211772209e84 Mon Sep 17 00:00:00 2001 From: popcorny Date: Mon, 17 Apr 2023 16:09:15 +0800 Subject: [PATCH 15/22] Add workflow for pr review Signed-off-by: popcorny Co-authored-by: wcchang --- .github/workflows/pr-compare.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/pr-compare.yml diff --git a/.github/workflows/pr-compare.yml b/.github/workflows/pr-compare.yml new file mode 100644 index 000000000..858fda7f3 --- /dev/null +++ b/.github/workflows/pr-compare.yml @@ -0,0 +1,21 @@ +name: PR Compare +on: + pull_request: + types: [opened, synchronize, reopened] + branches: ['main'] + paths: + - models/** + - seeds/** + - tests/** + +jobs: + piperider-compare: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v3 + + - name: PipeRider Compare + uses: InfuseAI/piperider-compare-action@v1 + From 48cc1d29cdec7ff8e7ef880a8b1fbb117c0ce71d Mon Sep 17 00:00:00 2001 From: popcorny Date: Thu, 20 Apr 2023 10:30:34 +0800 Subject: [PATCH 16/22] upload compare to cloud Signed-off-by: popcorny --- .github/workflows/pr-compare.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-compare.yml b/.github/workflows/pr-compare.yml index 858fda7f3..f72005358 100644 --- a/.github/workflows/pr-compare.yml +++ b/.github/workflows/pr-compare.yml @@ -18,4 +18,8 @@ jobs: - name: PipeRider Compare uses: InfuseAI/piperider-compare-action@v1 - + with: + cloud_api_token: ${{ secrets.PIPERIDER_CLOUD_TOKEN_ID }} + cloud_project: ${{ secrets.PIPERIDER_API_PROJECT }} + upload: true + share: true From 602c91356e2f94bf5cd7378346cba9007d216e95 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Thu, 3 Aug 2023 15:49:49 +0800 Subject: [PATCH 17/22] Add dbt version restriction Signed-off-by: Wei-Chun, Chang --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad3da7e71..72bc6886e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -dbt-duckdb +dbt-duckdb<1.6 piperider[duckdb] From 5e5d0f89eedd26014462dea955ea65249d1bd998 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 23 Aug 2023 12:30:22 +0800 Subject: [PATCH 18/22] Rewrite dbt metrics to semantic layer form Signed-off-by: Wei-Chun, Chang --- models/marts/average_order_amount.yml | 16 ------ models/marts/expenses.yml | 21 -------- models/marts/orders.yml | 73 +++++++++++++++++++++++++++ models/marts/profit.yml | 15 ------ models/marts/revenue.yml | 21 -------- models/metricflow_time_spine.sql | 24 +++++++++ packages.yml | 3 ++ 7 files changed, 100 insertions(+), 73 deletions(-) delete mode 100644 models/marts/average_order_amount.yml delete mode 100644 models/marts/expenses.yml delete mode 100644 models/marts/profit.yml delete mode 100644 models/marts/revenue.yml create mode 100644 models/metricflow_time_spine.sql create mode 100644 packages.yml diff --git a/models/marts/average_order_amount.yml b/models/marts/average_order_amount.yml deleted file mode 100644 index d7179d3c2..000000000 --- a/models/marts/average_order_amount.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 - -metrics: - - name: average_order_amount - label: Average Order Amount - model: ref('orders') - description: "The average size of a jaffle order" - - calculation_method: average - expression: amount - - timestamp: order_date - time_grains: [day, week, month] - - tags: ['piperider'] - diff --git a/models/marts/expenses.yml b/models/marts/expenses.yml deleted file mode 100644 index 584e309ce..000000000 --- a/models/marts/expenses.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 - -metrics: - - name: expenses - label: Expenses - model: ref('orders') - description: "The total expenses of our jaffle business" - - calculation_method: sum - expression: amount / 4 - - timestamp: order_date - time_grains: [day, week, month, year] - - filters: - - field: status - operator: '=' - value: "'completed'" - - tags: ['piperider'] - diff --git a/models/marts/orders.yml b/models/marts/orders.yml index 055a491c7..59a0f74ba 100644 --- a/models/marts/orders.yml +++ b/models/marts/orders.yml @@ -71,3 +71,76 @@ models: - name: total_order_amount description: Total value (AUD) of a customer's orders +semantic_models: + - name: orders + defaults: + agg_time_dimension: order_date + description: | + Order fact table. This table is at the order grain with one row per order. + model: ref('orders') + entities: + - name: order_id + type: primary + - name: customer + type: foreign + expr: customer_id + dimensions: + - name: order_date + expr: order_date + type: time + type_params: + time_granularity: day + - name: status + type: categorical + measures: + - name: expense + description: "The total expenses of our jaffle business" + agg: sum + expr: amount / 4 + - name: revenue + description: "The total revenue of our jaffle business" + agg: sum + expr: amount + - name: average_order_amount + description: "The average size of a jaffle order" + agg: average + expr: amount + + +metrics: + - name: expenses + description: "The total expenses of our jaffle business" + type: simple + label: Expenses + type_params: + measure: expense + filter: | + {{ Dimension('order_id__status') }} = 'completed' + - name: revenue + description: "The total revenue of our jaffle business" + type: simple + label: Revenue + type_params: + measure: revenue + filter: | + {{ Dimension('order_id__status') }} = 'completed' + - name: average_order_amount + description: "The average size of a jaffle order" + type: simple + label: Average Order Amount + type_params: + measure: average_order_amount + - name: profit + description: "The total money we get to take home from our jaffle business" + type: derived + label: Profit + type_params: + expr: revenue - expenses + metrics: + - name: revenue + filter: | + {{ Dimension('order_id__status') }} = 'completed' + - name: expenses + filter: | + {{ Dimension('order_id__status') }} = 'completed' + diff --git a/models/marts/profit.yml b/models/marts/profit.yml deleted file mode 100644 index b6103e3f6..000000000 --- a/models/marts/profit.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: 2 - -metrics: - - name: profit - label: Profit - description: "The total money we get to take home from our jaffle business" - - calculation_method: derived - expression: "{{metric('revenue')}} - {{metric('expenses')}}" - - timestamp: order_date - time_grains: [hour, day, week, month, year] - - tags: ['piperider'] - diff --git a/models/marts/revenue.yml b/models/marts/revenue.yml deleted file mode 100644 index be4be6c9b..000000000 --- a/models/marts/revenue.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 - -metrics: - - name: revenue - label: Revenue - model: ref('orders') - description: "The total revenue of our jaffle business" - - calculation_method: sum - expression: amount - - timestamp: order_date - time_grains: [day, week, month, year] - - filters: - - field: status - operator: '=' - value: "'completed'" - - tags: ['piperider'] - diff --git a/models/metricflow_time_spine.sql b/models/metricflow_time_spine.sql new file mode 100644 index 000000000..af185aa10 --- /dev/null +++ b/models/metricflow_time_spine.sql @@ -0,0 +1,24 @@ +{{ + config( + materialized = 'table', + ) +}} + +with days as ( + + {{ + dbt_utils.date_spine( + 'day', + "strptime('01/01/2000','%m/%d/%Y')", + "strptime('01/01/2027','%m/%d/%Y')" + ) + }} + +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * from final diff --git a/packages.yml b/packages.yml new file mode 100644 index 000000000..60f8dfb7a --- /dev/null +++ b/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.1.1 From e74e9f91e45e65fe9799f5c61e57ae7a0ba94e98 Mon Sep 17 00:00:00 2001 From: popcorny Date: Tue, 5 Sep 2023 14:36:05 +0800 Subject: [PATCH 19/22] Remove piperider tag Signed-off-by: popcorny --- .piperider/config.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.piperider/config.yml b/.piperider/config.yml index ad6356366..5878c9faf 100644 --- a/.piperider/config.yml +++ b/.piperider/config.yml @@ -1,13 +1,2 @@ -dataSources: [] -dbt: - projectDir: . - tag: 'piperider' - -profiler: -# table: -# # the maximum row count to profile. (Default unlimited) -# limit: 1000000 -# duplicateRows: false - telemetry: id: 2e470b721f5a4114aa7cf9b73eca760d From efdff1e0a7c006d25798953a7ad5dbe1a17e0cc2 Mon Sep 17 00:00:00 2001 From: popcorny Date: Tue, 5 Sep 2023 14:37:21 +0800 Subject: [PATCH 20/22] Bump dbt to 1.6+ Signed-off-by: popcorny --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72bc6886e..c9fa48cc7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -dbt-duckdb<1.6 +dbt-duckdb>=1.6 piperider[duckdb] From 40d0988135f7ef6990130435b297cfeab312316d Mon Sep 17 00:00:00 2001 From: popcorny Date: Tue, 5 Sep 2023 16:10:27 +0800 Subject: [PATCH 21/22] Fix piperider conifg Signed-off-by: popcorny --- .piperider/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.piperider/config.yml b/.piperider/config.yml index 5878c9faf..f20bfaab9 100644 --- a/.piperider/config.yml +++ b/.piperider/config.yml @@ -1,2 +1,6 @@ +dataSources: [] +dbt: + projectDir: . + telemetry: id: 2e470b721f5a4114aa7cf9b73eca760d From 7ceefc5d5b6cb22327d9c2081bc4469198e766c0 Mon Sep 17 00:00:00 2001 From: popcorny Date: Thu, 15 Jun 2023 16:32:29 +0800 Subject: [PATCH 22/22] Source migration Signed-off-by: popcorny --- dbt_project.yml | 4 +- models/staging/stg_customers.sql | 4 +- seeds/raw_customers_v2.csv | 102 +++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 seeds/raw_customers_v2.csv diff --git a/dbt_project.yml b/dbt_project.yml index 7e02c9c35..8d69027de 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -22,8 +22,6 @@ require-dbt-version: [">=1.0.0", "<2.0.0"] models: jaffle_shop: materialized: table + +tags: piperider staging: materialized: view - +tags: piperider - marts: - +tags: piperider diff --git a/models/staging/stg_customers.sql b/models/staging/stg_customers.sql index cad047269..5b8b77a80 100644 --- a/models/staging/stg_customers.sql +++ b/models/staging/stg_customers.sql @@ -4,14 +4,14 @@ with source as ( Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_customers') }} + select * from {{ ref('raw_customers_v2') }} ), renamed as ( select - id as customer_id, + customer_id, first_name, last_name diff --git a/seeds/raw_customers_v2.csv b/seeds/raw_customers_v2.csv new file mode 100644 index 000000000..7f94e6f66 --- /dev/null +++ b/seeds/raw_customers_v2.csv @@ -0,0 +1,102 @@ +customer_id,first_name,last_name +1,Michael,P. +2,Shawn,M. +3,Kathleen,P. +4,Jimmy,C. +5,Katherine,R. +6,Sarah,R. +7,Martin,M. +8,Frank,R. +9,Jennifer,F. +10,Henry,W. +11,Fred,S. +12,Amy,D. +13,Kathleen,M. +14,Steve,F. +15,Teresa,H. +16,Amanda,H. +17,Kimberly,R. +18,Johnny,K. +19,Virginia,F. +20,Anna,A. +21,Willie,H. +22,Sean,H. +23,Mildred,A. +24,David,G. +25,Victor,H. +26,Aaron,R. +27,Benjamin,B. +28,Lisa,W. +29,Benjamin,K. +30,Christina,W. +31,Jane,G. +32,Thomas,O. +33,Katherine,M. +34,Jennifer,S. +35,Sara,T. +36,Harold,O. +37,Shirley,J. +38,Dennis,J. +39,Louise,W. +40,Maria,A. +41,Gloria,C. +42,Diana,S. +43,Kelly,N. +44,Jane,R. +45,Scott,B. +46,Norma,C. +47,Marie,P. +48,Lillian,C. +49,Judy,N. +50,Billy,L. +51,Howard,R. +52,Laura,F. +53,Anne,B. +54,Rose,M. +55,Nicholas,R. +56,Joshua,K. +57,Paul,W. +58,Kathryn,K. +59,Adam,A. +60,Norma,W. +61,Timothy,R. +62,Elizabeth,P. +63,Edward,G. +64,David,C. +65,Brenda,W. +66,Adam,W. +67,Michael,H. +68,Jesse,E. +69,Janet,P. +70,Helen,F. +71,Gerald,C. +72,Kathryn,O. +73,Alan,B. +74,Harry,A. +75,Andrea,H. +76,Barbara,W. +77,Anne,W. +78,Harry,H. +79,Jack,R. +80,Phillip,H. +81,Shirley,H. +82,Arthur,D. +83,Virginia,R. +84,Christina,R. +85,Theresa,M. +86,Jason,C. +87,Phillip,B. +88,Adam,T. +89,Margaret,J. +90,Paul,P. +91,Todd,W. +92,Willie,O. +93,Frances,R. +94,Gregory,H. +95,Lisa,P. +96,Jacqueline,A. +97,Shirley,D. +98,Nicole,M. +99,Mary,G. +100,Jean,M. +101,CL,K.