diff --git a/dbt_project.yml b/dbt_project.yml index acdce4c57..1b65784ff 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -21,6 +21,12 @@ require-dbt-version: [">=1.0.0", "<2.0.0"] models: jaffle_shop: - materialized: table - staging: + + materialized: table + staging: + materialized: view + airbnb: + raw: materialized: view + schema: airbnb + diff --git a/macros/generate_schema_name.sql b/macros/generate_schema_name.sql new file mode 100644 index 000000000..6b926c50a --- /dev/null +++ b/macros/generate_schema_name.sql @@ -0,0 +1,14 @@ +{% macro generate_schema_name(custom_schema_name, node) -%} + + {%- set default_schema = target.schema -%} + {%- if custom_schema_name is none -%} + + {{ default_schema }} + + {%- else -%} + + {{ custom_schema_name | trim }} + + {%- endif -%} + +{%- endmacro %} \ No newline at end of file diff --git a/macros/not_null_columns.sql b/macros/not_null_columns.sql new file mode 100644 index 000000000..95e9d36fc --- /dev/null +++ b/macros/not_null_columns.sql @@ -0,0 +1,7 @@ +{% macro no_nulls_in_columns(model) %} + SELECT * FROM {{ model }} WHERE + {% for col in adapter.get_columns_in_relation(model) -%} + {{ col.column }} IS NULL OR + {% endfor %} + FALSE +{% endmacro %} \ No newline at end of file diff --git a/models/airbnb/marts/dim_listings_with_hosts.sql b/models/airbnb/marts/dim_listings_with_hosts.sql new file mode 100644 index 000000000..d26f9d49c --- /dev/null +++ b/models/airbnb/marts/dim_listings_with_hosts.sql @@ -0,0 +1,20 @@ +use warehouse DEFAULT_USER_WH; +with dim_listings_cleansed as ( + select * from {{ ref('stg_listings')}} +), +dim_hosts_cleansed as ( + select * from {{ ref('stg_hosts')}} +) +SELECT l.listing_id, +l.listing_name, +l.room_type, +l.minimum_nights, +l.price, +h.host_id, +h.created_at, +h.host_name, +h.is_superhost +from dim_listings_cleansed l + LEFT JOIN + dim_hosts_cleansed h + on l.host_id =h.host_id diff --git a/models/airbnb/marts/fct_reviews.sql b/models/airbnb/marts/fct_reviews.sql new file mode 100644 index 000000000..92cdc6274 --- /dev/null +++ b/models/airbnb/marts/fct_reviews.sql @@ -0,0 +1,16 @@ +{{ + config( + materialized = 'incremental', + on_schema_change='fail' + ) +}} +WITH stg_reviews AS ( + SELECT * FROM {{ ref('stg_reviews') }} +) +SELECT + * +FROM stg_reviews +WHERE review_text is not null +{% if is_incremental() %} + AND review_date > (select max(review_date) from {{ this }}) +{% endif %} \ No newline at end of file diff --git a/models/airbnb/marts/jinja_practice.sql b/models/airbnb/marts/jinja_practice.sql new file mode 100644 index 000000000..725af6c43 --- /dev/null +++ b/models/airbnb/marts/jinja_practice.sql @@ -0,0 +1,2 @@ +{% set my_cool_string = 'wow! cool!' %} +{{my_cool_string}} \ No newline at end of file diff --git a/models/airbnb/marts/reporting_mdl.sql b/models/airbnb/marts/reporting_mdl.sql new file mode 100644 index 000000000..ed21a157b --- /dev/null +++ b/models/airbnb/marts/reporting_mdl.sql @@ -0,0 +1,20 @@ + +with dim_listings_with_hosts as ( + select * from {{ ref('dim_listings_with_hosts')}} +), +fct_reviews as ( + select * from {{ ref('fct_reviews')}} +) +SELECT l.listing_id, +l.listing_name, +l.room_type, +l.minimum_nights, +l.price, +h.host_id, +h.created_at, +h.host_name, +h.is_superhost +from dim_listings_with_hosts l + LEFT JOIN + fct_reviews h + on l.host_id =h.host_id diff --git a/models/airbnb/raw/src_airbnb.yml b/models/airbnb/raw/src_airbnb.yml new file mode 100644 index 000000000..cf13723b4 --- /dev/null +++ b/models/airbnb/raw/src_airbnb.yml @@ -0,0 +1,10 @@ +version: 2 + +sources: + - name: airbnb + database: user_pchauhan + schema: airbnb + tables: + - name: raw_hosts + - name: raw_listings + - name: raw_reviews \ No newline at end of file diff --git a/models/airbnb/raw/src_hosts.sql b/models/airbnb/raw/src_hosts.sql new file mode 100644 index 000000000..8b1bb6fa2 --- /dev/null +++ b/models/airbnb/raw/src_hosts.sql @@ -0,0 +1,24 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ source('airbnb','raw_hosts') }} + +), + +renamed as ( + + select + id as host_id, + name as host_name, + is_superhost, + created_at, + updated_at + + from source + +) + +select * from renamed diff --git a/models/airbnb/raw/src_listings.sql b/models/airbnb/raw/src_listings.sql new file mode 100644 index 000000000..6bf0cc8a3 --- /dev/null +++ b/models/airbnb/raw/src_listings.sql @@ -0,0 +1,26 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ source('airbnb','raw_listings') }} + +), + +renamed as ( + + select + id as listing_id, + listing_url, + name as listing_name, + room_type, + minimum_nights, + host_id, + price as price_str, + CREATED_AT, + updated_at + from source +) + +select * from renamed diff --git a/models/airbnb/raw/src_reviews.sql b/models/airbnb/raw/src_reviews.sql new file mode 100644 index 000000000..05cd39293 --- /dev/null +++ b/models/airbnb/raw/src_reviews.sql @@ -0,0 +1,23 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ source('airbnb','raw_reviews') }} + +), + +renamed as ( + + select + LISTING_ID, + DATE as review_date, + REVIEWER_NAME, + COMMENTS as review_text, + SENTIMENT as review_sentiment + + from source +) + +select * from renamed diff --git a/models/staging/stg_customers.sql b/models/airbnb/staging/stg_customers.sql similarity index 77% rename from models/staging/stg_customers.sql rename to models/airbnb/staging/stg_customers.sql index cad047269..a829e9f8f 100644 --- a/models/staging/stg_customers.sql +++ b/models/airbnb/staging/stg_customers.sql @@ -4,14 +4,14 @@ with source as ( Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_customers') }} + select * from {{ ref('customers1') }} ), renamed as ( select - id as customer_id, + customer_id, first_name, last_name diff --git a/models/airbnb/staging/stg_hosts.sql b/models/airbnb/staging/stg_hosts.sql new file mode 100644 index 000000000..33851a492 --- /dev/null +++ b/models/airbnb/staging/stg_hosts.sql @@ -0,0 +1,10 @@ + + +with cte as ( + select * from {{ ref('src_hosts')}} +) +select +host_id, NVL(host_name, 'Anonymous') as host_name , +IS_SUPERHOST,CREATED_AT,UPDATED_AT, +current_timestamp as staged_at +from cte diff --git a/models/airbnb/staging/stg_listings.sql b/models/airbnb/staging/stg_listings.sql new file mode 100644 index 000000000..18de97ac0 --- /dev/null +++ b/models/airbnb/staging/stg_listings.sql @@ -0,0 +1,11 @@ + +with cte as ( + select * from {{ ref('src_listings')}} +) +select +listing_id, listing_name, room_type, host_id, +case when minimum_nights=0 then 1 +when minimum_nights >1 then minimum_nights end as minimum_nights, +cast(replace(price_str,'$','') as decimal) as price , +CREATED_AT,UPDATED_AT +,current_timestamp as staged_at from cte diff --git a/models/staging/stg_orders.sql b/models/airbnb/staging/stg_orders.sql similarity index 71% rename from models/staging/stg_orders.sql rename to models/airbnb/staging/stg_orders.sql index a654dcb94..f65412da8 100644 --- a/models/staging/stg_orders.sql +++ b/models/airbnb/staging/stg_orders.sql @@ -4,15 +4,15 @@ with source as ( Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_orders') }} + select * from user_pchauhan.jaffle_shop.orders ), renamed as ( select - id as order_id, - user_id as customer_id, + order_id, + customer_id, order_date, status diff --git a/models/staging/stg_payments.sql b/models/airbnb/staging/stg_payments.sql similarity index 74% rename from models/staging/stg_payments.sql rename to models/airbnb/staging/stg_payments.sql index 700cf7f4f..6b9e92f3d 100644 --- a/models/staging/stg_payments.sql +++ b/models/airbnb/staging/stg_payments.sql @@ -4,7 +4,7 @@ with source as ( Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_payments') }} + select * from {{ source('stripe','payment') }} ), @@ -12,8 +12,8 @@ renamed as ( select id as payment_id, - order_id, - payment_method, + orderid as order_id, + paymentmethod as payment_method , -- `amount` is currently stored in cents, so we convert it to dollars amount / 100 as amount diff --git a/models/airbnb/staging/stg_reviews.sql b/models/airbnb/staging/stg_reviews.sql new file mode 100644 index 000000000..c250dda93 --- /dev/null +++ b/models/airbnb/staging/stg_reviews.sql @@ -0,0 +1,9 @@ + + + +with cte as ( + select * from {{ ref('src_reviews')}} +) +SELECT listing_id, review_date, reviewer_name, review_text, +review_sentiment , + current_timestamp as staged_at from cte diff --git a/models/customers.sql b/models/customers.sql index 016a004fe..ca94ab6ac 100644 --- a/models/customers.sql +++ b/models/customers.sql @@ -1,47 +1,38 @@ with customers as ( - select * from {{ ref('stg_customers') }} + select + customer_id, + first_name, + last_name + + from {{ ref('stg_customers') }} ), orders as ( - select * from {{ ref('stg_orders') }} - -), - -payments as ( + select + order_id, + customer_id, + order_date, + status - select * from {{ ref('stg_payments') }} + from user_pchauhan.jaffle_shop.orders ), customer_orders as ( - select + select customer_id, - min(order_date) as first_order, - max(order_date) as most_recent_order, + min(order_date) as first_order_date, + max(order_date) as most_recent_order_date, count(order_id) as number_of_orders - from orders - group by customer_id - -), - -customer_payments as ( - - select - orders.customer_id, - sum(amount) as total_amount - - from payments - - left join orders on - payments.order_id = orders.order_id + from orders - group by orders.customer_id + group by 1 ), @@ -51,19 +42,14 @@ final as ( customers.customer_id, customers.first_name, customers.last_name, - customer_orders.first_order, - customer_orders.most_recent_order, - customer_orders.number_of_orders, - customer_payments.total_amount as customer_lifetime_value + customer_orders.first_order_date, + customer_orders.most_recent_order_date, + coalesce(customer_orders.number_of_orders, 0) as number_of_orders from customers - left join customer_orders - on customers.customer_id = customer_orders.customer_id - - left join customer_payments - on customers.customer_id = customer_payments.customer_id + left join customer_orders using (customer_id) ) -select * from final +select * from final \ No newline at end of file diff --git a/models/schema.yml b/models/schema.yml index 381349cfd..ebaec87c3 100644 --- a/models/schema.yml +++ b/models/schema.yml @@ -1,82 +1,56 @@ -version: 2 - -models: - - name: customers - description: This table has basic information about a customer, as well as some derived facts based on a customer's orders - - columns: - - name: customer_id - description: This is a unique identifier for a customer - tests: - - unique - - not_null - - - name: first_name - description: Customer's first name. PII. - - - name: last_name - description: Customer's last name. PII. - - - name: first_order - description: Date (UTC) of a customer's first order - - - name: most_recent_order - description: Date (UTC) of a customer's most recent order - - - name: number_of_orders - description: Count of the number of orders a customer has placed - - - name: total_order_amount - description: Total value (AUD) of a customer's orders - - - name: orders - description: This table has basic information about orders, as well as some derived facts based on payments - - columns: - - name: order_id - tests: - - unique - - not_null - description: This is a unique identifier for an order - - - name: customer_id - description: Foreign key to the customers table - tests: - - not_null - - relationships: - to: ref('customers') - field: customer_id - - - name: order_date - description: Date (UTC) that the order was placed - - - name: status - description: '{{ doc("orders_status") }}' - tests: - - accepted_values: - values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - - name: amount - description: Total amount (AUD) of the order - tests: - - not_null - - - name: credit_card_amount - description: Amount of the order (AUD) paid for by credit card - tests: - - not_null - - - name: coupon_amount - description: Amount of the order (AUD) paid for by coupon - tests: - - not_null - - - name: bank_transfer_amount - description: Amount of the order (AUD) paid for by bank transfer - tests: - - not_null - - - name: gift_card_amount - description: Amount of the order (AUD) paid for by gift card - tests: - - not_null +version: 2 + +models: + - name: stg_listings + description: contains clean listing records + columns: + - name: listing_id + description: Primary key + tests: + - unique + - not_null + - name: host_id + description: Refrences stg_hosts view + tests: + - not_null + - relationships: + to: ref('stg_hosts') + field: host_id + - name: room_type + description: accepted room type + tests: + - accepted_values: + values: + - ['Entire home/apt','Private room','Shared room','Hotel room'] + - name: customers + description: One record per customer + columns: + - name: customer_id + description: Primary key + tests: + - unique + - not_null + - name: first_order_date + description: NULL when a customer has not yet placed an order. + + - name: stg_customers + description: This model cleans up customer data + columns: + - name: customer_id + description: Primary key + tests: + - unique + - not_null + + - name: stg_orders + description: This model cleans up order data + columns: + - name: order_id + description: Primary key + tests: + - unique + - not_null + - name: status + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 000000000..82a4382c6 --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,19 @@ +version: 2 + +sources: + - name: jaffle_shop + database: user_pchauhan + schema: jaffle_shop + tables: + - name: orders + - name: customers + + - name: stripe + database: user_pchauhan + #schema: stripe + tables: + - name: payment + + + + diff --git a/models/staging/schema.yml b/models/staging/schema.yml deleted file mode 100644 index c207e4cf5..000000000 --- a/models/staging/schema.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 2 - -models: - - name: stg_customers - columns: - - name: customer_id - tests: - - unique - - not_null - - - name: stg_orders - columns: - - name: order_id - tests: - - unique - - not_null - - name: status - tests: - - accepted_values: - values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - - name: stg_payments - columns: - - name: payment_id - tests: - - unique - - not_null - - name: payment_method - tests: - - accepted_values: - values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/seeds/raw_customers.csv b/seeds/customers1.csv similarity index 100% rename from seeds/raw_customers.csv rename to seeds/customers1.csv diff --git a/seeds/raw_payments.csv b/seeds/payments.csv similarity index 100% rename from seeds/raw_payments.csv rename to seeds/payments.csv diff --git a/snapshots/scd_raw_hosts.sql b/snapshots/scd_raw_hosts.sql new file mode 100644 index 000000000..9680e1892 --- /dev/null +++ b/snapshots/scd_raw_hosts.sql @@ -0,0 +1,14 @@ +{% snapshot scd_raw_hosts %} + +{{ + config( + target_schema='airbnb', + unique_key='id', + strategy='timestamp', + updated_at='updated_at', + invalidate_hard_deletes=True + ) +}} + +select * FROM {{ source('airbnb', 'raw_hosts') }} +{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/scd_raw_listings.sql b/snapshots/scd_raw_listings.sql new file mode 100644 index 000000000..094fbbc12 --- /dev/null +++ b/snapshots/scd_raw_listings.sql @@ -0,0 +1,15 @@ +{% snapshot scd_raw_listings %} + +{{ + config( + target_schema='airbnb', + unique_key='id', + strategy='timestamp', + updated_at='updated_at', + invalidate_hard_deletes=True + ) +}} + +select * FROM {{ source('airbnb', 'raw_listings') }} + +{% endsnapshot %} \ No newline at end of file diff --git a/test/consistent_created_at.sql b/test/consistent_created_at.sql new file mode 100644 index 000000000..3ffa45c4d --- /dev/null +++ b/test/consistent_created_at.sql @@ -0,0 +1,5 @@ +select * from {{ ref('stg_listings')}} t1 +inner join {{ ref('stg_reviews')}} t2 +on t1.listing_id = t2.listing_id +where t2.review_date < =t1.created_at + diff --git a/test/non_nulls_listings.sql b/test/non_nulls_listings.sql new file mode 100644 index 000000000..b09b5c462 --- /dev/null +++ b/test/non_nulls_listings.sql @@ -0,0 +1 @@ +{{ no_nulls_in_columns(ref('stg_listings')) }} diff --git a/test/stg_listing_minimum_nights.sql b/test/stg_listing_minimum_nights.sql new file mode 100644 index 000000000..51c19c81b --- /dev/null +++ b/test/stg_listing_minimum_nights.sql @@ -0,0 +1,2 @@ +select * from {{ ref('stg_listings')}} +where minimum_nights<1 \ No newline at end of file diff --git a/test1 b/test1 new file mode 100644 index 000000000..af45d5cef --- /dev/null +++ b/test1 @@ -0,0 +1,5 @@ +SELECT * FROM USER_PCHAUHAN.jaffle_shop.fct_reviews WHERE listing_id = 3176; + +INSERT INTO USER_PCHAUHAN.airbnb.RAW_REVIEWS VALUES (3176, CURRENT_TIMESTAMP(), 'Zoltan', 'excellent stay!', 'positive'); + +SELECT * FROM USER_PCHAUHAN.jaffle_shop.fct_reviews WHERE listing_id = 3176;