Skip to content

[MBA-2202] Snowflake DDL migration for source, staging and data product tables #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hrabbani wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[MBA-2202] Snowflake DDL migration for source, staging and data product tables #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions ddl/snowflake/00_databases_and_schemas.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
-- =============================================================================
-- Snowflake Database and Schema Layout - Retail Banking Customer Analytics
-- =============================================================================
-- Migrated from Teradata (see ddl/00_source_tables.sql, ddl/01_staging_tables.sql,
-- ddl/02_data_product_tables.sql for the original definitions).
--
-- Teradata "databases" are containers of tables and are mapped to Snowflake
-- SCHEMAS inside a single per-environment database:
--
-- CORE_BANKING_DB -> RETAIL_BANKING_<ENV>.CORE_BANKING
-- TXN_PROCESSING_DB -> RETAIL_BANKING_<ENV>.TXN_PROCESSING
-- ETL_STAGING_DB -> RETAIL_BANKING_<ENV>.ETL_STAGING
-- DATA_PRODUCTS_DB -> RETAIL_BANKING_<ENV>.DATA_PRODUCTS
--
-- <ENV> is one of DEV, UAT, PROD and is supplied at run time through the
-- SnowSQL variable `env`:
--
-- snowsql -f ddl/snowflake/00_databases_and_schemas.sql -D env=DEV
--
-- All statements are idempotent and safe to re-run in any environment.
-- Warehouses, roles and grants are provisioned separately (TICKET-02); this
-- script only creates the database/schema containers the DDL depends on.
-- =============================================================================

!set variable_substitution=true

CREATE DATABASE IF NOT EXISTS RETAIL_BANKING_&{env}
COMMENT = 'Retail banking customer analytics platform (&{env})';

CREATE SCHEMA IF NOT EXISTS RETAIL_BANKING_&{env}.CORE_BANKING
COMMENT = 'Operational customer/account data (was CORE_BANKING_DB in Teradata)';

CREATE SCHEMA IF NOT EXISTS RETAIL_BANKING_&{env}.TXN_PROCESSING
COMMENT = 'Transaction processing tables (was TXN_PROCESSING_DB in Teradata)';

CREATE SCHEMA IF NOT EXISTS RETAIL_BANKING_&{env}.ETL_STAGING
COMMENT = 'ETL staging/intermediate tables (was ETL_STAGING_DB in Teradata)';

CREATE SCHEMA IF NOT EXISTS RETAIL_BANKING_&{env}.DATA_PRODUCTS
COMMENT = 'Certified analytical data products (was DATA_PRODUCTS_DB in Teradata)';
126 changes: 126 additions & 0 deletions ddl/snowflake/01_source_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
-- =============================================================================
-- Snowflake Source Table DDL - Retail Banking Operational Systems
-- =============================================================================
-- Migrated from ddl/00_source_tables.sql (Teradata).
-- These tables represent the upstream operational sources that feed the
-- analytics pipeline. They are owned by the core banking platform team.
-- This DDL is for DOCUMENTATION / lower-environment bootstrap only.
--
-- Usage: snowsql -f ddl/snowflake/01_source_tables.sql -D env=DEV
-- Idempotent: CREATE TABLE IF NOT EXISTS never drops existing operational data.
-- =============================================================================

!set variable_substitution=true

USE DATABASE RETAIL_BANKING_&{env};

-- -----------------------------------------------------------------------------
-- CORE_BANKING.CUSTOMERS
-- Master customer record from the core banking platform.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS CORE_BANKING.CUSTOMERS
(
CUSTOMER_ID NUMBER(19,0) NOT NULL,
FIRST_NAME VARCHAR(60),
LAST_NAME VARCHAR(60),
DATE_OF_BIRTH DATE,
SSN_HASH VARCHAR(64),
EMAIL VARCHAR(120),
PHONE_PRIMARY VARCHAR(20),
CUSTOMER_SINCE DATE,
CUSTOMER_STATUS VARCHAR(1), -- A=Active, I=Inactive, C=Closed
SEGMENT_CODE VARCHAR(10),
BRANCH_ID NUMBER(10,0),
CREATED_TS TIMESTAMP_NTZ(6),
UPDATED_TS TIMESTAMP_NTZ(6)
)
COMMENT = 'Master customer record (migrated from CORE_BANKING_DB.CUSTOMERS)';

-- -----------------------------------------------------------------------------
-- CORE_BANKING.ACCOUNTS
-- Account-level detail (checking, savings, credit, loan).
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS CORE_BANKING.ACCOUNTS
(
ACCOUNT_ID NUMBER(19,0) NOT NULL,
CUSTOMER_ID NUMBER(19,0) NOT NULL,
ACCOUNT_TYPE VARCHAR(20), -- CHECKING, SAVINGS, CREDIT, LOAN
ACCOUNT_STATUS VARCHAR(1), -- O=Open, C=Closed, F=Frozen
OPEN_DATE DATE,
CLOSE_DATE DATE,
CURRENT_BALANCE NUMBER(15,2),
AVAILABLE_BALANCE NUMBER(15,2),
CREDIT_LIMIT NUMBER(15,2),
INTEREST_RATE NUMBER(5,4),
BRANCH_ID NUMBER(10,0),
CREATED_TS TIMESTAMP_NTZ(6),
UPDATED_TS TIMESTAMP_NTZ(6)
)
COMMENT = 'Account-level detail (migrated from CORE_BANKING_DB.ACCOUNTS)';

-- -----------------------------------------------------------------------------
-- CORE_BANKING.ADDRESSES
-- Customer mailing and residential addresses.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS CORE_BANKING.ADDRESSES
(
ADDRESS_ID NUMBER(19,0) NOT NULL,
CUSTOMER_ID NUMBER(19,0) NOT NULL,
ADDRESS_TYPE VARCHAR(10), -- MAIL, HOME, WORK
ADDRESS_LINE_1 VARCHAR(100),
ADDRESS_LINE_2 VARCHAR(100),
CITY VARCHAR(60),
STATE_CODE VARCHAR(2),
ZIP_CODE VARCHAR(10),
COUNTRY_CODE VARCHAR(2) DEFAULT 'US',
IS_PRIMARY VARCHAR(1) DEFAULT 'N',
EFFECTIVE_DATE DATE,
EXPIRATION_DATE DATE,
CREATED_TS TIMESTAMP_NTZ(6),
UPDATED_TS TIMESTAMP_NTZ(6)
)
COMMENT = 'Customer addresses (migrated from CORE_BANKING_DB.ADDRESSES)';

-- -----------------------------------------------------------------------------
-- TXN_PROCESSING.TRANSACTIONS
-- Individual financial transactions across all account types.
-- Teradata PARTITION BY RANGE_N(TRANSACTION_DATE ... EACH INTERVAL '1' MONTH)
-- is replaced by a Snowflake clustering key on TRANSACTION_DATE: every
-- downstream BTEQ/SAS read filters on a transaction date window.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS TXN_PROCESSING.TRANSACTIONS
(
TRANSACTION_ID NUMBER(19,0) NOT NULL,
ACCOUNT_ID NUMBER(19,0) NOT NULL,
TRANSACTION_TYPE_CD VARCHAR(10),
TRANSACTION_DATE DATE,
TRANSACTION_TS TIMESTAMP_NTZ(6),
AMOUNT NUMBER(15,2),
RUNNING_BALANCE NUMBER(15,2),
MERCHANT_NAME VARCHAR(100),
MERCHANT_CATEGORY VARCHAR(60),
CHANNEL_CODE VARCHAR(10), -- ATM, POS, WEB, MOB, ACH, WIRE
REFERENCE_NUM VARCHAR(40),
STATUS_CODE VARCHAR(1), -- P=Posted, R=Reversed, H=Hold
CREATED_TS TIMESTAMP_NTZ(6)
)
CLUSTER BY (TRANSACTION_DATE)
COMMENT = 'Financial transactions (migrated from TXN_PROCESSING_DB.TRANSACTIONS)';

-- -----------------------------------------------------------------------------
-- TXN_PROCESSING.TRANSACTION_TYPES
-- Reference/lookup table for transaction type codes.
-- The Teradata UNIQUE PRIMARY INDEX is expressed as an (unenforced) Snowflake
-- primary key so the uniqueness contract stays documented in the catalogue.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS TXN_PROCESSING.TRANSACTION_TYPES
(
TRANSACTION_TYPE_CD VARCHAR(10) NOT NULL,
DESCRIPTION VARCHAR(60),
CATEGORY VARCHAR(30), -- DEBIT, CREDIT, FEE, INTEREST
IS_REVENUE VARCHAR(1) DEFAULT 'N',
EFFECTIVE_DATE DATE,
EXPIRATION_DATE DATE,
CONSTRAINT PK_TRANSACTION_TYPES PRIMARY KEY (TRANSACTION_TYPE_CD)
)
COMMENT = 'Transaction type reference data (migrated from TXN_PROCESSING_DB.TRANSACTION_TYPES)';
111 changes: 111 additions & 0 deletions ddl/snowflake/02_staging_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
-- =============================================================================
-- Snowflake Staging Table DDL - ETL Intermediate Tables
-- =============================================================================
-- Migrated from ddl/01_staging_tables.sql (Teradata).
-- These tables are populated by the migrated ETL layer and consumed by the
-- migrated analytics layer. They live in the ETL_STAGING schema and are
-- refreshed each run.
--
-- Usage: snowsql -f ddl/snowflake/02_staging_tables.sql -D env=DEV
--
-- Teradata COLLECT STATISTICS statements are dropped: Snowflake maintains
-- micro-partition metadata automatically, so there is no manual equivalent.
-- =============================================================================

!set variable_substitution=true

USE DATABASE RETAIL_BANKING_&{env};

-- -----------------------------------------------------------------------------
-- ETL_STAGING.STG_CUSTOMER_360
-- Denormalized customer view joining customer, account, and address data.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ETL_STAGING.STG_CUSTOMER_360
(
CUSTOMER_ID NUMBER(19,0) NOT NULL,
FIRST_NAME VARCHAR(60),
LAST_NAME VARCHAR(60),
DATE_OF_BIRTH DATE,
AGE NUMBER(5,0),
CUSTOMER_SINCE DATE,
TENURE_MONTHS NUMBER(10,0),
CUSTOMER_STATUS VARCHAR(1),
SEGMENT_CODE VARCHAR(10),
BRANCH_ID NUMBER(10,0),
PRIMARY_ADDRESS VARCHAR(200),
CITY VARCHAR(60),
STATE_CODE VARCHAR(2),
ZIP_CODE VARCHAR(10),
NUM_ACCOUNTS NUMBER(5,0),
NUM_ACTIVE_ACCOUNTS NUMBER(5,0),
HAS_CHECKING VARCHAR(1) DEFAULT 'N',
HAS_SAVINGS VARCHAR(1) DEFAULT 'N',
HAS_CREDIT VARCHAR(1) DEFAULT 'N',
HAS_LOAN VARCHAR(1) DEFAULT 'N',
TOTAL_BALANCE NUMBER(18,2),
TOTAL_CREDIT_LIMIT NUMBER(18,2),
CREDIT_UTILIZATION_PCT NUMBER(5,2),
LOAD_TS TIMESTAMP_NTZ(6)
)
COMMENT = 'Denormalized customer 360 staging table (migrated from ETL_STAGING_DB.STG_CUSTOMER_360)';

-- -----------------------------------------------------------------------------
-- ETL_STAGING.STG_TXN_SUMMARY
-- Aggregated transaction metrics per customer over configurable lookback.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ETL_STAGING.STG_TXN_SUMMARY
(
CUSTOMER_ID NUMBER(19,0) NOT NULL,
ACCOUNT_ID NUMBER(19,0) NOT NULL,
ACCOUNT_TYPE VARCHAR(20),
SUMMARY_PERIOD_START DATE,
SUMMARY_PERIOD_END DATE,
TXN_COUNT_TOTAL NUMBER(10,0),
TXN_COUNT_DEBIT NUMBER(10,0),
TXN_COUNT_CREDIT NUMBER(10,0),
TXN_COUNT_FEE NUMBER(10,0),
AMT_TOTAL_DEBIT NUMBER(18,2),
AMT_TOTAL_CREDIT NUMBER(18,2),
AMT_TOTAL_FEES NUMBER(18,2),
AMT_AVG_DEBIT NUMBER(15,2),
AMT_AVG_CREDIT NUMBER(15,2),
AMT_MAX_SINGLE_DEBIT NUMBER(15,2),
AMT_MAX_SINGLE_CREDIT NUMBER(15,2),
DISTINCT_MERCHANTS NUMBER(10,0),
TOP_MERCHANT_CATEGORY VARCHAR(60),
PCT_ATM NUMBER(5,2),
PCT_POS NUMBER(5,2),
PCT_WEB NUMBER(5,2),
PCT_MOBILE NUMBER(5,2),
DAYS_SINCE_LAST_TXN NUMBER(10,0),
LOAD_TS TIMESTAMP_NTZ(6)
)
COMMENT = 'Per-account transaction summary staging table (migrated from ETL_STAGING_DB.STG_TXN_SUMMARY)';

-- -----------------------------------------------------------------------------
-- ETL_STAGING.STG_RISK_FACTORS
-- Pre-computed risk indicator features per customer for scoring models.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ETL_STAGING.STG_RISK_FACTORS
(
CUSTOMER_ID NUMBER(19,0) NOT NULL,
ACCOUNT_OVERDRAFT_CNT NUMBER(10,0),
NSF_FEE_TOTAL NUMBER(15,2),
LARGE_WITHDRAWAL_CNT NUMBER(10,0),
LARGE_WITHDRAWAL_AMT NUMBER(18,2),
AVG_DAILY_BALANCE_30D NUMBER(15,2),
AVG_DAILY_BALANCE_90D NUMBER(15,2),
BALANCE_VOLATILITY NUMBER(10,4),
CREDIT_UTIL_RATIO NUMBER(5,4),
PAYMENT_ONTIME_PCT NUMBER(5,2),
PAYMENT_LATE_CNT NUMBER(10,0),
MONTHS_SINCE_LAST_LATE NUMBER(10,0),
EXTERNAL_CREDIT_SCORE NUMBER(10,0),
DEBIT_VELOCITY_7D NUMBER(15,2),
DEBIT_VELOCITY_30D NUMBER(15,2),
NEW_MERCHANT_CNT_30D NUMBER(10,0),
INTERNATIONAL_TXN_CNT NUMBER(10,0),
HIGH_RISK_MERCHANT_CNT NUMBER(10,0),
LOAD_TS TIMESTAMP_NTZ(6)
)
COMMENT = 'Risk feature staging table (migrated from ETL_STAGING_DB.STG_RISK_FACTORS)';
Loading