Roar reproduce

data/catalog/_data_main/process_data/process_german_data.py

@@ -59,6 +59,48 @@ def load_german_data():
     return processed_df.astype("float64")
 
 
+def load_german_data_modified():
+    # input vars
+    raw_data_file = os.path.join(os.path.dirname(__file__), "corrected_german.csv")
+    processed_file = os.path.join(os.path.dirname(__file__), "corrected_german_processed.csv")
+
+    # German Data Processing
+    raw_df = pd.read_csv(raw_data_file)  # , index_col = 0)
+    processed_df = pd.DataFrame()
+
+    processed_df["GoodCustomer (label)"] = raw_df["credit_risk"]
+    processed_df["Sex"] = raw_df["personal_status_sex"]
+    processed_df["Age"] = raw_df["age"]
+    processed_df["Credit"] = raw_df["amount"]
+    processed_df["LoanDuration"] = raw_df["duration"]
+
+    # # order important, more balance can overwrite less balance!
+    # processed_df.loc[raw_df['CheckingAccountBalance_geq_0'] == 1, 'CheckingAccountBalance'] = 2
+    # processed_df.loc[raw_df['CheckingAccountBalance_geq_200'] == 1, 'CheckingAccountBalance'] = 3
+    # processed_df = processed_df.fillna(1) # all other categories...
+
+    # # order important, more balance can overwrite less balance!
+    # processed_df.loc[raw_df['SavingsAccountBalance_geq_100'] == 1, 'SavingsAccountBalance'] = 2
+    # processed_df.loc[raw_df['SavingsAccountBalance_geq_500'] == 1, 'SavingsAccountBalance'] = 3
+    # processed_df = processed_df.fillna(1) # all other categories...
+
+    # # 2: owns house, 1: rents house, 0: neither
+    # processed_df.loc[raw_df['OwnsHouse'] == 1, 'HousingStatus'] = 3
+    # processed_df.loc[raw_df['RentsHouse'] == 1, 'HousingStatus'] = 2
+    # processed_df = processed_df.fillna(1) # all other categories...
+
+    # Save to CSV
+    processed_df = processed_df + 0  # convert boolean values to numeric
+    processed_df = processed_df.reset_index(drop=True)
+    processed_df = (
+        processed_df.dropna()
+    )  # drop all rows that include NAN (some exist in isMarried column, possibly elsewhere as well)
+    processed_df.to_csv(processed_file, header=True, index=False)
+    assert processed_df.shape[0] == 1000
+
+    return processed_df.astype("float64")
+
+
 # import numpy as np
 # import pandas as pd
 

data/catalog/_data_main/process_data/process_sba_data.py

@@ -0,0 +1,196 @@
+import os
+from random import seed
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+import numpy as np
+from random import shuffle
+
+import process_data.process_utils_data as ut
+
+RANDOM_SEED = 54321
+seed(
+    RANDOM_SEED
+)  # set the random seed so that the random permutations can be reproduced again
+
+def get_feat_types(df):
+		cat_feat = []
+		num_feat = []
+		for key in list(df):
+			if df[key].dtype==object:
+				cat_feat.append(key)
+			elif len(set(df[key]))>2:
+				num_feat.append(key)
+		return cat_feat,num_feat
+
+def load_sba_data():
+    # Define attributes of interest
+    attrs = [
+        'Zip', 'NAICS', 'ApprovalDate', 'ApprovalFY', 'Term', 'NoEmp',
+        'NewExist', 'CreateJob', 'RetainedJob', 'FranchiseCode', 'UrbanRural',
+        'RevLineCr', 'ChgOffDate', 'DisbursementDate', 'DisbursementGross',
+        'ChgOffPrinGr', 'GrAppv', 'SBA_Appv', 'New', 'RealEstate', 'Portion',
+        'Recession', 'daysterm', 'xx'
+    ]
+    sensitive_attrs = []   # just an example, pick what matters for fairness
+    attrs_to_ignore = [] # IDs or very sparse high-cardinality
+
+    # Path to raw SBA file
+    this_files_directory = os.path.dirname(os.path.realpath(__file__))
+    file_name = os.path.join(this_files_directory, "..", "raw_data", "SBAcase.11.13.17.csv")
+
+    # Load file
+    df = pd.read_csv(file_name)
+    df = df.fillna(-1)  # replace NaNs with sentinel
+    df = df.sample(frac=1, random_state=RANDOM_SEED).reset_index(drop=True)
+
+    # print(df['RevLineCr'].value_counts())
+
+    # Define target
+    y = 1 - df["Default"].values
+
+    # Dicts for storage
+    x_control = {}
+    attrs_to_vals = {}
+
+    for k in attrs:
+        if k in sensitive_attrs:
+            x_control[k] = df[k].tolist()
+        elif k in attrs_to_ignore:
+            pass
+        else:
+            attrs_to_vals[k] = df[k].tolist()
+
+    # Combine
+    all_attrs_to_vals = attrs_to_vals
+    for k in sensitive_attrs:
+        all_attrs_to_vals[k] = x_control[k]
+    all_attrs_to_vals["label"] = y
+
+    df_all = pd.DataFrame.from_dict(all_attrs_to_vals)
+
+    _, num_feat = get_feat_types(df_all)
+
+    # for key in num_feat:
+    #     scaler = StandardScaler()
+    #     df_all[key] = scaler.fit_transform(df_all[key].values.reshape(-1,1))
+
+    # ---- Create processed dataframe with integer encodings ----
+    processed_df = pd.DataFrame()
+
+    # Numeric attributes: keep directly
+    num_attrs = [
+        'Zip', 'NAICS', 'ApprovalDate', 'ApprovalFY', 'Term', 'NoEmp',
+        'NewExist', 'CreateJob', 'RetainedJob', 'FranchiseCode', 'UrbanRural'
+    ]
+    for a in num_attrs:
+        processed_df[a] = df_all[a]
+
+    # RevLineCr ("Y"/"N"/other) → 1,2,3
+    processed_df.loc[df_all["RevLineCr"] == "Y", "RevLineCr"] = 1
+    processed_df.loc[df_all["RevLineCr"] == "N", "RevLineCr"] = 2
+    processed_df.loc[df_all["RevLineCr"] == "T", "RevLineCr"] = 3
+    processed_df.loc[df_all["RevLineCr"] == "0", "RevLineCr"] = 4
+    # processed_df.loc[df_all["RevLineCr"] == -1, "RevLineCr"] = 5
+
+    # print(processed_df['RevLineCr'].value_counts())
+    # cant think of what to do, can just drop the Nas actaully.
+
+    # processed_df['RevLineCr'] = pd.Categorical(processed_df['RevLineCr'])
+
+    # Add recession, real estate, portion, etc. directly
+    for a in ['ChgOffDate', 'DisbursementDate', 'DisbursementGross',
+        'ChgOffPrinGr', 'GrAppv', 'SBA_Appv', 'New', 'RealEstate', 'Portion',
+        'Recession', 'daysterm', 'xx']:
+        processed_df[a] = df_all[a]
+
+    processed_df["Label"] = df_all["label"]
+
+    processed_df = processed_df[processed_df["ApprovalFY"]<2006]
+
+    processed_df = processed_df[processed_df['RevLineCr'].notna()] 
+
+    return processed_df.astype("float64")
+
+def load_sba_data_modified():
+    # Define attributes of interest
+    attrs = [
+        'Zip', 'NAICS', 'ApprovalDate', 'ApprovalFY', 'Term', 'NoEmp',
+        'NewExist', 'CreateJob', 'RetainedJob', 'FranchiseCode', 'UrbanRural',
+        'RevLineCr', 'ChgOffDate', 'DisbursementDate', 'DisbursementGross',
+        'ChgOffPrinGr', 'GrAppv', 'SBA_Appv', 'New', 'RealEstate', 'Portion',
+        'Recession', 'daysterm', 'xx'
+    ]
+    sensitive_attrs = []   # just an example, pick what matters for fairness
+    attrs_to_ignore = [] # IDs or very sparse high-cardinality
+
+    # Path to raw SBA file
+    this_files_directory = os.path.dirname(os.path.realpath(__file__))
+    file_name = os.path.join(this_files_directory, "..", "raw_data", "SBAcase.11.13.17.csv")
+
+    # Load file
+    df = pd.read_csv(file_name)
+    df = df.fillna(-1)  # replace NaNs with sentinel
+    df = df.sample(frac=1, random_state=RANDOM_SEED).reset_index(drop=True)
+
+
+    # Define target
+    y = 1 - df["Default"].values
+
+    # Dicts for storage
+    x_control = {}
+    attrs_to_vals = {}
+
+    for k in attrs:
+        if k in sensitive_attrs:
+            x_control[k] = df[k].tolist()
+        elif k in attrs_to_ignore:
+            pass
+        else:
+            attrs_to_vals[k] = df[k].tolist()
+
+    # Combine
+    all_attrs_to_vals = attrs_to_vals
+    for k in sensitive_attrs:
+        all_attrs_to_vals[k] = x_control[k]
+    all_attrs_to_vals["label"] = y
+
+    df_all = pd.DataFrame.from_dict(all_attrs_to_vals)
+
+    _, num_feat = get_feat_types(df_all)
+
+    # for key in num_feat:
+    #     scaler = StandardScaler()
+    #     df_all[key] = scaler.fit_transform(df_all[key].values.reshape(-1,1))
+
+    # ---- Create processed dataframe with integer encodings ----
+    processed_df = pd.DataFrame()
+
+    # Numeric attributes: keep directly
+    num_attrs = [
+        'Zip', 'NAICS', 'ApprovalDate', 'ApprovalFY', 'Term', 'NoEmp',
+        'NewExist', 'CreateJob', 'RetainedJob', 'FranchiseCode', 'UrbanRural'
+    ]
+    for a in num_attrs:
+        processed_df[a] = df_all[a]
+
+    # RevLineCr ("Y"/"N"/other) → 1,2,3
+    processed_df.loc[df_all["RevLineCr"] == "Y", "RevLineCr"] = 1
+    processed_df.loc[df_all["RevLineCr"] == "N", "RevLineCr"] = 2
+    processed_df.loc[df_all["RevLineCr"] == "T", "RevLineCr"] = 3
+    processed_df.loc[df_all["RevLineCr"] == "0", "RevLineCr"] = 4
+    # processed_df.loc[df_all["RevLineCr"] == -1, "RevLineCr"] = 5
+    # cant think of what to do, can just drop the Nas actaully.
+
+    # processed_df['RevLineCr'] = pd.Categorical(processed_df['RevLineCr'])
+
+    # Add recession, real estate, portion, etc. directly
+    for a in ['ChgOffDate', 'DisbursementDate', 'DisbursementGross',
+        'ChgOffPrinGr', 'GrAppv', 'SBA_Appv', 'New', 'RealEstate', 'Portion',
+        'Recession', 'daysterm', 'xx']:
+        processed_df[a] = df_all[a]
+
+    processed_df["Label"] = df_all["label"]
+
+    processed_df = processed_df[processed_df['RevLineCr'].notna()]
+
+    return processed_df.astype("float64")