Merge branch 'main' into fix--Fix-REVISE

Author: Chenghao-Tan

README.md

@@ -112,6 +112,8 @@ This folder houses all datasets and their cached versions. It also contains the
 
 This folder contains the implementation of all evaluation and benchmark metrics used to compare recourse methods in the repository. This includes metrics such as `distance`, `redundancy`, `success rate`, `time`, `violations`, and `y nearest neighbors`.
 
+**WARNING: Current success rate implementation (to be specific, `methods.processing.check_counterfactuals()` embedded in `get_counterfactuals()`) will only consider 0->1 flips and mark 0->0 counterfactuals to nan!**
+
 ### Live Site Folder
 
 This folder contains the implementation of the frontend UI interface, which displays results stored in `results.csv` from executing `./experiments/run_experiment.py`.
@@ -158,7 +160,7 @@ benchmark = Benchmark(model, gs, factuals)
 evaluation_measures = [
       evaluation_catalog.YNN(benchmark.mlmodel, {"y": 5, "cf_label": 1}),
       evaluation_catalog.Distance(benchmark.mlmodel),
-      evaluation_catalog.SuccessRate(),
+      evaluation_catalog.SuccessRate(),  # Will only consider 0->1 flips!!!
       evaluation_catalog.Redundancy(benchmark.mlmodel, {"cf_label": 1}),
       evaluation_catalog.ConstraintViolation(benchmark.mlmodel),
       evaluation_catalog.AvgTime({"time": benchmark.timer}),

_deprecated/test/test_cfmodel.py

@@ -8,7 +8,7 @@
 from carla.recourse_methods.catalog.cchvae import CCHVAE
 from carla.recourse_methods.catalog.cem import CEM
 from carla.recourse_methods.catalog.clue import Clue
-from carla.recourse_methods.catalog.crud import CRUD
+from carla.recourse_methods.catalog.cruds import CRUDS
 from carla.recourse_methods.catalog.dice import Dice
 from carla.recourse_methods.catalog.face import Face
 from carla.recourse_methods.catalog.feature_tweak import FeatureTweak
@@ -367,7 +367,7 @@ def test_cchvae(model_type):
 
 
 @pytest.mark.parametrize("model_type", testmodel)
-def test_crud(model_type):
+def test_cruds(model_type):
     # Build data and mlmodel
     data_name = "adult"
     data = OnlineCatalog(data_name)
@@ -384,8 +384,8 @@ def test_crud(model_type):
         },
     }
 
-    crud = CRUD(model, hyperparams)
-    df_cfs = crud.get_counterfactuals(test_factual)
+    cruds = CRUDS(model, hyperparams)
+    df_cfs = cruds.get_counterfactuals(test_factual)
 
     assert test_factual.shape[0] == df_cfs.shape[0]
     assert isinstance(df_cfs, pd.DataFrame)

data/catalog/_data_main/_cached/breast_cancer_one_hot

@@ -103,6 +103,10 @@ def load_adult_data(load_data_size=None):
     for f in data_files:
         check_data_file(f)
 
+        f = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "..", "raw_data", f
+        )
+
         for line in open(f):
             line = line.strip()
             if line == "":
@@ -227,7 +231,6 @@ def load_adult_data_new():
     ]  # sex and race are sensitive feature so we will not use them in classification, we will not consider fnlwght for classification since its computed externally and it highly predictive for the class (for details, see documentation of the adult data)
 
     # adult data comes in two different files, one for training and one for testing, however, we will combine data from both the files
-    this_files_directory = os.path.dirname(os.path.realpath(__file__))
     data_files = ["adult.data", "adult.test"]
 
     y = []
@@ -244,7 +247,9 @@ def load_adult_data_new():
 
     for file_name in data_files:
         check_data_file(file_name)
-        full_file_name = os.path.join(this_files_directory, file_name)
+        full_file_name = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "..", "raw_data", file_name
+        )
         print(full_file_name)
 
         for line in open(full_file_name):

data/catalog/_data_main/_cached/mortgage_one_hot

@@ -59,8 +59,12 @@ def load_compas_data():
     COMPAS_INPUT_FILE = "compas-scores-two-years.csv"
     check_data_file(COMPAS_INPUT_FILE)
 
+    f = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "..", "raw_data", COMPAS_INPUT_FILE
+    )
+
     # load the data and get some stats
-    df = pd.read_csv(COMPAS_INPUT_FILE)
+    df = pd.read_csv(f)
     df = df.dropna(subset=["days_b_screening_arrest"])  # dropping missing vals
 
     # convert to np array
@@ -185,9 +189,10 @@ def load_compas_data_new():
     CLASS_FEATURE = "two_year_recid"  # the decision variable
 
     file_name = "compas-scores-two-years.csv"
-    this_files_directory = os.path.dirname(os.path.realpath(__file__))
-    full_file_name = os.path.join(this_files_directory, file_name)
     check_data_file(file_name)
+    full_file_name = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "..", "raw_data", file_name
+    )
 
     # load the data and get some stats
     df = pd.read_csv(full_file_name)