Skip to content

Commit 6b0111e

Browse files
zyliang2001zyliang2001
committed
update with LMDI+ experiments
1 parent a6f797d commit 6b0111e

File tree

165 files changed

+82279
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

165 files changed

+82279
-0
lines changed

‎local_feature_importance/00_ranking_script.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
command="00_run_ranking.py --nreps 1 --config mdi_local.real_data_${1}_${2}_${3} --split_seed 42 --error_seed 0 --feature_seed ${4} --sample_seed ${5} --ignore_cache --create_rmd --folder_name ${2}_${3}2"
4+
5+
python $command

‎local_feature_importance/00_ranking_submit_all.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
slurm_script="00_ranking_script.sh"
4+
5+
for data_name in "openml_361242" "openml_361243" "openml_361253" "openml_361254" "openml_361259" "openml_361260"; do
6+
for dgp in "linear" "linear_lss" "interaction"; do
7+
for feature_seed in {1..10}; do
8+
for sample_seed in {1..3}; do
9+
sbatch $slurm_script "regression" $data_name $dgp $feature_seed $sample_seed
10+
done
11+
done
12+
done
13+
done
14+
15+
# for data_name in "openml_43" "openml_9978" "openml_361062" "openml_361063" "openml_361069" "openml_361071"; do
16+
# for dgp in "logistic_linear" "logistic_linear_lss" "logistic_interaction"; do
17+
# for feature_seed in {1..10}; do
18+
# for sample_seed in {1..3}; do
19+
# sbatch $slurm_script "classification" $data_name $dgp $feature_seed $sample_seed
20+
# done
21+
# done
22+
# done
23+
# done

‎local_feature_importance/00_results_ablation.ipynb‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"id": "d14a6520",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import pandas as pd\n",
11+
"import os\n",
12+
"pd.set_option('display.max_columns', None)\n",
13+
"import sys\n",
14+
"sys.path.append('..')\n",
15+
"sys.path.append('../..')\n",
16+
"sys.path.append('.')\n",
17+
"sys.path.append('./scripts')"
18+
]
19+
},
20+
{
21+
"cell_type": "code",
22+
"execution_count": null,
23+
"id": "af3cd88f",
24+
"metadata": {},
25+
"outputs": [],
26+
"source": [
27+
"def load_selection_results(task=\"regression\"):\n",
28+
" if task == \"regression\":\n",
29+
" datasets = ['openml_361242', 'openml_361243', 'openml_361253', 'openml_361254', 'openml_361259', 'openml_361260']\n",
30+
" result_cols = [\"RF_Regressor_R2_keep_0.1\", \"RF_Regressor_R2_keep_0.2\", \"RF_Regressor_R2_keep_0.3\", \"RF_Regressor_R2_keep_0.4\"]\n",
31+
" elif task == \"classification\":\n",
32+
" datasets = ['openml_361062', 'openml_361063', 'openml_361069', 'openml_361071', 'openml_43', 'openml_9978']\n",
33+
" result_cols = [\"RF_Classifier_AUROC_keep_0.1\", \"RF_Classifier_AUROC_keep_0.2\", \"RF_Classifier_AUROC_keep_0.3\", \"RF_Classifier_AUROC_keep_0.4\"]\n",
34+
" combined_df = pd.DataFrame()\n",
35+
" split_seeds = [1,2,3,4]\n",
36+
" sample_seeds = [1,2,3,4,5]\n",
37+
" for data in datasets:\n",
38+
" ablation_directory =f\"./results_new/mdi_local.real_data_{task}_{data}/{data}_selection/varying_sample_row_n\"\n",
39+
" for split_seed in split_seeds:\n",
40+
" for sample_seed in sample_seeds:\n",
41+
" df = pd.read_csv(os.path.join(ablation_directory, f\"seed_{split_seed}_{sample_seed}/results.csv\"))\n",
42+
" df[\"data\"] = data\n",
43+
" combined_df = pd.concat([combined_df, df], ignore_index=True)\n",
44+
" combined_df = combined_df.groupby(['fi', 'data'])[result_cols].mean().reset_index()\n",
45+
" return combined_df\n",
46+
"\n",
47+
"def load_stability_results(task=\"regression\"):\n",
48+
" if task == \"regression\":\n",
49+
" datasets = ['openml_361242', 'openml_361243', 'openml_361253', 'openml_361254', 'openml_361259', 'openml_361260']\n",
50+
" elif task == \"classification\":\n",
51+
" datasets = ['openml_361062', 'openml_361063', 'openml_361069', 'openml_361071', 'openml_43', 'openml_9978']\n",
52+
" result_cols = [\"avg_10_features\", \"avg_20_features\", \"avg_30_features\", \"avg_40_features\"]\n",
53+
" combined_df = pd.DataFrame()\n",
54+
" split_seeds = [1,2,3]\n",
55+
" sample_seeds = [1,2,3,4,5]\n",
56+
" for data in datasets:\n",
57+
" ablation_directory =f\"./results_new/mdi_local.real_data_{task}_{data}_stability/{data}_stability/varying_sample_row_n\"\n",
58+
" for split_seed in split_seeds:\n",
59+
" for sample_seed in sample_seeds:\n",
60+
" df = pd.read_csv(os.path.join(ablation_directory, f\"seed_{split_seed}_{sample_seed}/results.csv\"))\n",
61+
" df[\"data\"] = data\n",
62+
" combined_df = pd.concat([combined_df, df], ignore_index=True)\n",
63+
" combined_df = combined_df.groupby(['fi', 'data'])[result_cols].mean().reset_index()\n",
64+
" return combined_df"
65+
]
66+
},
67+
{
68+
"cell_type": "code",
69+
"execution_count": null,
70+
"id": "1bdcd077",
71+
"metadata": {},
72+
"outputs": [],
73+
"source": [
74+
"def compute_selection_ablation_rank_results(regression_results_df, classification_results_df, method=\"lmdi+\", baselines=['LIME', 'Treeshap']):\n",
75+
" baselines.append(method)\n",
76+
" \n",
77+
" regression_results_df = regression_results_df[regression_results_df[\"fi\"].isin(baselines)]\n",
78+
" datasets = ['openml_361242', 'openml_361243', 'openml_361253', 'openml_361254', 'openml_361259', 'openml_361260']\n",
79+
" result_cols = [\"RF_Regressor_R2_keep_0.1\", \"RF_Regressor_R2_keep_0.2\", \"RF_Regressor_R2_keep_0.3\", \"RF_Regressor_R2_keep_0.4\"]\n",
80+
" rank_results_regression = pd.DataFrame()\n",
81+
" for data in datasets:\n",
82+
" df_data = regression_results_df[regression_results_df[\"data\"] == data]\n",
83+
" ranked_df = df_data.copy()\n",
84+
" for col in result_cols:\n",
85+
" ranked_df[col] = df_data[col].rank(ascending=False, method='min')\n",
86+
" rank_results_regression = pd.concat([rank_results_regression, ranked_df], ignore_index=True)\n",
87+
" rank_results_regression.columns = ['fi', 'data', 'top_0.1', 'top_0.2', 'top_0.3', 'top_0.4']\n",
88+
"\n",
89+
" classification_results_df = classification_results_df[classification_results_df[\"fi\"].isin(baselines)]\n",
90+
" datasets = ['openml_361062', 'openml_361063', 'openml_361069', 'openml_361071', 'openml_43', 'openml_9978']\n",
91+
" result_cols = [\"RF_Classifier_AUROC_keep_0.1\", \"RF_Classifier_AUROC_keep_0.2\", \"RF_Classifier_AUROC_keep_0.3\", \"RF_Classifier_AUROC_keep_0.4\"]\n",
92+
" rank_results_classification = pd.DataFrame()\n",
93+
" for data in datasets:\n",
94+
" df_data = classification_results_df[classification_results_df[\"data\"] == data]\n",
95+
" ranked_df = df_data.copy()\n",
96+
" for col in result_cols:\n",
97+
" ranked_df[col] = df_data[col].rank(ascending=False, method='min')\n",
98+
" rank_results_classification = pd.concat([rank_results_classification, ranked_df], ignore_index=True)\n",
99+
" rank_results_classification.columns = ['fi', 'data', 'top_0.1', 'top_0.2', 'top_0.3', 'top_0.4']\n",
100+
" \n",
101+
" rank_results = pd.concat([rank_results_regression, rank_results_classification], ignore_index=True)\n",
102+
" rank_results_avg = rank_results.groupby('fi')[['top_0.1', 'top_0.2', 'top_0.3', 'top_0.4']].mean().sort_values(by='top_0.1').reset_index()\n",
103+
" rank_results_avg = rank_results_avg[rank_results_avg[\"fi\"] == method]\n",
104+
" return rank_results, rank_results_avg\n",
105+
"\n",
106+
"def compute_stability_ablation_rank_results(regression_results_df, classification_results_df, method=\"lmdi+\", baselines=['LIME', 'Treeshap']):\n",
107+
" baselines.append(method)\n",
108+
" \n",
109+
" regression_results_df = regression_results_df[regression_results_df[\"fi\"].isin(baselines)]\n",
110+
" datasets = ['openml_361242', 'openml_361243', 'openml_361253', 'openml_361254', 'openml_361259', 'openml_361260']\n",
111+
" result_cols = [\"avg_10_features\", \"avg_20_features\", \"avg_30_features\", \"avg_40_features\"]\n",
112+
" rank_results_regression = pd.DataFrame()\n",
113+
" for data in datasets:\n",
114+
" df_data = regression_results_df[regression_results_df[\"data\"] == data]\n",
115+
" ranked_df = df_data.copy()\n",
116+
" for col in result_cols:\n",
117+
" ranked_df[col] = df_data[col].rank(ascending=True, method='min')\n",
118+
" rank_results_regression = pd.concat([rank_results_regression, ranked_df], ignore_index=True)\n",
119+
" rank_results_regression.columns = ['fi', 'data', 'top_0.1', 'top_0.2', 'top_0.3', 'top_0.4']\n",
120+
"\n",
121+
" classification_results_df = classification_results_df[classification_results_df[\"fi\"].isin(baselines)]\n",
122+
" datasets = ['openml_361062', 'openml_361063', 'openml_361069', 'openml_361071', 'openml_43', 'openml_9978']\n",
123+
" result_cols = [\"avg_10_features\", \"avg_20_features\", \"avg_30_features\", \"avg_40_features\"]\n",
124+
" rank_results_classification = pd.DataFrame()\n",
125+
" for data in datasets:\n",
126+
" df_data = classification_results_df[classification_results_df[\"data\"] == data]\n",
127+
" ranked_df = df_data.copy()\n",
128+
" for col in result_cols:\n",
129+
" ranked_df[col] = df_data[col].rank(ascending=True, method='min')\n",
130+
" rank_results_classification = pd.concat([rank_results_classification, ranked_df], ignore_index=True)\n",
131+
" rank_results_classification.columns = ['fi', 'data', 'top_0.1', 'top_0.2', 'top_0.3', 'top_0.4']\n",
132+
" \n",
133+
" rank_results = pd.concat([rank_results_regression, rank_results_classification], ignore_index=True)\n",
134+
" rank_results_avg = rank_results.groupby('fi')[['top_0.1', 'top_0.2', 'top_0.3', 'top_0.4']].mean().sort_values(by='top_0.1').reset_index()\n",
135+
" rank_results_avg = rank_results_avg[rank_results_avg[\"fi\"] == method]\n",
136+
" return rank_results, rank_results_avg"
137+
]
138+
},
139+
{
140+
"cell_type": "code",
141+
"execution_count": null,
142+
"id": "8b8e639a",
143+
"metadata": {},
144+
"outputs": [],
145+
"source": [
146+
"regression_results_df = load_selection_results(task=\"regression\")\n",
147+
"classification_results_df = load_selection_results(task=\"classification\")\n",
148+
"compute_selection_ablation_rank_results(regression_results_df, classification_results_df, method=\"lmdi\", baselines=['LIME', 'Treeshap'])"
149+
]
150+
},
151+
{
152+
"cell_type": "code",
153+
"execution_count": null,
154+
"id": "f6877a17",
155+
"metadata": {},
156+
"outputs": [],
157+
"source": [
158+
"compute_selection_ablation_rank_results(regression_results_df, classification_results_df, method=\"Ablation_model0\", baselines=['LIME', 'Treeshap'])"
159+
]
160+
},
161+
{
162+
"cell_type": "code",
163+
"execution_count": null,
164+
"id": "1ceb8528",
165+
"metadata": {},
166+
"outputs": [],
167+
"source": [
168+
"compute_selection_ablation_rank_results(regression_results_df, classification_results_df, method=\"Ablation_model1\", baselines=['LIME', 'Treeshap'])"
169+
]
170+
},
171+
{
172+
"cell_type": "code",
173+
"execution_count": null,
174+
"id": "d9fb85f3",
175+
"metadata": {},
176+
"outputs": [],
177+
"source": [
178+
"compute_selection_ablation_rank_results(regression_results_df, classification_results_df, method=\"lmdi+\", baselines=['LIME', 'Treeshap'])"
179+
]
180+
},
181+
{
182+
"cell_type": "code",
183+
"execution_count": null,
184+
"id": "9771e9d8",
185+
"metadata": {},
186+
"outputs": [],
187+
"source": [
188+
"regression_results_df = load_stability_results(task=\"regression\")\n",
189+
"classification_results_df = load_stability_results(task=\"classification\")\n",
190+
"compute_stability_ablation_rank_results(regression_results_df, classification_results_df, method=\"lmdi\", baselines=['LIME', 'Treeshap'])"
191+
]
192+
},
193+
{
194+
"cell_type": "code",
195+
"execution_count": null,
196+
"id": "6ca28024",
197+
"metadata": {},
198+
"outputs": [],
199+
"source": [
200+
"compute_stability_ablation_rank_results(regression_results_df, classification_results_df, method=\"Ablation_model0\", baselines=['LIME', 'Treeshap'])"
201+
]
202+
},
203+
{
204+
"cell_type": "code",
205+
"execution_count": null,
206+
"id": "6e2a1ed1",
207+
"metadata": {},
208+
"outputs": [],
209+
"source": [
210+
"compute_stability_ablation_rank_results(regression_results_df, classification_results_df, method=\"Ablation_model1\", baselines=['LIME', 'Treeshap'])"
211+
]
212+
},
213+
{
214+
"cell_type": "code",
215+
"execution_count": null,
216+
"id": "c6b6aa35",
217+
"metadata": {},
218+
"outputs": [],
219+
"source": [
220+
"compute_stability_ablation_rank_results(regression_results_df, classification_results_df, method=\"lmdi+\", baselines=['LIME', 'Treeshap'])"
221+
]
222+
}
223+
],
224+
"metadata": {
225+
"kernelspec": {
226+
"display_name": "mdi",
227+
"language": "python",
228+
"name": "python3"
229+
},
230+
"language_info": {
231+
"codemirror_mode": {
232+
"name": "ipython",
233+
"version": 3
234+
},
235+
"file_extension": ".py",
236+
"mimetype": "text/x-python",
237+
"name": "python",
238+
"nbconvert_exporter": "python",
239+
"pygments_lexer": "ipython3",
240+
"version": "3.10.14"
241+
}
242+
},
243+
"nbformat": 4,
244+
"nbformat_minor": 5
245+
}

0 commit comments

Comments
 (0)