Skip to content

Commit 49bf66e

Browse files
SVC04SVC04
authored
Add files via upload
1 parent 5054962 commit 49bf66e

3 files changed

Lines changed: 342 additions & 0 deletions

File tree

‎scripts/anova_relativeranking_metrics.ipynb‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "3ed5e13e-1b3d-4283-a553-5949e0cfe24b",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"\"\"\"\n",
11+
"ANOVA Significance Testing for Spearman Correlation\n",
12+
"---------------------------------------------------\n",
13+
"This script performs one-way ANOVA across batches for Spearman correlation scores.\n",
14+
"\n",
15+
"Usage:\n",
16+
"- Place all global_spearman_article_level_scores_batchX.csv files in the same folder.\n",
17+
"- Each file must have columns of the form \"<method_pair>\" (no _k suffix).\n",
18+
"- Run: python anova_relativeranking_metrics\n",
19+
"- Output: anova_relative_ranking_results.csv\n",
20+
"\"\"\"\n",
21+
"\n",
22+
"import pandas as pd\n",
23+
"from scipy.stats import f_oneway\n",
24+
"import os\n",
25+
"\n",
26+
"# List of method pairs\n",
27+
"method_pairs = [\n",
28+
" 'attention_vs_deeplift', 'attention_vs_lime', 'attention_vs_gradient_shap',\n",
29+
" 'deeplift_vs_lime', 'deeplift_vs_gradient_shap', 'lime_vs_gradient_shap'\n",
30+
"]\n",
31+
"\n",
32+
"# Input batch files\n",
33+
"batch_files = [f\"global_spearman_article_level_scores_batch{i}.csv\" for i in range(1, 12)]\n",
34+
"\n",
35+
"anova_results = []\n",
36+
"\n",
37+
"for method in method_pairs:\n",
38+
" batch_scores = []\n",
39+
"\n",
40+
" for file in batch_files:\n",
41+
" if not os.path.exists(file):\n",
42+
" print(f\" File not found: {file}\")\n",
43+
" continue\n",
44+
"\n",
45+
" df = pd.read_csv(file)\n",
46+
" if method in df.columns:\n",
47+
" values = df[method].dropna().tolist()\n",
48+
" if values:\n",
49+
" batch_scores.append(values)\n",
50+
" else:\n",
51+
" print(f\" Empty column {method} in {file}\")\n",
52+
" else:\n",
53+
" print(f\" Column {method} not found in {file}\")\n",
54+
"\n",
55+
" if len(batch_scores) >= 2:\n",
56+
" try:\n",
57+
" f_stat, p_val = f_oneway(*batch_scores)\n",
58+
" anova_results.append({\n",
59+
" \"Method_Pair\": method,\n",
60+
" \"F_statistic\": f_stat,\n",
61+
" \"p_value\": p_val,\n",
62+
" \"Significant\": p_val < 0.05\n",
63+
" })\n",
64+
" except Exception as e:\n",
65+
" print(f\" Error running ANOVA for {method}: {e}\")\n",
66+
" else:\n",
67+
" print(f\" Skipped ANOVA for {method} (insufficient groups: {len(batch_scores)})\")\n",
68+
"\n",
69+
"# Save results\n",
70+
"anova_df = pd.DataFrame(anova_results)\n",
71+
"anova_df.to_csv(\"anova_spearman_results.csv\", index=False)\n",
72+
"print(\"✅ ANOVA results saved to 'anova_spearman_results.csv'\")\n"
73+
]
74+
}
75+
],
76+
"metadata": {
77+
"kernelspec": {
78+
"display_name": "Python 3 (ipykernel)",
79+
"language": "python",
80+
"name": "python3"
81+
},
82+
"language_info": {
83+
"codemirror_mode": {
84+
"name": "ipython",
85+
"version": 3
86+
},
87+
"file_extension": ".py",
88+
"mimetype": "text/x-python",
89+
"name": "python",
90+
"nbconvert_exporter": "python",
91+
"pygments_lexer": "ipython3",
92+
"version": "3.10.13"
93+
}
94+
},
95+
"nbformat": 4,
96+
"nbformat_minor": 5
97+
}

‎scripts/anova_topk_metrics.ipynb‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "e979cd5c-36e4-469c-9d5a-4db853cc7106",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"\"\"\"\n",
11+
"ANOVA Significance Testing for Top-k Based Metrics\n",
12+
"--------------------------------------------------\n",
13+
"This script performs one-way ANOVA across batches for top-k based metrics\n",
14+
"(Feature Agreement, Rank Agreement, Semantic Alignment Score, Pairwise Rank Agreement, CAS).\n",
15+
"\n",
16+
"Usage:\n",
17+
"- Place all global_*_article_level_scores_batchX.csv files in the same folder.\n",
18+
"- Each file must have columns of the form \"<method_pair>_k<k>\".\n",
19+
"- Run: anova_topk_metrics.py\n",
20+
"- Output: anova_topk_results.csv\n",
21+
"\"\"\"\n",
22+
"\n",
23+
"import pandas as pd\n",
24+
"from scipy.stats import f_oneway\n",
25+
"import os\n",
26+
"\n",
27+
"# List of method pairs\n",
28+
"method_pairs = [\n",
29+
" 'attention_vs_deeplift', 'attention_vs_lime', 'attention_vs_gradient_shap',\n",
30+
" 'deeplift_vs_lime', 'deeplift_vs_gradient_shap', 'lime_vs_gradient_shap'\n",
31+
"]\n",
32+
"\n",
33+
"# k values (e.g., top-2 to top-10)\n",
34+
"k_values = range(2, 11)\n",
35+
"\n",
36+
"# Input batch files (adjust path if needed)\n",
37+
"batch_files = [f\"global_fa_article_level_scores_batch{i}.csv\" for i in range(1, 12)]\n",
38+
"\n",
39+
"anova_results = []\n",
40+
"\n",
41+
"for method in method_pairs:\n",
42+
" for k in k_values:\n",
43+
" col_name = f\"{method}_k{k}\"\n",
44+
" batch_scores = []\n",
45+
"\n",
46+
" for file in batch_files:\n",
47+
" if not os.path.exists(file):\n",
48+
" print(f\" File not found: {file}\")\n",
49+
" continue\n",
50+
"\n",
51+
" df = pd.read_csv(file)\n",
52+
" if col_name in df.columns:\n",
53+
" values = df[col_name].dropna().tolist()\n",
54+
" if values:\n",
55+
" batch_scores.append(values)\n",
56+
" else:\n",
57+
" print(f\" Empty column {col_name} in {file}\")\n",
58+
" else:\n",
59+
" print(f\" Column '{col_name}' not found in {file}\")\n",
60+
"\n",
61+
" if len(batch_scores) >= 2:\n",
62+
" try:\n",
63+
" f_stat, p_val = f_oneway(*batch_scores)\n",
64+
" anova_results.append({\n",
65+
" \"Method_Pair\": method,\n",
66+
" \"k\": k,\n",
67+
" \"F_statistic\": f_stat,\n",
68+
" \"p_value\": p_val,\n",
69+
" \"Significant\": p_val < 0.05\n",
70+
" })\n",
71+
" except Exception as e:\n",
72+
" print(f\" Error running ANOVA for {col_name}: {e}\")\n",
73+
" else:\n",
74+
" print(f\" Skipped ANOVA for {col_name} (insufficient groups: {len(batch_scores)})\")\n",
75+
"\n",
76+
"# Save results\n",
77+
"anova_df = pd.DataFrame(anova_results)\n",
78+
"anova_df.to_csv(\"anova_topk_results.csv\", index=False)\n",
79+
"print(\" ANOVA results saved to 'anova_topk_results.csv'\")\n"
80+
]
81+
}
82+
],
83+
"metadata": {
84+
"kernelspec": {
85+
"display_name": "Python 3 (ipykernel)",
86+
"language": "python",
87+
"name": "python3"
88+
},
89+
"language_info": {
90+
"codemirror_mode": {
91+
"name": "ipython",
92+
"version": 3
93+
},
94+
"file_extension": ".py",
95+
"mimetype": "text/x-python",
96+
"name": "python",
97+
"nbconvert_exporter": "python",
98+
"pygments_lexer": "ipython3",
99+
"version": "3.10.13"
100+
}
101+
},
102+
"nbformat": 4,
103+
"nbformat_minor": 5
104+
}

‎scripts/wilcoxon_testing.ipynb‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "7c126ceb-e4a6-47d8-8a83-08bf9159d09f",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"# Wilcoxon Testing to check if the agreement scores before and after (imprrovement of scores) is significant or not\n",
11+
"\n",
12+
"\"\"\"\n",
13+
"wilcoxon_test.py\n",
14+
"\n",
15+
"This script performs Wilcoxon signed-rank tests to compare\n",
16+
"regional RXAI disagreement scores (segmented) against\n",
17+
"global disagreement scores (non-segmented).\n",
18+
"\n",
19+
"Inputs:\n",
20+
" - regional_article_level_fa_batch{n}.csv\n",
21+
" - global_fa_article_level_scores_batch{n}.csv\n",
22+
"\n",
23+
"Outputs:\n",
24+
" - wilcoxon_all_batches_k234.csv with p-values\n",
25+
"\"\"\"\n",
26+
"\n",
27+
"\n",
28+
"\n",
29+
"# Perform Wilcoxon signed-rank tests to compare RXAI (regional) vs Global disagreement scores.\n",
30+
"\n",
31+
"# Note: This script applies only to top-k based metrics\n",
32+
"# (Feature Agreement, Rank Agreement, Semantic Alignment Score).\n",
33+
"# It is not intended for relative ranking metrics such as\n",
34+
"# Spearman correlation or Pairwise Rank Agreement.\n",
35+
"\n",
36+
"\n",
37+
"import pandas as pd\n",
38+
"from scipy.stats import wilcoxon\n",
39+
"\n",
40+
"# Configuration\n",
41+
"method_pairs = [\n",
42+
" 'attention_vs_deeplift',\n",
43+
" 'attention_vs_lime',\n",
44+
" 'attention_vs_gradient_shap',\n",
45+
" 'deeplift_vs_lime',\n",
46+
" 'deeplift_vs_gradient_shap',\n",
47+
" 'lime_vs_gradient_shap'\n",
48+
"]\n",
49+
"k_values = [2, 3, 4] # Top-k values\n",
50+
"total_batches = 11 # Number of CSV batches\n",
51+
"\n",
52+
"all_batch_results = [] # Collect all test results\n",
53+
"\n",
54+
"for batch in range(1, total_batches + 1):\n",
55+
" print(f\"\\n Processing Batch {batch}...\")\n",
56+
"\n",
57+
" try:\n",
58+
" # File paths\n",
59+
" regional_path = f\"regional_article_level_fa_batch{batch}.csv\"\n",
60+
" global_path = f\"global_fa_article_level_scores_batch{batch}.csv\"\n",
61+
"\n",
62+
" # Load CSVs\n",
63+
" regional_df = pd.read_csv(regional_path)\n",
64+
" global_df = pd.read_csv(global_path)\n",
65+
" except Exception as e:\n",
66+
" print(f\" Failed to load batch {batch}: {e}\")\n",
67+
" continue\n",
68+
"\n",
69+
" # Compare each method pair at each k\n",
70+
" for pair in method_pairs:\n",
71+
" for k in k_values:\n",
72+
" print(f\" Method: {pair}, k={k}\")\n",
73+
"\n",
74+
" # Regional FA column (article-level scores with segmentation)\n",
75+
" if pair not in regional_df.columns:\n",
76+
" print(f\" Column '{pair}' not in regional batch {batch}\")\n",
77+
" continue\n",
78+
" regional_k = regional_df[regional_df['k'] == k][pair]\n",
79+
"\n",
80+
" # Global FA column (article-level scores without segmentation)\n",
81+
" global_col_name = f\"{pair}_k{k}\"\n",
82+
" if global_col_name not in global_df.columns:\n",
83+
" print(f\" Column '{global_col_name}' not in global batch {batch}\")\n",
84+
" continue\n",
85+
" global_k = global_df[global_col_name]\n",
86+
"\n",
87+
" # Align lengths\n",
88+
" min_len = min(len(regional_k), len(global_k))\n",
89+
" if min_len == 0:\n",
90+
" print(f\" No data to compare for k={k}\")\n",
91+
" continue\n",
92+
" regional_k = regional_k.iloc[:min_len]\n",
93+
" global_k = global_k.iloc[:min_len]\n",
94+
"\n",
95+
" # Wilcoxon signed-rank test\n",
96+
" try:\n",
97+
" _, p_value = wilcoxon(regional_k, global_k)\n",
98+
" except ValueError:\n",
99+
" print(f\" All differences zero, assigning p=1.0\")\n",
100+
" p_value = 1.0\n",
101+
"\n",
102+
" print(f\" p-value: {p_value:.3e}\")\n",
103+
"\n",
104+
" # Store results\n",
105+
" all_batch_results.append({\n",
106+
" 'batch': batch,\n",
107+
" 'method_pair': pair,\n",
108+
" 'k': k,\n",
109+
" 'p_value': p_value\n",
110+
" })\n",
111+
"\n",
112+
"# Save results across all batches\n",
113+
"df_results = pd.DataFrame(all_batch_results)\n",
114+
"df_results.to_csv(\"wilcoxon_all_batches_k234.csv\", index=False)\n",
115+
"\n",
116+
"print(\"\\n All batches processed. Results saved to 'wilcoxon_all_batches_k234.csv'.\")\n"
117+
]
118+
}
119+
],
120+
"metadata": {
121+
"kernelspec": {
122+
"display_name": "Python 3 (ipykernel)",
123+
"language": "python",
124+
"name": "python3"
125+
},
126+
"language_info": {
127+
"codemirror_mode": {
128+
"name": "ipython",
129+
"version": 3
130+
},
131+
"file_extension": ".py",
132+
"mimetype": "text/x-python",
133+
"name": "python",
134+
"nbconvert_exporter": "python",
135+
"pygments_lexer": "ipython3",
136+
"version": "3.10.13"
137+
}
138+
},
139+
"nbformat": 4,
140+
"nbformat_minor": 5
141+
}

0 commit comments

Comments
 (0)