Write train test subset for from_mean analysis

Author: youennmereljourdan

postprocess.sh

@@ -12,4 +12,7 @@ RESULTS="$BASE/results/auditory"
 
 g5k-setup-docker -t
 docker build . -t $TAG
-docker run -u root -v "$RESULTS:/results" $TAG python -u postprocess.py --results "/results"
+docker run -u root -v "$RESULTS:/results" $TAG python -u postprocess.py --results "/results"
+
+sudo-g5k chown -R ymerel:empenn $RESULTS/*.csv
+sudo-g5k chown -R ymerel:empenn $RESULTS/*.nii

postprocess/postprocess_service.py

@@ -5,7 +5,11 @@
 import pandas as pd
 from typing import List
 
+from pandas import DataFrame
+
+from core.file_service import FileService
 from postprocess.correlation_service import CorrelationService
+from sklearn.model_selection import train_test_split
 
 
 class PostprocessService:
@@ -48,6 +52,7 @@ def get_all_correlations(self, path, ids: List[str]) -> pd.DataFrame:
             data.append((ids[i], 'mean', corr))
             data.append(('mean', ids[i], corr))
             print(f"Processed correlations for [{i+1} / {n}] result")
+        data.append(('mean', 'mean', 1.0))
         dataframe = pd.DataFrame(data, columns=['source', 'target', 'correlation'])
         return dataframe.sort_values(by='correlation', ascending=False)
 
@@ -84,3 +89,29 @@ def get_mean_image(self, inputs: list, batch_size: int) -> nib.Nifti1Image:
         mean_nifti = nib.Nifti1Image(mean_image, affine=nib.load(inputs[0]).affine)
         print("Mean image created.")
         return mean_nifti
+
+    def get_train_test(self, path: str, dataset: pd.DataFrame, train_size: float, iteration: int):
+        print(f"Iteration [{iter}] - Training size [{train_size}]")
+        X = dataset['id']
+        y = dataset['id']
+        X_id_train, X_id_test, y_id_train, y_id_test = train_test_split(X, y, train_size=train_size)
+
+        self.write_subset(X_id_train, dataset, path, f'train_{iteration}')
+        self.write_subset(X_id_test, dataset, path, f'test_{iteration}')
+
+    def write_subset(self, ids: [], dataset: DataFrame, path: str, name: str):
+        size = len(ids)
+        ds_name = f'sub_dataset_{size}_{name}.csv'
+        mean_path = os.path.join(path, 'tmp_mean_result.nii')
+        files = []
+        for conf_id in ids:
+            files.append(os.path.join(path, conf_id, '_subject_id_01', 'result.nii'))
+        mean_img = self.get_mean_image(files, 10)
+        nib.save(mean_img, mean_path)
+        print(f"Computing correlations to mean image for [{size}] results...")
+        for index, row in dataset.iterrows():
+            img = os.path.join(path, row['id'], '_subject_id_01', 'result.nii')
+            dataset.at[index, 'from_mean'] = self.corr_srv.get_correlation_coefficient(mean_path, img, 'spearman')
+        dataset.to_csv(os.path.join(path, ds_name),
+                       index=False, sep=';')
+        print(f"Written to [{ds_name}].")

requirements.txt

@@ -2,4 +2,5 @@ nipype~=1.8.6
 pandas~=2.0.3
 nibabel~=5.2.1
 numpy~=1.24.4
-scipy~=1.10.1
+scipy~=1.10.1
+scikit-learn~=1.3.2

train_test.py

@@ -0,0 +1,23 @@
+import os
+from argparse import ArgumentParser
+
+import numpy as np
+import pandas as pd
+
+from postprocess.postprocess_service import PostprocessService
+
+
+def train_test():
+    postproc_srv = PostprocessService()
+
+    parser = ArgumentParser(description='Post processing of results')
+    parser.add_argument('--results', required=True, type=str, help='path to results')
+    parser.add_argument('--iter', required=True, type=int, help='iteration number')
+    args = parser.parse_args()
+    basedir = args.results
+    iteration = args.iter
+    dataset = pd.read_csv(os.path.join(basedir, 'extended_dataset.csv'), delimiter=';').drop(columns=['mean_corr'])
+    train_sizes = np.linspace(0.1, 0.7, 7).tolist()
+    for train_size in train_sizes:
+        postproc_srv.get_train_test(basedir, dataset, train_size, iteration)
+

train_test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#OAR -l walltime=5
+#OAR -O ./log/train_test_log_%jobid%.stdout
+#OAR -E ./log/train_test_log_%jobid%.stderr
+#OAR -q production
+
+TAG="fmri-confs-runner"
+
+BASE="/home/ymerel/empenn_group_storage/private/ymerel"
+RESULTS="$BASE/results/auditory"
+
+g5k-setup-docker -t
+docker build . -t $TAG
+docker run -u root -v "$RESULTS:/results" $TAG python -u train_test.py --results "/results" --iter 3
+
+sudo-g5k chown -R ymerel:empenn $RESULTS/*.csv