Correct subsets

Author: youennmereljourdan

postprocess/postprocess_service.py

@@ -101,6 +101,7 @@ def get_train_test(self, path: str, dataset: pd.DataFrame, train_size: float, it
 
     def write_subset(self, ids: [], dataset: DataFrame, path: str, name: str):
         size = len(ids)
+        filtered_ds = dataset[dataset['id'].isin(ids)]
         ds_name = f'sub_dataset_{size}_{name}.csv'
         mean_path = os.path.join(path, f'tmp_mean_result_{name}.nii')
         files = []
@@ -109,9 +110,9 @@ def write_subset(self, ids: [], dataset: DataFrame, path: str, name: str):
         mean_img = self.get_mean_image(files, 10)
         nib.save(mean_img, mean_path)
         print(f"Computing correlations to mean image for [{size}] results...")
-        for index, row in dataset.iterrows():
+        for index, row in filtered_ds.iterrows():
             img = os.path.join(path, row['id'], '_subject_id_01', 'result.nii')
-            dataset.at[index, 'from_mean'] = self.corr_srv.get_correlation_coefficient(mean_path, img, 'spearman')
-        dataset.to_csv(os.path.join(path, ds_name),
+            filtered_ds.at[index, 'from_mean'] = self.corr_srv.get_correlation_coefficient(mean_path, img, 'spearman')
+        filtered_ds.to_csv(os.path.join(path, ds_name),
                        index=False, sep=';')
         print(f"Written to [{ds_name}].")

train_test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-#OAR -l walltime=5
+#OAR --array 10
+#OAR -l walltime=4
 #OAR -O ./log/train_test_log_%jobid%.stdout
 #OAR -E ./log/train_test_log_%jobid%.stderr
 #OAR -q production
@@ -12,6 +13,6 @@ RESULTS="$BASE/results/auditory"
 
 g5k-setup-docker -t
 docker build . -t $TAG
-docker run -u root -v "$RESULTS:/results" $TAG python -u train_test.py --results "/results" --iter 3
+docker run -u root -v "$RESULTS:/results" $TAG python -u train_test.py --results "/results" --iter "$OAR_ARRAY_INDEX"
 
 sudo-g5k chown -R ymerel:empenn $RESULTS/*.csv