Merge pull request #11 from MELDProject/dev_v1.0.1

release V1.0.1

Author: kwagstyl

README.md

@@ -13,6 +13,13 @@ Note:
 - You will need the following demographic information (age at scan & sex) to run AID-HS on your patient's T1 MRI scan.
 - AID-HS has been developed on T1w scans acquired at 3T. It has not yet been thoroughly evaluated on 1.5T and 7T data
 
+**<span style="color: red;">SIGN UP TO THE AID-HS MAILING LIST</span>**:
+We request that all AID-HS users sign up to the mailing list. If you are using AID-HS, please send an email to `[email protected]` with the subject 'Request to be added to the AID-HS mailing list' and provide use with your name and institute. This will ensure that we can update you about bug fixs and new releases. 
+
+**<span style="color: red;">EXISTING USERS: PLEASE UPDATE TO VERSION V1.0.1</span>**:
+We have released AID-HS V1.0.1 which fixes a couple of issues found by users. For more information about the release please see [AID-HS V1.0.1](https://github.com/MELDProject/AID-HS/releases/tag/v1.0.1). To update your code please follow the guidelines [Updating AID-HS to V1.0.1](https://aid-hs.readthedocs.io/en/latest/FAQs.html#Updating-AID-HS-to-V1.0.1) from our FAQ.
+
+
 Pipeline overview:\
 <img src="https://raw.githubusercontent.com//MELDProject/AID-HS/main/docs/images/overview_pipeline.jpg " height="500" />
 
@@ -30,6 +37,10 @@ You can install and use the AID-HS pipeline with :
 
 **YouTube tutorial available for the [docker and singularity installation](https://www.youtube.com/watch?v=RRAET7r05ys&t=11s&ab_channel=MELDproject)**
 
+**FAQs** 
+If you have a question or if you are running into issues at any stage (installation/use/interpretation), have a look at our [FAQs](https://aid-hs.readthedocs.io/en/latest/FAQs.html) page as we may have already have a solution. 
+
+
 ### Running the pipeline 
 Once installed you will be able to use the AID-HS pipeline on your data following the steps:
 1. Prepare your data : [guidelines](https://aid-hs.readthedocs.io/en/latest/prepare_data.html)

aidhs/__init__.py

@@ -1,3 +1,3 @@
 __author__ = __maintainer__ = "Mathilde Ripart"
 __email__ = "[email protected]"
-__version__ = "0.1.0"
+__version__ = "1.0.1"

aidhs/aidhs_cohort_hip.py

@@ -329,7 +329,7 @@ def get_subject_ids(self, **kwargs):
         if isinstance(site_codes, str):
             site_codes = [site_codes]
         # get scanners
-        scanners = kwargs.get("scanners", ["3T", "15T"])
+        scanners = kwargs.get("scanners", ["3T", "15T", "XT"])
         if not isinstance(scanners, list):
             scanners = [scanners]
 
@@ -447,16 +447,16 @@ def __init__(self, subject_id, cohort):
     #adapted for hippo
     @property
     def scanner(self):
+        # Note: no need to specify scanner strength with AIDHS pipeline, but still need it to be compatible with previous AIDHS dataset
         scanner = self.get_demographic_features('Scanner')
+        if scanner is None:
+            scanner="XT" #no need to specify
         if scanner in ("15T" , "1.5T" , "15t" , "1.5t" ):
-            scanner="15T"
+            scanner="15T" # to be compatible with old way
         elif scanner in ("3T" , "3t" ):
-            scanner="3T"
+            scanner="3T" # to be compatible with old way
         else:
-            print(
-                f"Error: incorrect scanner for {self.subject_id}. Unable to determine if scanner 1.5T or 3T "
-            )
-            sys.exit()
+            scanner="XT" #no need to specify 
         return scanner
 
     #adapted for hippo
@@ -480,6 +480,11 @@ def surf_dir_path(self, hemi):
         """return path to features dir (surf_dir)"""
         return os.path.join(self.site_code, self.scanner, self.group, self.subject_id, hemi)
 
+    def find_path(self, name):
+         """ Find the first object with the subject id in the hdf5"""
+         if self.subject_id in name:
+             return name 
+         
     @property
     def is_patient(self):
         return self.group == "patient"
@@ -499,10 +504,10 @@ def get_lesion_hemisphere(self):
             return None
 
         with self.cohort._site_hdf5(self.site_code, self.group) as f:
-            surf_dir_lh = f.require_group(self.surf_dir_path("lh"))
+            surf_dir_lh = f[os.path.join(self.site_code, f[self.site_code].visit(self.find_path), "lh")]
             if ".on_lh.lesion.mgh" in surf_dir_lh.keys():
                 return "lh"
-            surf_dir_rh = f.require_group(self.surf_dir_path("rh"))
+            surf_dir_rh = f[os.path.join(self.site_code, f[self.site_code].visit(self.find_path), "rh")]
             if ".on_lh.lesion.mgh" in surf_dir_rh.keys():
                 return "rh"
         return None
@@ -514,7 +519,8 @@ def has_features(self, features):
     def get_feature_list(self, hemi="lh"):
         """Outputs a list of the features a participant has for each hemisphere"""
         with self.cohort._site_hdf5(self.site_code, self.group) as f:
-            keys = list(f[self.surf_dir_path(hemi)].keys())
+            surf_dir_path = os.path.join(self.site_code, f[self.site_code].visit(self.find_path), hemi)
+            keys =  list(f[surf_dir_path].keys())
             if ".on_lh.lesion.mgh" in keys:
                 keys.remove(".on_lh.lesion.mgh")
         return keys
@@ -573,6 +579,8 @@ def get_demographic_features(
 
                 if "urfer" in desired_name:
                     matched_name = 'Freesurfer_nul'
+                elif "Scanner" in desired_name:
+                     return None
                 else:
                     self.log.warning(f"Unable to find column matching {desired_name}, please double check for typos")
                     return None
@@ -616,7 +624,7 @@ def load_feature_values(self, feature, hemi="lh"):
         feature_values = np.zeros(NVERT, dtype=np.float32)
         # read data from hdf5
         with self.cohort._site_hdf5(self.site_code, self.group) as f:
-            surf_dir = f[self.surf_dir_path(hemi)]
+            surf_dir = f[os.path.join(self.site_code, f[self.site_code].visit(self.find_path), hemi)]
             if feature in surf_dir.keys():
                 feature_values[:] = surf_dir[feature][:]
             else:

aidhs/data_preprocessing.py

@@ -216,7 +216,7 @@ def load_covars(self, subject_ids=None, demographic_file=DEMOGRAPHIC_FEATURES_FI
             else:
                 print(f'ERROR: There is an issue with the coded sex of subject {subject}')
             group.append(subj.is_patient)
-            sites_scanners.append(subj.site_code + "_" + subj.scanner)
+            sites_scanners.append(subj.site_code) # just site code now
 
         covars["ages"] = ages
         covars["sex"] = sex
@@ -263,8 +263,8 @@ def compute_avg_feature(self, feature):
                 vals_avg.append(np.hstack([vals_avg_lh, vals_avg_rh]))
                 subject_include[k] = True
             else:
-                print("exclude")
                 subject_include[k] = False
+        print(f"INFO: exclude subjects {np.array(self.subject_ids)[~subject_include]}")
         if vals_avg:
             vals_avg = np.array(vals_avg)  
             self.save_cohort_features(feature.format('avg'), vals_avg, np.array(self.subject_ids)[np.array(subject_include)])
@@ -294,8 +294,8 @@ def extract_volumes_avg(self, feature):
                 vals.append(np.hstack([vals_lh, vals_rh]))
                 subject_include[k] = True
             else:
-                print("exclude")
                 subject_include[k] = False
+        print(f"INFO: exclude subjects {np.array(self.subject_ids)[~subject_include]}")
         if vals:
             vals = np.array(vals)  
             self.save_cohort_features(feature, vals, np.array(self.subject_ids)[np.array(subject_include)])
@@ -493,8 +493,8 @@ def combat_whole_cohort(self, feature_name, outliers_file=None, combat_params_fi
                 precombat_features.append(combined_hemis)
                 combat_subject_include[k] = True
             else:
-                print("exclude")
                 combat_subject_include[k] = False
+        print(f"INFO: exclude subjects {np.array(self.subject_ids)[~combat_subject_include]}")
         if precombat_features:
             precombat_features = np.array(precombat_features)
             # load in covariates - age, sex, group, site and scanner unless provided
@@ -677,12 +677,13 @@ def combat_new_subject(self, feature_name, combat_params_file):
                 rh = subj.load_feature_values(feature_name, hemi="rh")[mask]
                 combined_hemis = np.hstack([lh, rh])
                 precombat_features.append(combined_hemis)
-                site_scanner.append(subj.site_code + "_" + subj.scanner)
+                site_scanner.append(subj.site_code) # just site code now
                 subjects_included.append(subject)
         #if matrix empty, pass
         if precombat_features:
             combat_estimates = self.read_norm_combat_parameters(feature_name, combat_params_file)
             combat_estimates = self.unshrink_combat_estimates(combat_estimates)
+            combat_estimates["batches"] = [x.split('_')[0] for x in combat_estimates["batches"]] # remove scanner strenght from the batch code if exist
             precombat_features = np.array(precombat_features)
             site_scanner = np.array(site_scanner)
             dict_combat = neuroCombatFromTraining(dat=precombat_features.T, batch=site_scanner, estimates=combat_estimates)
@@ -818,8 +819,8 @@ def asymmetry_internorm_subject(self, feature, cohort_for_norm=None, params_norm
                 vals_asym = self.compute_asym(intra_norm)
                 vals_asym_array.append(vals_asym)
             else:
-                print('exlude subject {}'.format(id_sub))
-                included_subjects[k] = False       
+                included_subjects[k] = False
+        print(f"INFO: exclude subjects {np.array(self.subject_ids)[~included_subjects]}")    
         vals_asym_array=np.array(vals_asym_array)
         # remove exclude subjects
         included_subjects = np.array(self.subject_ids)[included_subjects]