finsihed orchestrate.py, made automate changes to match.py, getting ready for testing

Author: miloswrath

README.md

@@ -7,11 +7,11 @@ This repo is meant to house all of the code necessary to transfer actigraphy sou
 This repo contains a minimal amount of hard-coding to allow for reproducibility for future studies, however, there are some parts specific to BOOST that are hard coded like Observational vs. Interventional subject numbers. If you are reproducing this, please check through some of the files to make sure they do not contain any hard codes.
 
 ## Tasks
-- [ ] Fix directory structure for Boost I/O folders on LSS -> Cannot because permissions
+- [x] Fix directory structure for Boost I/O folders on LSS -> Cannot because permissions
     - [x] add bids folder to observational -> see above
 - [x] Find out how to change output naming convention for GGIR
 - [x] Add .csvs to BIDS not sourcedata
     - [x] GGIR is formatted as GGIR-#.# with no more version numbers after the second. Full version number should be put into the readme
- - [ ] Finish adding all required code to move to linux
+ - [x] Finish adding all required code to move to linux
  - [ ] move to linux?
 

code/orchestrate.py

@@ -12,12 +12,14 @@
 import sys
 
 OBSDIR = '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-Experiment/data/bids'
-INTDIR = '/Volumes/vosslabhpc/Projects/BOOST/ObersvationalStudy/3-Experiment/data'
+INTDIR = '/Volumes/vosslabhpc/Projects/BOOST/ObersvationalStudy/3-Experiment/data/bids'
+RDSSDIR = '/Volumes/VossLab/Repositories/Accelerometer_Data/'
+TXT = './resources/files.txt'
+
 
 def parse_args():
     parser = argparse.ArgumentParser(description="move act files across locations and run GGIR on them")
     parser.add_argument('-i', type=str, required=True, help='Input Directory (RDSS)')
-    parser.add_argument('-t', type=str, required=True, help='Text file of all the already ran files')
     parser.add_argument('-token', type=str, required=True, help='Token for RedCap API')
     args = parser.parse_args()
     return args
@@ -30,29 +32,28 @@ def init_servers():
         os.system("sudo mount -t cifs //itf-rs-store24.hpc.uiowa.edu/vosslabhpc /home/vosslab-svc/tmp/vosslabhpc -o uid=vosslab-svc,username=vosslab-svc,vers=3.0")
     except Exception as e:
         print(f'An error occured trying to connect to LSS: {e}')
-
+        sys.exit(1)
     try:
         os.system("sudo mount -t cifs //rdss.iowa.uiowa.edu/rdss_mwvoss/VossLab /mnt/nfs/rdss/rdss_vosslab -o user=vosslab-svc,uid=2418317,gid=900001021")
     except Exception as e:
         print(f'An error occured trying to connect to RDSS: {e}')
-
+        sys.exit(1)
     return None
 
 def check_files():
     from src.match import get_files, compare
 
-    rdss_files = get_files(#placeholder_RDSS_dir
-                           )
+    rdss_files = get_files(RDSSDIR)
 
     if len(rdss_files) == 0:
         print("error: RDSS files were not grabbed - find out why")
 
-    need = compare(rdss_files, placeholder_text_file)
+    need = compare(rdss_files, TXT)
 
 
     if len(need) == 0:
         print("no files need GGIR at the moment. quitting...")
-        exit()
+        sys.exit(1)
 
     return need
 
@@ -68,12 +69,12 @@ def create_comparable_dataframe(need):
     errors = []
     for index, row in lab_id_file.iterrows():
         #check if first column is string of four numbers
-        if not isinstance(row[0], str) or not row[0].isdigit() or len(row[0]) != 4:
-            errors.append(f"Row {index} has an invalid lab id: {row[0]}")
+        if not isinstance(str(row[0]), str) or not str(row[0]).isdigit() or len(str(row[0])) != 4:
+            errors.append(f"Row {index} has an invalid lab id: {str(row[0])}")
 
         #check if second column is valid filepath
-        if not os.path.isfile(row[1]):
-            errors.append(f"Row {index} has an invalid filepath: {row[1]}")
+        if not os.path.isfile(str(row[1])):
+            errors.append(f"Row {index} has an invalid filepath: {str(row[1])}")
 
         if errors:
             for error in errors:
@@ -85,12 +86,12 @@ def create_comparable_dataframe(need):
     return lab_id_file
 
 
-def get_redcap_list_and_compare(token, files, list):
+def get_redcap_list_and_compare(token, files):
 
     from src.match import get_list, compare_ids, add_sub_to_sublist, evaluate_run
     sub_lab_id = get_list(token)
 
-    matched_df = compare_ids(files, list)
+    matched_df = compare_ids(files, sub_lab_id)
 
     add_sub_to_sublist(matched_df)
 
@@ -106,21 +107,31 @@ def save_n_rename(matched, indir):
 
     return outputs
 
-def main():
-    args = parse_args
-
+def GGIR(outs):
+    #this is where we run GGIR using the GGIR function in matched
+    from src.match import GGIR
+    GGIR(outs)
+    return None
 
 
-    return None
+def main():
+    args = parse_args()
 
-if __name__ == 'main':
-    main()
+    init_servers()
 
+    need = check_files()
 
+    labid_df = create_comparable_dataframe(need)
 
+    matched_df = get_redcap_list_and_compare(args.token, labid_df)
 
+    outputs = save_n_rename(matched_df, args.i)
 
+    GGIR(outputs)
 
+    return None
 
+if __name__ == 'main':
+    main()
 
 

code/src/match.py

@@ -176,21 +176,47 @@ def save_n_rename_files(matched, dir):
         outputs.append(dest)
 
     return outputs
-    
-def GGIR(matched):
 
+def GGIR(matched):
+    import subprocess
     os.system('R') #start R term
 
-    for index, row in matched.iterrows():
-        # Use regex to extract the year from the file name
-        match = re.search(r'\((\d{4})-\d{2}-\d{2}\)', row['raw_file'])
-
-        if row['subject_id']<7000:
-            outdir = os.path.join(OBS_DIR, 'derivatives', 'GGIR-3.1.4', f"sub-{row['subject_id']")
-        else:
-            outdir = os.path.join(INT_DIR, 'derivatives', 'GGIR-3.1.4', f"sub-{row['subject_id']")
-        #run the GGIR script
-        os.system(f"Rscript code/src/accel.R --project_dir {outdir} --project_deriv_dir {outdir}/derivatives --files {row['raw_file']} --verbose TRUE")
+    if isinstance(matched, list):
+        for item in matched:
+            if os.path.exists(item):
+                try:
+                    result = subprocess.run(
+                           [f"Rscript code/src/accel.R --project_dir {item.split('/')[:-1]} --project_deriv_dir {item.split('/')[:-1]}/derivatives --files {item} --verbose TRUE"], 
+                            check=True,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE
+                            )
+                    print(f"Command output: {result.stdout.decode().strip()}")
+                except subprocess.CalledProcessError as e:
+                    print(f"Command failed with exit status: {e.returncode}")
+                    print(f"Error output: {e.stderr.decode().strip()}")
+
+    elif isinstance(matched, pd.DataFrame):
+        for index, row in matched.iterrows():
+            # Use regex to extract the year from the file name
+            match = re.search(r'\((\d{4})-\d{2}-\d{2}\)', row['raw_file'])
+            if row['subject_id']<7000:
+                outdir = os.path.join(OBS_DIR, 'derivatives', 'GGIR-3.1.4', f"sub-{row['subject_id']}")
+            else:
+                outdir = os.path.join(INT_DIR, 'derivatives', 'GGIR-3.1.4', f"sub-{row['subject_id']}")
+            try:
+                result = subprocess.run(
+                        [f"Rscript code/src/accel.R --project_dir {outdir} --project_deriv_dir {outdir}/derivatives --files {row['raw_file']} --verbose TRUE"],
+                        check=True,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE
+                        )
+                print(f"Command output: {result.stdout.decode().strip()}")
+
+            except subprocess.CalledProcessError as e:
+               print(f"Command failed with exit status: {e.returncode}")
+               print(f"Error output: {e.stderr.decode().strip()}") 
+            os.system(f"Rscript code/src/accel.R --project_dir {outdir} --project_deriv_dir {outdir}/derivatives --files {row['raw_file']} --verbose TRUE")
     os.system('q()')
     return None