made changes to orchestrate and match for handling conditional cases better

Author: miloswrath

README.md

@@ -13,5 +13,5 @@ This repo contains a minimal amount of hard-coding to allow for reproducibility
 - [x] Add .csvs to BIDS not sourcedata
     - [x] GGIR is formatted as GGIR-#.# with no more version numbers after the second. Full version number should be put into the readme
  - [x] Finish adding all required code to move to linux
- - [ ] move to linux?
-
+ - [x] move to linux?
+- [ ] Test on linux and prepare prelims

code/src/cron-act.sh

@@ -4,10 +4,8 @@
 
 #connect to LSS/RDSS
 
-sudo mount -t cifs "//itf-rs-store24.hpc.uiowa.edu/vosslabhpc" /home/vosslab-svc/tmp/vosslabhpc -o uid=vosslab-svc,username=vosslab-svc,vers=3.0
 
 
-sudo mount -t cifs //rdss.iowa.uiowa.edu/rdss_mwvoss/VossLab /mnt/nfs/rdss/rdss_vosslab -o user=vosslab-svc,uid=2418317,gid=900001021
 
 
 cd /path/to/code
@@ -17,7 +15,6 @@ source /path/to/venv/bin/activate
 
 python3 src/match.py --indir /Volumes/VossLab/Repositories/Accelerometer_Data --txt ./code/resources/files.txt --token DE4E2DB72778DACA9B8848574107D2F5
 
-place
 
 #deconnect from RDSS and LSS
 #deactivate the virtual environment

code/src/match.py

@@ -8,7 +8,6 @@
 
 INT_DIR = '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-experiment/data/bids'
 OBS_DIR = '/Volumes/vosslabhpc/Projects/BOOST/ObservationalStudy/3-experiment/data/'
-    
 
 def parse_args():
     argparser = argparse.ArgumentParser(description='Match files to REDCap')
@@ -29,7 +28,6 @@ def save_files(dir, txt):
         f.close()
     return txt
 
-        
 def get_files(dir):
     data = os.listdir(dir)
     return data
@@ -39,7 +37,6 @@ def compare(files, txt):
     with open(txt, 'r') as f:
         data = f.read().splitlines()
     f.close()
-    
     for file in files:
         if file not in data:
             need.append(file)
@@ -72,26 +69,54 @@ def get_list(token):
     df = pd.read_csv(StringIO(r.text))
     return df
 
+
 def compare_ids(files, list):
+    """
+    Compare IDs between two DataFrames and log matched and unmatched entries.
+    Args:
+        files (pd.DataFrame): DataFrame containing 'labid' and 'file' columns.
+        list (pd.DataFrame): DataFrame containing 'lab_id' and 'boost_id' columns.
+    Returns:
+        pd.DataFrame: Matched rows with columns 'lab_id', 'raw_file', and 'subject_id'.
+    """
     # Ensure both columns are of the same type before merging
     files['labid'] = files['labid'].astype(str)
     list['lab_id'] = list['lab_id'].astype(str)
-    
     # Merge files and list dataframes with validation
     matched = pd.merge(
-        files, 
-        list, 
-        left_on='labid', 
-        right_on='lab_id', 
-        how='inner', 
+        files,
+        list,
+        left_on='labid',
+        right_on='lab_id',
+        how='inner',
         validate='many_to_one'
     )
 
     # Rename columns to match the desired output
     matched = matched[['lab_id', 'file', 'boost_id']].rename(columns={'file': 'raw_file', 'boost_id': 'subject_id'})
 
-    # Print matched lab_ids
-    print('Matched lab_ids:', matched['lab_id'].unique())
+    # Identify unmatched files and subs
+    unmatched_files = files[~files['labid'].isin(matched['lab_id'])]
+    unmatched_subs = list[~list['lab_id'].isin(matched['lab_id'])]
+    
+    # Log matched IDs
+    print("Matched IDs:")
+    print(f"- Total Matched: {len(matched)}")
+    print(f"- Matched Lab IDs: {', '.join(matched['lab_id'].unique())}")
+    
+    # Log unmatched files
+    if not unmatched_files.empty:
+        print("Unmatched Files:")
+        print(unmatched_files[['labid', 'file']].to_string(index=False))
+    else:
+        print("No unmatched files found.")
+    
+    # Log unmatched subs
+    if not unmatched_subs.empty:
+        print("Unmatched Subs:")
+        print(unmatched_subs[['lab_id', 'boost_id']].to_string(index=False))
+    else:
+        print("No unmatched subs found.")
 
     return matched
 
@@ -140,7 +165,7 @@ def evaluate_run(matched):
 import re
 
 def save_n_rename_files(matched, dir):
-    outputs = []    
+    outputs = []
     for index, row in matched.iterrows():
         # Use regex to extract the year from the file name
         match = re.search(r'\((\d{4})-\d{2}-\d{2}\)', row['raw_file'])
@@ -149,7 +174,7 @@ def save_n_rename_files(matched, dir):
             outdir = OBS_DIR
         else:
             outdir = INT_DIR
-        
+
         if match:
             year = int(match.group(1))
         else:

docs/.gitkeeper

@@ -0,0 +1,156 @@
+anyio==4.6.2.post1
+appnope==0.1.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+astroid==3.3.5
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==24.2.0
+autopep8==2.0.4
+babel==2.16.0
+beautifulsoup4==4.12.3
+black==24.10.0
+bleach==6.2.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+click==8.1.7
+comm==0.2.2
+contourpy==1.3.1
+cycler==0.12.1
+debugpy==1.8.8
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.9
+docstring-to-markdown==0.15
+executing==2.1.0
+fastjsonschema==2.20.0
+flake8==7.1.1
+fonttools==4.55.0
+fqdn==1.5.1
+greenlet==3.1.1
+h11==0.14.0
+httpcore==1.0.7
+httpx==0.27.2
+idna==3.10
+ipykernel==6.29.5
+ipython==8.29.0
+ipywidgets==8.1.5
+isoduration==20.11.0
+isort==5.13.2
+jedi==0.19.2
+Jinja2==3.1.4
+joblib==1.4.2
+json5==0.9.28
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.14.2
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.6
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+jupytext==1.16.4
+kiwisolver==1.4.7
+lxml==5.3.0
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+matplotlib==3.9.2
+matplotlib-inline==0.1.7
+mccabe==0.7.0
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+mistune==3.0.2
+msgpack==1.1.0
+mypy==1.13.0
+mypy-extensions==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+nibabel==5.3.2
+nilearn==0.10.4
+nodeenv==1.9.1
+notebook==7.2.2
+notebook_shim==0.2.4
+numpy==2.1.3
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pathspec==0.12.1
+pexpect==4.9.0
+pillow==11.0.0
+platformdirs==4.3.6
+pluggy==1.5.0
+prometheus_client==0.21.0
+prompt_toolkit==3.0.48
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyarrow==18.0.0
+pycodestyle==2.12.1
+pycparser==2.22
+pydocstyle==6.3.0
+pyflakes==3.2.0
+Pygments==2.18.0
+pylance==0.19.2
+pylint==3.3.1
+pylsp-mypy==0.6.9
+pynvim==0.5.2
+pyparsing==3.2.0
+pyright==1.1.389
+python-dateutil==2.9.0.post0
+python-json-logger==2.0.7
+python-lsp-black==2.0.0
+python-lsp-isort==0.2.0
+python-lsp-jsonrpc==1.1.2
+python-lsp-server==1.12.0
+pytoolconfig==1.3.1
+pytz==2024.2
+PyYAML==6.0.2
+pyzmq==26.2.0
+referencing==0.35.1
+requests==2.32.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rope==1.13.0
+rpds-py==0.21.0
+scikit-learn==1.5.2
+scipy==1.14.1
+seaborn==0.13.2
+Send2Trash==1.8.3
+setuptools==75.5.0
+six==1.16.0
+sniffio==1.3.1
+snowballstemmer==2.2.0
+soupsieve==2.6
+stack-data==0.6.3
+terminado==0.18.1
+threadpoolctl==3.5.0
+tinycss2==1.4.0
+tomlkit==0.13.2
+tornado==6.4.1
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+tzdata==2024.2
+ujson==5.10.0
+uri-template==1.3.0
+urllib3==2.2.3
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+whatthepatch==1.0.7
+widgetsnbextension==4.0.13
+yapf==0.43.0