pushing for linux automation

Author: miloswrath

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
 
       - name: Install R packages
         run: |
-          Rscript -e 'install.packages(c("ggplot2", "plyr", "optparse", "tidyr", "GGIR"), repos="https://cloud.r-project.org")'
+          Rscript -e 'install.packages(c("ggplot2", "plyr", "optparse", "tidyr", "GGIR"), repos="https://cloud.r-project.org", dependencies=TRUE)'
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v3

code/main.py

@@ -1,19 +1,20 @@
 from utils.mnt import create_symlinks
 from utils.save import Save
 from core.gg import GG
+import sys
 
 INT_DIR = '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-experiment/data/act-int-test'
 OBS_DIR = '/Volumes/vosslabhpc/Projects/BOOST/ObservationalStudy/3-experiment/data/act-obs-test'
 RDSS_DIR = '/Volumes/VossLab/Repositories/Accelerometer_Data'
 
 class pipe:
-    def __init__(self):
-        self.funny = False # i happen to be pretty lame
+    def __init__(self, daysago=None):
+        self.daysago = daysago
 
     def run_pipe(self):
         self._create_syms()
-        matched = Save(intdir=INT_DIR, obsdir=OBS_DIR, rdssdir=RDSS_DIR).save()
-        with open('res/data.json', 'w') as file:
+        matched = Save(intdir=INT_DIR, obsdir=OBS_DIR, rdssdir=RDSS_DIR, daysago=self.daysago).save()
+        with open('code/res/data.json', 'w') as file:
             file.write('{\n}')
             file.write(',\n'.join(f'   "{key}": "{value}"' for key, value in matched.items()))
         #GG(matched=matched, intdir=INT_DIR, obsdir=OBS_DIR).run_gg()
@@ -24,5 +25,14 @@ def _create_syms(self):
 
 
 if __name__ == '__main__':
-    p = pipe()
+    if len(sys.argv) > 1:  # Ensure there's an argument
+        try:
+            daysago = int(sys.argv[1])  # Convert argument to integer
+        except ValueError:
+            print("Error: daysago must be an integer")
+            sys.exit(1)  # Exit script with error
+    else:
+        daysago = None  # Default to None if no argument is provided
+
+    p = pipe(daysago=daysago)
     p.run_pipe()

code/res/data.json

@@ -1,6 +1,3 @@
 {
-   "8022": "[{'filename': '1023 (2022-06-15)RAW.csv', 'labID': '1023', 'date': '2022-06-15', 'run': 1, 'study': 'int', 'file_path': '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-experiment/data/act-int-test/sub-8022/accel/sub-8022_ses-1_accel.csv'}]",
-   "8002": "[{'filename': '1124 (2024-11-07)RAW.csv', 'labID': '1124', 'date': '2024-11-07', 'run': 1, 'study': 'int', 'file_path': '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-experiment/data/act-int-test/sub-8002/accel/sub-8002_ses-1_accel.csv'}]",
-   "8018": "[{'filename': '1167 (2025-01-22)RAW.csv', 'labID': '1167', 'date': '2025-01-22', 'run': 1, 'study': 'int', 'file_path': '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-experiment/data/act-int-test/sub-8018/accel/sub-8018_ses-1_accel.csv'}]",
-   "7043": "[{'filename': '1174 (2025-01-28)RAW.csv', 'labID': '1174', 'date': '2025-01-28', 'run': 1, 'study': 'obs', 'file_path': '/Volumes/vosslabhpc/Projects/BOOST/ObservationalStudy/3-experiment/data/act-obs-test/sub-7043/accel/sub-7043_ses-1_accel.csv'}]"
-}
+}   "7069": "[{'filename': '1207 (2025-03-01)RAW.csv', 'labID': '1207', 'date': Timestamp('2025-03-01 00:00:00'), 'run': 1, 'study': 'obs', 'file_path': '/Volumes/vosslabhpc/Projects/BOOST/ObservationalStudy/3-experiment/data/act-obs-test/sub-7069/accel/sub-7069_ses-1_accel.csv'}]",
+   "7073": "[{'filename': '1211 (2025-03-01)RAW.csv', 'labID': '1211', 'date': Timestamp('2025-03-01 00:00:00'), 'run': 1, 'study': 'obs', 'file_path': '/Volumes/vosslabhpc/Projects/BOOST/ObservationalStudy/3-experiment/data/act-obs-test/sub-7073/accel/sub-7073_ses-1_accel.csv'}]"

code/utils/comparison_utils.py

@@ -3,16 +3,18 @@
 import logging
 import pandas as pd
 import requests
+from datetime import datetime, timedelta
 from io import StringIO
 
 
 class ID_COMPARISONS:
 
-    def __init__(self, mnt_dir) -> None:
+    def __init__(self, mnt_dir, daysago=None) -> None:
        self.token = 'DE4E2DB72778DACA9B8848574107D2F5'
        self.mnt_dir = mnt_dir
        self.INT_DIR = '/Volumes/vosslabhpc/Projects/BOOST/InterventionStudy/3-experiment/data/act-int-test'
        self.OBS_DIR = '/Volumes/vosslabhpc/Projects/BOOST/ObservationalStudy/3-experiment/data/act-obs-test'
+       self.daysago = daysago
        logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
     def compare_ids(self):
@@ -26,7 +28,7 @@ def compare_ids(self):
         # Retrieve the RedCap report and duplicates from report
         report, report_duplicates = self._return_report()
         # Retrieve the full RDSS file list and duplicate files merged with duplicates from report
-        rdss, file_duplicates = self._rdss_file_list(report_duplicates)
+        rdss, file_duplicates = self._rdss_file_list(report_duplicates, self.daysago)
 
         # Initialize the result dictionary for normal (non-duplicate) matches
         result = {}
@@ -110,7 +112,7 @@ def _return_report(self):
 
         return df_cleaned, duplicate_rows
 
-    def _rdss_file_list(self, duplicates):
+    def _rdss_file_list(self, duplicates, daysago=None):
         """
         Extracts the first string before the space and the date from filenames ending with .csv
         in the specified folder and stores them in a DataFrame.
@@ -138,7 +140,12 @@ def _rdss_file_list(self, duplicates):
 
         if not df.empty:
             df['Date'] = pd.to_datetime(df['Date'], errors='coerce')
-            df = df[df['Date'] >= '2024-08-05']  # Filter out rows before the threshold date
+
+            if daysago:
+                cutoff_date = datetime.today() - timedelta(days=daysago)
+                df = df[df['Date'] >= cutoff_date]  # Filter files within the last `daysago` days
+            else:
+                df = df[df['Date'] >= '2024-08-05']  # Filter out rows before the threshold date
 
         # Filter the file list to only include rows where ID is in the duplicate report (if any)
         if not duplicates.empty:

code/utils/save.py

@@ -5,8 +5,8 @@
 
 class Save:
 
-    def __init__(self, intdir, obsdir, rdssdir):
-        results = ID_COMPARISONS('../mnt').compare_ids()
+    def __init__(self, intdir, obsdir, rdssdir, daysago=None):
+        results = ID_COMPARISONS('../mnt', daysago).compare_ids()
         self.matches = results['matches']
         self.dupes = results['duplicates']
         print(f"Type of Dupes: {type(self.dupes)}")
@@ -25,7 +25,7 @@ def save(self):
             matches = self._handle_and_merge_duplicates(self.dupes)
 
         # Move the files based on the final matches.
-        self._move_files(matches=matches)
+        #self._move_files(matches=matches)
         return matches
 
     def _move_files_test(self, matches):