added github actions yml

Author: miloswrath

.github/workflows/main.yml

@@ -0,0 +1,39 @@
+name: temp name
+
+on:
+  push:
+    branches: [ "main", "dev" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+  run-main:
+    runs-on: ubuntu-latest
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10" # Specify one Python version for `run-main`
+    - name: Run main handler
+      run: |
+        python code/main_handler.py all

code/data_processing/__pycache__/save_utils.cpython-39.pyc

@@ -4,7 +4,7 @@
 
 class SAVE_EVERYTHING:
     def __init__(self):
-        self.datadir = './data/test'
+        self.datadir = './data'
         self.sessions = {}  # Dictionary to track session numbers for each subjectID
 
     def _get_folder(self, subjID):

code/data_processing/__pycache__/utils.cpython-39.pyc

@@ -3,6 +3,7 @@
 import pandas as pd
 import numpy as np
 
+from termcolor import cprint
 
 class CONVERT_TO_CSV:
     def __init__(self, task, init_path='./data/raw'):
@@ -20,8 +21,12 @@ def convert_to_csv(self, txt_dfs):
             tweets = []
 
             for line in lines:
+                print(line)
                 if line.strip():
-                    tweets.append(json.loads(line))
+                    try:
+                        tweets.append(json.loads(line))
+                    except json.JSONDecodeError as e:
+                        cprint(f"JSONDecodeError: {e} on line: {line}", 'red')
 
             if not tweets:
                 # If file is empty or has no valid lines, skip

code/data_processing/save_utils.py

@@ -38,7 +38,7 @@ def pull(self, task):
             proxy=False
         )
 
-        txt_dfs = pull_instance.load(days_ago=32)
+        txt_dfs = pull_instance.load(days_ago=127)
         return self.convert_to_csv(txt_dfs, task)
 
     def convert_to_csv(self, txt_dfs, task):