Refactor code and fix imports

initial_conf.py

@@ -1,7 +1,8 @@
 
-import pandas as pd
-from getpass import getpass
 import argparse
+from getpass import getpass
+
+import pandas as pd
 
 
 def initial_conf(save_user=True, replace_user=False, global_config_flag=True):

ruff.toml

@@ -18,3 +18,5 @@ ignore = [
     "E501"
 ]
 
+[lint.per-file-ignores]
+"__init__.py" = ["F401"]

scripts/conf_file_finding.py

@@ -2,6 +2,7 @@
 import os
 import pathlib
 
+
 def chdir_to_root():
 
     root_dir_found = 0

scripts/create_schemas.py

@@ -1 +0,0 @@
-from u19_pipeline import lab, reference, subject, task, action, acquisition, behavior, puffs

scripts/drop_schemas.py

@@ -1,6 +1,18 @@
-from u19_pipeline import lab, subject, acquisition, behavior, task, reference, recording, recording_process, ephys_pipeline, imaging_pipeline, ephys_sync
-from u19_pipeline.imaging_pipeline import scan_element, imaging_element
-from u19_pipeline.ephys_pipeline import probe_element, ephys_element
+from u19_pipeline import (
+    acquisition,
+    behavior,
+    ephys_pipeline,
+    ephys_sync,
+    imaging_pipeline,
+    lab,
+    recording,
+    recording_process,
+    reference,
+    subject,
+    task,
+)
+from u19_pipeline.ephys_pipeline import ephys_element, probe_element
+from u19_pipeline.imaging_pipeline import imaging_element, scan_element
 
 behavior.schema.drop()
 recording_process.schema.drop()

scripts/ingest_hdf5_to_db.py

@@ -4,17 +4,20 @@
 # Folder structure for hdf5 files on bucket will be:
 # /jukebox/braininit/puffs/{netid}/{project_name}/{cohort}/{rig}/{hdf5_filename}
 
+import glob
+import json
+import os
+import sys
+from datetime import timedelta
+
 import datajoint as dj
-import os, sys, glob, json
-import pandas as pd
 import numpy as np
-import time
-from datetime import datetime, timedelta
+import pandas as pd
 
 dj.config['database.host'] = 'datajoint00.pni.princeton.edu'
 # load dj creds from file
 credfile = '/jukebox/wang/ahoag/.djenv'
-with open(credfile,'r') as infile:
+with open(credfile) as infile:
     cred_dict = json.load(infile)
 
 dj.config['database.user'] = cred_dict.get('DJ_DB_USER')
@@ -314,18 +317,9 @@
             this_session_notes_key = f'sessions/{session_compressed_str}/notes'
             if this_session_notes_key in stored_session_keys:
                 session_notes_dict = json.loads(data[this_session_notes_key].iloc[0])
-                if 'notes' in session_notes_dict:
-                    session_notes = session_notes_dict['notes']
-                else:
-                    session_notes = None
-                if 'stdout' in session_notes_dict:
-                    session_stdout = session_notes_dict['stdout']
-                else:
-                    session_stdout = None
-                if 'stderr' in session_notes_dict:
-                    session_stderr = session_notes_dict['stderr']
-                else:
-                    session_stderr = None
+                session_notes = session_notes_dict.get('notes', None)
+                session_stdout = session_notes_dict.get('stdout', None)
+                session_stderr = session_notes_dict.get('stderr', None)
             else:
                 session_notes = None
                 session_stdout = None

scripts/populate_imaging.py

@@ -1,5 +1,4 @@
-from u19_pipeline_python import lab, reference, subject, task, action, acquisition, imaging
-
+from u19_pipeline_python import acquisition
 
 kargs = dict(supress_errors=True, display_progress=True)
 acquisition.Scan.populate(**kargs)

setup.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
-from setuptools import setup, find_packages
 from os import path
 
+from setuptools import find_packages, setup
+
 here = path.abspath(path.dirname(__file__))
 with open(path.join(here, 'requirements.txt')) as f:
     requirements = f.read().splitlines()

u19_pipeline/__init__.py

@@ -1,3 +1,4 @@
-from os.path import join, dirname
 import os
-import datajoint as dj
+from os.path import dirname, join
+
+import datajoint as dj

u19_pipeline/acquisition.py

@@ -1,5 +1,4 @@
 import datajoint as dj
-from u19_pipeline import lab, task, subject
 
 schema = dj.schema(dj.config['custom']['database.prefix'] + 'acquisition')
 

u19_pipeline/action.py

@@ -2,7 +2,6 @@
 
 
 import datajoint as dj
-from . import lab, reference, subject
 
 schema = dj.schema(dj.config['custom']['database.prefix'] + 'action')
 

u19_pipeline/alert_system/alert_code_skeleton.py

@@ -1,11 +1,5 @@
 
-import pandas as pd
-import datajoint as dj
-import datetime
-import numpy as np
 
-import u19_pipeline.alert_system.behavior_metrics as bm
-import u19_pipeline.alert_system.alert_system_utility as asu
 
 
 # Slack Configuration dictionary

u19_pipeline/alert_system/alert_system_utility.py

@@ -1,11 +1,13 @@
 
 
-import pandas as pd
-import datajoint as dj
 import datetime
 
+import datajoint as dj
+import pandas as pd
+
 import u19_pipeline.utils.dj_shortcuts as djs
 
+
 def get_acquisition_data_alert_system(type='subject_fullname', data_days=60, min_sessions=20):
     '''
     Get and filter data for alert system

u19_pipeline/alert_system/behavior_metrics.py

@@ -1,9 +1,7 @@
 
-import pandas as pd
-import datajoint as dj
 
 
-class BehaviorMetrics():
+class BehaviorMetrics:
 
     @staticmethod
     def get_bias_from_trial_df(trials_df, return_all_metrics=False):

u19_pipeline/alert_system/cronjob_alert.py

@@ -1,12 +1,13 @@
 
 import time
+
+import u19_pipeline.alert_system.main_alert_system as mas
 from scripts.conf_file_finding import try_find_conf_file
+
 try_find_conf_file()
 
 time.sleep(1)
 
-import datajoint as dj
-import u19_pipeline.alert_system.main_alert_system as mas
 
 mas.main_alert_system()
 

u19_pipeline/alert_system/custom_alerts/__init__.py

@@ -1,2 +1,3 @@
 import pkgutil
+
 __path__ = pkgutil.extend_path(__path__, __name__)

u19_pipeline/alert_system/custom_alerts/braininit_storage.py

@@ -1,14 +1,9 @@
 
-import pandas as pd
-import datajoint as dj
-
-import u19_pipeline.lab as lab
-
-import u19_pipeline.alert_system.behavior_metrics as bm
-import u19_pipeline.alert_system.alert_system_utility as asu
-import os
 import subprocess
 
+import datajoint as dj
+import pandas as pd
+
 # Slack Configuration dictionary
 slack_configuration_dictionary = {
     'slack_notification_channel': ['custom_alerts'],

u19_pipeline/alert_system/custom_alerts/rig_bias.py

@@ -1,12 +1,12 @@
 
-import pandas as pd
-import datajoint as dj
 import datetime
+
+import datajoint as dj
 import numpy as np
+import pandas as pd
 
-import u19_pipeline.alert_system.behavior_metrics as bm
 import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
 
 # Slack Configuration dictionary
 slack_configuration_dictionary = {

u19_pipeline/alert_system/custom_alerts/rig_trial.py

@@ -1,12 +1,10 @@
 
-import pandas as pd
-import datajoint as dj
 import datetime
+
 import numpy as np
 
-import u19_pipeline.alert_system.behavior_metrics as bm
 import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
 
 # Slack Configuration dictionary
 slack_configuration_dictionary = {

u19_pipeline/alert_system/custom_alerts/subject_bias.py

@@ -1,12 +1,12 @@
 
-import pandas as pd
-import datajoint as dj
 import datetime
+
+import datajoint as dj
 import numpy as np
+import pandas as pd
 
-import u19_pipeline.alert_system.behavior_metrics as bm
 import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
 
 # Slack Configuration dictionary
 slack_configuration_dictionary = {

u19_pipeline/alert_system/custom_alerts/subject_trial.py

@@ -1,12 +1,10 @@
 
-import pandas as pd
-import datajoint as dj
 import datetime
+
 import numpy as np
 
-import u19_pipeline.alert_system.behavior_metrics as bm
 import u19_pipeline.alert_system.alert_system_utility as asu
-
+import u19_pipeline.alert_system.behavior_metrics as bm
 
 # Slack Configuration dictionary
 slack_configuration_dictionary = {

u19_pipeline/alert_system/main_alert_system.py

@@ -1,17 +1,14 @@
 
-import pandas as pd
-import datajoint as dj
-import pkgutil
-import importlib
 import datetime
+import importlib
+import pkgutil
+import time
 import traceback
 
 import u19_pipeline.alert_system.custom_alerts as ca
 import u19_pipeline.lab as lab
 import u19_pipeline.utils.slack_utils as su
 
-import time
-
 # Slack Configuration dictionary
 slack_configuration_dictionary = {
     'slack_notification_channel': ['custom_alerts']
@@ -88,7 +85,7 @@ def slack_alert_message_format(alert_dictionaty, alert_module_name):
     m2_1["type"] = "mrkdwn"
 
     m2_1["text"] = ''
-    for key in alert_dictionaty.keys():
+    for key in alert_dictionaty:
         m2_1["text"] += '*' + key + '* : ' + str(alert_dictionaty[key]) + '\n'
     m2['text'] = m2_1
 

u19_pipeline/automatic_job/clusters_paths_and_transfers.py

@@ -1,15 +1,13 @@
-import datajoint as dj
-import pathlib
-import subprocess
 import json
-import re
 import os
+import pathlib
+import subprocess
 import time
 
-from datetime import datetime
-from element_interface.utils import dict_to_uuid
+import datajoint as dj
 
 import u19_pipeline.automatic_job.params_config as config
+
 #Functions to transfer files (globus, scp, smbclient)
 
 #FOR PNI endpoint
@@ -166,7 +164,7 @@ def request_globus_transfer(job_id_str, source_ep, dest_ep, source_filepath, des
                 dict_output = json.loads(p.stdout.decode('UTF-8'))
                 transfer_request['status'] = config.system_process['SUCCESS']
                 transfer_request['task_id'] = dict_output['task_id']
-            except Exception as e:
+            except Exception:
                 print('stdout is not a valid json, probably an error')
                 transfer_request['status'] = config.system_process['ERROR']
                 transfer_request['error_info'] = p.stdout.decode('UTF-8')
@@ -271,7 +269,7 @@ def get_error_log_str(recording_process_id):
     log_file_local_path = pathlib.Path(local_log_file_dir,log_filename).as_posix()
 
     if os.path.exists(log_file_local_path):
-        with open(log_file_local_path, 'r') as error_log_file:
+        with open(log_file_local_path) as error_log_file:
             error_log_data = ' '.join(error_log_file.readlines())
 
         error_log_data = error_log_data.replace("activate the default environment with 'conda activate' or create a new environment to customize with 'conda create'.\n",'')

u19_pipeline/automatic_job/create_missing_lfps_script.py

@@ -1,11 +1,11 @@
 
 import pathlib
 import sys
-import datajoint as dj
 
-from u19_pipeline import recording, ephys_pipeline, recording, recording_process
+import datajoint as dj
 
 import u19_pipeline.automatic_job.params_config as config
+from u19_pipeline import ephys_pipeline, recording, recording_process
 
 
 def main(recording_id):

u19_pipeline/automatic_job/cronjob_automatic_job.py

@@ -1,12 +1,14 @@
 
 import time
+
+import u19_pipeline.automatic_job.recording_handler as rec_handler
+import u19_pipeline.automatic_job.recording_process_handler as rec_process_handler
 from scripts.conf_file_finding import try_find_conf_file
+
 try_find_conf_file()
 
 time.sleep(1)
 
-import u19_pipeline.automatic_job.recording_handler as rec_handler
-import u19_pipeline.automatic_job.recording_process_handler as rec_process_handler
 
 # Check recordings and then jobs
 rec_handler.RecordingHandler.pipeline_handler_main()

u19_pipeline/automatic_job/ephys_element_ingest.py

@@ -1,13 +1,17 @@
-import re
 import pathlib
-from u19_pipeline import ephys_pipeline
+import re
 
-from u19_pipeline.ephys_pipeline import (probe_element, ephys_element,
-                                        get_ephys_root_data_dir, get_session_directory)
 from element_array_ephys.readers import spikeglx
-
 from element_interface.utils import find_full_path
 
+from u19_pipeline import ephys_pipeline
+from u19_pipeline.ephys_pipeline import (
+    ephys_element,
+    get_ephys_root_data_dir,
+    get_session_directory,
+    probe_element,
+)
+
 """
 The ingestion routine for imaging element includes: