v0.3.3 0 - Merge pull request #35 from wehr-lab/dev

v0.3.3

Bugfixes
* Fix layout in batch reassign from python 3 float division
* Cleaner close by catching KeyboardInterrupt in networking modules
* Fixing audioserver boot options -- if 'AUDIOSERVER' is set even if 'AUDIO' isn't set in prefs, should still start server. Not full fixed, need to make single plugin handler, single point of enabling/disabling optional services like audio server
* Fix conflict between polarity and pull in initializing `pulls` in pilot
* Catch `tables.HDF5ExtError` if local .h5 file corrupt in pilot
* For some reason 'fs' wasn't being replaced in the jackd string, reinstated.
* Fix comparison in LED_RGB that caused '0' to turn on full becuse 'value' was being checked for its truth value (0 is false) rather than checking if value is None. 
* `obj.next()` to `next(obj)` in jackdserver


Improvements
* Better internal handling of pigpiod -- you're now able to import and use hardware modules without needing to explicitly start pigpiod!!
* Hopefully better killing of processes on exit, though still should work into unified process manager so don't need to reimplement everything (eg. as is done with launching pigpiod and jackd)
* Environment scripts have been split out into `setup/scripts.py` and you can now run them with `python -m autopilot.setup.run_script` (use `--help` to see how!) 
* Informative error when setup is run with too narrow terminal: #23
* More loggers, but increased need to unify logger creation!!!


Cleanup
* remove unused imports in main `__init__.py` that made cyclical imports happen more frequently than necessary
* single-sourcing version number from `__init__.py`
* more cleanup of unnecessary meta and header stuff left from early days
* more debugging flags
* filter `NaturalNameWarning` from pytables
* quieter cleanups for hardware objects

Author: sneakers-the-rat

autopilot/__init__.py

@@ -1,6 +1,2 @@
-__version__ = '0.3'
-__author__  = 'Jonny Saunders <[email protected]>'
-
-from autopilot import core
-from autopilot.tasks import Task
-from autopilot.setup import setup_autopilot
+__author__  = 'Jonny Saunders <[email protected]>'
+__version__ = '0.3.3'

autopilot/core/gui.py

@@ -21,6 +21,7 @@
 import json
 import copy
 import datetime
+import time
 from collections import OrderedDict as odict
 import numpy as np
 import ast
@@ -32,21 +33,17 @@
 import threading
 import logging
 from operator import ior
-import pdb
 
 # adding autopilot parent directory to path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from autopilot.core.subject import Subject
 from autopilot import tasks, prefs
 from autopilot.stim.sound import sounds
 from autopilot.core.networking import Net_Node
 from functools import wraps
 from autopilot.core.utils import InvokeEvent
-#from autopilot.core.plots import gui_event
 from autopilot.core import styles
 
-import pdb
-import time
+
 
 
 def gui_event(fn):
@@ -2428,9 +2425,9 @@ def init_ui(self):
             step_box.currentIndexChanged.connect(self.set_step)
 
             # add to layout
-            self.grid.addWidget(subject_lab, i%25, 0+(i/25)*3)
-            self.grid.addWidget(protocol_box, i%25, 1+(i/25)*3)
-            self.grid.addWidget(step_box, i%25, 2+(i/25)*3)
+            self.grid.addWidget(subject_lab, i%25, 0+(np.floor(i/25))*3)
+            self.grid.addWidget(protocol_box, i%25, 1+(np.floor(i/25))*3)
+            self.grid.addWidget(step_box, i%25, 2+(np.floor(i/25))*3)
 
 
 

autopilot/core/networking.py

@@ -162,35 +162,41 @@ def run(self):
 
         The process is kept open by the :class:`tornado.IOLoop` .
         """
-        # init zmq objects
-        self.context = zmq.Context()
-        self.loop = IOLoop()
-
-        # Our networking topology is treelike:
-        # each Station object binds one Router to
-        # send and receive messages from its descendants
-        # each Station object may have one Dealer that
-        # connects it with its antecedents.
-        self.listener  = self.context.socket(zmq.ROUTER)
-        #self.listener.identity = self.id.encode('utf-8')
-        #self.listener.identity = self.id
-        self.listener.setsockopt_string(zmq.IDENTITY, self.id)
-        self.listener.bind('tcp://*:{}'.format(self.listen_port))
-        self.listener = ZMQStream(self.listener, self.loop)
-        self.listener.on_recv(self.handle_listen)
-
-        if self.pusher is True:
-            self.pusher = self.context.socket(zmq.DEALER)
-            #self.pusher.identity = self.id.encode('utf-8')
-            #self.pusher.identity = self.id
-            self.pusher.setsockopt_string(zmq.IDENTITY, self.id)
-            self.pusher.connect('tcp://{}:{}'.format(self.push_ip, self.push_port))
-            self.pusher = ZMQStream(self.pusher, self.loop)
-            self.pusher.on_recv(self.handle_listen)
-            # TODO: Make sure handle_listen knows how to handle ID-less messages
-
-        self.logger.info('Starting IOLoop')
-        self.loop.start()
+        try:
+            # init zmq objects
+            self.context = zmq.Context()
+            self.loop = IOLoop()
+
+            # Our networking topology is treelike:
+            # each Station object binds one Router to
+            # send and receive messages from its descendants
+            # each Station object may have one Dealer that
+            # connects it with its antecedents.
+            self.listener  = self.context.socket(zmq.ROUTER)
+            #self.listener.identity = self.id.encode('utf-8')
+            #self.listener.identity = self.id
+            self.listener.setsockopt_string(zmq.IDENTITY, self.id)
+            self.listener.bind('tcp://*:{}'.format(self.listen_port))
+            self.listener = ZMQStream(self.listener, self.loop)
+            self.listener.on_recv(self.handle_listen)
+
+            if self.pusher is True:
+                self.pusher = self.context.socket(zmq.DEALER)
+                #self.pusher.identity = self.id.encode('utf-8')
+                #self.pusher.identity = self.id
+                self.pusher.setsockopt_string(zmq.IDENTITY, self.id)
+                self.pusher.connect('tcp://{}:{}'.format(self.push_ip, self.push_port))
+                self.pusher = ZMQStream(self.pusher, self.loop)
+                self.pusher.on_recv(self.handle_listen)
+                # TODO: Make sure handle_listen knows how to handle ID-less messages
+
+            self.logger.info('Starting IOLoop')
+            self.loop.start()
+        except KeyboardInterrupt:
+            # normal quitting behavior
+            pass
+        finally:
+            self.release()
 
     def prepare_message(self, to, key, value, repeat=True, flags=None):
         """
@@ -689,8 +695,12 @@ def get_ip(self):
         return unwrap2
 
     def release(self):
-        self.closing.set()
-        self.terminate()
+        try:
+            self.closing.set()
+            self.terminate()
+        except AttributeError:
+            # already been called, NoneTypes have no attribute terminate
+            pass
 
         # Stopping the loop should kill the process, as it's what's holding us in run()
         #self.loop.stop()

autopilot/core/pilot.py

@@ -1,12 +1,7 @@
-#!/usr/bin/python2.7
-
 """
 
 """
 
-__version__ = '0.2'
-__author__ = 'Jonny Saunders <[email protected]>'
-
 import os
 import sys
 import datetime
@@ -18,11 +13,13 @@
 import json
 import base64
 import subprocess
+import warnings
 import numpy as np
 import pandas as pd
 from scipy.stats import linregress
 
 import tables
+warnings.simplefilter('ignore', category=tables.NaturalNameWarning)
 
 from autopilot import prefs
 
@@ -46,10 +43,10 @@
 
     prefs.init(prefs_file)
 
-    if hasattr(prefs, 'AUDIOSERVER') and 'AUDIO' in prefs.CONFIG:
+    if hasattr(prefs, 'AUDIOSERVER') or 'AUDIO' in prefs.CONFIG:
         if prefs.AUDIOSERVER == 'pyo':
             from autopilot.stim.sound import pyoserver
-        elif prefs.AUDIOSERVER == 'jack':
+        elif prefs.AUDIOSERVER in ('jack', True):
             from autopilot.stim.sound import jackclient
 
 from autopilot.core.networking import Pilot_Station, Net_Node, Message
@@ -172,7 +169,7 @@ def __init__(self, splash=True):
         self.init_pigpio()
 
         # Init audio server
-        if hasattr(prefs, 'AUDIOSERVER') and 'AUDIO' in prefs.CONFIG:
+        if hasattr(prefs, 'AUDIOSERVER') or 'AUDIO' in prefs.CONFIG:
             self.init_audio()
 
         # Init Station
@@ -199,10 +196,10 @@ def __init__(self, splash=True):
         self.pulls = []
         if hasattr(prefs, 'PULLUPS'):
             for pin in prefs.PULLUPS:
-                self.pulls.append(gpio.Digital_Out(int(pin), pull='U'))
+                self.pulls.append(gpio.Digital_Out(int(pin), pull='U', polarity=0))
         if hasattr(prefs, 'PULLDOWNS'):
             for pin in prefs.PULLDOWNS:
-                self.pulls.append(gpio.Digital_Out(int(pin), pull='D'))
+                self.pulls.append(gpio.Digital_Out(int(pin), pull='D', polarity=1))
 
         # check if the calibration file needs to be updated
 
@@ -234,7 +231,11 @@ def init_logging(self):
         self.log_formatter = logging.Formatter("%(asctime)s %(levelname)s : %(message)s")
         self.log_handler.setFormatter(self.log_formatter)
         self.logger.addHandler(self.log_handler)
-        self.logger.setLevel(logging.INFO)
+        if hasattr(prefs, 'LOGLEVEL'):
+            loglevel = getattr(logging, prefs.LOGLEVEL)
+        else:
+            loglevel = logging.WARNING
+        self.logger.setLevel(loglevel)
         self.logger.info('Pilot Logging Initiated')
 
     #################################################################
@@ -593,7 +594,7 @@ def init_audio(self):
         if prefs.AUDIOSERVER == 'pyo':
             self.server = pyoserver.pyo_server()
             self.logger.info("pyo server started")
-        elif prefs.AUDIOSERVER == 'jack':
+        elif prefs.AUDIOSERVER in ('jack', True):
             self.jackd = external.start_jackd()
             self.server = jackclient.JackClient()
             self.server.start()
@@ -625,14 +626,18 @@ def open_file(self):
         Opens `prefs.DATADIR/local.h5`, creates a group for the current subject,
         a new table for the current day.
 
+        .. todo::
+
+            This needs to be unified with a general file constructor abstracted from :class:`.Subject` so it doesn't reimplement file creation!!
+
         Returns:
             (:class:`tables.File`, :class:`tables.Table`,
             :class:`tables.tableextension.Row`): The file, table, and row for the local data table
         """
         local_file = os.path.join(prefs.DATADIR, 'local.h5')
         try:
             h5f = tables.open_file(local_file, mode='a')
-        except IOError as e:
+        except (IOError, tables.HDF5ExtError) as e:
             self.logger.warning("local file was broken, making new")
             self.logger.warning(e)
             os.remove(local_file)
@@ -695,10 +700,11 @@ def run_task(self, task_class, task_params):
         h5f, table, row = self.open_file()
 
         # TODO: Init sending continuous data here
-
+        self.logger.debug('Starting task loop')
         while True:
             # Calculate next stage data and prep triggers
             data = next(self.task.stages)() # Double parens because next just gives us the function, we still have to call it
+            self.logger.debug('called stage method')
 
             if data:
                 data['pilot'] = self.name
@@ -718,16 +724,20 @@ def run_task(self, task_class, task_params):
                 if 'TRIAL_END' in data.keys():
                     row.append()
                     table.flush()
+                self.logger.debug('sent data')
+
 
             # Wait on the stage lock to clear
             self.stage_block.wait()
+            self.logger.debug('stage lock passed')
 
             # If the running flag gets set, we're closing.
             if not self.running.is_set():
                 self.task.end()
                 self.task = None
                 row.append()
                 table.flush()
+                self.logger.debug('stopping task')
                 break
 
         h5f.flush()

autopilot/core/plots.py

@@ -17,7 +17,6 @@
 import os
 import numpy as np
 import PySide2 # have to import to tell pyqtgraph to use it
-#import PySide
 import pandas as pd
 from PySide2 import QtGui
 from PySide2 import QtCore
@@ -35,7 +34,6 @@
 pg.setConfigOptions(antialias=True)
 # from pyqtgraph.widgets.RawImageWidget import RawImageWidget, RawImageGLWidget
 
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from autopilot import tasks, prefs
 from autopilot.core import styles
 from .utils import InvokeEvent, Invoker

autopilot/core/subject.py

@@ -18,8 +18,8 @@
 import pandas as pd
 import warnings
 import typing
+import warnings
 from copy import copy
-# sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from autopilot.tasks import GRAD_LIST, TASK_LIST
 from autopilot import prefs
 from autopilot.stim.sound.sounds import STRING_PARAMS
@@ -29,6 +29,9 @@
 else:
     import Queue as queue
 
+# suppress pytables natural name warnings
+warnings.simplefilter('ignore', category=tables.NaturalNameWarning)
+
 import pdb
 import numpy as np
 
@@ -501,7 +504,7 @@ def assign_protocol(self, protocol, step_n=0):
                     # if this task has sounds, make columns for them
                     # TODO: Make stim managers return a list of properties for their sounds
                     if 'stim' in step.keys():
-                        if 'manager' in step['stim'].keys():
+                        if 'groups' in step['stim'].keys():
                             # managers have stim nested within groups, but this is still really ugly
                             sound_params = {}
                             for g in step['stim']['groups']:

autopilot/core/terminal.py

@@ -1,6 +1,3 @@
-__version__ = '0.3'
-__author__  = 'Jonny Saunders <[email protected]>'
-
 import argparse
 import json
 import sys

autopilot/core/utils.py

@@ -1,4 +1,3 @@
-# import numpy as np
 from autopilot import prefs
 # if prefs.AGENT in ("TERMINAL", "DOCS"):
 HAVE_PYSIDE = False