Merge pull request #1165 from aaronwmorris/dev

Base image URLs from S3 on information from camera

Author: aaronwmorris

indi_allsky/allsky.py

@@ -40,6 +40,8 @@
 from .flask.models import IndiAllSkyDbKeogramTable
 from .flask.models import IndiAllSkyDbStarTrailsTable
 from .flask.models import IndiAllSkyDbStarTrailsVideoTable
+from .flask.models import IndiAllSkyDbPanoramaImageTable
+from .flask.models import IndiAllSkyDbPanoramaVideoTable
 from .flask.models import IndiAllSkyDbTaskQueueTable
 
 from sqlalchemy import or_
@@ -521,7 +523,8 @@ def _stopFileUploadWorkers(self):
             active_worker_list.append(upload_worker_dict)
 
             # need to put the stops in the queue before waiting on workers to join
-            self.upload_q.put({'stop' : True})
+            #self.upload_q.put({'stop' : True})
+            upload_worker_dict['worker'].stop()
 
 
         for upload_worker_dict in active_worker_list:
@@ -670,6 +673,8 @@ def _dbImportImages(self):
                 'latitude'    : 0.0,
                 'longitude'   : 0.0,
                 'elevation'   : 0,
+                'alt'         : 0,
+                'az'          : 0,
             }
             camera = self._miscDb.addCamera(camera_metadata)
             camera_id = camera.id
@@ -709,7 +714,7 @@ def _dbImportImages(self):
             d_createDate = datetime.fromtimestamp(f.stat().st_mtime)
 
             video_dict = {
-                'filename'   : str(f),
+                'filename'   : str(f.relative_to(self.image_dir)),
                 'createDate' : d_createDate,
                 'dayDate'    : d_dayDate,
                 'night'      : night,
@@ -765,7 +770,7 @@ def _dbImportImages(self):
             d_createDate = datetime.fromtimestamp(f.stat().st_mtime)
 
             keogram_dict = {
-                'filename'   : str(f),
+                'filename'   : str(f.relative_to(self.image_dir)),
                 'createDate' : d_createDate,
                 'dayDate'    : d_dayDate,
                 'night'      : night,
@@ -815,7 +820,7 @@ def _dbImportImages(self):
             d_createDate = datetime.fromtimestamp(f.stat().st_mtime)
 
             startrail_dict = {
-                'filename'   : str(f),
+                'filename'   : str(f.relative_to(self.image_dir)),
                 'createDate' : d_createDate,
                 'dayDate'    : d_dayDate,
                 'night'      : night,
@@ -866,7 +871,7 @@ def _dbImportImages(self):
             d_createDate = datetime.fromtimestamp(f.stat().st_mtime)
 
             startrail_video_dict = {
-                'filename'   : str(f),
+                'filename'   : str(f.relative_to(self.image_dir)),
                 'createDate' : d_createDate,
                 'dayDate'    : d_dayDate,
                 'night'      : night,
@@ -887,17 +892,70 @@ def _dbImportImages(self):
             db.session.rollback()
 
 
+        ### Panorama Videos
+        file_list_panorama_video_tl = filter(lambda p: 'timelapse' in p.name, file_list_videos)
+        file_list_panorama_video = filter(lambda p: 'panorama' in p.name, file_list_panorama_video_tl)
+
+        #/var/www/html/allsky/images/20210915/allsky-panorama_timelapse_ccd1_20210915_night.mp4
+        re_panorama_video = re.compile(r'(?P<dayDate_str>\d{8})\/.+panorama_timelapse_ccd(?P<ccd_id_str>\d+)_\d{8}_(?P<timeofday_str>[a-z]+)\.[a-z0-9]+$')
+
+        panorama_video_entries = list()
+        for f in file_list_panorama_video:
+            #logger.info('Panorama timelapse: %s', f)
+
+            m = re.search(re_panorama_video, str(f))
+            if not m:
+                logger.error('Regex did not match file: %s', f)
+                continue
+
+            #logger.info('dayDate string: %s', m.group('dayDate_str'))
+            #logger.info('Time of day string: %s', m.group('timeofday_str'))
+
+            d_dayDate = datetime.strptime(m.group('dayDate_str'), '%Y%m%d').date()
+            #logger.info('dayDate: %s', str(d_dayDate))
+
+            if m.group('timeofday_str') == 'night':
+                night = True
+            else:
+                night = False
+
+            d_createDate = datetime.fromtimestamp(f.stat().st_mtime)
+
+            panorama_video_dict = {
+                'filename'   : str(f.relative_to(self.image_dir)),
+                'createDate' : d_createDate,
+                'dayDate'    : d_dayDate,
+                'night'      : night,
+                'uploaded'   : False,
+                'camera_id'  : camera_id,
+            }
+
+            panorama_video_entries.append(panorama_video_dict)
+
+
+        try:
+            db.session.bulk_insert_mappings(IndiAllSkyDbPanoramaVideoTable, panorama_video_entries)
+            db.session.commit()
+
+            logger.warning('*** Panorama timelapses inserted: %d ***', len(panorama_video_entries))
+        except IntegrityError as e:
+            logger.warning('Integrity error: %s', str(e))
+            db.session.rollback()
+
+
         ### Images
         # Exclude keograms and star trails
         file_list_images_nok = filter(lambda p: 'keogram' not in p.name, file_list_images)
         file_list_images_nok_nost = filter(lambda p: 'startrail' not in p.name, file_list_images_nok)
         file_list_images_nok_nost_noraw = filter(lambda p: 'raw' not in p.name, file_list_images_nok_nost)
+        file_list_images_nok_nost_noraw_nopan = filter(lambda p: 'panorama' not in p.name, file_list_images_nok_nost_noraw)
+        file_list_images_nok_nost_noraw_nopan_nothumb = filter(lambda p: 'thumbnail' not in p.name, file_list_images_nok_nost_noraw_nopan)
 
         #/var/www/html/allsky/images/20210825/night/26_02/ccd1_20210826_020202.jpg
         re_image = re.compile(r'(?P<dayDate_str>\d{8})\/(?P<timeofday_str>[a-z]+)\/\d{2}_\d{2}\/ccd(?P<ccd_id_str>\d+)_(?P<createDate_str>[0-9_]+)\.[a-z]+$')
 
         image_entries = list()
-        for f in file_list_images_nok_nost_noraw:
+        for f in file_list_images_nok_nost_noraw_nopan_nothumb:
             #logger.info('Image: %s', f)
 
             m = re.search(re_image, str(f))
@@ -923,7 +981,7 @@ def _dbImportImages(self):
 
 
             image_dict = {
-                'filename'   : str(f),
+                'filename'   : str(f.relative_to(self.image_dir)),
                 'camera_id'  : camera_id,
                 'createDate' : d_createDate,
                 'dayDate'    : d_dayDate,
@@ -951,6 +1009,63 @@ def _dbImportImages(self):
             db.session.rollback()
 
 
+        ### Panorama images
+        file_list_panorama_images = filter(lambda p: 'panoram' in p.name, file_list_images)
+
+        #/var/www/html/allsky/images/20210825/night/26_02/panorama_ccd1_20210826_020202.jpg
+        re_image = re.compile(r'(?P<dayDate_str>\d{8})\/(?P<timeofday_str>[a-z]+)\/\d{2}_\d{2}\/panorama_ccd(?P<ccd_id_str>\d+)_(?P<createDate_str>[0-9_]+)\.[a-z]+$')
+
+        panorama_image_entries = list()
+        for f in file_list_panorama_images:
+            #logger.info('Image: %s', f)
+
+            m = re.search(re_image, str(f))
+            if not m:
+                logger.error('Regex did not match file: %s', f)
+                continue
+
+            #logger.info('dayDate string: %s', m.group('dayDate_str'))
+            #logger.info('Time of day string: %s', m.group('timeofday_str'))
+            #logger.info('createDate string: %s', m.group('createDate_str'))
+
+            d_dayDate = datetime.strptime(m.group('dayDate_str'), '%Y%m%d').date()
+            #logger.info('dayDate: %s', str(d_dayDate))
+
+            if m.group('timeofday_str') == 'night':
+                night = True
+            else:
+                night = False
+
+            #d_createDate = datetime.strptime(m.group('createDate_str'), '%Y%m%d_%H%M%S')
+            d_createDate = datetime.fromtimestamp(f.stat().st_mtime)
+            #logger.info('createDate: %s', str(d_createDate))
+
+
+            panorama_image_dict = {
+                'filename'   : str(f.relative_to(self.image_dir)),
+                'camera_id'  : camera_id,
+                'createDate' : d_createDate,
+                'dayDate'    : d_dayDate,
+                'exposure'   : 0.0,
+                'gain'       : -1,
+                'binmode'    : 1,
+                'night'      : night,
+                'uploaded'   : False,
+            }
+
+
+            panorama_image_entries.append(panorama_image_dict)
+
+        try:
+            db.session.bulk_insert_mappings(IndiAllSkyDbPanoramaImageTable, panorama_image_entries)
+            db.session.commit()
+
+            logger.warning('*** Panorama images inserted: %d ***', len(panorama_image_entries))
+        except IntegrityError as e:
+            logger.warning('Integrity error: %s', str(e))
+            db.session.rollback()
+
+
     def _getFolderFilesByExt(self, folder, file_list, extension_list=None):
         if not extension_list:
             extension_list = [self.config['IMAGE_FILE_TYPE']]

indi_allsky/capture.py

@@ -618,6 +618,27 @@ def _initialize(self):
             cfa_pattern = ccd_info['CCD_CFA']['CFA_TYPE'].get('text')
 
 
+        # populate S3 data
+        s3_data = {
+            'host'      : self.config['S3UPLOAD'].get('HOST', ''),
+            'bucket'    : self.config['S3UPLOAD'].get('BUCKET', ''),
+            'region'    : self.config['S3UPLOAD'].get('REGION', ''),
+            'namespace' : self.config['S3UPLOAD'].get('NAMESPACE', ''),
+        }
+
+
+        try:
+            s3_prefix = self.config['S3UPLOAD']['URL_TEMPLATE'].format(**s3_data)
+        except KeyError as e:
+            app.logger.error('Failure to generate S3 prefix: %s', str(e))
+            s3_prefix = ''
+        except ValueError as e:
+            app.logger.error('Failure to generate S3 prefix: %s', str(e))
+            s3_prefix = ''
+
+
+        now = datetime.now()
+
         # need to get camera info before adding to DB
         camera_metadata = {
             'type'        : constants.CAMERA,
@@ -641,6 +662,9 @@ def _initialize(self):
             'longitude'   : self.longitude_v.value,
             'elevation'   : self.elevation_v.value,
 
+            'tz'          : str(now.astimezone().tzinfo),
+            'utc_offset'  : now.astimezone().utcoffset().total_seconds(),
+
             'owner'           : self.config['OWNER'],
             'lensName'        : self.config['LENS_NAME'],
             'lensFocalLength' : self.config['LENS_FOCAL_LENGTH'],
@@ -649,8 +673,13 @@ def _initialize(self):
             'alt'             : self.config['LENS_ALTITUDE'],
             'az'              : self.config['LENS_AZIMUTH'],
             'nightSunAlt'     : self.config['NIGHT_SUN_ALT_DEG'],
+
+            's3_prefix'             : s3_prefix,
+            'web_nonlocal_images'   : self.config.get('WEB_NONLOCAL_IMAGES', False),
+            'web_local_images_admin': self.config.get('WEB_LOCAL_IMAGES_ADMIN', False),
         }
 
+
         camera = self._miscDb.addCamera(camera_metadata)
         self.camera_id = camera.id
 

indi_allsky/darks.py

@@ -829,7 +829,7 @@ def _run(self, stacking_class):
     def _take_exposures(self, exposure, dark_filename_t, bpm_filename_t, ccd_bits, stacking_class):
         exposure_f = float(exposure)
 
-        tmp_fit_dir = tempfile.TemporaryDirectory()
+        tmp_fit_dir = tempfile.TemporaryDirectory()    # context manager automatically deletes files when finished
         tmp_fit_dir_p = Path(tmp_fit_dir.name)
 
         logger.info('Temp folder: %s', tmp_fit_dir_p)

indi_allsky/flask/base_views.py

@@ -55,30 +55,29 @@ def __init__(self, **kwargs):
 
         self._miscDb = miscDb(self.indi_allsky_config)
 
-        self.s3_prefix = self.getS3Prefix()
+        self.camera = None  # set in setupSession()
 
+        self.setupSession()
 
-    def getS3Prefix(self):
-        s3_data = {
-            'host'      : self.indi_allsky_config['S3UPLOAD']['HOST'],
-            'bucket'    : self.indi_allsky_config['S3UPLOAD']['BUCKET'],
-            'region'    : self.indi_allsky_config['S3UPLOAD']['REGION'],
-            'namespace' : self.indi_allsky_config['S3UPLOAD'].get('NAMESPACE', ''),
-        }
+        self.s3_prefix = self.camera.s3_prefix
+        self.web_nonlocal_images = self.camera.web_nonlocal_images
+        self.web_local_images_admin = self.camera.web_local_images_admin
+
+        camera_time_offset = self.camera.utc_offset - datetime.now().astimezone().utcoffset().total_seconds()
+        self.camera_now = datetime.now() + timedelta(seconds=camera_time_offset)
 
-        try:
-            prefix = self.indi_allsky_config['S3UPLOAD']['URL_TEMPLATE'].format(**s3_data)
-        except KeyError as e:
-            app.logger.error('Failure to generate S3 prefix: %s', str(e))
-            return ''
-        except ValueError as e:
-            app.logger.error('Failure to generate S3 prefix: %s', str(e))
-            return ''
 
+    def setupSession(self):
+        if session.get('camera_id'):
+            self.camera = self.getCameraById(session['camera_id'])
+            return
 
-        #app.logger.info('S3 Prefix: %s', prefix)
+        try:
+            self.camera = self.getLatestCamera()
+        except NoResultFound:
+            self.camera = FakeCamera()
 
-        return prefix
+        session['camera_id'] = self.camera.id
 
 
     def getLatestCamera(self):
@@ -179,10 +178,6 @@ def __init__(self, template_name, **kwargs):
         super(TemplateView, self).__init__(**kwargs)
         self.template_name = template_name
 
-        self.camera = None  # set in setupSession()
-
-        self.setupSession()
-
         self.local_indi_allsky = self.camera.local
 
         self.check_config(self._indi_allsky_config_obj.config_id)
@@ -193,19 +188,6 @@ def __init__(self, template_name, **kwargs):
         self.night = True
 
 
-    def setupSession(self):
-        if session.get('camera_id'):
-            self.camera = self.getCameraById(session['camera_id'])
-            return
-
-        try:
-            self.camera = self.getLatestCamera()
-        except NoResultFound:
-            self.camera = FakeCamera()
-
-        session['camera_id'] = self.camera.id
-
-
     def render_template(self, context):
         return render_template(self.template_name, **context)
 
@@ -358,10 +340,20 @@ def get_camera_info(self):
             'location' : str(self.camera.location),
             'owner' : str(self.camera.owner),
             'lens_name' : str(self.camera.lensName),
-            'alt' : float(self.camera.alt),
-            'az' : float(self.camera.az),
         }
 
+
+        if isinstance(self.camera.alt, type(None)):
+            data['alt'] = 0
+        else:
+            data['alt'] = float(self.camera.alt)
+
+        if isinstance(self.camera.az, type(None)):
+            data['az'] = 0
+        else:
+            data['az'] = float(self.camera.az)
+
+
         return data
 
 
@@ -707,5 +699,8 @@ class FakeCamera(object):
     lensName = ''
     name = ''
     friendlyName = ''
+    s3_prefix = ''
+    web_nonlocal_images = False
+    web_local_images_admin = False
     data = {}