Skip to content

Commit c2cffee

Browse files
aaronwmorrisaaronwmorris
authored
Merge pull request #904 from aaronwmorris/dev
Tag images and videos with additional metadata
2 parents 5207690 + e1174fa commit c2cffee

16 files changed

Lines changed: 545 additions & 125 deletions

File tree

‎docker/nginx.local.conf‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ server {
1414

1515
rewrite ^/$ /indi-allsky;
1616

17+
client_max_body_size 1024M;
18+
1719

1820
location /indi-allsky/images {
1921
alias %INDIALLSKY_IMAGE_FOLDER%;
@@ -70,7 +72,9 @@ server {
7072
# 1 week HSTS header
7173
add_header Strict-Transport-Security "max-age=604800; includeSubDomains" always;
7274

73-
rewrite ^/$ /indi-allsky;
75+
rewrite ^/$ https://$host/indi-allsky;
76+
77+
client_max_body_size 1024M;
7478

7579

7680
location /indi-allsky/images {

‎indi_allsky/constants.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,31 @@
6161
None : 'None',
6262
}
6363

64+
65+
# Leaving gaps for addtional classifications
66+
SMOKE_RATING_CLEAR = 1
67+
SMOKE_RATING_LIGHT = 30
68+
SMOKE_RATING_MEDIUM = 60
69+
SMOKE_RATING_HEAVY = 90
70+
71+
SMOKE_RATING_MAP_STR = {
72+
SMOKE_RATING_CLEAR : 'Clear',
73+
SMOKE_RATING_LIGHT : 'Light',
74+
SMOKE_RATING_MEDIUM : 'Medium',
75+
SMOKE_RATING_HEAVY : 'Heavy',
76+
None : 'No Data',
77+
'Clear' : 'Clear', # legacy
78+
'Light' : 'Light', # legacy
79+
'Medium' : 'Medium', # legacy
80+
'Heavy' : 'Heavy', # legacy
81+
'No Data' : 'No Data', # legacy
82+
83+
}
84+
85+
SMOKE_RATING_STR_MAP = {
86+
'clear' : SMOKE_RATING_CLEAR,
87+
'light' : SMOKE_RATING_LIGHT,
88+
'medium' : SMOKE_RATING_MEDIUM,
89+
'heavy' : SMOKE_RATING_HEAVY,
90+
'no data' : None,
91+
}

‎indi_allsky/darks.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,9 @@ def _take_exposures(self, exposure, dark_filename_t, bpm_filename_t, ccd_bits, s
848848
'width' : image_width,
849849
}
850850

851+
bpm_metadata['data'] = {}
852+
853+
851854
dark_metadata = {
852855
'type' : constants.DARK_FRAME,
853856
'createDate' : exp_date.timestamp(),
@@ -861,6 +864,9 @@ def _take_exposures(self, exposure, dark_filename_t, bpm_filename_t, ccd_bits, s
861864
'width' : image_width,
862865
}
863866

867+
dark_metadata['data'] = {}
868+
869+
864870
self._miscDb.addBadPixelMap(
865871
full_bpm_filename_p,
866872
self.camera_id,

‎indi_allsky/flask/base_views.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
from pathlib import Path
88
import ephem
99

10+
from .. import constants
11+
1012
from flask import request
1113
from flask import session
1214
from flask import render_template
@@ -482,7 +484,9 @@ def get_smoke_info(self):
482484
return data
483485

484486

485-
smoke_rating = str(camera_data.get('SMOKE_RATING', 'No Data'))
487+
#app.logger.info('Smoke data: %s', camera_data)
488+
489+
smoke_rating = constants.SMOKE_RATING_MAP_STR.get(camera_data.get('SMOKE_RATING'), 'No Data')
486490

487491

488492
now = datetime.now()

‎indi_allsky/flask/forms.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111

1212
from passlib.hash import argon2
1313

14+
from .. import constants
15+
1416
from flask_wtf import FlaskForm
1517
from wtforms import IntegerField
1618
from wtforms import FloatField
@@ -2734,10 +2736,22 @@ def getVideos(self, year, month, timeofday):
27342736
app.logger.error('Error determining relative file name: %s', str(e))
27352737
continue
27362738

2739+
2740+
if v.data:
2741+
data = v.data
2742+
else:
2743+
data = {}
2744+
27372745
entry = {
2738-
'url' : str(url),
2739-
'dayDate' : v.dayDate.strftime('%B %d, %Y'),
2740-
'night' : v.night,
2746+
'url' : str(url),
2747+
'dayDate' : v.dayDate.strftime('%B %d, %Y'),
2748+
'night' : v.night,
2749+
'max_smoke_rating' : constants.SMOKE_RATING_MAP_STR[data.get('max_smoke_rating')],
2750+
'max_kpindex' : data.get('max_kpindex', 0.0),
2751+
'max_ovation_max' : data.get('max_ovation_max', 0),
2752+
'max_moonphase' : data.get('max_moonphase', 0), # might be null
2753+
'avg_stars' : int(data.get('avg_stars', 0)),
2754+
'avg_sqm' : int(data.get('avg_sqm', 0)),
27412755
}
27422756
videos_data.append(entry)
27432757

‎indi_allsky/flask/miscDb.py‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ def addImage(self, filename, camera_id, metadata):
163163
# 'stars'
164164
# 'detections'
165165
# 'process_elapsed'
166+
# 'data'
166167
# 'width'
167168
# 'height'
168169
#}
@@ -195,18 +196,11 @@ def addImage(self, filename, camera_id, metadata):
195196
temp_val = None
196197

197198

198-
# if moonmode is 0, moonphase is Null
199-
if metadata['moonmode']:
200-
moonphase_val = float(metadata['moonmode'])
201-
else:
202-
moonphase_val = None
203-
204199
moonmode_val = bool(metadata['moonmode'])
205200

206201
night_val = bool(metadata['night']) # integer to boolean
207202
adu_roi_val = bool(metadata['adu_roi'])
208203

209-
210204
image = IndiAllSkyDbImageTable(
211205
camera_id=camera_id,
212206
filename=str(filename_p),
@@ -223,15 +217,19 @@ def addImage(self, filename, camera_id, metadata):
223217
adu_roi=adu_roi_val,
224218
stable=metadata['stable'],
225219
moonmode=moonmode_val,
226-
moonphase=moonphase_val,
220+
moonphase=metadata['moonphase'],
227221
sqm=metadata['sqm'],
228222
stars=metadata['stars'],
229223
detections=metadata['detections'],
230224
process_elapsed=metadata['process_elapsed'],
231225
height=metadata['height'],
232226
width=metadata['width'],
227+
kpindex=metadata.get('kpindex'),
228+
ovation_max=metadata.get('ovation_max'),
229+
smoke_rating=metadata.get('smoke_rating'),
233230
remote_url=metadata.get('remote_url'),
234231
s3_key=metadata.get('s3_key'),
232+
data=metadata.get('data', {}),
235233
)
236234

237235
db.session.add(image)
@@ -294,6 +292,7 @@ def addDarkFrame(self, filename, camera_id, metadata):
294292
adu=metadata['adu'],
295293
height=metadata['height'],
296294
width=metadata['width'],
295+
data=metadata.get('data', {}),
297296
)
298297

299298
db.session.add(dark)
@@ -355,6 +354,7 @@ def addBadPixelMap(self, filename, camera_id, metadata):
355354
adu=metadata['adu'],
356355
height=metadata['height'],
357356
width=metadata['width'],
357+
data=metadata.get('data', {}),
358358
)
359359

360360
db.session.add(bpm)
@@ -370,6 +370,7 @@ def addVideo(self, filename, camera_id, metadata):
370370
# 'createDate' # datetime or timestamp
371371
# 'dayDate' # date or string
372372
# 'night'
373+
# 'data'
373374
#}
374375

375376

@@ -400,6 +401,9 @@ def addVideo(self, filename, camera_id, metadata):
400401
filename=str(filename_p),
401402
dayDate=dayDate,
402403
night=metadata['night'],
404+
height=metadata.get('height'), # optional
405+
width=metadata.get('width'), # optional
406+
data=metadata.get('data', {}),
403407
remote_url=metadata.get('remote_url'),
404408
s3_key=metadata.get('s3_key'),
405409
)
@@ -451,6 +455,7 @@ def addKeogram(self, filename, camera_id, metadata):
451455
night=metadata['night'],
452456
height=metadata.get('height'), # optional
453457
width=metadata.get('width'), # optional
458+
data=metadata.get('data', {}),
454459
remote_url=metadata.get('remote_url'),
455460
s3_key=metadata.get('s3_key'),
456461
)
@@ -503,6 +508,7 @@ def addStarTrail(self, filename, camera_id, metadata):
503508
night=metadata['night'],
504509
height=metadata.get('height'), # optional
505510
width=metadata.get('width'), # optional
511+
data=metadata.get('data', {}),
506512
remote_url=metadata.get('remote_url'),
507513
s3_key=metadata.get('s3_key'),
508514
)
@@ -551,6 +557,9 @@ def addStarTrailVideo(self, filename, camera_id, metadata):
551557
filename=str(filename_p),
552558
dayDate=dayDate,
553559
night=metadata['night'],
560+
height=metadata.get('height'), # optional
561+
width=metadata.get('width'), # optional
562+
data=metadata.get('data', {}),
554563
remote_url=metadata.get('remote_url'),
555564
s3_key=metadata.get('s3_key'),
556565
)
@@ -607,6 +616,7 @@ def addFitsImage(self, filename, camera_id, metadata):
607616
night=metadata['night'],
608617
height=metadata['height'],
609618
width=metadata['width'],
619+
data=metadata.get('data', {}),
610620
remote_url=metadata.get('remote_url'),
611621
s3_key=metadata.get('s3_key'),
612622
)
@@ -663,6 +673,7 @@ def addRawImage(self, filename, camera_id, metadata):
663673
night=metadata['night'],
664674
height=metadata['height'],
665675
width=metadata['width'],
676+
data=metadata.get('data', {}),
666677
remote_url=metadata.get('remote_url'),
667678
s3_key=metadata.get('s3_key'),
668679
)

‎indi_allsky/flask/models.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ class IndiAllSkyDbCameraTable(db.Model):
6464
lensFocalRatio = db.Column(db.Float, nullable=True)
6565
lensImageCircle = db.Column(db.Integer, nullable=True) # pixels
6666

67-
data = db.Column(db.JSON) # this is only for local data, will not be synced
67+
data = db.Column(db.JSON, index=True)
6868

6969
local = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
7070
sync_id = db.Column(db.Integer, nullable=True, index=True)
@@ -187,6 +187,10 @@ class IndiAllSkyDbImageTable(IndiAllSkyDbFileBase):
187187
sync_id = db.Column(db.Integer, nullable=True, index=True)
188188
calibrated = db.Column(db.Boolean, server_default=expression.false(), nullable=False)
189189
detections = db.Column(db.Integer, server_default='0', nullable=False, index=True)
190+
kpindex = db.Column(db.Float, nullable=True, index=True)
191+
ovation_max = db.Column(db.Integer, nullable=True, index=True)
192+
smoke_rating = db.Column(db.Integer, nullable=True, index=True)
193+
data = db.Column(db.JSON, index=True)
190194
width = db.Column(db.Integer, nullable=True, index=True)
191195
height = db.Column(db.Integer, nullable=True, index=True)
192196
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
@@ -220,6 +224,7 @@ class IndiAllSkyDbDarkFrameTable(IndiAllSkyDbFileBase):
220224
adu = db.Column(db.Float, nullable=True)
221225
width = db.Column(db.Integer, nullable=True, index=True)
222226
height = db.Column(db.Integer, nullable=True, index=True)
227+
data = db.Column(db.JSON, index=True)
223228
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
224229
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='darkframes')
225230

@@ -254,6 +259,7 @@ class IndiAllSkyDbBadPixelMapTable(IndiAllSkyDbFileBase):
254259
adu = db.Column(db.Float, nullable=True)
255260
width = db.Column(db.Integer, nullable=True, index=True)
256261
height = db.Column(db.Integer, nullable=True, index=True)
262+
data = db.Column(db.JSON, index=True)
257263
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
258264
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='badpixelmaps')
259265

@@ -287,6 +293,10 @@ class IndiAllSkyDbVideoTable(IndiAllSkyDbFileBase):
287293
uploaded = db.Column(db.Boolean, server_default=expression.false(), nullable=False)
288294
sync_id = db.Column(db.Integer, nullable=True, index=True)
289295
success = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
296+
#kpindex = db.Column(db.Float, nullable=True, index=True)
297+
#ovation_max = db.Column(db.Integer, nullable=True, index=True)
298+
#smoke_rating = db.Column(db.Integer, nullable=True, index=True)
299+
data = db.Column(db.JSON, index=True)
290300
width = db.Column(db.Integer, nullable=True, index=True) # this may never be populated
291301
height = db.Column(db.Integer, nullable=True, index=True) # this may never be populated
292302
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
@@ -319,6 +329,7 @@ class IndiAllSkyDbKeogramTable(IndiAllSkyDbFileBase):
319329
success = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
320330
width = db.Column(db.Integer, nullable=True, index=True)
321331
height = db.Column(db.Integer, nullable=True, index=True)
332+
data = db.Column(db.JSON, index=True)
322333
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
323334
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='keograms')
324335

@@ -341,6 +352,7 @@ class IndiAllSkyDbStarTrailsTable(IndiAllSkyDbFileBase):
341352
success = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
342353
width = db.Column(db.Integer, nullable=True, index=True)
343354
height = db.Column(db.Integer, nullable=True, index=True)
355+
data = db.Column(db.JSON, index=True)
344356
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
345357
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='startrails')
346358

@@ -363,6 +375,7 @@ class IndiAllSkyDbStarTrailsVideoTable(IndiAllSkyDbFileBase):
363375
success = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
364376
width = db.Column(db.Integer, nullable=True, index=True) # this may never be populated
365377
height = db.Column(db.Integer, nullable=True, index=True) # this may never be populated
378+
data = db.Column(db.JSON, index=True)
366379
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
367380
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='startrailvideos')
368381

@@ -387,6 +400,7 @@ class IndiAllSkyDbFitsImageTable(IndiAllSkyDbFileBase):
387400
sync_id = db.Column(db.Integer, nullable=True, index=True)
388401
width = db.Column(db.Integer, nullable=True, index=True)
389402
height = db.Column(db.Integer, nullable=True, index=True)
403+
data = db.Column(db.JSON, index=True)
390404
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
391405
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='fitsimages')
392406

@@ -411,6 +425,7 @@ class IndiAllSkyDbRawImageTable(IndiAllSkyDbFileBase):
411425
sync_id = db.Column(db.Integer, nullable=True, index=True)
412426
width = db.Column(db.Integer, nullable=True, index=True)
413427
height = db.Column(db.Integer, nullable=True, index=True)
428+
data = db.Column(db.JSON, index=True)
414429
camera_id = db.Column(db.Integer, db.ForeignKey('camera.id'), nullable=False)
415430
camera = db.relationship('IndiAllSkyDbCameraTable', back_populates='rawimages')
416431

@@ -511,7 +526,7 @@ class IndiAllSkyDbConfigTable(db.Model):
511526
level = db.Column(db.String(length=12), nullable=False)
512527
encrypted = db.Column(db.Boolean, server_default=expression.false(), nullable=False, index=True)
513528
note = db.Column(db.String(length=255), nullable=False)
514-
data = db.Column(db.JSON)
529+
data = db.Column(db.JSON, index=True)
515530
user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=True) # users can be deleted
516531
user = db.relationship('IndiAllSkyDbUserTable', back_populates='configs')
517532

@@ -541,6 +556,7 @@ class IndiAllSkyDbUserTable(db.Model):
541556
active = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
542557
staff = db.Column(db.Boolean, server_default=expression.true(), nullable=False, index=True)
543558
admin = db.Column(db.Boolean, server_default=expression.false(), nullable=False, index=True)
559+
data = db.Column(db.JSON, index=True)
544560
configs = db.relationship('IndiAllSkyDbConfigTable', back_populates='user')
545561

546562

0 commit comments

Comments
 (0)