L5 script now writes real-time L1 data from NOAA to pickle file (last three months are kept), so that there is always data available for a 27-day recurrence model.

Author: R Bailey

predstorm/data.py

@@ -2378,9 +2378,9 @@ def merge_Data(satdata1, satdata2, keys=None):
             raise Exception("Dataset1 contains key ({}) not available in Dataset2!".format(k))
 
     # Find num of points for addition:
-    if (satdata1.h['SamplingRate'] - 1./24.) < 0.0001:
+    if np.abs((satdata1.h['SamplingRate'] - 1./24.)) < 0.0001:
         timestep = 1./24.
-    elif (satdata1.h['SamplingRate'] - 1./24./60.) < 0.0001:
+    elif np.abs((satdata1.h['SamplingRate'] - 1./24./60.)) < 0.0001:
         timestep = 1./24./60.
     else:
         timestep = satdata1.h['SamplingRate']
@@ -2398,16 +2398,21 @@ def merge_Data(satdata1, satdata2, keys=None):
 
     MergedData = SatData(datadict, source=satdata1.source+'+'+satdata2.source)
     tf = "%Y-%m-%d %H:%M:%S"
-    MergedData.h['DataSource'] = '{} ({} - {}) & {} ({} - {})'.format(satdata1.h['DataSource'],
-                                                datetime.strftime(num2date(satdata1['time'][0]), tf),
-                                                datetime.strftime(num2date(satdata1['time'][-1]), tf),
-                                                satdata2.h['DataSource'],
-                                                datetime.strftime(num2date(new_time[0]), tf),
-                                                datetime.strftime(num2date(new_time[-1]), tf))
+    if satdata1.h['DataSource'] == satdata2.h['DataSource']:
+        MergedData.h['DataSource'] = satdata1.h['DataSource']
+        MergedData.h['Instruments'] = satdata1.h['Instruments']
+        MergedData.h['FileVersion'] = satdata1.h['FileVersion']
+    else:
+        MergedData.h['DataSource'] = '{} ({} - {}) & {} ({} - {})'.format(satdata1.h['DataSource'],
+                                                    datetime.strftime(num2date(satdata1['time'][0]), tf),
+                                                    datetime.strftime(num2date(satdata1['time'][-1]), tf),
+                                                    satdata2.h['DataSource'],
+                                                    datetime.strftime(num2date(new_time[0]), tf),
+                                                    datetime.strftime(num2date(new_time[-1]), tf))
+        MergedData.h['Instruments'] = satdata1.h['Instruments'] + satdata2.h['Instruments']
+        MergedData.h['FileVersion'] = {**satdata1.h['FileVersion'], **satdata2.h['FileVersion']}
     MergedData.h['SamplingRate'] = timestep
     MergedData.h['ReferenceFrame'] = satdata1.h['ReferenceFrame']
-    MergedData.h['Instruments'] = satdata1.h['Instruments'] + satdata2.h['Instruments']
-    MergedData.h['FileVersion'] = {**satdata1.h['FileVersion'], **satdata2.h['FileVersion']}
 
     logger.info("merge_Data: Finished merging data.")
 

predstorm_l5.py

@@ -158,6 +158,22 @@ def main():
         timenow = dism['time'][-1]
         # Get UTC time now
         timeutc = date2num(timestamp)
+        # Save to pickle file for recurrence model:
+        rec_dism = copy.deepcopy(dism)
+        # Cut by three days so data is always 3 days old at least, in case it's updated
+        rec_dism.cut(endtime=num2date(timenow-3))
+        rec_model_path = 'recurrence_l1_27days.pickle'
+        if not os.path.exists(rec_model_path):
+            with open(rec_model_path, 'wb') as f:
+                pickle.dump(rec_dism, f)
+        with open(rec_model_path, 'rb') as f:
+            rec_27days = pickle.load(f)
+        if rec_dism['time'][-1] > rec_27days['time'][-1]:
+            rec_dism.cut(starttime=num2date(rec_27days['time'][-1]))
+            rec_new = ps.merge_Data(rec_27days, rec_dism)
+            rec_new.cut(starttime=num2date(timenow-90))  # don't keep more than three months
+            with open(rec_model_path, 'wb') as f:
+                pickle.dump(rec_new, f)
     elif run_mode == 'historic':
         timestamp = historic_date
         if (datetime.utcnow() - historic_date).days < (7.-plot_past_days):