Merge pull request #1350 from spedas/t_1346

Add support for METOP data sets archived at NCEI

Author: jameswilburlewis

pyspedas/projects/poes/config.py

@@ -4,6 +4,7 @@
     "local_data_dir": "poes_data/",
     "remote_data_dir": "https://spdf.gsfc.nasa.gov/pub/data/noaa/",
     "ncei_remote_data_dir": "https://www.ncei.noaa.gov/data/poes-metop-space-environment-monitor/access/l2/v01r00/cdf/",
+    "ncei_l1b_remote_data_dir": "https://www.ncei.noaa.gov/data/poes-metop-space-environment-monitor/access/l1b/v01r00/"
 }
 
 # override local data directory with environment variables

pyspedas/projects/poes/load.py

@@ -2,6 +2,8 @@
 from pyspedas.utilities.download import download
 from pyspedas.tplot_tools import time_clip as tclip
 from pyspedas.tplot_tools import cdf_to_tplot
+from pyspedas.tplot_tools import netcdf_to_tplot
+import re
 
 from .config import CONFIG
 
@@ -15,6 +17,7 @@ def load(
     suffix="",
     get_support_data=False,
     ncei_server=False,
+    ncei_l1b_server=False,
     varformat=None,
     varnames=[],
     downloadonly=False,
@@ -91,6 +94,14 @@ def load(
             remote_path = CONFIG["ncei_remote_data_dir"]
             num = prb[-2:]
             pathformat = "%Y/" + prb + "/poes_n" + num + "_%Y%m%d.cdf"
+        elif ncei_l1b_server:
+            remote_path = CONFIG["ncei_l1b_remote_data_dir"]
+            num = re.search("[0-9]+",prb)[0]
+            prb_name=prb.split(num)[0]
+            prb_l = prb[0:1]
+
+            prb_name_num_formatted=prb_name + num.zfill(2)
+            pathformat = "%Y/" + prb_name_num_formatted + "/poes_" + prb_l + num.zfill(2) + "_%Y%m%d_proc.nc"
         else:
             remote_path = CONFIG["remote_data_dir"]
             if instrument == "sem":
@@ -120,15 +131,22 @@ def load(
     if downloadonly:
         return out_files
 
-    tvars = cdf_to_tplot(
-        out_files,
-        prefix=prefix,
-        suffix=suffix,
-        get_support_data=get_support_data,
-        varformat=varformat,
-        varnames=varnames,
-        notplot=notplot,
-    )
+    if ncei_l1b_server:
+        tvars = netcdf_to_tplot(
+            out_files,
+            prefix=prefix,
+            suffix=suffix,
+        )
+    else:
+        tvars = cdf_to_tplot(
+            out_files,
+            prefix=prefix,
+            suffix=suffix,
+            get_support_data=get_support_data,
+            varformat=varformat,
+            varnames=varnames,
+            notplot=notplot,
+        )
 
     if notplot:
         return tvars

pyspedas/projects/poes/tests/test_poes.py

@@ -40,6 +40,13 @@ def test_ncei_server(self):
         vars = load(trange=['1999-01-03', '1999-01-04'], probe=['noaa15'], ncei_server=True, time_clip=True)
         self.assertTrue('geogLL' in vars)
 
+    def test_ncei_l1b_server(self):
+        vars = load(trange=['2026-04-05', '2026-04-06'], probe=['metop03'], ncei_l1b_server=True, time_clip=True)
+        self.assertTrue('ted_ele_tel30_hi_eflux' in vars)    
+
+    def test_ncei_l1b_server_metop_varient(self):
+        vars = load(trange=['2026-04-05', '2026-04-06'], probe=['metop3'], ncei_l1b_server=True, time_clip=True)
+        self.assertTrue('ted_ele_tel30_hi_eflux' in vars)
 
 if __name__ == '__main__':
     unittest.main()

pyspedas/tplot_tools/importers/netcdf_to_tplot.py

@@ -23,7 +23,15 @@ def change_time_to_unix_time(time_var):
     # ICON uses nonstandard units strings
     if units == "ms":
         units = "milliseconds since 1970-01-01 00:00:00"
-    dates = num2date(time_var[:], units=units)
+    # Check if the long_name attribute has stored the epoch description (POES/METOP):
+    elif hasattr(time_var, "long_name"):
+        if time_var.long_name == "milliseconds since 1970":
+            units = "milliseconds since 1970-01-01 00:00:00"
+
+    time_data=time_var[:]
+    if hasattr(time_data,"data"):
+        time_data=time_var[:].data
+    dates = num2date(time_data, units=units)
     unix_times = list()
     for date in dates:
         unix_time = calendar.timegm(date.timetuple()) + date.microsecond/1e6
@@ -130,17 +138,27 @@ def netcdf_to_tplot(
                     # If multiple matching keys are found, the one that appears latest in the above list
                     # will take precedence
                     var_fill_value = atts_dict[key]
+            
+            if hasattr(reg_var[:],"get_fill_value"):
+                var_fill_value=reg_var[:].get_fill_value()
 
             # If var_fill_value is None, or already NaN, there's nothing to do here.
             # Integer arrays can't be NaN-filled, so if var_fill_value is any kind of integer, skip those too.
             # Some missions have strings defined as fill values.  (ICON)
             if var_fill_value is not None and not isinstance(var_fill_value, np.integer) and not isinstance(var_fill_value, str) and not np.isnan(var_fill_value):
                 # We want to force missing values to be nan so that plots don't look strange
-                var_mask = np.ma.masked_where(
-                    reg_var == np.float32(var_fill_value), reg_var
-                )
-                var_filled = np.ma.filled(var_mask, np.nan)
-                masked_vars[var] = var_filled
+                if hasattr(reg_var[:],"data"):
+                        var_mask = np.ma.masked_where(
+                            reg_var[:].data == np.float32(var_fill_value), reg_var[:].data
+                        )
+                        var_filled = np.ma.filled(var_mask, np.nan)
+                        masked_vars[var] = var_filled
+                else:    
+                    var_mask = np.ma.masked_where(
+                        reg_var == np.float32(var_fill_value), reg_var
+                    )
+                    var_filled = np.ma.filled(var_mask, np.nan)
+                    masked_vars[var] = var_filled
             else:
                 var_filled = reg_var
                 masked_vars[var] = var_filled
@@ -168,8 +186,19 @@ def netcdf_to_tplot(
                     # If this_time does not exist, we can't save this as tplot variable.
                     continue
                 elif this_time == var:
-                    # If this_time has the same name as the current variable, do not save it.
-                    continue
+                    # The time the variable depends on may not have been set.
+                    # Check if time is a variable:
+                    if 'time' in vars_and_atts.keys():
+                        # If it is, check if the sizes match:
+                        if vfile[var].size == vfile['time'].size:
+                            # If they do, we can infer that the time variable is the one we want here.
+                            this_time = "time"
+                        else:
+                            # If not, it probably depends on something else / nothing.    
+                            continue
+                    else:        
+                        # If this_time has the same name as the current variable, do not save it.
+                        continue
 
                 # Find the time values (as unix times).
                 if this_time in times_dict: