process for new sea0 gliders

Author: callumrollo

all_nrt_plots.py

@@ -3,6 +3,9 @@
 from glidertools_plots import public_plots
 import matplotlib.pyplot as plt
 import logging
+
+from gridded_plots import create_plots, make_map
+
 _log = logging.getLogger(__name__)
 bad_missions = ((57, 75), (70, 29), (66, 45))
 
@@ -17,19 +20,28 @@
     ts_files = list(pathlib.Path('/data/data_l0_pyglider/nrt/').rglob("mission_timeseries.nc"))
     for ts_file in ts_files:
         parts = ts_file.parts
-        glider = int(parts[-4][3:])
+        platform_serial = parts[-4]
         mission = int(parts[-3][1:])
-        if (glider, mission) in bad_missions:
-            _log.info(f"this missions is to be skipped SEA{glider} M{mission}")
+        if (int(platform_serial[-3:]), mission) in bad_missions:
+            _log.info(f"this missions is to be skipped {platform_serial} M{mission}")
             continue
-        outdir = pathlib.Path(f"/data/plots/nrt/SEA{glider}/M{mission}/")
-        _log.info(f"process for SEA{glider}, M{mission}")
+        outdir = pathlib.Path(f"/data/plots/nrt/{platform_serial}/M{mission}/")
+        _log.info(f"Check {platform_serial}, M{mission}")
         if not outdir.exists():
             outdir.mkdir(parents=True)
-        filename = outdir / f'SEA{glider}_M{mission}_gt.png'
+        files_parts = list(ts_file.parts)
+        files_parts[-1] = "gridded.nc"
+        files_parts[-2] = "gridfiles"
+        nc_file = pathlib.Path("/".join(files_parts))
+        filename = outdir / f'{platform_serial}_M{mission}_gt.png'
         if not filename.exists() or reprocess:
+            _log.info(f"Process {platform_serial}, M{mission}")
             public_plots(ts_file, outdir)
-        filename = outdir / f'SEA{glider}_M{mission}_cmd_log.png'
+            _log.info(f'start plotting {nc_file} ')
+            image_file = create_plots(nc_file, outdir, False)
+            make_map(nc_file, image_file)
+        filename = outdir / f'{platform_serial}_M{mission}_cmd_log.png'
         if not filename.exists() or reprocess:
-            command_cosole_log_plots(glider, mission, outdir)
+            _log.info(f"Process command console plots {platform_serial}, M{mission}")
+            command_cosole_log_plots(platform_serial, mission, outdir)
         plt.close('all')

cmd_data_plots.py

@@ -227,14 +227,14 @@ def make_all_plots(path_to_cmdlog):
     return fig
 
 
-def command_cosole_log_plots(glider, mission, plots_dir):
-    _log.info(f"Make command console plots for SEA{glider} M{mission}")
-    cmd_log = Path(f"/data/data_raw/nrt/SEA{str(glider).zfill(3)}/{str(mission).zfill(6)}/G-Logs/sea{str(glider).zfill(3)}.{mission}.com.raw.log")
+def command_cosole_log_plots(platform_serial, mission, plots_dir):
+    _log.info(f"Make command console plots for {platform_serial} M{mission}")
+    cmd_log = Path(f"/data/data_raw/nrt/{platform_serial}/{str(mission).zfill(6)}/G-Logs/{platform_serial.lower()}.{mission}.com.raw.log")
     if not cmd_log.exists():
         _log.warning(f"No command log found at {cmd_log}")
         return
     make_all_plots(cmd_log)
-    filename = plots_dir / f'SEA{glider}_M{mission}_cmd_log.png'
+    filename = plots_dir / f'{platform_serial}_M{mission}_cmd_log.png'
     plt.savefig(filename, format='png', transparent=True)
     _log.debug(f"saved figure to {filename}")
 
@@ -246,4 +246,4 @@ def command_cosole_log_plots(glider, mission, plots_dir):
                         format='%(asctime)s %(levelname)-8s %(message)s',
                         level=logging.WARNING,
                         datefmt='%Y-%m-%d %H:%M:%S')
-    command_cosole_log_plots(45, 87, Path("."))
+    command_cosole_log_plots("SEA045", 87, Path("."))

complete_mission_plots.py

@@ -10,13 +10,13 @@
 _log = logging.getLogger(__name__)
 
 
-def complete_plots(glider, mission):
+def complete_plots(platform_serial, mission):
     logging.basicConfig(filename='/data/log/complete_mission/plots.log',
                         filemode='a',
                         format='%(asctime)s %(levelname)-8s %(message)s',
                         level=logging.INFO,
                         datefmt='%Y-%m-%d %H:%M:%S')
-    mission_dir = Path(f'/data/data_l0_pyglider/complete_mission/SEA{glider}/M{mission}/gridfiles/')
+    mission_dir = Path(f'/data/data_l0_pyglider/complete_mission/{platform_serial}/M{mission}/gridfiles/')
     netcdf = None
     for fn in ['gridded_adcp_gliderad2cp.nc', 'gridded_gliderad2cp.nc']:
         if (mission_dir / fn).exists():
@@ -27,7 +27,7 @@ def complete_plots(glider, mission):
         except IndexError:
             _log.error(f"nc file in {mission_dir} not found")
             return
-    outdir = Path(f'/data/plots/complete_mission/SEA{glider}/M{mission}/')
+    outdir = Path(f'/data/plots/complete_mission/{platform_serial}/M{mission}/')
     if not outdir.exists():
         outdir.mkdir(parents=True)
     if 'scatter' in sys.argv:

glidertools_plots.py

@@ -71,7 +71,7 @@ def public_plots(nc, plots_dir):
         ax.set(xlabel='', ylabel='Depth (m)', ylim=(ymin, 0))
         plt.colorbar(mappable=im, ax=ax, label=label_replace(ds[variable].units), aspect=13, pad=0.02)
     plt.tight_layout()
-    filename = plots_dir / f'SEA{ds.glider_serial}_M{ds.deployment_id}_gt.png'
+    filename = plots_dir / f'{ds.glider_serial}_M{ds.deployment_id}_gt.png'
     plt.savefig(filename, format='png', transparent=True)
 
 

gridded_plots.py

@@ -289,7 +289,7 @@ def multiplotter(dataset, variables, plots_dir, glider='', mission='', grid=True
         ax.set(xlabel='', ylabel='Depth (m)', ylim=(ymin, 0))
         plt.colorbar(mappable=pcol, ax=ax, label=label_replace(ds.units), aspect=13, pad=0.02)
     plt.tight_layout()
-    filename = plots_dir / f'SEA{glider}_M{mission}.png'
+    filename = plots_dir / f'{glider}_M{mission}.png'
     _log.info(f'writing figure to {filename}')
     fig.savefig(filename, format='png', transparent=True)
     return filename
@@ -354,7 +354,7 @@ def make_map(nc, filename):
     fig = plt.figure(figsize=(12, 6))
     ax = fig.add_subplot(111, projection=coord)
     attrs = dataset.attrs
-    fig.suptitle(f"SEA{attrs['glider_serial']} {attrs['glider_name']} mission {attrs['deployment_id']}")
+    fig.suptitle(f"{attrs['platform_serial']} {attrs['glider_name']} mission {attrs['deployment_id']}")
     ax.scatter(lons, lats, transform=pc, s=10)
     lon_extend = 3
     lat_extend = 1

nrt_process.py

@@ -23,50 +23,51 @@
 
 def main():
     _log.info("Start nrt processing")
-    all_glider_paths = pathlib.Path(f"/data/data_raw/nrt").glob("SEA*")
+    all_glider_paths = pathlib.Path(f"/data/data_raw/nrt").glob("S*")
     for glider_path in all_glider_paths:
         glider = str(glider_path)[-3:].lstrip("0")
         if int(glider) in glider_no_proc:
             _log.info(f"SEA{glider} is not to be processed. Skipping")
             continue
-        _log.info(f"Checking SEA{glider}")
+        platform_serial = glider_path.parts[-1]
+        _log.info(f"Checking {platform_serial}")
         mission_paths = list(glider_path.glob("00*"))
         if not mission_paths:
-            _log.warning(f"No missions found for SEA{glider}. Skipping")
+            _log.warning(f"No missions found for {platform_serial}. Skipping")
             continue
         mission_paths.sort()
         mission = str(mission_paths[-1])[-3:].lstrip("0")
-        _log.info(f"Checking SEA{glider} M{mission}")
-        mission_dir = f'/data/data_l0_pyglider/nrt/SEA{glider}/M{mission}/gridfiles/'
+        _log.info(f"Checking {platform_serial} M{mission}")
+        mission_dir = f'/data/data_l0_pyglider/nrt/{platform_serial}/M{mission}/gridfiles/'
         try:
             nc_file = list(pathlib.Path(mission_dir).glob('*.nc'))[0]
         except IndexError:
             _log.error(f"nc file in {mission_dir} not found")
             continue
         nc_time = nc_file.lstat().st_mtime
         infile_time = 0
-        in_files = list(pathlib.Path(f'/data/plots/nrt/SEA{glider}/M{mission}/').glob('*.png'))
+        in_files = list(pathlib.Path(f'/data/plots/nrt/{platform_serial}/M{mission}/').glob('*.png'))
         for file in in_files:
             if file.lstat().st_mtime > infile_time:
                 infile_time = file.lstat().st_mtime
         if nc_time < infile_time:
-            _log.info(f"SEA{glider} M{mission} unchanged. No plotting")
+            _log.info(f"{platform_serial} M{mission} unchanged. No plotting")
             continue
-        _log.info(f"Processing SEA{glider} M{mission}")
-        outdir = pathlib.Path(f'/data/plots/nrt/SEA{glider}/M{mission}/')
+        _log.info(f"Processing {platform_serial} M{mission}")
+        outdir = pathlib.Path(f'/data/plots/nrt/{platform_serial}/M{mission}/')
         if not outdir.exists():
             outdir.mkdir(parents=True)
         _log.info(f'start plotting {nc_file} ')
         image_file = create_plots(nc_file, outdir, False)
         make_map(nc_file, image_file)
         _log.info("start glidertools plots")
-        ts_dir = f'/data/data_l0_pyglider/nrt/SEA{glider}/M{mission}/timeseries/'
+        ts_dir = f'/data/data_l0_pyglider/nrt/{platform_serial}/M{mission}/timeseries/'
         ts_file = list(pathlib.Path(ts_dir).glob('*.nc'))[0]
         public_plots(ts_file, outdir)
         #plot_qc(ts_file, outdir)
         _log.info("start pilot plots")
-        command_cosole_log_plots(glider, mission, outdir)
-        combi_nav_files = list(pathlib.Path(f'/data/data_l0_pyglider/nrt/SEA{glider}/M{mission}/rawnc/').glob("*rawgli.parquet"))
+        command_cosole_log_plots(platform_serial, mission, outdir)
+        combi_nav_files = list(pathlib.Path(f'/data/data_l0_pyglider/nrt/{platform_serial}/M{mission}/rawnc/').glob("*rawgli.parquet"))
         if combi_nav_files:
             battery_plots(combi_nav_files[0], outdir)
             _log.info("Finished pilot plots")

pilot_plots.py

@@ -10,9 +10,9 @@
 
 def battery_plots(combined_nav_file, out_dir):
     parts = combined_nav_file.parts
-    glider = parts[-4][3:]
+    platform_serial = parts[-4]
     mission = parts[-3][1:]
-    title = f"SEA{glider.zfill(3)} M{mission}"
+    title = f"{platform_serial} M{mission}"
     df_polar = pl.read_parquet(combined_nav_file)
     df = pd.read_parquet(combined_nav_file)
     df.index = df_polar.select("time").to_numpy()[:, 0]
@@ -87,7 +87,7 @@ def battery_plots(combined_nav_file, out_dir):
         ax.set(ylabel="Voltage (v)", title=title)
         plt.xticks(rotation=45)
         ax.legend(loc=1)
-        dline = f"{datetime.datetime.now()},{glider},{mission},{v_per_days[1]},{recoveries[0]},{end}\n"
+        dline = f"{datetime.datetime.now()},{platform_serial},{mission},{v_per_days[1]},{recoveries[0]},{end}\n"
         with open("/data/plots/nrt/battery_prediction.csv", "a") as file:
             file.write(dline)
     plt.tight_layout()