Clean up splitting dual profile files

Author: jmcvey3

examples/data/dolfyn/Sig500_dp_all.ad2cp examples/data/dolfyn/Sig500_dp_ice.ad2cpexamples/data/dolfyn/Sig500_dp_all.ad2cp renamed to examples/data/dolfyn/Sig500_dp_ice.ad2cp

@@ -164,9 +164,14 @@ def _create_dataset(data):
     """
 
     tag = ["_avg", "_b5", "_echo", "_bt", "_gps", "_altraw", "_altraw_avg", "_sl"]
-    # dual profile where burst velocity is not measured but echo sounder is
-    if ("vel" not in data["data_vars"]) and ("vel_avg" in data["data_vars"]):
-        data["coords"]["time"] = data["coords"]["time_avg"]
+    # If burst velocity not measured
+    if "vel" not in data["data_vars"]:
+        # dual profile where burst velocity is not measured but echo sounder is
+        if "vel_avg" in data["data_vars"]:
+            data["coords"]["time"] = data["coords"]["time_avg"]
+        else:
+            t_vars = [t for t in data["coords"] if "time" in t]
+            data["coords"]["time"] = data["coords"][t_vars[0]]
 
     ds_dict = {}
     for key in data["coords"]:

…data/dolfyn/test_data/Sig1000_dp_echo.nc …ata/dolfyn/test_data/Sig1000_dp_echo1.ncexamples/data/dolfyn/test_data/Sig1000_dp_echo.nc renamed to examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc examples/data/dolfyn/test_data/Sig1000_dp_echo.nc renamed to examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc

@@ -769,41 +769,29 @@ def _reduce(data):
             tm[irow, icol] = tmat["M" + str(irow + 1) + str(icol + 1)]
     dv["beam2inst_orientmat"] = tm
 
-    # If burst velocity isn't used, need to copy one for 'time'
-    if "time" not in dc:
-        for val in dc:
-            if "time" in val:
-                time = val
-        dc["time"] = dc[time]
-
 
 def split_dp_datasets(ds):
     """
     Splits a dataset containing dual profiles into individual profiles
     """
 
     ds2 = type(ds)()
-    # Figure out which variables belong to which profile based on length of time variables
-    t_dict = {}
-    for t in ds.coords:
-        if "time" in t:
-            t_dict[t] = ds[t].size
-
-    other_coords = []
-    for key, val in t_dict.items():
-        if val != t_dict["time"]:
-            if key.endswith("altraw"):
-                # altraw goes with burst, altraw_avg goes with avg
-                continue
-            other_coords.append(key)
+    # Get averaged time variables
+    other_coords = [t for t in ds.coords if "_avg" in t]
+    # Check if ice tracking was used
+    if ("vel" in ds.data_vars) and ("time_bt" in ds.coords):
+        if ds["time"].size != ds["time_bt"].size:
+            other_coords.append("time_bt")
+
     if other_coords:
         # Fetch variables, coordinates, and attrs for second profiling configuration
         other_vars = [
-            v for v in ds.data_vars if any(x in ds[v].coords for x in other_coords)
+            v
+            for v in ds.data_vars
+            if any(x in ds[v].coords for x in other_coords)
+            or any(ds[v].shape[-1] == ds[x].size for x in other_coords)
         ]
-        other_tags = [s.split("_")[-1] for s in other_coords]
-        other_coords += [v for v in ds.coords if any(x in v for x in other_tags)]
-        other_attrs = [s for s in ds.attrs if any(x in s for x in other_tags)]
+        other_attrs = [s for s in ds.attrs if "_avg" in s]
         critical_attrs = [
             "inst_model",
             "inst_make",
@@ -825,19 +813,13 @@ def split_dp_datasets(ds):
         ds2.attrs["rotate_vars"] = rotate_vars2
         # Set orientation matricies
         ds2["beam2inst_orientmat"] = ds["beam2inst_orientmat"]
-        # drop echo sounder tag for following code
-        if "echo" in other_tags:
-            other_tags.remove("echo")
-        if ds.attrs["has_imu"] and other_tags:
-            ds2 = ds2.rename({"orientmat_" + other_tags[0]: "orientmat"})
+        if ds.attrs["has_imu"]:
+            ds2 = ds2.rename({"orientmat_avg": "orientmat"})
         else:
             ds2["orientmat"] = ds["orientmat"]
         # Set original coordinate system
-        if other_tags:
-            cy = ds2.attrs["coord_sys_axes_" + other_tags[0]]
-            ds2.attrs["coord_sys"] = {"XYZ": "inst", "ENU": "earth", "beam": "beam"}[cy]
-        else:
-            ds2.attrs["coord_sys"] = ds.attrs["coord_sys"]
+        cy = ds2.attrs["coord_sys_axes_avg"]
+        ds2.attrs["coord_sys"] = {"XYZ": "inst", "ENU": "earth", "beam": "beam"}[cy]
         # Need to add beam coordinate
         if "beam" not in ds2.dims:
             ds2 = ds2.assign_coords({"beam": ds["beam"], "dir": ds["dir"]})
@@ -849,4 +831,7 @@ def split_dp_datasets(ds):
         for itm in rotate_vars2:
             ds.attrs["rotate_vars"].remove(itm)
 
-    return ds, ds2
+    if ds2:
+        return ds, ds2
+    else:
+        return ds

examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc

@@ -62,7 +62,7 @@ def test_clean_upADCP(self):
         td_awac = tp.dat_awac.copy(deep=True)
         td_sig = tp.dat_sig_tide.copy(deep=True)
         td_rdi = tp.dat_rdi.copy(deep=True)
-        td_dual = tp.dat_sig_dp2.copy(deep=True)
+        td_dual = tp.dat_sig_dp1_ice.copy(deep=True)
 
         apm.clean.water_depth_from_pressure(td_awac, salinity=30)
         apm.clean.remove_surface_interference(td_awac, beam_angle=20, inplace=True)
@@ -95,7 +95,7 @@ def test_clean_upADCP(self):
 
     def test_clean_downADCP(self):
         td = tp.dat_sig_ie.copy(deep=True)
-        td_dual = tp.dat_sig_dp2.copy(deep=True)
+        td_dual = tp.dat_sig_dp1_ice.copy(deep=True)
 
         # First remove bad data
         td["vel"] = apm.clean.val_exceeds_thresh(td.vel, thresh=3)

examples/data/dolfyn/test_data/Sig500_dp_ice1.nc

@@ -38,8 +38,10 @@
 dat_sig_skip = load("Sig_SkippedPings01.nc")
 dat_sig_badt = load("Sig1000_BadTime01.nc")
 dat_sig5_leiw = load("Sig500_last_ensemble_is_whole.nc")
-dat_sig_dp2_all = load("Sig500_dp_all.nc")
-dat_sig_dp2_echo = load("Sig1000_dp_echo.nc")
+dat_sig_dp1_all = load("Sig500_dp_ice1.nc")
+dat_sig_dp1_ice = load("Sig500_dp_ice2.nc")
+dat_sig_dp2_echo = load("Sig1000_dp_echo1.nc")
+dat_sig_dp2_avg = load("Sig1000_dp_echo2.nc")
 
 
 class io_adp_testcase(unittest.TestCase):
@@ -111,9 +113,8 @@ def test_io_nortek2(self):
         td_sig_raw_avg = read("Sig100_raw_avg.ad2cp", nens=nens, rebuild_index=True)
         td_sig_avg = read("Sig100_avg.ad2cp", nens=nens, rebuild_index=True)
         td_sig_rt = read("Sig1000_online.ad2cp", nens=nens, rebuild_index=True)
-        # Only need to test 2nd dataset
-        td_sig_dp1_all, td_sig_dp2_all = read("Sig500_dp_all.ad2cp", rebuild_index=True)
-        td_sig_dp1_avg, td_sig_dp2_echo = read(
+        td_sig_dp1_all, td_sig_dp1_ice = read("Sig500_dp_ice.ad2cp", rebuild_index=True)
+        td_sig_dp2_echo, td_sig_dp2_avg = read(
             "Sig1000_dp_echo.ad2cp", rebuild_index=True
         )
 
@@ -139,7 +140,7 @@ def test_io_nortek2(self):
         os.remove(tb.exdt("Sig_SkippedPings01.ad2cp.index"))
         os.remove(tb.exdt("Sig500_last_ensemble_is_whole.ad2cp.index"))
         os.remove(tb.rfnm("Sig1000_BadTime01.ad2cp.index"))
-        os.remove(tb.exdt("Sig500_dp_all.ad2cp.index"))
+        os.remove(tb.exdt("Sig500_dp_ice.ad2cp.index"))
         os.remove(tb.exdt("Sig1000_dp_echo.ad2cp.index"))
 
         if make_data:
@@ -155,8 +156,10 @@ def test_io_nortek2(self):
             save(td_sig_skip, "Sig_SkippedPings01.nc")
             save(td_sig_badt, "Sig1000_BadTime01.nc")
             save(td_sig5_leiw, "Sig500_last_ensemble_is_whole.nc")
-            save(td_sig_dp2_all, "dual_profile.nc")
-            save(td_sig_dp2_echo, "Sig1000_dp_echo.nc")
+            save(td_sig_dp1_all, "Sig500_dp_ice1.nc")
+            save(td_sig_dp1_ice, "Sig500_dp_ice2.nc")
+            save(td_sig_dp2_echo, "Sig1000_dp_echo1.nc")
+            save(td_sig_dp2_avg, "Sig1000_dp_echo2.nc")
             return
 
         assert_allclose(td_sig, dat_sig, atol=1e-6)
@@ -171,8 +174,10 @@ def test_io_nortek2(self):
         assert_allclose(td_sig5_leiw, dat_sig5_leiw, atol=1e-6)
         assert_allclose(td_sig_skip, dat_sig_skip, atol=1e-6)
         assert_allclose(td_sig_badt, dat_sig_badt, atol=1e-6)
-        assert_allclose(td_sig_dp2_all, dat_sig_dp2_all, atol=1e-6)
+        assert_allclose(td_sig_dp1_all, dat_sig_dp1_all, atol=1e-6)
+        assert_allclose(td_sig_dp1_ice, dat_sig_dp1_ice, atol=1e-6)
         assert_allclose(td_sig_dp2_echo, dat_sig_dp2_echo, atol=1e-6)
+        assert_allclose(td_sig_dp2_avg, dat_sig_dp2_avg, atol=1e-6)
 
     def test_nortek2_crop(self):
         # Test file cropping function

…s/data/dolfyn/test_data/Sig500_dp_all.nc …/data/dolfyn/test_data/Sig500_dp_ice2.ncexamples/data/dolfyn/test_data/Sig500_dp_all.nc renamed to examples/data/dolfyn/test_data/Sig500_dp_ice2.nc examples/data/dolfyn/test_data/Sig500_dp_all.nc renamed to examples/data/dolfyn/test_data/Sig500_dp_ice2.nc

@@ -118,6 +118,8 @@ def test_rotate_earth2inst(self):
         rotate2(td_sig, "inst", inplace=True)
         td_sig_i = load("Sig1000_IMU_rotate_inst2earth.nc")
         rotate2(td_sig_i, "inst", inplace=True)
+        td_sig_avg = load("Sig100_avg")
+        rotate2(td_sig_avg, "inst", inplace=True)
 
         cd_rdi = load("RDI_test01_rotate_beam2inst.nc")
         cd_wr2 = tr.dat_wr2