use existing timesteps written to a file to figure out the remaining file outputs before rotation

Author: Chrismarsh

src/chkpt_op.hpp

@@ -73,7 +73,6 @@ class chkptOp
     {
         std::string base_name;
         std::string path;
-        boost::optional<size_t> rotate_offset;
     };
 
     // Cached output rotation state from checkpoint metadata (used during resume).

src/core.cpp

@@ -490,11 +490,6 @@ void core::config_checkpoint( pt::ptree& value)
                     chkptOp::ugrid_output_state state;
                     state.base_name = itr.second.get<std::string>("base_name", "");
                     state.path = itr.second.get<std::string>("path", "");
-                    auto offset = itr.second.get_optional<size_t>("rotate_offset");
-                    if (offset)
-                    {
-                        state.rotate_offset = *offset;
-                    }
                     if (!state.base_name.empty() || !state.path.empty())
                     {
                         _checkpoint_opts.ugrid_outputs.push_back(state);
@@ -539,10 +534,14 @@ void core::config_checkpoint( pt::ptree& value)
                         writer->set_store_path(match->path);
                         SPDLOG_DEBUG("Resuming ugrid output from checkpoint file {}", match->path);
                     }
-                    if (match->rotate_offset)
+                    if (out.rotate_frequency && !match->path.empty())
                     {
-                        out.rotate_offset = match->rotate_offset;
-                        SPDLOG_DEBUG("Resuming ugrid rotation with offset {}", *match->rotate_offset);
+                        auto& writer = boost::get<std::shared_ptr<ugrid_writer>>(out.writer);
+                        // Derive rotation offset from the existing ugrid file instead of checkpoint JSON.
+                        size_t time_len = writer->probe_time_index();
+                        out.rotate_offset = time_len % *out.rotate_frequency;
+                        SPDLOG_DEBUG("Resuming ugrid rotation with offset {} ({} timesteps already in file)",
+                                     *out.rotate_offset, time_len);
                     }
                 }
             }
@@ -2587,13 +2586,6 @@ void core::run()
                 auto& writer = boost::get<std::shared_ptr<ugrid_writer>>(out.writer);
                 entry.put("path", writer->store_path());
 
-                if (out.rotate_frequency)
-                {
-                    // Store the offset for the next timestep after this checkpoint.
-                    size_t offset = (current_ts + 1) % *out.rotate_frequency;
-                    entry.put("rotate_offset", offset);
-                }
-
                 ugrid_outputs.push_back(std::make_pair("", entry));
             }
             if (!ugrid_outputs.empty())

src/mesh/ugrid_writer.cpp

@@ -98,6 +98,59 @@ const std::string& ugrid_writer::store_path() const
     return _store_path;
 }
 
+size_t ugrid_writer::probe_time_index()
+{
+    if (!store_exists())
+    {
+        return 0;
+    }
+
+    // Open the existing ugrid file read-only to determine how many timesteps are present.
+    int fid = -1;
+    MPI_Info info_used;
+    MPI_Comm_get_info(_comm_world, &info_used);
+
+    nc_chk_ret(nc_open_par(_fname.c_str(), NC_NOWRITE, _comm_world, info_used, &fid));
+    size_t time_len = read_time_index(fid);
+
+    nc_chk_ret(nc_close(fid));
+    MPI_Info_free(&info_used);
+
+    return time_len;
+}
+
+size_t ugrid_writer::read_time_index(int fid) const
+{
+    int unlimdimidp;
+    nc_chk_ret(nc_inq_unlimdim(fid, &unlimdimidp));
+
+    size_t time_len = 0;
+    nc_chk_ret(nc_inq_dimlen(fid, unlimdimidp, &time_len));
+
+    if (_global->from_checkpoint() && time_len > 0)
+    {
+        // Trim to the restart time so we overwrite any outputs past the checkpoint.
+        int time_varid = -1;
+        int status = nc_inq_varid(fid, "time", &time_varid);
+        if (status == NC_NOERR)
+        {
+            std::vector<double> time_vals(time_len, 0.0);
+            nc_chk_ret(nc_get_var_double(fid, time_varid, time_vals.data()));
+
+            const double restart_time_minutes = _global->posix_time_double() / 60.0;
+            auto it = std::lower_bound(time_vals.begin(), time_vals.end(), restart_time_minutes);
+            auto idx = static_cast<size_t>(std::distance(time_vals.begin(), it));
+
+            if (idx < time_len)
+            {
+                time_len = idx;
+            }
+        }
+    }
+
+    return time_len;
+}
+
 std::string ugrid_writer::build_store_uri(const std::string& store_path) const
 {
     if (!_use_zarr)
@@ -394,30 +447,7 @@ void ugrid_writer::open_ugrid(const std::vector<std::string>& output_variables)
         nc_chk_ret(nc_var_par_access(_ugrid_fid, p.second, NC_COLLECTIVE));
     }
 
-    int unlimdimidp;
-
-    nc_chk_ret(nc_inq_unlimdim(_ugrid_fid, &unlimdimidp)); // get the time /dimension/. it's the only unlimited
-    nc_chk_ret(nc_inq_dimlen(_ugrid_fid, unlimdimidp, &_time_index));
-
-    // If we are resuming from a checkpoint, make sure we overwrite any timesteps
-    // beyond the checkpoint instead of blindly appending to the existing file.
-    if (_global->from_checkpoint() && _time_index > 0)
-    {
-        // chm outputs minutes since as time unit
-        const double restart_time_minutes = _global->posix_time_double() / 60.0;
-        std::vector<double> time_vals(_time_index, 0.0);
-
-        nc_chk_ret(nc_get_var_double(_ugrid_fid, _ugrid_id_var["time"], time_vals.data()));
-
-        auto it = std::lower_bound(time_vals.begin(), time_vals.end(), restart_time_minutes);
-        auto idx = static_cast<size_t>(std::distance(time_vals.begin(), it));
-
-        if (idx < _time_index)
-        {
-            SPDLOG_DEBUG("Resuming ugrid at time index {} (was {}).", idx, _time_index);
-            _time_index = idx;
-        }
-    }
+    _time_index = read_time_index(_ugrid_fid);
 
     SPDLOG_DEBUG("Existing ugrid output has {} timesteps already", _time_index);
 

src/mesh/ugrid_writer.hpp

@@ -57,13 +57,16 @@ class ugrid_writer
                                const boost::optional<double>& chunk_target_mb);
     void set_store_path(std::string store_path);
     const std::string& store_path() const;
+    // Probe the existing ugrid file to determine how many timesteps are already written.
+    size_t probe_time_index();
 
     bool bitgroom;
     bool compress;
 private:
     std::string build_store_uri(const std::string& store_path) const;
     bool store_exists() const;
     size_t compute_time_chunk_len(size_t max_faces_per_rank, size_t num_output_vars) const;
+    size_t read_time_index(int fid) const;
 
     //holds the file id for the ugrid output netcdf
     int _ugrid_fid;