storage shapes in paged data structures for improved performance on unordered files (#149)

felixguendling · web-flow · commit 06fc5ef490a2 · 2024-10-25T16:23:08.000+02:00
* wip

* wip

* wip

* increase max page size

* sort by shape seq

* fix tests

* fix nigiri

* remove unnecessary include

* move seq to shape.cc

* wip
diff --git a/.pkg b/.pkg
@@ -5,15 +5,15 @@
 [cista]
   url=git@github.com:felixguendling/cista.git
   branch=master
-  commit=48210e6925658163952e458a4d13629ffdb0cea3
+  commit=fabfc0cbcf07343a2e0c2def3009555b22f0ae46
 [geo]
   url=git@github.com:motis-project/geo.git
   branch=master
   commit=0a14addf42e91b267906a156c9c2564935c03eaf
 [utl]
   url=git@github.com:motis-project/utl.git
   branch=master
-  commit=77aac494c45d2b070e65fe712abc34ac74a91d0f
+  commit=8bfa7fc4edc23f716173bdfbcab8294fcc31e457
 [miniz]
   url=git@github.com:motis-project/miniz.git
   branch=master
diff --git a/exe/import.cc b/exe/import.cc
@@ -10,7 +10,7 @@
 #include "nigiri/loader/load.h"
 #include "nigiri/loader/loader_interface.h"
 #include "nigiri/common/parse_date.h"
-#include "nigiri/shape.h"
+#include "nigiri/shapes_storage.h"
 
 namespace fs = std::filesystem;
 namespace bpo = boost::program_options;
@@ -108,7 +108,8 @@ int main(int ac, char** av) {
 
   auto shapes = std::unique_ptr<shapes_storage>{};
   if (vm.contains("shapes")) {
-    shapes = std::make_unique<shapes_storage>(out_shapes);
+    shapes = std::make_unique<shapes_storage>(out_shapes,
+                                              cista::mmap::protection::WRITE);
   }
 
   auto const start = parse_date(start_date);
diff --git a/include/nigiri/loader/gtfs/load_timetable.h b/include/nigiri/loader/gtfs/load_timetable.h
@@ -2,7 +2,6 @@
 
 #include "nigiri/loader/dir.h"
 #include "nigiri/loader/loader_interface.h"
-#include "nigiri/shape.h"
 #include "nigiri/types.h"
 
 namespace nigiri {
diff --git a/include/nigiri/loader/gtfs/loader.h b/include/nigiri/loader/gtfs/loader.h
@@ -1,7 +1,6 @@
 #pragma once
 
 #include "nigiri/loader/loader_interface.h"
-#include "nigiri/shape.h"
 
 namespace nigiri::loader::gtfs {
 
diff --git a/include/nigiri/loader/gtfs/shape.h b/include/nigiri/loader/gtfs/shape.h
@@ -2,19 +2,17 @@
 
 #include <string_view>
 
-#include "nigiri/shape.h"
 #include "nigiri/types.h"
 
-namespace nigiri::loader::gtfs {
+namespace nigiri {
+struct shapes_storage;
+}
 
-struct shape_state {
-  shape_idx_t index_{};
-  std::size_t last_seq_{};
-};
+namespace nigiri::loader::gtfs {
 
 struct shape_loader_state {
-  hash_map<std::string, shape_state> id_map_{};
-  vecvec<shape_idx_t, double> distances_{};
+  hash_map<std::string, shape_idx_t> id_map_{};
+  std::vector<std::vector<double>> distances_{};
   shape_idx_t index_offset_;
 };
 
diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h
@@ -2,8 +2,11 @@
 
 #include "nigiri/loader/gtfs/shape.h"
 #include "nigiri/loader/gtfs/trip.h"
-#include "nigiri/shape.h"
-#include "nigiri/timetable.h"
+
+namespace nigiri {
+struct shapes_storage;
+struct timetable;
+}  // namespace nigiri
 
 namespace nigiri::loader::gtfs {
 
diff --git a/include/nigiri/loader/hrd/loader.h b/include/nigiri/loader/hrd/loader.h
@@ -2,7 +2,6 @@
 
 #include "nigiri/loader/hrd/parser_config.h"
 #include "nigiri/loader/loader_interface.h"
-#include "nigiri/shape.h"
 
 namespace nigiri::loader::hrd {
 
diff --git a/include/nigiri/loader/load.h b/include/nigiri/loader/load.h
@@ -7,10 +7,13 @@
 
 #include "nigiri/loader/build_footpaths.h"
 #include "nigiri/common/interval.h"
-#include "nigiri/shape.h"
 #include "nigiri/timetable.h"
 #include "nigiri/types.h"
 
+namespace nigiri {
+struct shapes_storage;
+}
+
 namespace nigiri::loader {
 
 struct assistance_times;
diff --git a/include/nigiri/loader/loader_interface.h b/include/nigiri/loader/loader_interface.h
@@ -5,12 +5,12 @@
 
 #include "nigiri/loader/assistance.h"
 #include "nigiri/loader/dir.h"
-#include "nigiri/shape.h"
 #include "nigiri/types.h"
 
 namespace nigiri {
+struct shapes_storage;
 struct timetable;
-}
+}  // namespace nigiri
 
 namespace nigiri::loader {
 
diff --git a/include/nigiri/rt/frun.h b/include/nigiri/rt/frun.h
@@ -8,11 +8,11 @@
 #include "nigiri/common/interval.h"
 #include "nigiri/location.h"
 #include "nigiri/rt/run.h"
-#include "nigiri/shape.h"
 #include "nigiri/stop.h"
 #include "nigiri/types.h"
 
 namespace nigiri {
+struct shapes_storage;
 struct rt_timetable;
 struct timetable;
 }  // namespace nigiri
diff --git a/include/nigiri/shapes_storage.h b/include/nigiri/shapes_storage.h
@@ -12,9 +12,10 @@
 namespace nigiri {
 
 struct shapes_storage {
-  explicit shapes_storage(
-      std::filesystem::path const&,
-      cista::mmap::protection = cista::mmap::protection::WRITE);
+  shapes_storage(std::filesystem::path, cista::mmap::protection);
+
+  cista::mmap mm(char const* file);
+
   std::span<geo::latlng const> get_shape(shape_idx_t) const;
   std::span<geo::latlng const> get_shape(trip_idx_t) const;
   std::span<geo::latlng const> get_shape(trip_idx_t,
@@ -23,8 +24,11 @@ struct shapes_storage {
   void add_trip_shape_offsets(
       trip_idx_t, cista::pair<shape_idx_t, shape_offset_idx_t> const&);
 
-  mm_vecvec<shape_idx_t, geo::latlng> data_;
-  mm_vecvec<shape_offset_idx_t, shape_offset_t> offsets_;
+  cista::mmap::protection mode_;
+  std::filesystem::path p_;
+
+  mm_paged_vecvec<shape_idx_t, geo::latlng> data_;
+  mm_vecvec<shape_offset_idx_t, shape_offset_t, std::uint64_t> offsets_;
   mm_vec_map<trip_idx_t, cista::pair<shape_idx_t, shape_offset_idx_t>>
       trip_offset_indices_;
 };
diff --git a/include/nigiri/types.h b/include/nigiri/types.h
@@ -122,14 +122,26 @@ template <typename Key, typename V, typename SizeType = cista::base_t<Key>>
 using mm_vecvec = cista::basic_vecvec<Key, mm_vec<V>, mm_vec<SizeType>>;
 
 template <typename Key, typename T>
-struct mmap_paged_vecvec_helper {
-  using data_t = cista::paged<vector<T>>;
+struct paged_vecvec_helper {
+  using data_t =
+      cista::paged<vector<T>, std::uint64_t, std::uint32_t, 4U, 1U << 31U>;
   using idx_t = vector<typename data_t::page_t>;
   using type = cista::paged_vecvec<idx_t, data_t, Key>;
 };
 
 template <typename Key, typename T>
-using paged_vecvec = mmap_paged_vecvec_helper<Key, T>::type;
+using paged_vecvec = paged_vecvec_helper<Key, T>::type;
+
+template <typename Key, typename T>
+struct mm_paged_vecvec_helper {
+  using data_t =
+      cista::paged<mm_vec<T>, std::uint64_t, std::uint32_t, 2U, 1U << 31U>;
+  using idx_t = mm_vec<typename data_t::page_t>;
+  using type = cista::paged_vecvec<idx_t, data_t, Key>;
+};
+
+template <typename Key, typename T>
+using mm_paged_vecvec = mm_paged_vecvec_helper<Key, T>::type;
 
 using bitfield_idx_t = cista::strong<std::uint32_t, struct _bitfield_idx>;
 using location_idx_t = cista::strong<std::uint32_t, struct _location_idx>;
diff --git a/src/abi.cc b/src/abi.cc
@@ -20,7 +20,7 @@
 #include "nigiri/rt/create_rt_timetable.h"
 #include "nigiri/rt/gtfsrt_update.h"
 #include "nigiri/rt/rt_timetable.h"
-#include "nigiri/shape.h"
+#include "nigiri/shapes_storage.h"
 #include "nigiri/timetable.h"
 #include "nigiri/types.h"
 
diff --git a/src/loader/gtfs/shape.cc b/src/loader/gtfs/shape.cc
@@ -7,9 +7,11 @@
 #include "utl/parser/line_range.h"
 #include "utl/pipes/for_each.h"
 #include "utl/progress_tracker.h"
+#include "utl/sort_by.h"
 
 #include "nigiri/common/cached_lookup.h"
 #include "nigiri/logging.h"
+#include "nigiri/shapes_storage.h"
 
 namespace nigiri::loader::gtfs {
 
@@ -20,7 +22,7 @@ shape_loader_state parse_shapes(std::string_view const data,
     utl::csv_col<utl::cstr, UTL_NAME("shape_id")> id_;
     utl::csv_col<double, UTL_NAME("shape_pt_lat")> lat_;
     utl::csv_col<double, UTL_NAME("shape_pt_lon")> lon_;
-    utl::csv_col<std::size_t, UTL_NAME("shape_pt_sequence")> seq_;
+    utl::csv_col<std::uint32_t, UTL_NAME("shape_pt_sequence")> seq_;
     utl::csv_col<double, UTL_NAME("shape_dist_traveled")> distance_;
   };
 
@@ -29,6 +31,7 @@ shape_loader_state parse_shapes(std::string_view const data,
       .index_offset_ = index_offset,
   };
   auto lookup = cached_lookup(states.id_map_);
+  auto seq = std::vector<std::vector<std::uint32_t>>{};
 
   auto const progress_tracker = utl::get_active_progress_tracker();
   progress_tracker->status("Parse Shapes")
@@ -37,32 +40,48 @@ shape_loader_state parse_shapes(std::string_view const data,
   utl::line_range{utl::make_buf_reader(data, progress_tracker->update_fn())}  //
       | utl::csv<shape_entry>()  //
       | utl::for_each([&](shape_entry const entry) {
-          auto& state = lookup(entry.id_->view(), [&] {
-            auto const index = static_cast<shape_idx_t>(shapes.size());
-            shapes.add_back_sized(0U);
-            states.distances_.add_back_sized(0U);
-            return shape_state{index, 0U};
+          auto const shape_idx = lookup(entry.id_->view(), [&] {
+            auto const idx = static_cast<shape_idx_t>(shapes.size());
+            shapes.emplace_back_empty();
+            states.distances_.emplace_back();
+            seq.emplace_back();
+            return idx;
           });
-          auto const seq = *entry.seq_;
-          auto bucket = shapes[state.index_];
-          if (!bucket.empty() && state.last_seq_ >= seq) {
-            log(log_lvl::error, "loader.gtfs.shape",
-                "Non monotonic sequence for shape_id '{}': Sequence number {} "
-                "followed by {}",
-                entry.id_->to_str(), state.last_seq_, seq);
-          }
-          bucket.push_back(geo::latlng{*entry.lat_, *entry.lon_});
-          state.last_seq_ = seq;
-          auto distances = states.distances_[state.index_ - index_offset];
+          auto polyline = shapes[shape_idx];
+          polyline.push_back(geo::latlng{*entry.lat_, *entry.lon_});
+          auto const state_idx = to_idx(shape_idx - index_offset);
+          seq[state_idx].push_back(*entry.seq_);
+          auto& distances = states.distances_[state_idx];
           if (distances.empty()) {
             if (*entry.distance_ != 0.0) {
-              distances.grow(bucket.size());
+              for (auto i = 0U; i != polyline.size(); ++i) {
+                distances.push_back(0.0);
+              }
               distances.back() = *entry.distance_;
             }
           } else {
             distances.push_back(*entry.distance_);
           }
         });
+
+  auto polyline = std::vector<geo::latlng>();
+  auto shape_idx = states.index_offset_;
+  for (auto i = 0U; i != states.distances_.size(); ++i) {
+    if (utl::is_sorted(seq[i], std::less<>{})) {
+      continue;
+    }
+
+    polyline.resize(shapes[shape_idx].size());
+    for (auto j = 0U; j != shapes[shape_idx].size(); ++j) {
+      polyline[j] = shapes[shape_idx][j];
+    }
+
+    std::tie(seq[i], states.distances_[i], polyline) =
+        utl::sort_by(seq[i], states.distances_[i], polyline);
+    std::copy(begin(polyline), end(polyline), begin(shapes[shape_idx]));
+    ++shape_idx;
+  }
+
   return states;
 }
 
diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc
@@ -12,6 +12,7 @@
 #include "utl/get_or_create.h"
 #include "utl/progress_tracker.h"
 
+#include "nigiri/shapes_storage.h"
 #include "nigiri/stop.h"
 #include "nigiri/types.h"
 
@@ -92,9 +93,9 @@ void calculate_shape_offsets(timetable const& tt,
         return h;
       };
   auto const key_compare =
-      [](std::pair<shape_idx_t, stop_seq_t const*> const& lhs,
-         std::pair<shape_idx_t, stop_seq_t const*> const& rhs) noexcept {
-        return (lhs.first == rhs.first) && (*lhs.second == *rhs.second);
+      [](std::pair<shape_idx_t, stop_seq_t const*> const& a,
+         std::pair<shape_idx_t, stop_seq_t const*> const& b) noexcept {
+        return (a.first == b.first) && (*a.second == *b.second);
       };
   auto shape_offsets_cache =
       hash_map<std::pair<shape_idx_t, stop_seq_t const*>, shape_offset_idx_t,
@@ -111,7 +112,8 @@ void calculate_shape_offsets(timetable const& tt,
             return shape_offset_idx_t::invalid();
           }
           auto const& shape_distances =
-              shape_states.distances_[shape_idx - shape_states.index_offset_];
+              shape_states
+                  .distances_[to_idx(shape_idx - shape_states.index_offset_)];
           if (!shape_distances.empty() && !trip.distance_traveled_.empty()) {
             auto const offsets = get_offsets_by_dist_traveled(
                 trip.distance_traveled_, shape_distances);
diff --git a/src/loader/gtfs/trip.cc b/src/loader/gtfs/trip.cc
@@ -303,7 +303,7 @@ trip_data read_trips(
           auto const shape_it = shapes.find(t.shape_id_->view());
           auto const shape_idx = (shape_it == end(shapes))
                                      ? shape_idx_t::invalid()
-                                     : shape_it->second.index_;
+                                     : shape_it->second;
 
           auto bikes_allowed = bikes_allowed_default[static_cast<std::size_t>(
               route_it->second->clasz_)];
diff --git a/src/rt/frun.cc b/src/rt/frun.cc
@@ -9,6 +9,7 @@
 
 #include "nigiri/lookup/get_transport_stop_tz.h"
 #include "nigiri/rt/rt_timetable.h"
+#include "nigiri/shapes_storage.h"
 #include "nigiri/timetable.h"
 
 namespace nigiri::rt {
diff --git a/src/shapes_storage.cc b/src/shapes_storage.cc
diff --git a/test/loader/gtfs/shape_test.cc b/test/loader/gtfs/shape_test.cc
diff --git a/test/loader/gtfs/trip_test.cc b/test/loader/gtfs/trip_test.cc
diff --git a/test/rt/frun_shape_test.cc b/test/rt/frun_shape_test.cc
diff --git a/test/shape_test.cc b/test/shape_test.cc
