MPI Read in Batches

CMakeLists.txt

@@ -186,7 +186,7 @@ if (GDSB_TEST)
 
   # GDSB test target
   add_executable(gdsb_test
-    test/batcher.cpp
+    test/batcher_tests.cpp
     test/experiment_tests.cpp
     test/graph_input_tests.cpp
     test/graph_test.cpp

include/gdsb/batcher.h

@@ -83,8 +83,41 @@ template <class EdgeContainer> class Batcher
     EIt m_end;
 };
 
+constexpr uint32_t count_of_batches(uint64_t const edge_count, uint64_t expected_batch_size)
+{
+    return edge_count / expected_batch_size;
+}
+
+constexpr size_t fair_batch_size(uint64_t const edge_count, uint64_t max_batch_size)
+{
+    if (edge_count <= max_batch_size)
+    {
+        return edge_count;
+    }
+
+    uint64_t const batch_count = edge_count / max_batch_size;
+    uint64_t const remaining = edge_count % max_batch_size;
+    uint64_t fair_size = max_batch_size + (remaining / batch_count);
+    return fair_size;
+}
+
+constexpr std::pair<uint64_t, uint64_t>
+fair_batch_offset(uint64_t const fair_size, uint64_t current_batch_num, uint64_t batches_count, uint64_t edge_count)
+{
+    uint64_t begin = current_batch_num * fair_size;
+
+    bool const is_last_batch = (current_batch_num + 1) == batches_count;
+    if (is_last_batch)
+    {
+        uint64_t const count = edge_count - begin;
+        return std::make_pair(begin, count);
+    }
+
+    uint64_t const count = fair_size;
+    return std::make_pair(begin, count);
+}
 
-inline uint64_t partition_batch_count(uint64_t const batch_count, uint32_t const partition_id, uint32_t const partition_size)
+constexpr uint64_t partition_batch_count(uint64_t const batch_count, uint32_t const partition_id, uint32_t const partition_size)
 {
     uint64_t partition_batch_count = batch_count / partition_size;
     if (partition_id == partition_size - 1)

include/gdsb/mpi_graph_io.h

@@ -132,6 +132,33 @@ std::tuple<Vertex64, uint64_t> all_read_binary_graph_partition(MPI_File const in
     return std::make_tuple(data.vertex_count, edge_count);
 }
 
+// Parameter edges must be a pointer to the first element of the array/vector
+// containing edges.
+template <typename Edges>
+void all_read_binary_graph_batch(MPI_File const input,
+                                 BinaryGraphHeader const& data,
+                                 Edges* const edges,
+                                 size_t const edge_size_in_bytes,
+                                 uint64_t const offset,
+                                 uint64_t const count,
+                                 MPI_Datatype const mpi_datatype)
+{
+    // Header offset should be implicit since input is already read until begin of edges
+    size_t const offset_in_bytes = offset * edge_size_in_bytes;
+    int const seek_error = MPI_File_seek(input, offset_in_bytes, MPI_SEEK_CUR);
+    if (seek_error != MPI_SUCCESS)
+    {
+        throw std::runtime_error("Could not seek to specified offset [" + std::to_string(offset) + "] within MPI file.");
+    }
+
+    MPI_Status status;
+    int const read_all_error = MPI_File_read_all(input, edges, count, mpi_datatype, &status);
+    if (read_all_error != MPI_SUCCESS)
+    {
+        throw std::runtime_error("Could not successfully read all edges from MPI file.");
+    }
+}
+
 namespace binary
 {