#558: output: add output to JSON for vt-tv for dervied stats

Author: lifflander

src/lbaf/IO/lbsVTDataWriter.py

@@ -44,6 +44,7 @@
 import multiprocessing as mp
 import os
 import sys
+import math
 from logging import Logger
 from typing import Optional
 
@@ -131,6 +132,20 @@ def __create_tasks(self, rank_id, objects, migratable):
             user_defined = o.get_user_defined()
             if user_defined:
                 task_data["user_defined"] = dict(sorted(user_defined.items()))
+            else:
+                task_data["user_defined"] = dict()
+
+            task_data["user_defined"]["object_memory"] = o.get_size()
+            task_data["user_defined"]["object_overhead_memory"] = o.get_overhead()
+            task_data["user_defined"]["rank_id"] = o.get_rank_id()
+            task_data["user_defined"]["sent_volume"] = o.get_sent_volume()
+            task_data["user_defined"]["received_volume"] = o.get_received_volume()
+            task_data["user_defined"]["max_volume"] = o.get_max_volume()
+
+            if o.get_shared_id() is not None:
+                task_data["user_defined"]["shared_id"] = o.get_shared_id()
+            else:
+                task_data["user_defined"]["shared_id"] = -1
 
             tasks.append(task_data)
 
@@ -243,7 +258,31 @@ def _json_serializer(self, rank_phases_double) -> str:
 
             # Create data to be outputted for current phase
             self.__logger.debug(f"Writing phase {p_id} for rank {r_id}")
+            rank_info : Rank = [r for r in current_phase.get_ranks() if r.get_id() == r_id][0]
+            # JSON can not handle nan so make this ratio -1 when it's not valid
+            homed_ratio = -1
+            if not math.isnan(rank_info.get_homed_blocks_ratio()):
+                homed_ratio = rank_info.get_homed_blocks_ratio()
             phase_data= {"id": p_id,
+                         "user_defined": {
+                             "id": rank_info.get_id(),
+                             "load": rank_info.get_load(),
+                             "num_shared_blocks": rank_info.get_number_of_shared_blocks(),
+                             "num_homed_blocks": rank_info.get_number_of_homed_blocks(),
+                             "num_uprooted_blocks": rank_info.get_number_of_uprooted_blocks(),
+                             "num_homed_ratio": homed_ratio,
+                             "num_homed_blocks": rank_info.get_number_of_homed_blocks(),
+                             "shared_memory": rank_info.get_shared_memory(),
+                             "num_objects": rank_info.get_number_of_objects(),
+                             "num_migratable_objects": len(rank_info.get_migratable_objects()),
+                             "num_sentinel_objects": len(rank_info.get_sentinel_objects()),
+                             "migratable_load": rank_info.get_migratable_load(),
+                             "sentinel_load": rank_info.get_sentinel_load(),
+                             "sent_volume": rank_info.get_sent_volume(),
+                             "received_volume": rank_info.get_received_volume(),
+                             "max_object_level_memory": rank_info.get_max_object_level_memory(),
+                             "max_memory_usage": rank_info.get_max_memory_usage()
+                         },
                          "tasks": sorted(
                             self.__create_tasks(
                                 r_id, rank.get_migratable_objects(), migratable=True) +

src/lbaf/Model/lbsBlock.py

@@ -54,10 +54,10 @@ def __init__(
             o_ids = set()
 
         # Block index
-        if not isinstance(b_id, int) or isinstance(b_id, bool):
-            raise TypeError(
-                f"b_id: incorrect type {type(b_id)}")
-        self.__index = b_id
+        # if not isinstance(b_id, int) or isinstance(b_id, bool):
+        #     raise TypeError(
+        #         f"b_id: incorrect type {type(b_id)}")
+        self.__index = int(b_id)
 
         # Rank to which block is initially assigned
         if not isinstance(h_id, int) or isinstance(h_id, bool):

src/lbaf/Model/lbsObject.py

@@ -205,15 +205,21 @@ def get_received(self) -> dict:
 
     def get_received_volume(self) -> float:
         """Return volume of communications received by object."""
-        return sum(v for v in self.__communicator.get_received().values()) if self.__communicator else 0
+        v : float = 0.0
+        v += sum(v for v in self.__communicator.get_received().values()) if self.__communicator else 0.0
+        return v
 
     def get_sent_volume(self) -> float:
         """Return volume of communications sent by object."""
-        return sum(v for v in self.__communicator.get_sent().values()) if self.__communicator else 0
+        v : float = 0.0
+        v += sum(v for v in self.__communicator.get_sent().values()) if self.__communicator else 0.0
+        return v
 
     def get_max_volume(self) -> float:
         """Return the maximum bytes received or sent by object."""
-        return self.__communicator.get_max_volume() if self.__communicator else 0
+        v : float = 0.0
+        v += self.__communicator.get_max_volume() if self.__communicator else 0.0
+        return v
 
     def set_rank_id(self, r_id: int) -> None:
         """Assign object to rank ID"""

src/lbaf/Model/lbsRank.py

@@ -247,16 +247,20 @@ def get_load(self) -> float:
 
     def get_migratable_load(self) -> float:
         """Return migratable load on rank."""
-        return sum(o.get_load() for o in self.__migratable_objects)
+        val : float = 0.0
+        val += sum(o.get_load() for o in self.__migratable_objects)
+        return val
 
     def get_sentinel_load(self) -> float:
         """Return sentinel load on rank."""
-        return sum(o.get_load() for o in self.__sentinel_objects)
+        val : float = 0.0
+        val += sum(o.get_load() for o in self.__sentinel_objects)
+        return val
 
-    def get_received_volume(self):
+    def get_received_volume(self) -> float:
         """Return volume received by objects assigned to rank from other ranks."""
         # Iterate over all objects assigned to rank
-        volume = 0
+        volume : float = 0.0
         obj_set = self.__migratable_objects.union(self.__sentinel_objects)
         for o in obj_set:
             # Skip objects without communication
@@ -269,10 +273,10 @@ def get_received_volume(self):
         # Return computed volume
         return volume
 
-    def get_sent_volume(self):
+    def get_sent_volume(self) -> float:
         """Return volume sent by objects assigned to rank to other ranks."""
         # Iterate over all objects assigned to rank
-        volume = 0
+        volume = 0.0
         obj_set = self.__migratable_objects.union(self.__sentinel_objects)
         for o in obj_set:
             # Skip objects without communication