feat(autoware_system_monitor): change diagnostic logic for mem_monitor (#11971)

* feat(autoware_system_monitor): change diagnostic logic for mem_monitor (#11971)

---------

Signed-off-by: Takayuki AKAMINE <takayuki.akamine@tier4.jp>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

Author: takam5f2

system/autoware_system_monitor/CMakeLists.txt

@@ -194,6 +194,7 @@ rclcpp_components_register_node(voltage_monitor_lib
 )
 
 if(BUILD_TESTING)
+  # Process Monitor Test
   ament_add_ros_isolated_gtest(test_process_monitor
     test/src/process_monitor/test_process_monitor.cpp
   )
@@ -211,6 +212,7 @@ if(BUILD_TESTING)
     COMMENT "Copying test data files to the build directory after build"
   )
 
+  # CPU Monitor Test
   ament_add_ros_isolated_gtest(test_cpu_monitor
     test/src/cpu_monitor/test_${CMAKE_CPU_PLATFORM}_cpu_monitor.cpp
     ${CPU_MONITOR_SOURCE}
@@ -222,6 +224,28 @@ if(BUILD_TESTING)
 
   target_link_libraries(test_cpu_monitor cpu_monitor_lib ${Boost_LIBRARIES} ${LIBRARIES})
 
+  # Mem Monitor Test
+  ament_add_ros_isolated_gtest(test_mem_monitor
+    test/src/mem_monitor/test_mem_monitor.cpp
+    src/mem_monitor/mem_monitor.cpp
+  )
+
+  target_include_directories(test_mem_monitor
+    PRIVATE "include"
+  )
+
+  target_link_libraries(test_mem_monitor mem_monitor_lib  ${Boost_LIBRARIES} ${LIBRARIES})
+
+  # helper free command to failed free command.
+  add_executable(free1 test/src/mem_monitor/free1.cpp)
+
+  add_custom_command(
+    TARGET test_mem_monitor
+    POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:free1> $<TARGET_FILE_DIR:test_mem_monitor>
+    COMMENT "Copying free1 dummy executable to the test directory"
+  )
+
 endif()
 
 # TODO(yunus.caliskan): Port the tests to ROS 2, robustify the tests.

system/autoware_system_monitor/config/mem_monitor.param.yaml

@@ -1,3 +1,5 @@
 /**:
   ros__parameters:
-    available_size: 1024 # MB
+    available_size: 1024 # MiB
+    warning_margin: 0 # MiB
+    swap_error_threshold: 1024 # MiB

system/autoware_system_monitor/docs/ros_parameters.md

@@ -52,10 +52,11 @@ hdd_monitor:
 
 mem_monitor:
 
-| Name        | Type  |  Unit   | Default | Notes                                                                             |
-| :---------- | :---: | :-----: | :-----: | :-------------------------------------------------------------------------------- |
-| usage_warn  | float | %(1e-2) |  0.95   | Generates warning when physical memory usage reaches a specified value or higher. |
-| usage_error | float | %(1e-2) |  0.99   | Generates error when physical memory usage reaches a specified value or higher.   |
+| Name                 | Type | Unit | Default |                                                                Notes                                                                 |
+| :------------------- | :--: | :--: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------: |
+| available_size       | int  | MiB  |  1024   |                  Generates error when physical memory usage reaches a specified value defined by `available_size`.                   |
+| warning_margin       | int  | MiB  |    0    | Generates warning when physical memory usage reaches a specified value defined by addition of `available_size` and `warning_margin`. |
+| swap_error_threshold | int  | MiB  |  1024   |                                 Generates error when swap usage reaches a specified value or higher.                                 |
 
 ## <u>Net Monitor</u>
 

system/autoware_system_monitor/include/system_monitor/mem_monitor/mem_monitor.hpp

@@ -68,6 +68,14 @@ class MemMonitor : public rclcpp::Node
    */
   std::string toHumanReadable(const std::string & str);
 
+  /**
+   * @brief determine diagnostic level based on available memory and swap usage
+   * @param [in] mem_available available memory size in bytes
+   * @param [in] swap_used swap usage size in bytes
+   * @return diagnostic level (OK, WARN, or ERROR)
+   */
+  uint8_t determineDiagnosticLevel(size_t mem_available, size_t swap_used);
+
   /**
    * @brief publish memory status
    * @param [in] usage memory usage
@@ -78,15 +86,17 @@ class MemMonitor : public rclcpp::Node
 
   char hostname_[HOST_NAME_MAX + 1];  //!< @brief host name
 
-  size_t available_size_;  //!< @brief Memory available size to generate error
+  size_t error_available_size_;    //!< @brief Memory available size to generate error
+  size_t warning_available_size_;  //!< @brief Memory available size to generate warning
+  size_t swap_error_threshold_;    //!< @brief Swap usage size to generate error
 
   rclcpp::Publisher<tier4_external_api_msgs::msg::MemoryStatus>::SharedPtr
     pub_memory_status_;  //!< @brief publisher
 
   /**
    * @brief Memory usage status messages
    */
-  const std::map<int, const char *> usage_dict_ = {
+  const std::map<uint8_t, const char *> usage_dict_ = {
     {DiagStatus::OK, "OK"}, {DiagStatus::WARN, "high load"}, {DiagStatus::ERROR, "very high load"}};
 };
 

system/autoware_system_monitor/src/mem_monitor/mem_monitor.cpp

@@ -34,8 +34,35 @@ namespace bp = boost::process;
 MemMonitor::MemMonitor(const rclcpp::NodeOptions & options)
 : Node("mem_monitor", options),
   updater_(this),
-  available_size_(declare_parameter<int>("available_size", 1024) * 1024 * 1024)
+  error_available_size_(
+    declare_parameter<int>(
+      "available_size", 1024,
+      rcl_interfaces::msg::ParameterDescriptor().set__read_only(true).set__description(
+        "Memory available size to generate error [MiB]. Cannot be changed after "
+        "initialization.")) *
+    1024 * 1024),
+  warning_available_size_(0),
+  swap_error_threshold_(
+    declare_parameter<int>(
+      "swap_error_threshold", 1024,
+      rcl_interfaces::msg::ParameterDescriptor().set__read_only(true).set__description(
+        "Swap usage size to generate error [MiB]. Cannot be changed after "
+        "initialization.")) *
+    1024 * 1024)
 {
+  // Define warning_available_size_
+  int warning_margin =
+    declare_parameter<int>(
+      "warning_margin", 0,
+      rcl_interfaces::msg::ParameterDescriptor().set__read_only(true).set__description(
+        "Warning margin with error threshold [MiB] to generate Warn [MiB]. Cannot be changed after "
+        "initialization.")) *
+    1024 * 1024;
+  if (warning_margin < 0) {
+    warning_margin = 0;
+  }
+  warning_available_size_ = error_available_size_ + warning_margin;
+
   gethostname(hostname_, sizeof(hostname_));
   updater_.setHardwareID(hostname_);
   updater_.add("Memory Usage", this, &MemMonitor::checkUsage);
@@ -99,9 +126,8 @@ void MemMonitor::checkUsage(diagnostic_updater::DiagnosticStatusWrapper & stat)
   std::vector<std::string> list;
   float usage = 0.0f;
   size_t mem_total = 0;
-  size_t mem_shared = 0;
   size_t mem_available = 0;
-  size_t used_plus = 0;
+  size_t swap_used = 0;
 
   /*
    Output example of `free -tb`
@@ -124,15 +150,18 @@ void MemMonitor::checkUsage(diagnostic_updater::DiagnosticStatusWrapper & stat)
     // Physical memory
     if (index == 1) {
       mem_total = std::atoll(list[1].c_str());
-      mem_shared = std::atoll(list[4].c_str());
       mem_available = std::atoll(list[6].c_str());
 
       // available divided by total is available memory including calculation for buff/cache,
       // so the subtraction of this from 1 gives real usage.
       usage = 1.0f - static_cast<double>(mem_available) / mem_total;
       stat.addf(fmt::format("{} usage", list[0]), "%.2f%%", usage * 1e+2);
+    } else if (index == 2) {
+      // Swap memory
+      swap_used = std::atoll(list[2].c_str());
     }
 
+    // Add additional information for each memory type and total
     stat.add(fmt::format("{} total", list[0]), toHumanReadable(list[1]));
     stat.add(fmt::format("{} used", list[0]), toHumanReadable(list[2]));
     stat.add(fmt::format("{} free", list[0]), toHumanReadable(list[3]));
@@ -142,26 +171,14 @@ void MemMonitor::checkUsage(diagnostic_updater::DiagnosticStatusWrapper & stat)
       stat.add(fmt::format("{} shared", list[0]), toHumanReadable(list[4]));
       stat.add(fmt::format("{} buff/cache", list[0]), toHumanReadable(list[5]));
       stat.add(fmt::format("{} available", list[0]), toHumanReadable(list[6]));
-    } else if (index == 3) {
-      // Total:used + Mem:shared
-      used_plus = std::atoll(list[2].c_str()) + mem_shared;
-      double giga = static_cast<double>(used_plus) / (1024 * 1024 * 1024);
-      stat.add(fmt::format("{} used+", list[0]), fmt::format("{:.1f}{}", giga, "G"));
     } else {
-      /* nothing */
+      // Swap (index 2) and Total (index 3) do not have their specific entry.
+      // So do nothing here.
     }
     ++index;
   }
 
-  int level;
-  if (mem_total > used_plus) {
-    level = DiagStatus::OK;
-  } else if (mem_available >= available_size_) {
-    level = DiagStatus::WARN;
-  } else {
-    level = DiagStatus::ERROR;
-  }
-
+  const uint8_t level = determineDiagnosticLevel(mem_available, swap_used);
   stat.summary(level, usage_dict_.at(level));
 
   publishMemoryStatus(usage);
@@ -235,6 +252,29 @@ std::string MemMonitor::toHumanReadable(const std::string & str)
   return fmt::format(format, size, units[count]);
 }
 
+uint8_t MemMonitor::determineDiagnosticLevel(size_t mem_available, size_t swap_used)
+{
+  uint8_t level = DiagStatus::OK;
+  if (mem_available < error_available_size_) {
+    level = DiagStatus::ERROR;
+  } else if (mem_available < warning_available_size_) {
+    level = DiagStatus::WARN;
+  }
+
+  // Swap usage checking
+  if (swap_used > 0) {
+    if (level == DiagStatus::OK) {
+      level = DiagStatus::WARN;
+    }
+
+    if (swap_used > swap_error_threshold_) {
+      level = DiagStatus::ERROR;
+    }
+  }
+
+  return level;
+}
+
 void MemMonitor::publishMemoryStatus(float usage)
 {
   tier4_external_api_msgs::msg::MemoryStatus memory_status;

system/autoware_system_monitor/test/src/mem_monitor/free1.cpp

@@ -19,5 +19,7 @@
 
 int main(int argc, char ** argv)
 {
+  (void)argc;
+  (void)argv;
   return -1;
 }