Merge pull request #192 from hexinw-nvidia/rank_monitor

fix: Initialize rank monitor client logger at the class level

Author: hexinw-nvidia

src/nvidia_resiliency_ext/fault_tolerance/launcher.py

@@ -877,7 +877,7 @@ class LaunchConfig:
     max_restarts: int = 3
     restart_policy: str = "any-failed"
     term_timeout: float = 1800
-    workers_stop_timeout: float = 30
+    workers_stop_timeout: float = 15
     monitor_interval: float = 30
     start_method: str = "spawn"
     log_line_prefix_template: Optional[str] = None
@@ -1592,7 +1592,7 @@ def get_args_parser() -> ArgumentParser:
         "--workers_stop_timeout",
         action=env,
         type=float,
-        default=30,
+        default=15,
         help="Interval, in seconds, between initial SIGTERM and rank termination with SIGKILL, when the launcher stops its ranks in order to restart them.",
     )
     parser.add_argument(

src/nvidia_resiliency_ext/fault_tolerance/rank_monitor_client.py

@@ -19,8 +19,6 @@
 import socket
 from typing import Any, Collection, Mapping, Optional
 
-from nvidia_resiliency_ext.shared_utils.log_manager import LogConfig
-
 from .data import (
     FT_LAUNCHER_IPC_SOCKET_ENV_VAR,
     FT_RANK_MONITOR_IPC_SOCKET_ENV_VAR,
@@ -40,9 +38,6 @@
 from .timeouts_calc import TimeoutsCalc
 from .utils import read_obj_from_ipc_socket, write_object_to_ipc_socket
 
-# Get the nvrx logger
-logger = logging.getLogger(LogConfig.name)
-
 
 class RankMonitorClientError(Exception):
     pass
@@ -104,12 +99,13 @@ def __init__(self):
         self.chkpt_manager = None
         self.iter_idx = 0
         self.cfg = None
+        self.logger = logging.getLogger(__name__)
         self.launcher_connector = None
         launcher_ipc_socket_path = os.getenv(FT_LAUNCHER_IPC_SOCKET_ENV_VAR, None)
         if launcher_ipc_socket_path is not None:
             self.launcher_connector = IpcConnector(launcher_ipc_socket_path)
         else:
-            logger.info(
+            self.logger.info(
                 f"{FT_LAUNCHER_IPC_SOCKET_ENV_VAR} env varialble is not set. "
                 "`.send_workload_control_request` wont work. This is normal if "
                 "this rank was not started with ft_launcher"
@@ -291,7 +287,7 @@ def init_workload_monitoring(
         if self.is_initialized:
             raise RankMonitorClientError("RankMonitorClient is already initialized")
 
-        logger.info(f"Initializing fault detection. Rank process PID={os.getpid()}")
+        self.logger.debug(f"Initializing fault detection. Rank process PID={os.getpid()}")
 
         self.rank_info = RankInfo.get_for_current_rank()