ITEP-84109 [Release 2025.2] Bugfix: Memory usage steadily increases over time

controller/config/tracker-config-time-chunking.json

@@ -4,5 +4,6 @@
   "non_measurement_frames_static": 8,
   "baseline_frame_rate": 30,
   "time_chunking_enabled": true,
-  "time_chunking_interval_milliseconds": 66
+  "time_chunking_interval_milliseconds": 66,
+  "suspended_track_timeout_secs": 60.0
 }

controller/config/tracker-config.json

@@ -4,5 +4,6 @@
   "non_measurement_frames_static": 16,
   "baseline_frame_rate": 30,
   "time_chunking_enabled": false,
-  "time_chunking_interval_milliseconds": 50
+  "time_chunking_interval_milliseconds": 50,
+  "suspended_track_timeout_secs": 60.0
 }

controller/docs/user-guide/How-to-configure-tracker.md

@@ -26,7 +26,8 @@ The default content of the `tracker-config.json` file is shown below. It is reco
   "non_measurement_frames_dynamic": 8,
   "non_measurement_frames_static": 16,
   "baseline_frame_rate": 30,
-  "time_chunking_enabled": false
+  "time_chunking_enabled": false,
+  "suspended_track_timeout_secs": 60.0
 }
 ```
 
@@ -99,7 +100,8 @@ The content of the `tracker-config-time-chunking.json` file is shown below.
   "non_measurement_frames_static": 8,
   "baseline_frame_rate": 30,
   "time_chunking_enabled": true,
-  "time_chunking_interval_milliseconds": 66
+  "time_chunking_interval_milliseconds": 66,
+  "suspended_track_timeout_secs": 60.0
 }
 ```
 
@@ -119,3 +121,13 @@ The time-chunking interval may be further increased beyond the recommended value
 The mechanism of time-based parameters described above still applies when time-chunking is enabled. What may change with time-chunking enabled is the track refresh rate, which is the rate at which a track is updated with new detections. When time-chunking is disabled, each track is refreshed at a rate equal to the cumulative FPS of cameras observing the object. With time-chunking enabled, each track is refreshed at the tracker processing rate, which is `1000 / time_chunking_interval_milliseconds` Hz.
 
 This means that if all cameras use comparable FPS and time-chunking is enabled with the interval set as recommended above, the time-based parameters may need adjustment depending on camera overlap. For example, if most of the scene is covered by two cameras, the track refresh rate may drop by a factor of two after enabling time-chunking. To compensate, the time-based parameters (`max_unreliable_frames`, `non_measurement_frames_dynamic`, `non_measurement_frames_static`) may need to be reduced by a factor of 2. However, it should always be experimentally verified which parameters work best for a given use case.
+
+## Suspended Track Timeout
+
+The tracker may accumulate suspended tracks for some time for re-tracking purposes (tracks that have been temporarily suspended rather than deleted). To avoid unbounded memory growth, suspended tracks are deleted after a configurable period. You can set an upper bound on how long suspended tracks are retained.
+
+- **Parameter:** `suspended_track_timeout_secs`
+- **Meaning:** Maximum age in seconds for suspended tracks before they are cleaned up. Default: `60.0` seconds.
+- **How to set it:**
+  - Add `"suspended_track_timeout_secs": <value>` to `controller/config/tracker-config.json` (or `tracker-config-time-chunking.json` for time-chunked mode).
+  - The parameter follows the same configuration flow as other tracker parameters like `max_unreliable_time`, `non_measurement_time_dynamic`, and `non_measurement_time_static`.

controller/src/controller/cache_manager.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: (C) 2024 - 2025 Intel Corporation
+# SPDX-FileCopyrightText: (C) 2024 - 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 from controller.scene import Scene
@@ -53,7 +53,8 @@ def refreshScenes(self):
                                       self.tracker_config_data["non_measurement_time_dynamic"],
                                       self.tracker_config_data["non_measurement_time_static"],
                                       self.tracker_config_data["time_chunking_enabled"],
-                                      self.tracker_config_data["time_chunking_interval_milliseconds"]]
+                                      self.tracker_config_data["time_chunking_interval_milliseconds"],
+                                      self.tracker_config_data["suspended_track_timeout_secs"]]
         scene_data["persist_attributes"] = self.tracker_config_data.get("persist_attributes", {})
 
       uid = scene_data['uid']

controller/src/controller/ilabs_tracking.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: (C) 2022 - 2025 Intel Corporation
+# SPDX-FileCopyrightText: (C) 2022 - 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import uuid
@@ -11,15 +11,17 @@
                                       DEFAULT_TRACKING_RADIUS)
 from controller.tracking import (MAX_UNRELIABLE_TIME,
                                  NON_MEASUREMENT_TIME_DYNAMIC,
-                                 NON_MEASUREMENT_TIME_STATIC, Tracking)
+                                 NON_MEASUREMENT_TIME_STATIC,
+                                 DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS,
+                                 Tracking)
 from scene_common import log
 from scene_common.geometry import Point
 from scene_common.timestamp import get_epoch_time
 
 
 class IntelLabsTracking(Tracking):
 
-  def __init__(self, max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, name=None):
+  def __init__(self, max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, suspended_track_timeout_secs=DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS, name=None):
     """Initialize the tracker with tracker configuration parameters"""
     super().__init__()
     self.name = name if name is not None else "IntelLabsTracking"
@@ -45,6 +47,13 @@ def __init__(self, max_unreliable_time, non_measurement_time_dynamic, non_measur
       tracker_config.non_measurement_time_dynamic = NON_MEASUREMENT_TIME_DYNAMIC
       tracker_config.non_measurement_time_static = NON_MEASUREMENT_TIME_STATIC
 
+    if suspended_track_timeout_secs is not None and suspended_track_timeout_secs > 0:
+      tracker_config.suspended_track_timeout_secs = suspended_track_timeout_secs
+    else:
+      log.error("The suspended_track_timeout_secs parameter needs to be positive and less than 3600 seconds. \
+                 Initiating the tracker with the default value.")
+      tracker_config.suspended_track_timeout_secs = DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS
+
     self.tracker = rv.tracking.MultipleObjectTracker(tracker_config)
     log.info(f"Multiple Object Tracker {self.__str__()} initialized")
     log.info("Tracker config: {}".format(tracker_config))
@@ -59,7 +68,6 @@ def check_valid_time_parameters(self, max_unreliable_time, non_measurement_time_
         return True
     return False
 
-
   def rv_classification(self, confidence=None):
     confidence = 1.0 if confidence is None else confidence
     return np.array([confidence, 1.0 - confidence])

controller/src/controller/scene.py

@@ -20,7 +20,8 @@
 from controller.time_chunking import TimeChunkedIntelLabsTracking, DEFAULT_CHUNKING_INTERVAL_MS
 from controller.tracking import (MAX_UNRELIABLE_TIME,
                                  NON_MEASUREMENT_TIME_DYNAMIC,
-                                 NON_MEASUREMENT_TIME_STATIC)
+                                 NON_MEASUREMENT_TIME_STATIC,
+                                 DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS)
 
 DEBOUNCE_DELAY = 0.5
 
@@ -42,14 +43,16 @@ def __init__(self, name, map_file, scale=None,
                non_measurement_time_dynamic = NON_MEASUREMENT_TIME_DYNAMIC,
                non_measurement_time_static = NON_MEASUREMENT_TIME_STATIC,
                time_chunking_enabled = False,
-               time_chunking_interval_milliseconds = DEFAULT_CHUNKING_INTERVAL_MS):
+               time_chunking_interval_milliseconds = DEFAULT_CHUNKING_INTERVAL_MS,
+               suspended_track_timeout_secs = DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS):
     log.info("NEW SCENE", name, map_file, scale, max_unreliable_time,
              non_measurement_time_dynamic, non_measurement_time_static)
     super().__init__(name, map_file, scale)
     self.ref_camera_frame_rate = None
     self.max_unreliable_time = max_unreliable_time
     self.non_measurement_time_dynamic = non_measurement_time_dynamic
     self.non_measurement_time_static = non_measurement_time_static
+    self.suspended_track_timeout_secs = suspended_track_timeout_secs
     self.tracker = None
     self.trackerType = None
     self.persist_attributes = {}
@@ -75,6 +78,7 @@ def _setTracker(self, trackerType):
             self.non_measurement_time_static)
     if trackerType == "time_chunked_intel_labs":
       args += (self.time_chunking_interval_milliseconds,)
+    args += (self.suspended_track_timeout_secs,)
     self.tracker = self.available_trackers[self.trackerType](*args)
     return
 

controller/src/controller/scene_controller.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: (C) 2021 - 2025 Intel Corporation
+# SPDX-FileCopyrightText: (C) 2021 - 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import orjson
@@ -21,6 +21,7 @@
 from scene_common.transform import applyChildTransform
 from controller.observability import metrics
 from controller.time_chunking import DEFAULT_CHUNKING_INTERVAL_MS
+from controller.tracking import DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS
 AVG_FRAMES = 100
 
 class SceneController:
@@ -67,6 +68,7 @@ def extractTrackerConfigData(self, tracker_config_file):
       self.tracker_config_data["max_unreliable_time"] = tracker_config["max_unreliable_frames"]/tracker_config["baseline_frame_rate"]
       self.tracker_config_data["non_measurement_time_dynamic"] = tracker_config["non_measurement_frames_dynamic"]/tracker_config["baseline_frame_rate"]
       self.tracker_config_data["non_measurement_time_static"] = tracker_config["non_measurement_frames_static"]/tracker_config["baseline_frame_rate"]
+      self.tracker_config_data["suspended_track_timeout_secs"] = tracker_config.get("suspended_track_timeout_secs", DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS)
       self._extractTimeChunkingEnabled(tracker_config)
       self._extractTimeChunkingInterval(tracker_config)
 

controller/src/controller/time_chunking.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: (C) 2025 Intel Corporation
+# SPDX-FileCopyrightText: (C) 2025 - 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 """
@@ -42,7 +42,7 @@
 
 from scene_common import log
 from controller.ilabs_tracking import IntelLabsTracking
-from controller.tracking import BATCHED_MODE, STREAMING_MODE
+from controller.tracking import BATCHED_MODE, STREAMING_MODE, DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS
 from controller.observability import metrics
 
 DEFAULT_CHUNKING_INTERVAL_MS = 50  # Default interval in milliseconds
@@ -145,10 +145,11 @@ def run(self):
 class TimeChunkedIntelLabsTracking(IntelLabsTracking):
   """Time-chunked version of IntelLabsTracking."""
 
-  def __init__(self, max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, time_chunking_interval_milliseconds):
+  def __init__(self, max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, time_chunking_interval_milliseconds, suspended_track_timeout_secs=DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS):
     # Call parent constructor to initialize IntelLabsTracking
-    super().__init__(max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static)
+    super().__init__(max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, suspended_track_timeout_secs)
     self.time_chunking_interval_milliseconds = time_chunking_interval_milliseconds
+    self.suspended_track_timeout_secs = suspended_track_timeout_secs
     log.info(f"Initialized TimeChunkedIntelLabsTracking {self.__str__()} with chunking interval: {self.time_chunking_interval_milliseconds} ms")
 
   def trackObjects(self, objects, already_tracked_objects, when, categories,
@@ -192,7 +193,7 @@ def _createIlabsTrackers(self, categories, max_unreliable_time, non_measurement_
     # delegate tracking to IntelLabsTracking
     for category in categories:
       if category not in self.trackers:
-        tracker = IntelLabsTracking(max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static)
+        tracker = IntelLabsTracking(max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, self.suspended_track_timeout_secs)
         self.trackers[category] = tracker
         tracker.start()
         log.info(f"Started IntelLabs tracker {tracker.__str__()} thread for category {category}")

controller/src/controller/tracking.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: (C) 2022 - 2025 Intel Corporation
+# SPDX-FileCopyrightText: (C) 2022 - 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 from queue import Queue
@@ -21,6 +21,7 @@
 MAX_UNRELIABLE_TIME = 0.3333
 NON_MEASUREMENT_TIME_DYNAMIC = 0.2666
 NON_MEASUREMENT_TIME_STATIC = 0.5333
+DEFAULT_SUSPENDED_TRACK_TIMEOUT_SECS = 60.0
 
 # Queue mode constants for tracking operation
 STREAMING_MODE = False  # (DEFAULT) Objects from one source (camera) at a time are put into the queue

controller/src/robot_vision/include/rv/tracking/TrackManager.hpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2017 - 2025 Intel Corporation
+// SPDX-FileCopyrightText: 2017 - 2026 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 
 #pragma once
@@ -30,6 +30,8 @@ struct TrackManagerConfig
   double mDefaultMeasurementNoise{1e-2};
   double mInitStateCovariance{1.};
 
+  double mSuspendedTrackMaxAgeSecs{60.0};
+
   std::vector<MotionModel> mMotionModels{MotionModel::CV, MotionModel::CA, MotionModel::CTRV};
 
   std::string toString() const
@@ -60,7 +62,7 @@ struct TrackManagerConfig
       + std::to_string(mMaxUnreliableTime) + ", reactivation_frames:" + std::to_string(mReactivationFrames)
       + ", default_process_noise:" + std::to_string(mDefaultProcessNoise) + ", default_measurement_noise:"
       + std::to_string(mDefaultMeasurementNoise) + ", init_state_covariance:"
-      + std::to_string(mInitStateCovariance) + motionModelsText + ")";
+      + std::to_string(mInitStateCovariance) + ", suspended_track_max_age_secs:" + std::to_string(mSuspendedTrackMaxAgeSecs) + motionModelsText + ")";
   }
 };
 
@@ -113,6 +115,13 @@ class TrackManager
    */
   void predict(double deltaT);
 
+  /**
+   * @brief Remove old suspended tracks to prevent unbounded accumulation
+   *
+   * @param maxAgeSecs Maximum age in seconds for suspended tracks before removal
+   */
+  void cleanupOldSuspendedTracks(double maxAgeSecs);
+
   /**
    * @brief Assign a measurement to an KalmanEstimator.
    *
@@ -198,6 +207,7 @@ class TrackManager
   std::unordered_map<Id, TrackedObject> mMeasurementMap;
   std::unordered_map<Id, uint32_t> mNonMeasurementFrames;
   std::unordered_map<Id, uint32_t> mNumberOfTrackedFrames;
+  std::unordered_map<Id, std::chrono::steady_clock::time_point> mSuspensionTimes;
 
   Id mCurrentId = 0;
 

controller/src/robot_vision/python/src/robot_vision/extensions/tracking.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: (C) 2019 - 2025 Intel Corporation
+// SPDX-FileCopyrightText: (C) 2019 - 2026 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 
 #include <opencv2/core.hpp>
@@ -196,6 +196,8 @@ py::class_<rv::tracking::Classification>(tracking, "Classification", "Classifica
      "Default init state covariance passed to the KalmanEstimator init function.")
     .def_readwrite("motion_models", &rv::tracking::TrackManagerConfig::mMotionModels,
      "List of motion models to use. It defaults to [CV, CA, CTRV]")
+     .def_readwrite("suspended_track_timeout_secs", &rv::tracking::TrackManagerConfig::mSuspendedTrackMaxAgeSecs,
+      "Maximum age (seconds) for a suspended track before cleanup. Configurable via Python.")
     .def("__repr__", &rv::tracking::TrackManagerConfig::toString, "String representation");
 
 

controller/src/robot_vision/src/rv/tracking/TrackManager.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: (C) 2017 - 2025 Intel Corporation
+// SPDX-FileCopyrightText: (C) 2017 - 2026 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 
 #include "rv/Utils.hpp"
@@ -40,6 +40,7 @@ void TrackManager::deleteTrack(const Id &id)
 void TrackManager::suspendTrack(const Id &id)
 {
   mSuspendedKalmanEstimators[id] = std::move(mKalmanEstimators.at(id));
+  mSuspensionTimes[id] = std::chrono::steady_clock::now();
   mKalmanEstimators.erase(id);
   mNonMeasurementFrames.erase(id);
 }
@@ -53,10 +54,38 @@ void TrackManager::reactivateTrack(const Id &id)
   mNumberOfTrackedFrames[id] = mConfig.mMaxNumberOfUnreliableFrames - mConfig.mReactivationFrames;
 
   mSuspendedKalmanEstimators.erase(id);
+  mSuspensionTimes.erase(id);
+}
+
+void TrackManager::cleanupOldSuspendedTracks(double maxAgeSecs)
+{
+  if (maxAgeSecs <= 0) { // Noop if max age is not positive
+    return;
+  }
+
+  auto now = std::chrono::steady_clock::now();
+  std::vector<Id> toDelete;
+
+  for (const auto& [id, suspensionTime] : mSuspensionTimes)
+  {
+    double ageSeconds = std::chrono::duration<double>(now - suspensionTime).count();
+    if (ageSeconds > maxAgeSecs)
+    {
+      toDelete.push_back(id);
+    }
+  }
+
+  for (const auto& id : toDelete)
+  {
+    mSuspendedKalmanEstimators.erase(id);
+    mSuspensionTimes.erase(id);
+  }
 }
 
 void TrackManager::predict(const std::chrono::system_clock::time_point &timestamp)
 {
+  cleanupOldSuspendedTracks(mConfig.mSuspendedTrackMaxAgeSecs);
+
   // Convert map to vector for parallel iteration
   std::vector<std::reference_wrapper<MultiModelKalmanEstimator>> estimators;
   estimators.reserve(mKalmanEstimators.size());
@@ -78,6 +107,8 @@ void TrackManager::predict(const std::chrono::system_clock::time_point &timestam
 
 void TrackManager::predict(double deltaT)
 {
+  cleanupOldSuspendedTracks(mConfig.mSuspendedTrackMaxAgeSecs);
+
   // Convert map to vector for parallel iteration
   std::vector<std::reference_wrapper<MultiModelKalmanEstimator>> estimators;
   estimators.reserve(mKalmanEstimators.size());

tests/system/metric/tc_tracker_metric.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# SPDX-FileCopyrightText: (C) 2023 - 2025 Intel Corporation
+# SPDX-FileCopyrightText: (C) 2023 - 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import json
@@ -68,6 +68,7 @@ def track(params):
   non_measurement_time_static = trackerConfigData["non_measurement_frames_static"]/trackerConfigData["baseline_frame_rate"]
   time_chunking_enabled = trackerConfigData["time_chunking_enabled"]
   time_chunking_interval_ms = trackerConfigData["time_chunking_interval_milliseconds"]
+  suspended_track_timeout_secs = trackerConfigData["suspended_track_timeout_secs"]
 
   camera_fps = []
   for input_file in params["input"]:
@@ -96,7 +97,8 @@ def track(params):
     non_measurement_time_dynamic=non_measurement_time_dynamic,
     non_measurement_time_static=non_measurement_time_static,
     time_chunking_enabled=time_chunking_enabled,
-    time_chunking_interval_milliseconds=time_chunking_interval_ms
+    time_chunking_interval_milliseconds=time_chunking_interval_ms,
+    suspended_track_timeout_secs=suspended_track_timeout_secs
   )
 
   if 'sensors' in scene_config:

tests/system/metric/test_data/tracker-config-time-chunking.json

@@ -4,5 +4,6 @@
   "non_measurement_frames_static": 16,
   "baseline_frame_rate": 30,
   "time_chunking_enabled": true,
-  "time_chunking_interval_milliseconds": 1
+  "time_chunking_interval_milliseconds": 1,
+  "suspended_track_timeout_secs": 60.0
 }

tests/system/metric/test_data/tracker-config.json

@@ -4,5 +4,6 @@
     "non_measurement_frames_static": 16,
     "baseline_frame_rate": 30,
     "time_chunking_enabled": false,
-    "time_chunking_interval_milliseconds": 50
+    "time_chunking_interval_milliseconds": 50,
+    "suspended_track_timeout_secs": 60.0
 }