Merge branch 'main' into ITEP-89519/visibility-avaliable-in-analytics-only

Author: dmytroye

.github/skills/testing.md

@@ -71,6 +71,16 @@ Run this workflow before executing any test command:
 - Do not report completion without runtime verification for the resolved target (unless blocked).
 - Always report: should-run target, whether it was run, exact command, and pass/fail summary (or blocker).
 
+## Test Configuration Hygiene (Mandatory)
+
+- Do not add new environment variables to `tools/scenescape-start` for test-only behavior.
+- Treat `tools/scenescape-start` as a stable shared launcher, not a per-test configuration surface.
+- For test scenario inputs, prefer one of these paths:
+  - test configuration files (for example scenario JSON, service config JSON)
+  - test runner arguments (for example pytest options)
+  - Makefile variables scoped to test targets
+- If a new environment variable is absolutely required for non-test runtime behavior, document and justify it in the related service docs; do not introduce it solely to satisfy a test matrix.
+
 ### Quick Mapping Examples
 
 - `tests/functional/tc_sensors_send_mqtt_messages.py` -> `make -C tests sensors-send-events`

controller/config/reid-config.json

@@ -2,6 +2,7 @@
   "stale_feature_timeout_secs": 5.0,
   "stale_feature_check_interval_secs": 1.0,
   "feature_accumulation_threshold": 12,
+  "minimum_bbox_area": 5000,
   "feature_slice_size": 10,
-  "similarity_threshold": 60
+  "similarity_threshold": 30.0
 }

controller/src/controller-cmd

@@ -94,7 +94,7 @@ def main():
                               args.brokerauth, args.resturl,
                               args.restauth, args.cert,
                               args.rootcert, args.ntp, args.tracker_config_file, args.schema_file,
-                              args.visibility_topic, args.data_source)
+                              args.visibility_topic, args.data_source, args.reid_config_file)
 
   # Start health check server if port is specified
   if args.healthcheck_port > 0:

controller/src/controller/cache_manager.py

@@ -63,7 +63,6 @@ def refreshScenes(self):
 
       uid = scene_data['uid']
       if uid not in self.cached_scenes_by_uid:
-        # Creating new scene - check if there was an old scene with sensor cache
         scene = Scene.deserialize(scene_data)
 
         old_scene = self._sensorNeedsRestoring(uid)

controller/src/controller/detections_builder.py

@@ -26,24 +26,38 @@ def buildDetectionsList(objects, scene, update_visibility=False, include_sensors
     result_list.append(obj_dict)
   return result_list
 
-def _get_region_entered_epoch(region_data):
+def _getRegionEnteredEpoch(region_data):
   entered_epoch = region_data.get('entered_epoch')
   if entered_epoch is None:
     entered_epoch = get_epoch_time(region_data['entered'])
     region_data['entered_epoch'] = entered_epoch
   return entered_epoch
 
-def _build_region_output(regions, include_region_dwell, current_time):
+def _buildRegionOutput(regions, include_region_dwell, current_time):
   serialized_regions = {}
   for region_name, region_data in regions.items():
     serialized_region = dict(region_data)
     serialized_region.pop('entered_epoch', None)
     if include_region_dwell and 'entered' in region_data:
-      entered = _get_region_entered_epoch(region_data)
+      entered = _getRegionEnteredEpoch(region_data)
       serialized_region['dwell'] = current_time - entered
     serialized_regions[region_name] = serialized_region
   return serialized_regions
 
+def _serializePreviousIdsChain(previous_ids_chain):
+  serialized_chain = []
+  for entry in previous_ids_chain:
+    serialized_entry = dict(entry)
+    timestamp = serialized_entry.get('timestamp')
+
+    # UUIDManager records chain timestamps as epoch floats; normalize to ISO 8601 in output.
+    if isinstance(timestamp, (int, float)):
+      serialized_entry['timestamp'] = get_iso_time(float(timestamp))
+
+    serialized_chain.append(serialized_entry)
+
+  return serialized_chain
+
 def prepareObjDict(scene, obj, update_visibility, include_sensors=False,
                    include_region_dwell=False, current_time=None):
   aobj = obj
@@ -114,7 +128,7 @@ def prepareObjDict(scene, obj, update_visibility, include_sensors=False,
       if include_region_dwell:
         if current_time is None:
           current_time = get_epoch_time()
-        obj_dict['regions'] = _build_region_output(chain_data.regions, include_region_dwell, current_time)
+        obj_dict['regions'] = _buildRegionOutput(chain_data.regions, include_region_dwell, current_time)
       else:
         obj_dict['regions'] = chain_data.regions
 
@@ -150,6 +164,15 @@ def prepareObjDict(scene, obj, update_visibility, include_sensors=False,
     obj_dict['similarity'] = aobj.similarity
   if hasattr(aobj, 'first_seen'):
     obj_dict['first_seen'] = get_iso_time(aobj.first_seen)
+
+  # Add reid state for downstream business logic to distinguish "never queried" from "query made"
+  if hasattr(aobj, 'reid_state'):
+    obj_dict['reid_state'] = aobj.reid_state.value  # Convert enum to string
+
+  # Add previous IDs chain for post-mortem object stitching analysis
+  if hasattr(aobj, 'previous_ids_chain') and aobj.previous_ids_chain:
+    obj_dict['previous_ids_chain'] = _serializePreviousIdsChain(aobj.previous_ids_chain)
+
   if isinstance(obj, TripwireEvent):
     obj_dict['direction'] = obj.direction
   if hasattr(aobj, 'asset_scale'):

controller/src/controller/moving_object.py

@@ -4,8 +4,10 @@
 import base64
 import binascii
 import datetime
+import uuid
 import warnings
 from dataclasses import dataclass, field
+from enum import Enum
 from threading import Lock
 from typing import Dict, List
 
@@ -16,6 +18,7 @@
 
 from scene_common.geometry import DEFAULTZ, Line, Point, Rectangle
 from scene_common.options import TYPE_1, TYPE_2
+from scene_common.timestamp import get_epoch_time
 from scene_common.transform import normalize, rotationToTarget
 from scene_common import log
 
@@ -110,6 +113,19 @@ def serializeReIDPayload(reid):
 
   return reid
 
+class ReidState(Enum):
+  """State of ReID query and matching for an object.
+
+  PENDING_COLLECTION: Collecting embeddings, query not yet made
+  QUERY_NO_MATCH: Query made but no match found (new object)
+  MATCHED: Successfully matched to previous object (reID)
+  REID_DISABLED: ReID system is disabled, no query will be made
+  """
+  PENDING_COLLECTION = "pending_collection"
+  QUERY_NO_MATCH = "query_no_match"
+  MATCHED = "matched"
+  REID_DISABLED = "reid_disabled"
+
 @dataclass
 class ChainData:
   regions: Dict
@@ -196,6 +212,9 @@ def __init__(self, info, when, camera):
     self.intersected = False
     self.reid = {}  # Initialize reid as empty dict
     self.metadata = {}  # Initialize metadata as empty dict
+    self.reid_state = ReidState.PENDING_COLLECTION  # Track reID state
+    self.similarity = None  # Similarity score from last reID match
+    self.previous_ids_chain = []  # Track object ID history: [{'id': gid, 'timestamp': ts, 'similarity_score': score}, ...]
     # Extract reid from metadata if present and preserve metadata attribute
     metadata_from_info = self.info.get('metadata', {})
     if metadata_from_info and isinstance(metadata_from_info, dict):
@@ -294,6 +313,9 @@ def setPrevious(self, otherObj):
     self.gid = otherObj.gid
     self.first_seen = otherObj.first_seen
     self.frameCount = otherObj.frameCount + 1
+    self.reid_state = otherObj.reid_state
+    self.similarity = otherObj.similarity
+    self.previous_ids_chain = otherObj.get_previous_ids()
 
     del self.chain_data.publishedLocations[LOCATION_LIMIT:]
 
@@ -379,6 +401,43 @@ def _projectBounds(self):
   def when(self):
     return self.location[0].when
 
+  def save_previous_object_id(self, previous_id, similarity_score=None, timestamp=None):
+    """Save the previous object ID for post-mortem analysis.
+
+    @param previous_id: The previous global ID assigned to this object
+    @param similarity_score: Similarity score from reID matching (if matched), or None if new object
+    @param timestamp: When the change occurred (epoch time), defaults to current time
+    """
+    try:
+      uuid.UUID(previous_id)
+    except (TypeError, ValueError, AttributeError) as err:
+      raise ValueError("previous_id must be a valid UUID") from err
+
+    if timestamp is None:
+      timestamp = get_epoch_time()
+
+    self.previous_ids_chain.append({
+      'id': previous_id,
+      'timestamp': timestamp,
+      'similarity_score': similarity_score
+    })
+    log.debug(f"MovingObject.save_previous_object_id: rv_id={getattr(self, 'rv_id', 'unknown')}, "
+              f"previous_id={previous_id}, similarity={similarity_score}, state={self.reid_state.value}")
+
+  def is_reidentified(self):
+    """Check if this object resulted from successful reID matching.
+
+    @return: True if object was matched to a previous object, False otherwise
+    """
+    return self.reid_state == ReidState.MATCHED
+
+  def get_previous_ids(self):
+    """Get chain of previous IDs for this object.
+
+    @return: List of dicts with 'id', 'timestamp', 'similarity_score' for post-mortem analysis
+    """
+    return self.previous_ids_chain.copy()
+
   def __repr__(self):
     return "%s: %s/%s %s %s vectors: %s" % \
       (self.__class__.__name__,

controller/src/controller/scene.py

@@ -58,6 +58,7 @@ def __init__(self, name, map_file, scale=None,
     self.non_measurement_time_static = non_measurement_time_static
     self.suspended_track_timeout_secs = suspended_track_timeout_secs
     self.reid_config_data = reid_config_data if reid_config_data else {}
+
     self.tracker = None
     self.trackerType = None
     self.persist_attributes = {}
@@ -89,6 +90,9 @@ def _setTracker(self, trackerType):
     self.trackerType = trackerType
     log.info("SETTING TRACKER TYPE", trackerType)
 
+    if self.tracker is not None:
+      self.tracker.join()
+
     args = (self.max_unreliable_time,
             self.non_measurement_time_dynamic,
             self.non_measurement_time_static)
@@ -99,25 +103,39 @@ def _setTracker(self, trackerType):
     self.tracker = self.available_trackers[self.trackerType](*args)
     return
 
-  def updateScene(self, scene_data):
+  def _hydrateFromSceneData(self, scene_data, reid_runtime_update=True):
+    reid_config_changed = False
+    if 'reid_config_data' in scene_data:
+      new_reid_config_data = scene_data['reid_config_data']
+      reid_config_changed = new_reid_config_data != self.reid_config_data
+      if reid_config_changed:
+        self.reid_config_data = new_reid_config_data
+
     self.parent = scene_data.get('parent', None)
     self.cameraPose = None
     if 'transform' in scene_data:
       self.cameraPose = CameraPose(scene_data['transform'], None)
-    self.use_tracker = scene_data.get('use_tracker', True)
+    self.use_tracker = scene_data.get('use_tracker', True) and not ControllerMode.isAnalyticsOnly()
     self.output_lla = scene_data.get('output_lla', False)
     self.map_corners_lla = scene_data.get('map_corners_lla', None)
+    self.retrack = scene_data.get('retrack', True)
+    self.persist_attributes = scene_data.get('persist_attributes', {})
     self._updateChildren(scene_data.get('children', []))
     self.updateCameras(scene_data.get('cameras', []))
     self._updateRegions(self.regions, scene_data.get('regions', []))
     self._updateTripwires(scene_data.get('tripwires', []))
     self._updateRegions(self.sensors, scene_data.get('sensors', []))
-    # Update reid config if provided
-    if 'reid_config_data' in scene_data:
-      self.reid_config_data = scene_data['reid_config_data']
+
     tracker_config = scene_data.get('tracker_config', None)
     if tracker_config:
       self.updateTracker(tracker_config[0], tracker_config[1], tracker_config[2])
+
+    # Apply ReID config changes in-place to preserve active tracks while
+    # updating UUID manager thresholds and timers.
+    if reid_runtime_update and reid_config_changed and self.trackerType and not ControllerMode.isAnalyticsOnly():
+      log.info(f"ReID config changed for scene={self.uid}; updating tracker ReID runtime config")
+      self.tracker.updateReidConfig(self.reid_config_data)
+
     self.name = scene_data['name']
     if 'scale' in scene_data:
       self.scale = scene_data['scale']
@@ -130,6 +148,10 @@ def updateScene(self, scene_data):
     _ = self.trs_xyz_to_lla
     return
 
+  def updateScene(self, scene_data):
+    self._hydrateFromSceneData(scene_data, reid_runtime_update=True)
+    return
+
   def updateTracker(self, max_unreliable_time, non_measurement_time_dynamic,
                     non_measurement_time_static):
     # Only update tracker if the values have changed to avoid losing tracking data
@@ -757,38 +779,14 @@ def _updateVisible(self, curObjects):
   @classmethod
   def deserialize(cls, data):
     tracker_config = data.get('tracker_config', [])
+    reid_config_data = data.get('reid_config_data', None)
     scale_from_data = data.get('scale', None)
     scene = cls(data['name'], data.get('map', None), scale_from_data,
-                *tracker_config)
+                *tracker_config, reid_config_data=reid_config_data)
     scene.uid = data['uid']
     scene.mesh_translation = data.get('mesh_translation', None)
     scene.mesh_rotation = data.get('mesh_rotation', None)
-    scene.use_tracker = data.get('use_tracker', True) and not ControllerMode.isAnalyticsOnly()
-    scene.output_lla = data.get('output_lla', None)
-    scene.map_corners_lla = data.get('map_corners_lla', None)
-    scene.retrack = data.get('retrack', True)
-    scene.regulated_rate = data.get('regulated_rate', None)
-    scene.external_update_rate = data.get('external_update_rate', None)
-    scene.persist_attributes = data.get('persist_attributes', {})
-    if 'cameras' in data:
-      scene.updateCameras(data['cameras'])
-    if 'regions' in data:
-      scene._updateRegions(scene.regions, data['regions'])
-    if 'tripwires' in data:
-      scene._updateTripwires(data['tripwires'])
-    if 'sensors' in data:
-      scene._updateRegions(scene.sensors, data['sensors'])
-    if 'children' in data:
-      scene.children = [x['name'] for x in data['children']]
-    if 'parent' in data:
-      scene.parent = data['parent']
-    if 'transform' in data:
-      scene.cameraPose = CameraPose(data['transform'], None)
-    if 'tracker_config' in data:
-      tracker_config = data['tracker_config']
-      scene.updateTracker(tracker_config[0], tracker_config[1], tracker_config[2])
-    # Access the property to trigger initialization
-    _ = scene.trs_xyz_to_lla
+    scene._hydrateFromSceneData(data, reid_runtime_update=False)
     return scene
 
   def _updateChildren(self, newChildren):

controller/src/controller/tracking.py

@@ -46,6 +46,14 @@ def getUniqueIDCount(self, category):
     log.warning("No tracker for category", category)
     return 0
 
+  def updateReidConfig(self, reid_config_data=None):
+    """Update ReID behavior in-place for this tracker and all child trackers."""
+    self.reid_config_data = reid_config_data if reid_config_data else {}
+    self.uuid_manager.updateReidConfig(self.reid_config_data)
+    for tracker in self.trackers.values():
+      tracker.updateReidConfig(self.reid_config_data)
+    return
+
   def trackObjects(self, objects, already_tracked_objects, when, categories, \
                    ref_camera_frame_rate, \
                    max_unreliable_time, \
@@ -90,7 +98,13 @@ def _createTrackers(self, categories, max_unreliable_time, non_measurement_time_
     """Create a tracker object for each category"""
     for category in categories:
       if category not in self.trackers:
-        tracker = self.__class__(max_unreliable_time, non_measurement_time_dynamic, non_measurement_time_static, ref_camera_frame_rate)
+        tracker = self.__class__(
+          max_unreliable_time,
+          non_measurement_time_dynamic,
+          non_measurement_time_static,
+          ref_camera_frame_rate,
+          reid_config_data=self.reid_config_data,
+        )
         self.trackers[category] = tracker
         tracker.start()
     return
@@ -199,6 +213,8 @@ def join(self):
       tracker.waitForComplete()
       log.debug(f"Joining tracker thread category {category}")
       tracker.join()
+      tracker.uuid_manager.shutdown()
+    self.uuid_manager.shutdown()
     return
 
   @staticmethod