ITEP-23442: Include dwell time in region updates topic (#1300)

Adds “live region dwell time” to region-scoped object outputs (region data + region events) so consumers can see how long an object has been inside a region at publish time.

Additionally, it updates the stability test to include memory leak checks.

Author: saratpoluri

controller/src/controller/detections_builder.py

@@ -7,24 +7,45 @@
 from scene_common import log
 from scene_common.earth_lla import convertXYZToLLA, calculateHeading
 from scene_common.geometry import DEFAULTZ, Point, Size
-from scene_common.timestamp import get_iso_time
+from scene_common.timestamp import get_epoch_time, get_iso_time
 
 
-def buildDetectionsDict(objects, scene, include_sensors=False):
+def buildDetectionsDict(objects, scene, include_sensors=False, include_region_dwell=False, current_time=None):
   result_dict = {}
   for obj in objects:
-    obj_dict = prepareObjDict(scene, obj, False, include_sensors)
+    obj_dict = prepareObjDict(scene, obj, False, include_sensors, include_region_dwell, current_time)
     result_dict[obj_dict['id']] = obj_dict
   return result_dict
 
-def buildDetectionsList(objects, scene, update_visibility=False, include_sensors=False):
+def buildDetectionsList(objects, scene, update_visibility=False, include_sensors=False,
+                        include_region_dwell=False, current_time=None):
   result_list = []
   for obj in objects:
-    obj_dict = prepareObjDict(scene, obj, update_visibility, include_sensors)
+    obj_dict = prepareObjDict(scene, obj, update_visibility, include_sensors,
+                              include_region_dwell, current_time)
     result_list.append(obj_dict)
   return result_list
 
-def prepareObjDict(scene, obj, update_visibility, include_sensors=False):
+def _get_region_entered_epoch(region_data):
+  entered_epoch = region_data.get('entered_epoch')
+  if entered_epoch is None:
+    entered_epoch = get_epoch_time(region_data['entered'])
+    region_data['entered_epoch'] = entered_epoch
+  return entered_epoch
+
+def _build_region_output(regions, include_region_dwell, current_time):
+  serialized_regions = {}
+  for region_name, region_data in regions.items():
+    serialized_region = dict(region_data)
+    serialized_region.pop('entered_epoch', None)
+    if include_region_dwell and 'entered' in region_data:
+      entered = _get_region_entered_epoch(region_data)
+      serialized_region['dwell'] = current_time - entered
+    serialized_regions[region_name] = serialized_region
+  return serialized_regions
+
+def prepareObjDict(scene, obj, update_visibility, include_sensors=False,
+                   include_region_dwell=False, current_time=None):
   aobj = obj
   if isinstance(obj, TripwireEvent):
     aobj = obj.object
@@ -89,7 +110,12 @@ def prepareObjDict(scene, obj, update_visibility, include_sensors=False):
   if hasattr(aobj, 'chain_data'):
     chain_data = aobj.chain_data
     if len(chain_data.regions):
-      obj_dict['regions'] = chain_data.regions
+      if include_region_dwell:
+        if current_time is None:
+          current_time = get_epoch_time()
+        obj_dict['regions'] = _build_region_output(chain_data.regions, include_region_dwell, current_time)
+      else:
+        obj_dict['regions'] = chain_data.regions
 
     if include_sensors:
       sensors_output = {}

controller/src/controller/scene_controller.py

@@ -276,13 +276,16 @@ def publishRegulatedDetections(self, scene_obj, msg_objects, otype, jdata, camer
     return
 
   def publishRegionDetections(self, scene, objects, otype, jdata):
+    current_time = get_epoch_time(jdata['timestamp'])
     for rname in scene.regions:
       robjects = []
       for obj in objects:
         if rname in obj.chain_data.regions:
           robjects.append(obj)
       # Region-specific detections: include sensor data
-      jdata['objects'] = buildDetectionsList(robjects, scene, False, include_sensors=True)
+      jdata['objects'] = buildDetectionsList(
+        robjects, scene, False, include_sensors=True,
+        include_region_dwell=True, current_time=current_time)
       olen = len(jdata['objects'])
       rid = scene.name + "/" + rname + "/" + otype
       if olen > 0 or rid not in scene.lastPubCount or scene.lastPubCount[rid] > 0:
@@ -346,7 +349,9 @@ def _buildAllRegionObjsList(self, scene, region, event_data):
       num_objects += counts[otype]
       all_objects += objects
     event_data['counts'] = counts
-    detections_dict = buildDetectionsDict(all_objects, scene, include_sensors=True)
+    detections_dict = buildDetectionsDict(
+      all_objects, scene, include_sensors=True,
+      include_region_dwell=True, current_time=get_epoch_time(event_data['timestamp']))
     event_data['objects'] = list(detections_dict.values())
     return detections_dict, num_objects
 
@@ -364,7 +369,9 @@ def _buildEnteredObjsList(self, scene, region, event_data, detections_dict):
 
     # Build any objects not in detections_dict (e.g., from sensor events)
     if missing_objs:
-      entered_objs = buildDetectionsList(missing_objs, scene, False, include_sensors=True)
+      entered_objs = buildDetectionsList(
+        missing_objs, scene, False, include_sensors=True,
+        include_region_dwell=True, current_time=get_epoch_time(event_data['timestamp']))
       event_data['entered'].extend(entered_objs)
 
   def _buildExitedObjsList(self, scene, region, event_data):
@@ -377,7 +384,9 @@ def _buildExitedObjsList(self, scene, region, event_data):
         exited_dict[exited_obj.gid] = dwell
         exited_objs.extend([exited_obj])
       # Exit events: include sensor data (timestamped readings and attribute events)
-      exited_objs = buildDetectionsList(exited_objs, scene, False, include_sensors=True)
+      exited_objs = buildDetectionsList(
+        exited_objs, scene, False, include_sensors=True,
+        include_region_dwell=True, current_time=get_epoch_time(event_data['timestamp']))
       exited_data = [{'object': exited_obj, 'dwell': exited_dict[exited_obj['id']]} for exited_obj in exited_objs]
       event_data['exited'].extend(exited_data)
     return

docs/user-guide/microservices/controller/_assets/scene-controller-api.yaml

@@ -199,13 +199,16 @@ channels:
               type: array
               items:
                 type: object
+              description: >-
+                In region-scoped payloads, objects currently inside a region include a live
+                regions.<region_id>.dwell value in seconds alongside the entered timestamp.
             entered:
               type: array
               items:
                 type: object
               description: >-
                 Objects that entered the region during this cycle. In region events, each element is a bare
-                track object. In tripwire events, this array is always empty. May be
+                track object and may include regions.<region_id>.dwell for the current region. In tripwire events, this array is always empty. May be
                 empty in region events when no entry occurred.
             exited:
               type: array

docs/user-guide/microservices/controller/data_formats.md

@@ -196,7 +196,7 @@ tracked object contains the following fields:
 | `velocity`      | array[3] of number | Velocity vector (`x`, `y`, `z`) in metres per second                                                                                                                                                                                                         |
 | `rotation`      | array[4] of number | Orientation quaternion                                                                                                                                                                                                                                       |
 | `visibility`    | array of string    | Camera IDs currently observing this object                                                                                                                                                                                                                   |
-| `regions`       | object             | Map of region/sensor IDs to entry timestamps (`{id: {entered: timestamp}}`)                                                                                                                                                                                  |
+| `regions`       | object             | Map of region/sensor IDs to membership metadata. By default this is `{id: {entered: timestamp}}`. In region-scoped outputs, objects currently inside a region also include a live dwell time as `{id: {entered: timestamp, dwell: seconds}}`.                |
 | `sensors`       | object             | Map of sensor IDs to timestamped readings (`{id: [[timestamp, value], ...]}`)                                                                                                                                                                                |
 | `similarity`    | number or null     | Re-ID similarity score; `null` when not computed                                                                                                                                                                                                             |
 | `first_seen`    | string (ISO 8601)  | Timestamp when the track was first created                                                                                                                                                                                                                   |
@@ -210,6 +210,12 @@ tracked object contains the following fields:
 > camera input it is a base64-encoded string. `metadata` is absent when no semantic
 > analytics pipeline is configured.
 
+> **Note on live region dwell**: In region data and region event payloads, objects that are
+> still inside a region include `regions.<region_id>.dwell`, which is the current elapsed
+> time in seconds since that object entered the region. Exit records continue to expose the
+> final dwell time separately as `{"object": <track>, "dwell": <seconds>}` in the
+> top-level `exited` array.
+
 ## Data Scene Output Message Format
 
 Published on MQTT topic: `scenescape/data/scene/{scene_id}/{thing_type}`
@@ -375,18 +381,18 @@ interest changes. The `{event_type}` segment is typically `objects`.
 
 ### Region Event Top-Level Fields
 
-| Field         | Type                  | Description                                                                                                                              |
-| ------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
-| `timestamp`   | string (ISO 8601 UTC) | Event timestamp                                                                                                                          |
-| `scene_id`    | string                | Scene identifier (UUID)                                                                                                                  |
-| `scene_name`  | string                | Scene name                                                                                                                               |
-| `region_id`   | string                | Region identifier (UUID)                                                                                                                 |
-| `region_name` | string                | Region name                                                                                                                              |
-| `counts`      | object                | Map of category to object count currently inside the region (e.g. `{"person": 2}`)                                                       |
-| `objects`     | array                 | Tracked objects currently inside the region (see [Common Output Track Fields](#common-output-track-fields))                              |
-| `entered`     | array                 | Objects that entered the region during this cycle; each element is a bare track object. Empty when no entry occurred                     |
-| `exited`      | array                 | Objects that exited the region during this cycle; each element is `{"object": <track>, "dwell": <seconds>}`. Empty when no exit occurred |
-| `metadata`    | object                | Region geometry: `title`, `uuid`, `points` (polygon vertices in metres), `area` (`"poly"`), `fromSensor` (boolean)                       |
+| Field         | Type                  | Description                                                                                                                                                                 |
+| ------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `timestamp`   | string (ISO 8601 UTC) | Event timestamp                                                                                                                                                             |
+| `scene_id`    | string                | Scene identifier (UUID)                                                                                                                                                     |
+| `scene_name`  | string                | Scene name                                                                                                                                                                  |
+| `region_id`   | string                | Region identifier (UUID)                                                                                                                                                    |
+| `region_name` | string                | Region name                                                                                                                                                                 |
+| `counts`      | object                | Map of category to object count currently inside the region (e.g. `{"person": 2}`)                                                                                          |
+| `objects`     | array                 | Tracked objects currently inside the region. Each object includes live `regions.<region_id>.dwell` in addition to [Common Output Track Fields](#common-output-track-fields) |
+| `entered`     | array                 | Objects that entered the region during this cycle; each element is a bare track object and may include live `regions.<region_id>.dwell`. Empty when no entry occurred       |
+| `exited`      | array                 | Objects that exited the region during this cycle; each element is `{"object": <track>, "dwell": <seconds>}`. Empty when no exit occurred                                    |
+| `metadata`    | object                | Region geometry: `title`, `uuid`, `points` (polygon vertices in metres), `area` (`"poly"`), `fromSensor` (boolean)                                                          |
 
 ### Example Region Event Message
 

tests/Makefile

@@ -270,7 +270,10 @@ build-time: # NEX-T12520
 
 system-stability: # NEX-T10411
 	$(eval HOURS ?= 24)
+	$(eval OLDSECRETSDIR := $(SECRETSDIR))
+	$(eval SECRETSDIR := $(PWD)/manager/secrets)
 	$(call common-recipe, $(COMPOSE)/dlstreamer/broker.yml:$(COMPOSE)/ntp.yml:$(COMPOSE)/pgserver.yml:$(COMPOSE)/dlstreamer/retail_video.yml:$(COMPOSE)/scene.yml:$(COMPOSE)/web.yml:$(COMPOSE)/cams.yml, tests/system/stability/tc_sscape_stability.py --hours=$(HOURS),'pgserver web scene',true)
+	$(eval SECRETSDIR := $(OLDSECRETSDIR))
 
 python-indent-check: # NEX-T10448
 	$(eval LOGFILE=$(TEST_DATA)/code_check/$@-$(shell date -u +"%F-%T").log)

tests/functional/common_scene_obj.py

@@ -70,6 +70,9 @@ def verifyRegionEvent(self, regionEvent):
             self.expectedExit.append(event['id'])
             self.expectedEnter.remove(event['id'])
             self.entered = True
+            # Track entry time for dwell verification
+            if event['id'] not in self.objectEntryTimes:
+              self.objectEntryTimes[event['id']] = get_epoch_time()
             print("object with id {} entered region\n".format(event['id']))
 
     if len(regionEvent['exited']) > 0:
@@ -129,6 +132,8 @@ def runSceneObjMqttInitialize(self):
     self.roiPoints = ((0.9, 4.0), (0.9, 2.4),
                       (8.1, 2.4), (8.1, 4.0))
     self.message_received_after_delete = False
+    self.objectEntryTimes = {}  # Track when each object entered region for dwell verification
+    self.previousDwellTimes = {}  # Track previous dwell times to verify monotonic increase
     if self.testName and self.recordXMLAttribute:
       self.recordXMLAttribute("name", self.testName)
 
@@ -152,6 +157,40 @@ def sceneReady(self, max_attempts, waitTopic, publishTopic, objData):
   def regulatedReceived(self, pahoClient, userdata, message):
     data = message.payload.decode("utf-8")
     self.sceneData = json.loads(data)
+    # Track that dwell data appears in scene messages when expected
+    self.verifyDwellPresenceInSceneData(self.sceneData)
+    return
+
+  def verifyDwellPresenceInSceneData(self, sceneData):
+    """Verify dwell values appear in scene object data when objects are in regions.
+
+    Unlike unit tests which validate calculation formulas, this checks integration:
+    that dwell data is present and available in the scene data stream.
+    """
+    if 'objects' not in sceneData or not sceneData['objects']:
+      return
+
+    for obj in sceneData['objects']:
+      obj_id = obj.get('id')
+      if not obj_id or obj_id not in self.objectEntryTimes:
+        continue
+
+      # Check that dwell data is present for objects known to be in regions
+      if 'regions' in obj and obj['regions']:
+        for region_name, region_data in obj['regions'].items():
+          if 'dwell' in region_data:
+            dwell = region_data['dwell']
+            # Basic sanity check: dwell should be non-negative
+            assert dwell >= 0, f"Object {obj_id} has negative dwell: {dwell}"
+
+            # Verify monotonic increase (no dwell decrease over time)
+            key = (obj_id, region_name)
+            if key in self.previousDwellTimes:
+              prev_dwell = self.previousDwellTimes[key]
+              assert dwell >= prev_dwell - 0.2, \
+                f"Object {obj_id} dwell in {region_name} decreased from {prev_dwell:.2f} to {dwell:.2f}"
+
+            self.previousDwellTimes[key] = dwell
     return
 
   def runSceneObjMqttPrepare(self):

tests/functional/tc_mqtt_sensor_roi.py

@@ -71,8 +71,27 @@ def runSceneObjMqttVerifyPassedExtra(self):
     assert self.checkedEntered > 0
     assert self.checkedExited > 0
     assert self.checkedValues > 0
+    # Verify dwell window calculation is sensible
+    self.verifyDwellWindowExists()
     return True
 
+  def verifyDwellWindowExists(self):
+    """Verify dwell window metadata exists for integration validation.
+
+    This validates integration aspects only - that entry/exit times were properly
+    captured. The actual dwell calculation formulas are already tested by unit tests.
+    """
+    if self.enteredTimestamp is None:
+      return  # No entry detected, nothing to verify
+
+    if self.exitedTimestamp is not None:
+      dwell_window = self.exitedTimestamp - self.enteredTimestamp
+      assert dwell_window >= 0, f"Invalid dwell window: exit before entry"
+      print(f"Dwell window validated: {dwell_window:.2f}s from entry to exit")
+    else:
+      print("Object still in region - no exit timestamp to validate")
+    return
+
   def sendDetections(self, objLocation, frame_rate):
     jdata = self.objData()
     start_time = get_epoch_time()