Tracker Service v0.4.0: scene config dynamic loading

Author: Józef Daniecki

.github/resources/.prettierignore

@@ -13,5 +13,4 @@
 **/CMakePresets.json
 **/CMakeFiles/
 
-**/tests/api/README.md
-**/tests/system/metric/dataset/*.json
+**/tests/api/README.md

controller/src/robot_vision/src/rv/tracking/ObjectMatching.cpp

@@ -11,8 +11,6 @@
 #include "rv/apollo/secure_matrix.hpp"
 #include "rv/tracking/Classification.hpp"
 
-#include <iostream>
-
 namespace rv {
 namespace tracking {
 

controller/src/robot_vision/src/rv/tracking/TrackManager.cpp

@@ -3,7 +3,6 @@
 
 #include "rv/Utils.hpp"
 #include "rv/tracking/TrackManager.hpp"
-#include <iostream>
 #include <omp.h>
 
 namespace rv {
@@ -354,10 +353,6 @@ void TrackManager::updateTrackerConfig(int camera_frame_rate)
   mConfig.mMaxNumberOfUnreliableFrames = std::ceil(camera_frame_rate*mConfig.mMaxUnreliableTime);
   mConfig.mNonMeasurementFramesDynamic = std::ceil(camera_frame_rate*mConfig.mNonMeasurementTimeDynamic);
   mConfig.mNonMeasurementFramesStatic = std::ceil(camera_frame_rate*mConfig.mNonMeasurementTimeStatic);
-  std::cout << "Updated parameters for reference camera frame rate = " << camera_frame_rate << "fps" << std::endl;
-  std::cout << "max_unreliable_frames = " << mConfig.mMaxNumberOfUnreliableFrames << std::endl;
-  std::cout << "non_measurement_frames_dynamic = " << mConfig.mNonMeasurementFramesDynamic << std::endl;
-  std::cout << "non_measurement_frames_static = " << mConfig.mNonMeasurementFramesStatic << std::endl;
 }
 
 } // namespace tracking

docs/design/tracker-evaluation-pipeline.md

@@ -394,13 +394,12 @@ function main() {
           sensorManager.sceneSensors[sensorId]
         ) {
           const sensor = sensorManager.sceneSensors[sensorId];
-          const sensorArea = sensor.region && sensor.region.area;
 
           // Only control lighting for sensors with area set to "scene"
           // Don't control lighting for localized sensors ("circle", "poly") or any other value
-          if (sensorArea !== "scene") {
+          if (sensor.area !== "scene") {
             console.log(
-              `Light sensor (${sensorId}): area="${sensorArea}" - not controlling scene lighting (only "scene" area sensors affect ambient light)`,
+              `Light sensor (${sensorId}): area="${sensor.area}" - not controlling scene lighting (only "scene" area sensors affect ambient light)`,
             );
             return;
           }

manager/src/static/js/scenescape3d.js

@@ -1542,7 +1542,7 @@ function setupGenerateMesh() {
 }
 
 async function pollMeshStatus(sceneId, requestId) {
-  const timeout = 15 * 60 * 1000; // 15 minutes
+  const timeout = 10 * 60 * 1000; // 10 minutes
   const start = Date.now();
 
   while (true) {

manager/src/static/js/sscape.js

@@ -110,7 +110,7 @@ RUN pip install --no-cache-dir -r $SCENESCAPE_HOME/requirements_api.txt && \
         pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r $SCENESCAPE_HOME/requirements_vggt.txt && \
         cd /workspace/vggt && \
         pip install --no-cache-dir -e . && \
-        pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r requirements.txt -r requirements_demo.txt; \
+        pip install --no-cache-dir -r requirements.txt -r requirements_demo.txt; \
     fi && \
     rm -rf /usr/local/lib/python3.11/site-packages/torch/test \
            /usr/local/lib/python3.11/site-packages/torch/share \

mapping/Dockerfile

@@ -11,7 +11,11 @@
 """
 
 import base64
+import math
+import os
+import subprocess
 import sys
+import tempfile
 from typing import Dict, Any, List, Optional, Tuple
 
 import numpy as np
@@ -116,6 +120,119 @@ def runInference(self, frames: List[Dict[str, Any]]) -> Dict[str, Any]:
       log.error(f"MapAnything inference (frames) failed: {e}")
       raise RuntimeError(f"MapAnything inference (frames) failed: {e}")
 
+  def _maxFramesForTimeBudget(
+    self,
+    time_budget_seconds: float,
+    overhead: float,
+  ) -> int:
+
+    cpu_sec_per_frame = float(os.getenv("MAPANYTHING_CPU_SEC_PER_FRAME", "10"))
+    cuda_sec_per_frame = float(os.getenv("MAPANYTHING_CUDA_SEC_PER_FRAME", "0.8"))
+    sec_per_frame = cpu_sec_per_frame
+    if self.device.startswith("cuda") and cuda_sec_per_frame:
+      sec_per_frame = cuda_sec_per_frame
+
+    usable = max(0.0, time_budget_seconds - overhead)
+    if usable <= 0:
+      return 0
+
+    # conservative: floor
+    max_frames = int(math.floor(usable / max(1e-6, sec_per_frame)))
+    return max_frames
+
+  # Put in ReconstructionModel base class
+  def _framesFromVideoAsBase64Dicts(
+    self,
+    video_path: str,
+    max_frames: int,
+    use_keyframes: bool = True,
+    sample_every_n: int = 10,
+    jpeg_quality: int = 85,
+    max_side: Optional[int] = 960,
+  ) -> List[Dict[str, Any]]:
+    """
+    Extract frames using ffmpeg and return:
+      [{"data": "<base64-encoded-jpeg>"}, ...]
+
+    Modes:
+      - use_keyframes=True: extract TRUE keyframes (I-frames)
+      - use_keyframes=False: sample every N frames using select filter
+    """
+    if max_frames < 1:
+      return []
+
+    if not os.path.isfile(video_path):
+      raise ValueError(f"Video file not found: {video_path}")
+
+    if sample_every_n < 1:
+      sample_every_n = 1
+
+    # Map jpeg_quality (1..100) -> ffmpeg mjpeg qscale (2..31), where 2 is best quality
+    qscale = int(round(31 - (np.clip(jpeg_quality, 1, 100) / 100.0) * 29))
+    qscale = int(np.clip(qscale, 2, 31))
+
+    vf_parts: List[str] = []
+
+    # If not keyframes, use select filter to sample frames
+    if not use_keyframes:
+      # keep frames where n % sample_every_n == 0
+      vf_parts.append(f"select='not(mod(n\\,{sample_every_n}))'")
+    else:
+      log.info("Using key frames")
+
+    # Optional downscale: keep aspect ratio, cap longest side
+    if max_side is not None and max_side > 0:
+      vf_parts.append(
+        f"scale='if(gte(iw,ih),min(iw,{max_side}),-2)':'if(lt(iw,ih),min(ih,{max_side}),-2)'"
+      )
+
+    vf = ",".join(vf_parts) if vf_parts else None
+
+    frames: List[Dict[str, Any]] = []
+
+    with tempfile.TemporaryDirectory(prefix="frames_") as tmpdir:
+      out_pattern = os.path.join(tmpdir, "frame_%06d.jpg")
+
+      cmd = [
+        "ffmpeg",
+        "-hide_banner",
+        "-loglevel", "error",
+      ]
+
+      # Keyframes mode: only decode keyframes
+      if use_keyframes:
+        cmd += ["-skip_frame", "nokey"]
+
+      cmd += ["-i", video_path]
+
+      if vf:
+        cmd += ["-vf", vf]
+
+      cmd += [
+        "-vsync", "vfr",
+        "-frames:v", str(max_frames),
+        "-q:v", str(qscale),
+        out_pattern,
+      ]
+
+      try:
+        subprocess.run(cmd, check=True)
+      except FileNotFoundError:
+        raise RuntimeError("ffmpeg not found. Install ffmpeg in the container/host.")
+      except subprocess.CalledProcessError as e:
+        mode = "keyframes" if use_keyframes else f"sample_every_n={sample_every_n}"
+        raise RuntimeError(f"ffmpeg failed extracting frames ({mode}): {e}")
+
+      # Read extracted frames back into base64
+      for i in range(1, max_frames + 1):
+        fpath = os.path.join(tmpdir, f"frame_{i:06d}.jpg")
+        if not os.path.exists(fpath):
+          break
+        with open(fpath, "rb") as f:
+          frames.append({"data": base64.b64encode(f.read()).decode("utf-8")})
+
+    return frames
+
   def getSupportedOutputs(self) -> List[str]:
     """Get supported output formats."""
     return ["mesh", "pointcloud"]

mapping/src/mapanything_model.py

@@ -12,15 +12,10 @@
 """
 
 from abc import ABC, abstractmethod
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List
 
-import base64
 import cv2
 import numpy as np
-import os
-import subprocess
-import math
-import tempfile
 
 from scene_common import log
 
@@ -292,115 +287,3 @@ def rotationMatrixToQuaternion(self, R: np.ndarray) -> np.ndarray:
       z = 0.25 * s
 
     return np.array([x, y, z, w])
-
-  def _maxFramesForTimeBudget(
-    self,
-    time_budget_seconds: float,
-    overhead: float,
-  ) -> int:
-
-    cpu_sec_per_frame = float(os.getenv("MAPPING_CPU_SEC_PER_FRAME", "10"))
-    cuda_sec_per_frame = float(os.getenv("MAPPING_CUDA_SEC_PER_FRAME", "0.8"))
-    sec_per_frame = cpu_sec_per_frame
-    if self.device.startswith("cuda") and cuda_sec_per_frame:
-      sec_per_frame = cuda_sec_per_frame
-
-    usable = max(0.0, time_budget_seconds - overhead)
-    if usable <= 0:
-      return 0
-
-    # conservative: floor
-    max_frames = int(math.floor(usable / max(1e-6, sec_per_frame)))
-    return max_frames
-
-  def _framesFromVideoAsBase64Dicts(
-    self,
-    video_path: str,
-    max_frames: int,
-    use_keyframes: bool = True,
-    sample_every_n: int = 10,
-    jpeg_quality: int = 85,
-    max_side: Optional[int] = 960,
-  ) -> List[Dict[str, Any]]:
-    """
-    Extract frames using ffmpeg and return:
-      [{"data": "<base64-encoded-jpeg>"}, ...]
-
-    Modes:
-      - use_keyframes=True: extract TRUE keyframes (I-frames)
-      - use_keyframes=False: sample every N frames using select filter
-    """
-    if max_frames < 1:
-      return []
-
-    if not os.path.isfile(video_path):
-      raise ValueError(f"Video file not found: {video_path}")
-
-    if sample_every_n < 1:
-      sample_every_n = 1
-
-    # Map jpeg_quality (1..100) -> ffmpeg mjpeg qscale (2..31), where 2 is best quality
-    qscale = int(round(31 - (np.clip(jpeg_quality, 1, 100) / 100.0) * 29))
-    qscale = int(np.clip(qscale, 2, 31))
-
-    vf_parts: List[str] = []
-
-    # If not keyframes, use select filter to sample frames
-    if not use_keyframes:
-      # keep frames where n % sample_every_n == 0
-      vf_parts.append(f"select='not(mod(n\\,{sample_every_n}))'")
-    else:
-      log.info("Using key frames")
-
-    # Optional downscale: keep aspect ratio, cap longest side
-    if max_side is not None and max_side > 0:
-      vf_parts.append(
-        f"scale='if(gte(iw,ih),min(iw,{max_side}),-2)':'if(lt(iw,ih),min(ih,{max_side}),-2)'"
-      )
-
-    vf = ",".join(vf_parts) if vf_parts else None
-
-    frames: List[Dict[str, Any]] = []
-
-    with tempfile.TemporaryDirectory(prefix="frames_") as tmpdir:
-      out_pattern = os.path.join(tmpdir, "frame_%06d.jpg")
-
-      cmd = [
-        "ffmpeg",
-        "-hide_banner",
-        "-loglevel", "error",
-      ]
-
-      # Keyframes mode: only decode keyframes
-      if use_keyframes:
-        cmd += ["-skip_frame", "nokey"]
-
-      cmd += ["-i", video_path]
-
-      if vf:
-        cmd += ["-vf", vf]
-
-      cmd += [
-        "-vsync", "vfr",
-        "-frames:v", str(max_frames),
-        "-q:v", str(qscale),
-        out_pattern,
-      ]
-
-      try:
-        subprocess.run(cmd, check=True)
-      except FileNotFoundError:
-        raise RuntimeError("ffmpeg not found. Install ffmpeg in the container/host.")
-      except subprocess.CalledProcessError as e:
-        mode = "keyframes" if use_keyframes else f"sample_every_n={sample_every_n}"
-        raise RuntimeError(f"ffmpeg failed extracting frames ({mode}): {e}")
-
-      # Read extracted frames back into base64
-      for i in range(1, max_frames + 1):
-        fpath = os.path.join(tmpdir, f"frame_{i:06d}.jpg")
-        if not os.path.exists(fpath):
-          break
-        with open(fpath, "rb") as f:
-          frames.append({"data": base64.b64encode(f.read()).decode("utf-8")})
-
-    return frames

mapping/src/model_interface.py

@@ -406,33 +406,35 @@ def _preprocessImages(self, pil_images: List[Image.Image]) -> tuple:
     Returns:
       Tuple of (processed_tensor, model_size)
     """
-    target = 518
-    n = len(pil_images)
+    processed_images = []
+    target_size = 518
 
-    # Preallocate on CPU, then move once
-    batch = torch.empty((n, 3, target, target), dtype=torch.float32)
+    for pil_image in pil_images:
+      # Apply VGGT preprocessing (similar to load_and_preprocess_images)
+      width, height = pil_image.size
 
-    for i, im in enumerate(pil_images):
-      w, h = im.size
-      new_w = target
-      new_h = round(h * (new_w / w) / 14) * 14
-      new_h = max(14, new_h)
+      # Set width to target_size, calculate height maintaining aspect ratio
+      new_width = target_size
+      new_height = round(height * (new_width / width) / 14) * 14  # Divisible by 14
 
-      im = im.resize((new_w, new_h), Image.Resampling.BICUBIC)
+      # Resize image
+      img_resized = pil_image.resize((new_width, new_height), Image.Resampling.BICUBIC)
 
-      if new_h > target:
-        top = (new_h - target) // 2
-        im = im.crop((0, top, target, top + target))
-      elif new_h < target:
-        pad_top = (target - new_h) // 2
-        canvas = Image.new(im.mode, (target, target))
-        canvas.paste(im, (0, pad_top))
-        im = canvas
+      # Convert to tensor
+      img_tensor = tvf.ToTensor()(img_resized)  # Shape: (3, H, W), values [0, 1]
 
-      batch[i] = tvf.ToTensor()(im)
+      # Center crop height if larger than target_size
+      if new_height > target_size:
+        start_y = (new_height - target_size) // 2
+        img_tensor = img_tensor[:, start_y:start_y + target_size, :]
 
-    images_tensor = batch.to(self.device, non_blocking=True)
-    return images_tensor, (target, target)
+      processed_images.append(img_tensor)
+
+    # Stack all images and move to device
+    images_tensor = torch.stack(processed_images).to(self.device)  # Shape: (N, 3, H, W)
+    model_size = images_tensor.shape[-2:]  # (height, width)
+
+    return images_tensor, model_size
 
   def _runModelInference(self, images_tensor: torch.Tensor) -> Dict[str, Any]:
     """