Feature/spatial analytics documentation

.github/resources/sdl/.trivyignore.yaml

@@ -11,19 +11,19 @@ misconfigurations:
   # Helm chart
   - id: AVD-KSV-0005
     paths:
-      - "kubernetes/scenescape-chart/templates/web-dep.yaml"
+      - "kubernetes/scenescape-chart/templates/web-app/deployment.yaml"
     statement: Current implementation requires admin capabilities
   - id: AVD-KSV-0014
     paths:
-      - "kubernetes/scenescape-chart/templates/camcalibration-dep.yaml"
-      - "kubernetes/scenescape-chart/templates/pgserver-dep.yaml"
-      - "kubernetes/scenescape-chart/templates/queuing-dep.yaml"
-      - "kubernetes/scenescape-chart/templates/retail-dep.yaml"
-      - "kubernetes/scenescape-chart/templates/vdms-dep.yaml"
-      - "kubernetes/scenescape-chart/templates/web-dep.yaml"
-      - "kubernetes/scenescape-chart/templates/kubeclient-dep.yaml"
+      - "kubernetes/scenescape-chart/templates/camcalibration/deployment.yaml"
+      - "kubernetes/scenescape-chart/templates/pgserver/deployment.yaml"
+      - "kubernetes/scenescape-chart/templates/dl-streamer-pipeline-server/queuing-deployment.yaml"
+      - "kubernetes/scenescape-chart/templates/dl-streamer-pipeline-server/retail-deployment.yaml"
+      - "kubernetes/scenescape-chart/templates/vdms/deployment.yaml"
+      - "kubernetes/scenescape-chart/templates/web-app/deployment.yaml"
+      - "kubernetes/scenescape-chart/templates/kubeclient/deployment.yaml"
     statement: Current implementation requires these containers to write to filesystem
   - id: AVD-KSV-0017
     paths:
-      - "kubernetes/scenescape-chart/templates/web-dep.yaml"
+      - "kubernetes/scenescape-chart/templates/web-app/deployment.yaml"
     statement: Current implementation requires privileged container

.github/resources/sdl/trivy_config.yml

@@ -14,3 +14,4 @@ misconfiguration:
   helm:
     set:
       - supass=demo
+      - pgserver.password=demo

Makefile

@@ -8,7 +8,8 @@ SHELL := /bin/bash
 
 # Build folders
 COMMON_FOLDER := scene_common
-IMAGE_FOLDERS := autocalibration controller manager mapping model_installer cluster_analytics
+CORE_IMAGE_FOLDERS := autocalibration controller manager model_installer
+IMAGE_FOLDERS := $(CORE_IMAGE_FOLDERS) mapping cluster_analytics
 
 # Build flags
 EXTRA_BUILD_FLAGS :=
@@ -24,7 +25,7 @@ COMPOSE_PROJECT_NAME ?= scenescape
 # - User can adjust build output folder (defaults to $PWD/build)
 BUILD_DIR ?= $(PWD)/build
 # - User can adjust folders being built (defaults to all)
-FOLDERS ?= $(IMAGE_FOLDERS)
+FOLDERS ?= $(CORE_IMAGE_FOLDERS)
 # - User can adjust number of parallel jobs (defaults to CPU count)
 JOBS ?= $(shell nproc)
 # - User can adjust the target branch
@@ -45,8 +46,8 @@ DLSTREAMER_DOCKER_COMPOSE_FILE := ./sample_data/docker-compose-dl-streamer-examp
 # Test variables
 TESTS_FOLDER := tests
 TEST_DATA_FOLDER := test_data
-TEST_IMAGE_FOLDERS := autocalibration controller manager mapping
-TEST_IMAGES := $(addsuffix -test, camcalibration controller manager mapping)
+TEST_IMAGE_FOLDERS := autocalibration controller manager mapping cluster_analytics
+TEST_IMAGES := $(addsuffix -test, camcalibration controller manager mapping cluster_analytics)
 DEPLOYMENT_TEST ?= 0
 
 # Observability variables
@@ -59,10 +60,13 @@ CONTROLLER_TRACING_SAMPLE_RATIO ?= 1.0
 
 # ========================= Default Target ===========================
 
-default: build-all
+default: build-core
+
+.PHONY: build-core
+build-core: init-secrets build-core-images install-models
 
 .PHONY: build-all
-build-all: init-secrets build-images install-models
+build-all: init-secrets build-all-images install-models
 
 # ============================== Help ================================
 
@@ -72,13 +76,16 @@ help:
 	@echo "Intel® SceneScape version $(VERSION)"
 	@echo ""
 	@echo "Available targets:"
-	@echo "  build-all         (default) Build secrets, all images, and install models"
-	@echo "  build-images                Build all microservice images in parallel"
+	@echo "  build-core        (default) Build secrets, core images (excluding mapping and cluster_analytics), and install models"
+	@echo "  build-all                   Build secrets, all images, and install models"
+	@echo "  build-core-images           Build core microservice images (excluding mapping and cluster_analytics) in parallel"
+	@echo "  build-all-images            Build all microservice images in parallel"
 	@echo "  init-secrets                Generate secrets and certificates"
 	@echo "  <image folder>              Build a specific microservice image (autocalibration, controller, etc.)"
 	@echo ""
-	@echo "  demo                        Start the SceneScape demo using Docker Compose"
-	@echo "                              (the demo target requires the SUPASS environment variable to be set"
+	@echo "  demo                        (default) Start the SceneScape demo with core services using Docker Compose"
+	@echo "  demo-all                    Start the SceneScape demo with all services using Docker Compose"
+	@echo "                              (the demo targets require the SUPASS environment variable to be set"
 	@echo "                              as the super user password for logging into Intel® SceneScape)"
 	@echo "  demo-k8s                    Start the SceneScape demo using Kubernetes"
 	@echo ""
@@ -89,11 +96,15 @@ help:
 	@echo "  upgrade-database            Backup and upgrade database to a newer PostgreSQL version"
 	@echo "                              (automatically transfers data to Docker volumes)"
 	@echo ""
-	@echo "  rebuild                     Clean and build all images"
+	@echo "  rebuild-core                Clean and build core images and create secrets and volumes"
+	@echo "  rebuild-core-images         Clean and build core images"
 	@echo "  rebuild-all                 Clean and build everything including secrets and volumes"
+	@echo "  rebuild-all-images          Clean and build all images"
 	@echo ""
-	@echo "  clean                       Clean images and build artifacts (logs etc.)"
+	@echo "  clean-core                  Clean core images and remove secrets, volumes and models"
+	@echo "  clean-core-images           Clean core images"
 	@echo "  clean-all                   Clean everything including volumes, secrets and models"
+	@echo "  clean-images                Clean all images"
 	@echo "  clean-volumes               Remove all project Docker volumes"
 	@echo "  clean-secrets               Remove all generated secrets"
 	@echo "  clean-models                Remove all installed models"
@@ -173,42 +184,81 @@ $(IMAGE_FOLDERS):
 	@echo "DONE ====> Building folder $@"
 
 # Dependency on the common base image
-autocalibration controller manager mapping: build-common
+autocalibration controller manager mapping cluster_analytics: build-common
+
+# Helper function to build images in parallel
+define parallel-build
+	@echo "==> Running parallel builds of folders: $(1)"
+	@set -e; trap 'grep --color=auto -i -r --include="*.log" "^error" $(BUILD_DIR) || true' EXIT; \
+	$(MAKE) -j$(JOBS) $(1)
+	@echo "DONE ==> Parallel builds of folders: $(1)"
+endef
 
 # Parallel wrapper handles parallel builds of folders specified in FOLDERS variable
-.PHONY: build-images
-build-images: $(BUILD_DIR)
-	@echo "==> Running parallel builds of folders: $(FOLDERS)"
+.PHONY: build-all-images
+build-all-images: $(BUILD_DIR)
+	$(call parallel-build, $(IMAGE_FOLDERS))
+
+# Parallel wrapper for core images (excluding mapping and cluster_analytics)
+.PHONY: build-core-images
+build-core-images: $(BUILD_DIR)
+	$(call parallel-build, $(CORE_IMAGE_FOLDERS))
+
+# Parallel wrapper for core images (excluding mapping and cluster_analytics)
+.PHONY: build-core-images
+build-core-images: $(BUILD_DIR)
+	@echo "==> Running parallel builds of core folders: $(CORE_IMAGE_FOLDERS)"
 # Use a trap to catch errors and print logs if any error occurs in parallel build
 	@set -e; trap 'grep --color=auto -i -r --include="*.log" "^error" $(BUILD_DIR) || true' EXIT; \
-	$(MAKE) -j$(JOBS) $(FOLDERS)
-	@echo "DONE ==> Parallel builds of folders: $(FOLDERS)"
+	$(MAKE) -j$(JOBS) $(CORE_IMAGE_FOLDERS)
+	@echo "DONE ==> Parallel builds of core folders: $(CORE_IMAGE_FOLDERS)"
 
 # ===================== Cleaning and Rebuilding =======================
+.PHONY: rebuild-core-images
+rebuild-core-images: clean-core-images build-core-images
 
-.PHONY: rebuild
-rebuild: clean build-images
+.PHONY: rebuild-core
+rebuild-core: clean-core build-core
+
+.PHONY: rebuild-all-images
+rebuild-all-images: clean-images build-all-images
 
 .PHONY: rebuild-all
 rebuild-all: clean-all build-all
 
-.PHONY: clean
-clean:
+define clean-image-folders
 	@echo "==> Cleaning up all build artifacts..."
-	@for dir in $(FOLDERS); do \
+	@for dir in $(1); do \
 		$(MAKE) -C $$dir clean 2>/dev/null; \
 	done
 	@echo "Cleaning common folder..."
 	@$(MAKE) -C $(COMMON_FOLDER) clean 2>/dev/null
 	@-rm -rf $(BUILD_DIR)
 	@echo "DONE ==> Cleaning up all build artifacts"
+endef
+
+.PHONY: clean-core-images
+clean-core-images:
+	$(call clean-image-folders,$(CORE_IMAGE_FOLDERS))
+
+.PHONY: clean-images
+clean-images:
+	$(call clean-image-folders,$(IMAGE_FOLDERS))
+
+.PHONY: clean-core
+clean-core: clean-core-images clean-secrets clean-volumes clean-models clean-tests
+	$(call clean-artifacts)
 
 .PHONY: clean-all
-clean-all: clean clean-secrets clean-volumes clean-models clean-tests
-	@echo "==> Cleaning all..."
+clean-all: clean-images clean-secrets clean-volumes clean-models clean-tests
+	$(call clean-artifacts)
+
+define clean-artifacts
+	@echo "==> Cleaning build artifacts..."
 	@-rm -f $(DLSTREAMER_SAMPLE_VIDEOS)
 	@-rm -f docker-compose.yml .env
-	@echo "DONE ==> Cleaning all"
+	@echo "DONE ==> Cleaning build artifacts"
+endef
 
 .PHONY: clean-models
 clean-models:
@@ -286,7 +336,7 @@ install-models:
 # =========================== Run Tests ==============================
 
 .PHONY: setup_tests
-setup_tests: build-images init-secrets .env
+setup_tests: build-all-images init-secrets .env
 	@echo "Setting up test environment..."
 	for dir in $(TEST_IMAGE_FOLDERS); do \
 		$(MAKE) -C $$dir test-build; \
@@ -486,14 +536,39 @@ init-sample-data: convert-dls-videos
 	fi
 	@echo "Sample data volume initialized."
 
+# Helper target to start demo with compose
+define start_demo
+	@$(MAKE) docker-compose.yml
+	@$(MAKE) .env
+	@if [ -z "$$SUPASS" ]; then \
+		echo "Please set the SUPASS environment variable before starting the demo for the first time."; \
+		echo "The SUPASS environment variable is the super user password for logging into Intel® SceneScape."; \
+		exit 1; \
+	fi
+	docker compose $(1) up -d
+	@echo ""
+	@echo "To stop SceneScape, type:"
+	@echo "    docker compose $(1) down"
+endef
+
 .PHONY: demo
-demo: docker-compose.yml .env init-sample-data
+demo: build-core init-sample-data
+	$(call start_demo,)
+
+.PHONY: demo-all
+demo-all: build-all init-sample-data
+	$(call start_demo,--profile experimental)
+
+.PHONY: demo-all
+demo-all: build-all init-sample-data
+	@$(MAKE) docker-compose.yml
+	@$(MAKE) .env
 	@if [ -z "$$SUPASS" ]; then \
 		echo "Please set the SUPASS environment variable before starting the demo for the first time."; \
 		echo "The SUPASS environment variable is the super user password for logging into Intel® SceneScape."; \
 		exit 1; \
 	fi
-	docker compose up -d
+	docker compose --profile experimental up -d
 	@echo ""
 	@echo "To stop SceneScape, type:"
 	@echo "    docker compose down"

cluster_analytics/Dockerfile

@@ -40,8 +40,6 @@ RUN : \
     && rm -rf scene_common \
     && rm -f /tmp/requirements-build.txt
 
-COPY ./tools/waitforbroker /tmp/tools/waitforbroker
-
 # -------------- Cluster Analytics Runtime Stage --------------
 FROM scenescape-common-base-24-04 AS scenescape-cluster-analytics-runtime
 
@@ -67,7 +65,6 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN : \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
-        gosu \
         libgl1 \
         libopencv-contrib406t64 \
         libpython3.12 \
@@ -99,10 +96,8 @@ RUN : \
 
 COPY --chown=$WSUSER:$WSUSER --from=scenescape-common-base-24-04 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/fast_geometry /usr/local/lib/python${PYTHON_VERSION}/dist-packages/fast_geometry
 COPY --chown=$WSUSER:$WSUSER --from=scenescape-common-base-24-04 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/scene_common /usr/local/lib/python${PYTHON_VERSION}/dist-packages/scene_common
-COPY --chown=$WSUSER:$WSUSER --from=scenescape-common-base-24-04 /tmp/tools/waitforbroker $SCENESCAPE_HOME/tools/waitforbroker
 
 USER $USER_ID:$GROUP_ID
-HEALTHCHECK CMD true
 
 # Copy source code
 COPY ./cluster_analytics/src /app

cluster_analytics/config/config.json

@@ -6,8 +6,8 @@
     },
     "category_specific": {
       "person": {
-        "eps": 1,
-        "min_samples": 3
+        "eps": 2,
+        "min_samples": 2
       },
       "vehicle": {
         "eps": 4.0,
@@ -30,39 +30,5 @@
         "min_samples": 2
       }
     }
-  },
-  "cluster_tracking": {
-    "state_transitions": {
-      "frames_to_activate": 3,
-      "frames_to_stable": 20,
-      "frames_to_fade": 15,
-      "frames_to_lost": 10
-    },
-    "confidence": {
-      "initial_confidence": 0.5,
-      "activation_threshold": 0.6,
-      "stability_threshold": 0.7,
-      "miss_penalty": 0.1,
-      "max_miss_penalty": 0.5,
-      "longevity_bonus_max": 0.2,
-      "longevity_frames": 100
-    },
-    "archival": {
-      "archive_time_threshold": 5.0
-    }
-  },
-  "shape_detection": {
-    "variance_threshold": 0.5,
-    "quadrant_angle": 1.5707963267948966,
-    "angle_distribution_threshold": 0.5,
-    "linear_formation_area_threshold": 0.5
-  },
-  "movement_analysis": {
-    "alignment_threshold": 0.5,
-    "convergence_divergence_ratio_threshold": 0.6
-  },
-  "velocity_analysis": {
-    "stationary_threshold": 0.1,
-    "velocity_coherence_threshold": 0.3
   }
 }

cluster_analytics/docs/user-guide/overview.md

@@ -15,11 +15,11 @@ This service processes real-time object detection data from SceneScape scenes, a
 
 #### Using Docker Compose (Recommended)
 
-The cluster analytics service is included in the main SceneScape demo docker-compose stack:
+The cluster analytics service is included in the extended SceneScape demo docker-compose stack:
 
 ```bash
 SUPASS=admin123 make
-SUPASS=admin123 make demo
+SUPASS=admin123 make demo-all
 ```
 
 ## Architecture