Default enable telemetry with default settings

docker/README.md

@@ -75,6 +75,7 @@ regarding FiftyOne Enterprise.
 - [Advanced Configuration](#advanced-configuration)
   - [Backup And Recovery](#backup-and-recovery)
   - [Secrets And Sensitive Data](#secrets-and-sensitive-data)
+  - [Telemetry](#telemetry)
   - [Snapshot Archival](#snapshot-archival)
   - [Static Banner Configuration](#static-banner-configuration)
   - [Storage Credentials and `FIFTYONE_ENCRYPTION_KEY`](#storage-credentials-and-fiftyone_encryption_key)
@@ -656,6 +657,17 @@ and
 [adding secrets](https://docs.voxel51.com/enterprise/secrets.html#adding-secrets)
 for questions regarding storage and encryption.
 
+### Telemetry
+
+FiftyOne Enterprise bundles a telemetry sidecar + Redis backend by
+default in every compose file. The Settings → Metrics page in teams-app
+exposes live per-service metrics (CPU, memory, FDs, thread counts) and
+tailed logs. No opt-in flags are required.
+
+Please refer to the
+[telemetry configuration documentation](./docs/configuring-telemetry.md)
+for full details.
+
 ### Snapshot Archival
 
 Since version v1.5, FiftyOne Enterprise supports

docker/common-services.yaml

@@ -17,6 +17,7 @@ services:
       FIFTYONE_MEDIA_CACHE_APP_IMAGES: false
       FIFTYONE_MEDIA_CACHE_SIZE_BYTES: -1
       FIFTYONE_SIGNED_URL_EXPIRATION: ${FIFTYONE_SIGNED_URL_EXPIRATION:-24}
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
       # If you are routing through a proxy server you will want to set
       #  HTTP_PROXY_URL, HTTPS_PROXY_URL, and NO_PROXY_LIST in your .env
       #  then add the following environment variables to your
@@ -43,6 +44,7 @@ services:
       FIFTYONE_ENV: ${FIFTYONE_ENV}
       FIFTYONE_INTERNAL_SERVICE: true
       FIFTYONE_LOGGING_FORMAT: ${FIFTYONE_LOGGING_FORMAT:-text}
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
       GRAPHQL_DEFAULT_LIMIT: ${GRAPHQL_DEFAULT_LIMIT}
       LOGGING_LEVEL: ${API_LOGGING_LEVEL:-INFO}
       MONGO_DEFAULT_DB: ${FIFTYONE_DATABASE_NAME}
@@ -72,6 +74,7 @@ services:
       FIFTYONE_SERVER_ADDRESS: ""
       FIFTYONE_SERVER_PATH_PREFIX: /api/proxy/fiftyone-teams
       FIFTYONE_TEAMS_PROXY_URL: ${FIFTYONE_TEAMS_PROXY_URL}
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
       NODE_ENV: production
       RECOIL_DUPLICATE_ATOM_KEY_CHECKING_ENABLED: false
       FIFTYONE_APP_ANONYMOUS_ANALYTICS_ENABLED: ${FIFTYONE_APP_ANONYMOUS_ANALYTICS_ENABLED:-true}
@@ -144,6 +147,7 @@ services:
       FIFTYONE_MEDIA_CACHE_APP_IMAGES: false
       FIFTYONE_MEDIA_CACHE_SIZE_BYTES: -1
       FIFTYONE_PLUGINS_DIR: /opt/plugins
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
       # If you are routing through a proxy server you will want to set
       #  HTTP_PROXY_URL, HTTPS_PROXY_URL, and NO_PROXY_LIST in your .env
       #  then add the following environment variables to your
@@ -158,8 +162,12 @@ services:
 
   teams-do-common:
     image: voxel51/fiftyone-teams-cv-full:v2.18.1
+    # Telemetry default-on requires teams-do to share its PID namespace
+    # with a single sidecar (compose's `pid: "service:<name>"` only joins
+    # one replica). Force replicas=1 to keep the 1:1 pairing deterministic.
+    # See docker/docs/configuring-telemetry.md for multi-worker patterns.
     deploy:
-      replicas: ${FIFTYONE_DELEGATED_OPERATOR_WORKER_REPLICAS:-3}
+      replicas: 1
     command: >
       /bin/sh -c "fiftyone delegated launch -t remote -m"
     environment:
@@ -171,6 +179,9 @@ services:
       FIFTYONE_INTERNAL_SERVICE: true
       FIFTYONE_MEDIA_CACHE_SIZE_BYTES: -1
       FIFTYONE_PLUGINS_DIR: /opt/plugins
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
+      TELEMETRY_SOCKET: /tmp/telemetry/agent.sock
     restart: always
     volumes:
       - plugins-vol:/opt/plugins:ro
+      - telemetry-socket:/tmp/telemetry

docker/docs/configuring-telemetry.md

@@ -0,0 +1,184 @@
+<!-- markdownlint-disable no-inline-html line-length -->
+<!-- markdownlint-disable-next-line first-line-heading -->
+<div align="center">
+<p align="center">
+
+<img alt="Voxel51 Logo" src="https://user-images.githubusercontent.com/25985824/106288517-2422e000-6216-11eb-871d-26ad2e7b1e59.png" height="55px"> &nbsp;
+<img alt="Voxel51 FiftyOne" src="https://user-images.githubusercontent.com/25985824/106288518-24bb7680-6216-11eb-8f10-60052c519586.png" height="50px">
+
+</p>
+</div>
+<!-- markdownlint-enable no-inline-html line-length -->
+
+---
+
+# Configuring FiftyOne Enterprise Telemetry
+
+Telemetry adds a lightweight per-service metrics collector (sidecar
+pattern) plus a Redis backend. The Settings → Metrics page in teams-app
+displays live CPU / memory / thread / file-descriptor samples and tailed
+stdout logs for each observed service.
+
+**Telemetry is enabled by default.** The base compose files bundle the
+`telemetry-redis` service and per-workload sidecars; no opt-in flags or
+overlay files are needed.
+
+## Default deployment
+
+```shell
+docker compose -f compose.yaml up -d
+```
+
+renders `fiftyone-app`, `teams-api`, `teams-app`, `teams-cas`,
+`telemetry-redis`, `fiftyone-app-telemetry`, and `teams-api-telemetry`.
+
+Combine with optional overlays as before — each carries its own bundled
+sidecar:
+
+```shell
+docker compose \
+  -f compose.yaml \
+  -f compose.dedicated-plugins.yaml \
+  -f compose.delegated-operators.yaml \
+  up -d
+```
+
+This adds `teams-plugins`, `teams-plugins-telemetry`, `teams-do`, and
+`teams-do-telemetry` in addition to the default set.
+
+For GPU-enabled delegated operators, layer
+`compose.delegated-operators.gpu.yaml` (which includes its own bundled
+telemetry sidecar) and activate the `gpu` profile:
+
+```shell
+docker compose --profile gpu \
+  -f compose.yaml \
+  -f compose.delegated-operators.yaml \
+  -f compose.delegated-operators.gpu.yaml \
+  up -d
+```
+
+## What's bundled by default
+
+- `telemetry-redis` — Redis 7 container that holds metric streams and log
+  entries. Data is capped by a maxmemory policy (`allkeys-lru`) so disk usage
+  stays bounded.
+- `fiftyone-app-telemetry`, `teams-api-telemetry` — sidecar containers, one
+  per observed service. Each joins the target's PID namespace via
+  `pid: "service:<target>"` so it can read `/proc/<pid>/fd/1` and use
+  psutil to sample CPU, memory, FDs, and thread counts. Sidecars run
+  with the `SYS_PTRACE` capability so py-spy can attach to the target.
+- `teams-plugins-telemetry` (only with `compose.dedicated-plugins.yaml`) —
+  sidecar for the dedicated `teams-plugins` service.
+- `teams-do-telemetry` (only with `compose.delegated-operators.yaml`) —
+  sidecar in `EXECUTOR_SIDECAR=true` mode that watches the executor for
+  per-operation child processes and records per-op metrics back to the
+  `delegated_ops` MongoDB document.
+- `teams-do-gpu` + `teams-do-gpu-telemetry` (only with
+  `compose.delegated-operators.gpu.yaml` and `--profile gpu`) — a GPU-
+  enabled delegated-operator worker registered as a distinct
+  orchestrator (`-n teams-do-gpu`) plus its paired sidecar. Sidecar
+  reads GPU metrics via NVML and requires its own GPU reservation.
+- `FIFTYONE_TELEMETRY_REDIS_URL` injected on `fiftyone-app`, `teams-api`,
+  `teams-app`, `teams-plugins`, and (when the DO overlay is used)
+  `teams-do` so the in-app telemetry blueprint and SSE endpoints can
+  read from Redis.
+
+## Opt out
+
+To run without telemetry, add a `compose.override.yaml` that scales the
+telemetry services to zero replicas:
+
+```yaml
+services:
+  telemetry-redis:
+    deploy:
+      replicas: 0
+  fiftyone-app-telemetry:
+    deploy:
+      replicas: 0
+  teams-api-telemetry:
+    deploy:
+      replicas: 0
+  # Only needed when running the corresponding overlay:
+  teams-plugins-telemetry:
+    deploy:
+      replicas: 0
+  teams-do-telemetry:
+    deploy:
+      replicas: 0
+```
+
+`docker compose -f compose.yaml -f compose.override.yaml up -d` will start
+the base services without the telemetry collector. The main containers
+will still have `FIFTYONE_TELEMETRY_REDIS_URL` set, but the in-app agent
+gracefully no-ops when Redis is unreachable.
+
+### Scaling teams-do with telemetry
+
+docker-compose's `pid: "service:<name>"` only joins a single replica's
+PID namespace. To keep the sidecar observation honest, `teams-do-common`
+**forces `teams-do` replicas to 1**, overriding any
+`FIFTYONE_DELEGATED_OPERATOR_WORKER_REPLICAS` setting.
+
+If you need more than one delegated-operator worker observed at the same
+time, either:
+
+1. Define additional explicit services in a compose override — e.g.
+   `teams-do-1`, `teams-do-2` — each with a paired `teams-do-N-telemetry`
+   sidecar using `pid: "service:teams-do-N"`.
+2. Deploy via the helm chart, which automatically adds a telemetry sidecar
+   to every pod in the delegated-operator deployment.
+
+### Sidecar lifecycle on workload restart
+
+Each sidecar joins its workload's PID namespace at container-create
+time. If the workload is recreated (force-recreate, image upgrade,
+configuration change) the namespace reference goes stale and Docker
+cannot re-attach the sidecar — it stays in `Exited (137)` until
+manually recreated.
+
+The overlays mitigate this with `depends_on.<target>.restart: true`
+(compose v2.17+), which tells compose to recreate the sidecar in
+lockstep with the workload. `docker compose version` must report
+v2.17 or newer for this to take effect. If the workload itself crash-
+loops, the sidecar follows it into the crash loop, matching the
+Kubernetes pod-restart behavior.
+
+## Environment overrides
+
+All knobs live in your `.env` — see `env.template` for the full list:
+
+| Variable                       | Default                             | Purpose                                              |
+| ------------------------------ | ----------------------------------- | ---------------------------------------------------- |
+| `FIFTYONE_TELEMETRY_REDIS_URL` | `redis://telemetry-redis:6379`      | Override to point at an external Redis if desired    |
+| `TELEMETRY_SIDECAR_IMAGE`      | `voxel51/telemetry-sidecar:latest`  | Pin sidecar to a specific tag for production         |
+| `TELEMETRY_REDIS_IMAGE`        | `redis:7-alpine`                    | Alternate redis image                                |
+| `TELEMETRY_REDIS_MAXMEMORY`    | `400mb`                             | Redis maxmemory budget                               |
+| `TELEMETRY_NAMESPACE`          | `docker`                            | Namespace label attached to each registered pod      |
+| `FIFTYONE_APP_TARGET_NAME`     | `hypercorn`                         | Substring used to locate the fiftyone-app process    |
+| `TEAMS_API_TARGET_NAME`        | `fiftyone-teams-api`                | Substring used to locate the teams-api process       |
+| `TEAMS_PLUGINS_TARGET_NAME`    | `hypercorn`                         | Substring used to locate the teams-plugins process   |
+| `TEAMS_DO_TARGET_NAME`         | `fiftyone delegated`                | Substring used to locate the teams-do process        |
+| `NVIDIA_GPU_COUNT`             | `1`                                 | GPU reservation for the GPU DO worker + sidecar      |
+| `NVIDIA_VISIBLE_DEVICES`       | `all`                               | Pass-through to teams-do-gpu / sidecar               |
+| `NVIDIA_DRIVER_CAPABILITIES`   | `compute,utility`                   | Must include `utility` so NVML is available          |
+
+## Access control
+
+The telemetry endpoints (`/telemetry/*` on teams-api; `/api/telemetry/stream`
+and `/api/telemetry/logs` on teams-app) require an authenticated user with
+the `ADMIN` role. Non-admin users and unauthenticated requests receive 401/403.
+
+## Verify
+
+```shell
+docker compose exec telemetry-redis redis-cli HGETALL active_pods
+docker compose exec telemetry-redis redis-cli XLEN metrics:docker:fiftyone-app
+```
+
+`XLEN` should increase over time. If it does not, check the sidecar logs:
+
+```shell
+docker compose logs fiftyone-app-telemetry teams-api-telemetry --tail 20
+```

docker/internal-auth/compose.dedicated-plugins.yaml

@@ -39,5 +39,80 @@ services:
     volumes:
       - plugins-vol:/opt/plugins:ro
 
+  telemetry-redis:
+    image: ${TELEMETRY_REDIS_IMAGE:-redis:7-alpine}
+    command:
+      - redis-server
+      - --save
+      - "60"
+      - "1"
+      - --dir
+      - /data
+      - --maxmemory
+      - ${TELEMETRY_REDIS_MAXMEMORY:-400mb}
+      - --maxmemory-policy
+      - allkeys-lru
+    restart: always
+    volumes:
+      - telemetry-redis-data:/data
+
+  fiftyone-app-telemetry:
+    image: ${TELEMETRY_SIDECAR_IMAGE:-voxel51/telemetry-sidecar:latest}
+    pid: "service:fiftyone-app"
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      POD_NAME: fiftyone-app
+      POD_NAMESPACE: ${TELEMETRY_NAMESPACE:-docker}
+      SERVICE_TYPE: fiftyone-app
+      TARGET_NAME: ${FIFTYONE_APP_TARGET_NAME:-hypercorn}
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
+    depends_on:
+      fiftyone-app:
+        condition: service_started
+        restart: true
+      telemetry-redis:
+        condition: service_started
+    restart: always
+
+  teams-api-telemetry:
+    image: ${TELEMETRY_SIDECAR_IMAGE:-voxel51/telemetry-sidecar:latest}
+    pid: "service:teams-api"
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      POD_NAME: teams-api
+      POD_NAMESPACE: ${TELEMETRY_NAMESPACE:-docker}
+      SERVICE_TYPE: teams-api
+      TARGET_NAME: ${TEAMS_API_TARGET_NAME:-fiftyone-teams-api}
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
+    depends_on:
+      teams-api:
+        condition: service_started
+        restart: true
+      telemetry-redis:
+        condition: service_started
+    restart: always
+
+  teams-plugins-telemetry:
+    image: ${TELEMETRY_SIDECAR_IMAGE:-voxel51/telemetry-sidecar:latest}
+    pid: "service:teams-plugins"
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      POD_NAME: teams-plugins
+      POD_NAMESPACE: ${TELEMETRY_NAMESPACE:-docker}
+      SERVICE_TYPE: teams-plugins
+      TARGET_NAME: ${TEAMS_PLUGINS_TARGET_NAME:-hypercorn}
+      FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
+    depends_on:
+      teams-plugins:
+        condition: service_started
+        restart: true
+      telemetry-redis:
+        condition: service_started
+    restart: always
+
 volumes:
   plugins-vol:
+  telemetry-redis-data: