fix: add driver-ops and docs warning

Author: kaixi-wang

docker/common-services.yaml

@@ -180,6 +180,9 @@ services:
       FIFTYONE_MEDIA_CACHE_SIZE_BYTES: -1
       FIFTYONE_PLUGINS_DIR: /opt/plugins
       FIFTYONE_TELEMETRY_REDIS_URL: ${FIFTYONE_TELEMETRY_REDIS_URL:-redis://telemetry-redis:6379}
+      # Flush stdout per line so `docker logs` shows registration/heartbeat
+      # output in near-real-time instead of block-buffered chunks.
+      PYTHONUNBUFFERED: "1"
       TELEMETRY_SOCKET: /tmp/telemetry/agent.sock
     restart: always
     volumes:

docker/docs/upgrading.md

@@ -21,6 +21,7 @@
   - [From FiftyOne Enterprise Version 2.0.0 and Later](#from-fiftyone-enterprise-version-200-and-later)
     - [FiftyOne Enterprise v2.19+ Telemetry Sidecars](#fiftyone-enterprise-v219-telemetry-sidecars)
       - [Host Requirements](#host-requirements)
+      - [Volume ownership (upgrading from v2.18 or earlier)](#volume-ownership-upgrading-from-v218-or-earlier)
       - [Opting out of Telemetry](#opting-out-of-telemetry)
       - [Scaling delegated-operator workers](#scaling-delegated-operator-workers)
     - [FiftyOne Enterprise v2.16+ Additional API Routes](#fiftyone-enterprise-v216-additional-api-routes)
@@ -149,6 +150,59 @@ Use an existing Redis instance instead of the bundled one by setting
 (e.g. `redis://my-managed-redis.example.com:6379`) and scaling
 `telemetry-redis` to `replicas: 0` as below.
 
+##### Volume ownership (upgrading from v2.18 or earlier)
+
+The telemetry sidecars and workload containers now run as UID/GID 1000
+(previously root). Fresh deploys are unaffected — the
+`telemetry-socket{,-2,-3,-gpu}` named volumes are declared with
+`driver_opts: type: tmpfs, o: "uid=1000,gid=1000,mode=0775"`, so Docker
+creates them already owned by UID 1000.
+
+Existing named volumes from a v2.18-or-earlier deploy retain their
+original `root:root` ownership across `up`/`down` cycles. Once the
+sidecar image flips to UID 1000, the first restart crash-loops with
+`PermissionError: [Errno 13]` when the sidecar tries to bind
+`agent.sock`. Pick one of the two paths below to remediate.
+
+**Option 1 — Drop and recreate all project volumes (simplest):**
+
+```shell
+docker compose -p <project> -f compose.yaml -f compose.delegated-operators.yaml down --volumes
+docker compose -p <project> -f compose.yaml -f compose.delegated-operators.yaml up -d
+```
+
+`--volumes` only removes volumes declared in this project's compose
+files; other Docker projects on the host are untouched.
+`telemetry-redis-data` is also wiped — telemetry archives in MongoDB
+are unaffected, but the un-archived ~10 minute dashboard backscroll is
+lost (same trade-off as `telemetry.redis.persistence.enabled: false`
+in the helm chart).
+
+**Option 2 — Selective wipe of only the socket volumes (preserves
+redis backscroll and plugins):**
+
+```shell
+docker compose -p <project> -f compose.yaml -f compose.delegated-operators.yaml down
+docker volume rm <project>_telemetry-socket <project>_telemetry-socket-2 <project>_telemetry-socket-3
+docker compose -p <project> -f compose.yaml -f compose.delegated-operators.yaml up -d
+```
+
+Recreated socket volumes come up 1000-owned via `driver_opts`. Persistent
+volumes (`plugins-vol`, `telemetry-redis-data`) are preserved as-is —
+the workload Dockerfile pre-chowns `/opt/plugins` and the upstream redis
+image owns its `/data`, so existing data continues to be readable by the
+new UID 1000 / UID 999 containers.
+
+If you maintain custom overrides that declare additional named volumes
+that non-root workloads write to, chown each manually before `up -d`:
+
+```shell
+docker run --rm -v <project>_<volume>:/v alpine chown 1000:1000 /v
+```
+
+Use UID `999` (not `1000`) when chowning `telemetry-redis-data` — the
+upstream `redis:7-alpine` image runs as the built-in `redis` user.
+
 ##### Opting out of Telemetry
 
 Telemetry is enabled by default.

docker/internal-auth/compose.delegated-operators.gpu.yaml

@@ -91,6 +91,12 @@ services:
         condition: service_started
     restart: always
 
+# Tmpfs-backed named volume with uid/gid set at mount time so the
+# non-root sidecar can bind agent.sock.
 volumes:
   plugins-vol:
   telemetry-socket-gpu:
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: "uid=1000,gid=1000,mode=0755"

docker/internal-auth/compose.delegated-operators.yaml

@@ -29,10 +29,11 @@
 # invisible to the telemetry sidecar. The cap of 3 is a deliberate
 # trade-off — for higher worker counts prefer the helm chart. If
 # docker compose is required, the slot-2/3 blocks below are templates:
-# duplicate them as `teams-do-4` / `teams-do-4-telemetry` etc.,
-# bumping the service name, `pid`, `POD_NAME`, `-n teams-do-N`, and
-# `telemetry-socket-N` volume on each copy. See
-# docker/docs/configuring-telemetry.md for details.
+# duplicate them as `teams-do-4` / `teams-do-4-telemetry`, bumping the
+# service name, `pid`, `POD_NAME`, `-n teams-do-N`, and
+# `telemetry-socket-N` volume on each copy, then add
+# `telemetry-socket-N: *telemetry-socket-spec` to the volumes block.
+# See docker/docs/configuring-telemetry.md for details.
 #
 # For Proxy Server instructions please see
 #  https://github.com/voxel51/fiftyone-teams-app-deploy/tree/main/docker#environment-proxies
@@ -186,8 +187,15 @@ services:
         condition: service_started
     restart: always
 
+# Tmpfs-backed named volumes with uid/gid set at mount time so the
+# non-root sidecar can bind agent.sock without any init container.
+# Add new slots with `telemetry-socket-N: *telemetry-socket-spec`.
 volumes:
   plugins-vol:
-  telemetry-socket:
-  telemetry-socket-2:
-  telemetry-socket-3:
+  telemetry-socket: &telemetry-socket-spec
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: "uid=1000,gid=1000,mode=0755"
+  telemetry-socket-2: *telemetry-socket-spec
+  telemetry-socket-3: *telemetry-socket-spec

docker/legacy-auth/compose.delegated-operators.gpu.yaml

@@ -91,6 +91,12 @@ services:
         condition: service_started
     restart: always
 
+# Tmpfs-backed named volume with uid/gid set at mount time so the
+# non-root sidecar can bind agent.sock.
 volumes:
   plugins-vol:
   telemetry-socket-gpu:
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: "uid=1000,gid=1000,mode=0755"

docker/legacy-auth/compose.delegated-operators.yaml

@@ -29,10 +29,11 @@
 # invisible to the telemetry sidecar. The cap of 3 is a deliberate
 # trade-off — for higher worker counts prefer the helm chart. If
 # docker compose is required, the slot-2/3 blocks below are templates:
-# duplicate them as `teams-do-4` / `teams-do-4-telemetry` etc.,
-# bumping the service name, `pid`, `POD_NAME`, `-n teams-do-N`, and
-# `telemetry-socket-N` volume on each copy. See
-# docker/docs/configuring-telemetry.md for details.
+# duplicate them as `teams-do-4` / `teams-do-4-telemetry`, bumping the
+# service name, `pid`, `POD_NAME`, `-n teams-do-N`, and
+# `telemetry-socket-N` volume on each copy, then add
+# `telemetry-socket-N: *telemetry-socket-spec` to the volumes block.
+# See docker/docs/configuring-telemetry.md for details.
 #
 # For Proxy Server instructions please see
 #  https://github.com/voxel51/fiftyone-teams-app-deploy/tree/main/docker#environment-proxies
@@ -188,8 +189,15 @@ services:
         condition: service_started
     restart: always
 
+# Tmpfs-backed named volumes with uid/gid set at mount time so the
+# non-root sidecar can bind agent.sock without any init container.
+# Add new slots with `telemetry-socket-N: *telemetry-socket-spec`.
 volumes:
   plugins-vol:
-  telemetry-socket:
-  telemetry-socket-2:
-  telemetry-socket-3:
+  telemetry-socket: &telemetry-socket-spec
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: "uid=1000,gid=1000,mode=0755"
+  telemetry-socket-2: *telemetry-socket-spec
+  telemetry-socket-3: *telemetry-socket-spec

helm/docs/upgrading.md

@@ -21,6 +21,7 @@
   - [From FiftyOne Enterprise Version 2.0.0 or Higher](#from-fiftyone-enterprise-version-200-or-higher)
     - [FiftyOne Enterprise v2.19+ Telemetry Sidecars](#fiftyone-enterprise-v219-telemetry-sidecars)
       - [Cluster Requirements](#cluster-requirements)
+      - [Volume ownership (upgrading from v2.18 or earlier)](#volume-ownership-upgrading-from-v218-or-earlier)
       - [Opting out of Telemetry](#opting-out-of-telemetry)
     - [FiftyOne Enterprise v2.16+ Additional API Routes](#fiftyone-enterprise-v216-additional-api-routes)
     - [FiftyOne Enterprise v2.15+ Additional API Routes](#fiftyone-enterprise-v215-additional-api-routes)
@@ -169,17 +170,16 @@ opt into a `PersistentVolumeClaim` with
 
 1. **`shareProcessNamespace: true`** is set on all four workloads so
    the sidecar can read `/proc/<pid>/fd/1` of the target container.
-1. **The teams-do sidecar runs as root with `SYS_PTRACE`** (required for
-   `py-spy` and `/proc` access).
-   Clusters that enforce
-   [Pod Security Admission][psa]
-   `restricted`, or admission policies (OPA/Gatekeeper, Kyverno) that
-   block `runAsUser: 0` or capability adds, will reject these pods at
-   admission.
-   On such clusters, either:
-    - allow the chart's `namespace.name` namespace at PSA `baseline`
-      (or relax the relevant admission policy), or
-    - disable telemetry with `telemetry.enabled: false`.
+1. **Sidecars run as UID/GID 1000** (matching the workload image's
+   user) with `cap_drop: ALL`. The delegated-operator sidecar
+   additionally has `SYS_PTRACE` added (required for `py-spy` stack
+   sampling); the other three sidecars run with no capabilities. This
+   posture is compatible with
+   [Pod Security Admission][psa] `restricted` out of the box.
+   Clusters enforcing additional admission policies (OPA/Gatekeeper,
+   Kyverno) that explicitly block `SYS_PTRACE` on the
+   delegated-operator namespace can either allow that single capability
+   or disable telemetry with `telemetry.enabled: false`.
 
 **External Redis:**
 To point at a managed Redis (ElastiCache, MemoryStore, an existing
@@ -195,6 +195,62 @@ telemetry:
 The chart skips the bundled Redis' `Deployment` and `Service` and wires
 every consumer at this URL.
 
+##### Volume ownership (upgrading from v2.18 or earlier)
+
+In v2.19+ all workloads and their telemetry sidecars run as UID/GID
+1000 (UID/GID 999 for the bundled Redis, matching the upstream
+`redis:7-alpine` image). The chart sets `podSecurityContext.fsGroup`
+on each pod so Kubernetes group-chowns the mounted volumes at attach
+time. For most CSI drivers this is automatic on chart upgrade — no
+operator action required.
+
+**Two edge cases need manual remediation:**
+
+1. **CSI drivers that ignore `fsGroup`** (notably NFS, some Filestore
+   configurations, certain `ReadWriteMany` volumes): the recursive
+   chown is skipped, leaving pre-existing data owned by whatever UID
+   the prior root-running containers wrote it as. The new UID-1000
+   container can't write back. Symptom: workload pod crash-loops with
+   `PermissionError: [Errno 13]` against `/opt/plugins` or
+   `/data` shortly after a chart upgrade.
+
+   Remediate by chowning the volume contents once, from a debug pod
+   with elevated privileges:
+
+   ```shell
+   kubectl run -n <namespace> chown-fix --rm -i --restart=Never \
+     --image=alpine \
+     --overrides='{"spec":{"containers":[{"name":"chown-fix","image":"alpine","command":["chown","-R","1000:1000","/v"],"volumeMounts":[{"name":"v","mountPath":"/v"}],"securityContext":{"runAsUser":0}}],"volumes":[{"name":"v","persistentVolumeClaim":{"claimName":"<your-pvc>"}}]}}'
+   ```
+
+   Use UID `999` (not `1000`) for the Redis data PVC if you opted into
+   `telemetry.redis.persistence.enabled: true`.
+
+2. **Very large pre-populated PVs** (e.g. multi-GB `plugins-vol`): the
+   recursive chown on first attach can stall pod startup for several
+   minutes — readiness probes may fail long enough for the rollout to
+   look stuck before recovering. Mitigate by setting:
+
+   ```yaml
+   apiSettings:
+     podSecurityContext:
+       fsGroupChangePolicy: OnRootMismatch
+   fiftyoneAppSettings:
+     podSecurityContext:
+       fsGroupChangePolicy: OnRootMismatch
+   pluginsSettings:
+     podSecurityContext:
+       fsGroupChangePolicy: OnRootMismatch
+   ```
+
+   With `OnRootMismatch` the recursive walk is skipped when the
+   volume's root directory already matches the target GID — effectively
+   a one-time cost rather than per-rollout. Requires Kubernetes 1.23+.
+
+Fresh installs are unaffected: empty PVs come up correctly via
+`fsGroup` regardless of CSI driver, and there is no pre-existing
+root-owned data to chown.
+
 ##### Opting out of Telemetry
 
 Telemetry is enabled by default.

tests/unit/compose/common_test.go

@@ -3,6 +3,14 @@
 
 package unit
 
+import "github.com/compose-spec/compose-go/v2/types"
+
 const (
 	envFixtureFilePath = "../../fixtures/docker/.env"
 )
+
+var telemetrySocketDriverOpts = types.Options{
+	"device": "tmpfs",
+	"o":      "uid=1000,gid=1000,mode=0755",
+	"type":   "tmpfs",
+}

tests/unit/compose/docker-compose-internal-auth_test.go

@@ -614,6 +614,7 @@ func (s *commonServicesInternalAuthDockerComposeTest) TestServiceEnvironment() {
 				"FIFTYONE_MEDIA_CACHE_SIZE_BYTES=-1",
 				"FIFTYONE_PLUGINS_DIR=/opt/plugins",
 				"FIFTYONE_TELEMETRY_REDIS_URL=redis://telemetry-redis:6379",
+				"PYTHONUNBUFFERED=1",
 				"TELEMETRY_SOCKET=/tmp/telemetry/agent.sock",
 			},
 		},
@@ -1111,17 +1112,20 @@ func (s *commonServicesInternalAuthDockerComposeTest) TestVolumes() {
 					Name: "fiftyone-compose-test_telemetry-redis-data",
 				},
 				"telemetry-socket": {
-					Name: "fiftyone-compose-test_telemetry-socket",
+					Name:       "fiftyone-compose-test_telemetry-socket",
+					DriverOpts: telemetrySocketDriverOpts,
 				},
 				// telemetry-socket-{2,3} are declared at the project level
 				// so they appear in project.Volumes regardless of which
 				// `do-N` profile is active. Slot 2/3 services only start
 				// when the matching profile is selected.
 				"telemetry-socket-2": {
-					Name: "fiftyone-compose-test_telemetry-socket-2",
+					Name:       "fiftyone-compose-test_telemetry-socket-2",
+					DriverOpts: telemetrySocketDriverOpts,
 				},
 				"telemetry-socket-3": {
-					Name: "fiftyone-compose-test_telemetry-socket-3",
+					Name:       "fiftyone-compose-test_telemetry-socket-3",
+					DriverOpts: telemetrySocketDriverOpts,
 				},
 			},
 		},

tests/unit/compose/docker-compose-legacy-auth_test.go

@@ -696,6 +696,7 @@ func (s *commonServicesLegacyAuthDockerComposeTest) TestServiceEnvironment() {
 				"FIFTYONE_MEDIA_CACHE_SIZE_BYTES=-1",
 				"FIFTYONE_PLUGINS_DIR=/opt/plugins",
 				"FIFTYONE_TELEMETRY_REDIS_URL=redis://telemetry-redis:6379",
+				"PYTHONUNBUFFERED=1",
 				"TELEMETRY_SOCKET=/tmp/telemetry/agent.sock",
 			},
 		},
@@ -1198,17 +1199,20 @@ func (s *commonServicesLegacyAuthDockerComposeTest) TestVolumes() {
 					Name: "fiftyone-compose-test_telemetry-redis-data",
 				},
 				"telemetry-socket": {
-					Name: "fiftyone-compose-test_telemetry-socket",
+					Name:       "fiftyone-compose-test_telemetry-socket",
+					DriverOpts: telemetrySocketDriverOpts,
 				},
 				// telemetry-socket-{2,3} are declared at the project level
 				// so they appear in project.Volumes regardless of which
 				// `do-N` profile is active. Slot 2/3 services only start
 				// when the matching profile is selected.
 				"telemetry-socket-2": {
-					Name: "fiftyone-compose-test_telemetry-socket-2",
+					Name:       "fiftyone-compose-test_telemetry-socket-2",
+					DriverOpts: telemetrySocketDriverOpts,
 				},
 				"telemetry-socket-3": {
-					Name: "fiftyone-compose-test_telemetry-socket-3",
+					Name:       "fiftyone-compose-test_telemetry-socket-3",
+					DriverOpts: telemetrySocketDriverOpts,
 				},
 			},
 		},