[Feature] Support adding an observer role in FE to improve query load balancing and scale read workloads

config/crd/bases/starrocks.com_starrocksclusters.yaml

@@ -13601,6 +13601,22 @@ spec:
                     description: (Optional) If specified, the pod's nodeSelector，displayName="Map
                       of nodeSelectors to match when scheduling pods on nodes"
                     type: object
+                  observerSpec:
+                    description: ObserverSpec configures optional FE observer pods.
+                      Observer pods reuse the generic FE resources.
+                    properties:
+                      enabled:
+                        description: Enabled controls whether FE observer should be
+                          reconciled.
+                        type: boolean
+                      observerNumber:
+                        default: 1
+                        description: ObserverNumber is the number of desired FE observer
+                          Pods.
+                        format: int32
+                        minimum: 0
+                        type: integer
+                    type: object
                   persistentVolumeClaimRetentionPolicy:
                     description: |-
                       PersistentVolumeClaimRetentionPolicy specifies the retention policy for PersistentVolumeClaims associated with the component.
@@ -15597,6 +15613,44 @@ spec:
                 required:
                 - phase
                 type: object
+              starRocksFeObserverStatus:
+                description: Represents the status of fe observer. the status have
+                  running, failed and creating pods.
+                properties:
+                  creatingInstances:
+                    description: CreatingInstances in creating pod names.
+                    items:
+                      type: string
+                    type: array
+                  failedInstances:
+                    description: FailedInstances failed pod names.
+                    items:
+                      type: string
+                    type: array
+                  phase:
+                    description: |-
+                      Phase the value from all pods of component status. If component have one failed pod phase=failed,
+                      also if fe have one creating pod phase=creating, also if component all running phase=running, others unknown.
+                    type: string
+                  reason:
+                    description: Reason represents the reason of not running.
+                    type: string
+                  resourceNames:
+                    description: ResourceNames the statefulset names of fe.
+                    items:
+                      type: string
+                    type: array
+                  runningInstances:
+                    description: RunningInstances in running status pod names.
+                    items:
+                      type: string
+                    type: array
+                  serviceName:
+                    description: the name of fe service exposed for user.
+                    type: string
+                required:
+                - phase
+                type: object
               starRocksFeProxyStatus:
                 description: Represents the status of fe proxy. the status have running,
                   failed and creating pods.

deploy/starrocks.com_starrocksclusters.yaml

@@ -6413,6 +6413,16 @@ spec:
                     additionalProperties:
                       type: string
                     type: object
+                  observerSpec:
+                    properties:
+                      enabled:
+                        type: boolean
+                      observerNumber:
+                        default: 1
+                        format: int32
+                        minimum: 0
+                        type: integer
+                    type: object
                   persistentVolumeClaimRetentionPolicy:
                     properties:
                       whenDeleted:
@@ -7326,6 +7336,33 @@ spec:
                 required:
                 - phase
                 type: object
+              starRocksFeObserverStatus:
+                properties:
+                  creatingInstances:
+                    items:
+                      type: string
+                    type: array
+                  failedInstances:
+                    items:
+                      type: string
+                    type: array
+                  phase:
+                    type: string
+                  reason:
+                    type: string
+                  resourceNames:
+                    items:
+                      type: string
+                    type: array
+                  runningInstances:
+                    items:
+                      type: string
+                    type: array
+                  serviceName:
+                    type: string
+                required:
+                - phase
+                type: object
               starRocksFeProxyStatus:
                 properties:
                   creatingInstances:

doc/scale_fe_observer_howto.md

@@ -0,0 +1,56 @@
+# Scale FE observer Howto
+
+## Overview
+
+FE observers are mainly used to increase the query concurrency of a StarRocks cluster. Unlike FE followers, observers do
+not participate in leader election, so adding observers does not add leader selection pressure to the cluster.
+
+Use `starRocksFeSpec.observerSpec` when the cluster needs more FE read capacity without changing the FE follower quorum.
+Observers can be scaled with the read workload, while FE followers should still be scaled carefully because they store
+metadata and participate in quorum.
+
+## How to correctly scale out FE read workloads by observer
+
+FE observers can be used to scale FE read workloads without changing the FE follower quorum. To use
+`starRocksFeSpec.observerSpec`, the FE image configured in `starRocksFeSpec.image` must be StarRocks 4.1.0 or later.
+The operator rejects FE observer deployment for older FE image versions because those images do not enable FE observer.
+
+To scale FE observers out, enable `observerSpec` and set the desired observer count:
+
+```yaml
+starRocksFeSpec:
+   image: "starrocks/fe-ubuntu:4.1.0"
+   observerSpec:
+      enabled: true
+      observerNumber: 2
+```
+
+## How to correctly scale in FE observer
+
+Known limitation: neither the operator nor `fe_entrypoint.sh` runs `ALTER SYSTEM DROP OBSERVER` when observers are
+removed. If you decrease `starRocksFeSpec.observerSpec.observerNumber` or set
+`starRocksFeSpec.observerSpec.enabled: false`, the observer Pods are removed from Kubernetes, but the removed observers
+can remain in StarRocks metadata and appear as permanent `Alive=false` rows in `SHOW FRONTENDS`. Because observers are
+intended for dynamic read-workload scaling, stale observer rows can accumulate after repeated scale-in operations.
+
+For planned observer scale-in, manually drop the observers that will be removed before lowering `observerNumber`.
+
+1. Execute `SHOW FRONTENDS` and find the `OBSERVER` rows for the observer Pods that will be removed.
+
+2. Drop each observer from StarRocks metadata by using its FE host and `EditLogPort` from `SHOW FRONTENDS`.
+   ```sql
+   mysql> ALTER SYSTEM DROP OBSERVER "<observer-host>:<edit-log-port>";
+   ```
+
+   For example:
+   ```sql
+   mysql> ALTER SYSTEM DROP OBSERVER "kube-starrocks-fe-observer-1.kube-starrocks-fe-observer-search.default.svc.cluster.local:9010";
+   ```
+
+3. Lower `starRocksFeSpec.observerSpec.observerNumber`, or set `starRocksFeSpec.observerSpec.enabled: false` if all
+   observers should be removed.
+
+4. Execute `SHOW FRONTENDS` again and verify the removed observer rows are gone.
+
+If the observer Pods were already removed from Kubernetes, connect to an FE node, run `SHOW FRONTENDS`, find `OBSERVER`
+rows with `Alive=false`, and run `ALTER SYSTEM DROP OBSERVER "<observer-host>:<edit-log-port>"` for each stale row.

helm-charts/charts/kube-starrocks/charts/starrocks/templates/starrockscluster.yaml

@@ -284,6 +284,11 @@ spec:
     persistentVolumeClaimRetentionPolicy:
       {{- toYaml .Values.starrocksFESpec.persistentVolumeClaimRetentionPolicy | nindent 6 }}
     {{- end }}
+    {{- if .Values.starrocksFESpec.observerSpec.enabled }}
+    observerSpec:
+      enabled: true
+      observerNumber: {{ .Values.starrocksFESpec.observerSpec.observerNumber }}
+    {{- end }}
   {{- if .Values.starrocksCluster.enabledBe }}
   starRocksBeSpec:
     {{- if .Values.starrocksBeSpec.initContainers }}

helm-charts/charts/kube-starrocks/charts/starrocks/values.yaml

@@ -513,6 +513,17 @@ starrocksFESpec:
   # When this is set, containers will be able to view and signal processes from other containers
   # in the same pod, and the first process in each container will not be assigned PID 1.
   shareProcessNamespace:
+  # Optional FE observer pods. Observer pods reuse the FE image, resources, scheduling,
+  # config, env, storage, and services from this starrocksFESpec.
+  # FE observer requires the FE image tag to be StarRocks 4.1.0 or later.
+  # Known limitation: scaling observerNumber down or disabling observerSpec removes
+  # Kubernetes pods, but does not run ALTER SYSTEM DROP OBSERVER. Manually drop
+  # observer nodes if SHOW FRONTENDS reports Alive = false.
+  observerSpec:
+    # specify the FE observer deployment or not.
+    enabled: false
+    # number of FE observer pods to deploy.
+    observerNumber: 1
 
 # spec for compute node, compute node provide compute function.
 starrocksCnSpec:

helm-charts/charts/kube-starrocks/values.yaml

@@ -629,6 +629,17 @@ starrocks:
     # When this is set, containers will be able to view and signal processes from other containers
     # in the same pod, and the first process in each container will not be assigned PID 1.
     shareProcessNamespace:
+    # Optional FE observer pods. Observer pods reuse the FE image, resources, scheduling,
+    # config, env, storage, and services from this starrocksFESpec.
+    # FE observer requires the FE image tag to be StarRocks 4.1.0 or later.
+    # Known limitation: scaling observerNumber down or disabling observerSpec removes
+    # Kubernetes pods, but does not run ALTER SYSTEM DROP OBSERVER. Manually drop
+    # observer nodes if SHOW FRONTENDS reports Alive = false.
+    observerSpec:
+      # specify the FE observer deployment or not.
+      enabled: false
+      # number of FE observer pods to deploy.
+      observerNumber: 1
 
   # spec for compute node, compute node provide compute function.
   starrocksCnSpec:

pkg/apis/starrocks/v1/labels.go

@@ -30,14 +30,17 @@ const (
 
 // the labels value. default statefulset name
 const (
-	DEFAULT_FE       = "fe"
-	DEFAULT_BE       = "be"
-	DEFAULT_CN       = "cn"
-	DEFAULT_FE_PROXY = "fe-proxy"
+	DEFAULT_FE          = "fe"
+	DEFAULT_BE          = "be"
+	DEFAULT_CN          = "cn"
+	DEFAULT_FE_PROXY    = "fe-proxy"
+	DEFAULT_FE_OBSERVER = "fe-observer"
 )
 
 // the env of container
 const (
-	COMPONENT_NAME  = "COMPONENT_NAME"
-	FE_SERVICE_NAME = "FE_SERVICE_NAME"
+	COMPONENT_NAME   = "COMPONENT_NAME"
+	FE_SERVICE_NAME  = "FE_SERVICE_NAME"
+	FE_OBSERVER_NAME = "FE_OBSERVER_NAME"
+	IS_FE_OBSERVER   = "IS_FE_OBSERVER"
 )

pkg/apis/starrocks/v1/load_type.go

@@ -85,7 +85,6 @@ type StarRocksLoadSpec struct {
 	// +kubebuilder:default=1
 	// +optional
 	Replicas *int32 `json:"replicas,omitempty"`
-
 	// Image for a starrocks deployment.
 	// +optional
 	Image string `json:"image"`
@@ -304,7 +303,6 @@ type StarRocksProbe struct {
 func (spec *StarRocksLoadSpec) GetReplicas() *int32 {
 	return spec.Replicas
 }
-
 func (spec *StarRocksLoadSpec) GetStorageVolumes() []StorageVolume {
 	return spec.StorageVolumes
 }