feat: add metric server performance metrics

CHANGELOG.md

@@ -78,6 +78,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
 
 ### Improvements
 
+- **General**: Add metric server performance metrics for HPA external metric requests and operator GetMetrics gRPC handling ([#3120](https://github.com/kedacore/keda/issues/3120))
 - **Kafka Scaler**: Add optional `fullMetadata` trigger metadata field to control Sarama's full cluster metadata refresh, reducing operator memory for topic scoped triggers ([#7453](https://github.com/kedacore/keda/issues/7453))
 - TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX))
 

cmd/adapter/main.go

@@ -42,6 +42,7 @@ import (
 	"sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	"github.com/kedacore/keda/v2/pkg/metricscollector"
 	"github.com/kedacore/keda/v2/pkg/metricsservice"
 	kedaprovider "github.com/kedacore/keda/v2/pkg/provider"
 	kedautil "github.com/kedacore/keda/v2/pkg/util"
@@ -65,6 +66,7 @@ var (
 	metricsServiceAddr          string
 	profilingAddr               string
 	metricsServiceGRPCAuthority string
+	enableOpenTelemetryMetrics  bool
 	logToSTDerr                 bool
 	verbosityLevel              int
 	stdErrThreshold             string
@@ -243,6 +245,7 @@ func main() {
 	cmd.Flags().Float32Var(&adapterClientRequestQPS, "kube-api-qps", 20.0, "Set the QPS rate for throttling requests sent to the apiserver")
 	cmd.Flags().IntVar(&adapterClientRequestBurst, "kube-api-burst", 30, "Set the burst for throttling requests sent to the apiserver")
 	cmd.Flags().BoolVar(&disableCompression, "disable-compression", true, "Disable response compression for k8s restAPI in client-go. ")
+	cmd.Flags().BoolVar(&enableOpenTelemetryMetrics, "enable-opentelemetry-metrics", false, "Enable OpenTelemetry export of keda-metrics-apiserver performance metrics.")
 
 	// legacy klogr flags handled for backwards compatibility. Default set to -1 so it doesn't override values set via zap options
 	cmd.Flags().IntVar(&verbosityLevel, "v", -1, "Logging level for Metrics Server. (DEPRECATED)")
@@ -314,6 +317,11 @@ func main() {
 
 	setupLog.Info(cmd.Message)
 
+	metricscollector.RegisterAdapterPerformancePromMetrics(legacyregistry.Registerer())
+	if enableOpenTelemetryMetrics {
+		metricscollector.InitAdapterOtelPerformanceMetrics()
+	}
+
 	RunMetricsServer(ctx)
 
 	if err = cmd.Run(ctx); err != nil {

pkg/metricscollector/export_test.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2026 The KEDA Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metricscollector
+
+import (
+	"go.opentelemetry.io/otel/sdk/metric"
+)
+
+// ResetAdapterOtelPerformanceMetricsForTest clears adapter OTEL performance metrics state.
+func ResetAdapterOtelPerformanceMetricsForTest() {
+	adapterOtelMetrics = nil
+}
+
+// InitAdapterOtelPerformanceMetricsForTest initializes adapter OTEL performance metrics with a test reader.
+func InitAdapterOtelPerformanceMetricsForTest(reader metric.Reader) {
+	ResetAdapterOtelPerformanceMetricsForTest()
+	initAdapterOtelPerformanceMetricsWithReader(reader)
+}

pkg/metricscollector/metricscollectors.go

@@ -89,6 +89,12 @@ type MetricsCollector interface {
 	// namespace, and scaledResource are provided explicitly by the caller; upstream
 	// instrumentation may derive them from context before invoking the collector.
 	RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bool, scaler, triggerName, metricName, namespace, scaledResource string)
+
+	// RecordExternalMetricRequest records an HPA external metric request handled by keda-metrics-apiserver.
+	RecordExternalMetricRequest(durationSeconds float64, err error, namespace, scaledObject, metricName string)
+
+	// RecordMetricsServiceGetMetricsRequest records a GetMetrics gRPC request handled by keda-operator.
+	RecordMetricsServiceGetMetricsRequest(durationSeconds float64, err error, namespace, scaledObject, metricName string)
 }
 
 func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) {
@@ -231,6 +237,18 @@ func RecordHTTPClientRequest(durationSeconds float64, statusCode int, isError bo
 	}
 }
 
+// RecordExternalMetricRequest records an HPA external metric request handled by keda-metrics-apiserver.
+func RecordExternalMetricRequest(durationSeconds float64, err error, namespace, scaledObject, metricName string) {
+	RecordAdapterExternalMetricRequest(durationSeconds, err, namespace, scaledObject, metricName)
+}
+
+// RecordMetricsServiceGetMetricsRequest records a GetMetrics gRPC request handled by keda-operator.
+func RecordMetricsServiceGetMetricsRequest(durationSeconds float64, err error, namespace, scaledObject, metricName string) {
+	for _, element := range collectors {
+		element.RecordMetricsServiceGetMetricsRequest(durationSeconds, err, namespace, scaledObject, metricName)
+	}
+}
+
 func httpStatusCodeLabel(code int, isError bool) string {
 	if isError {
 		return "error"

pkg/metricscollector/metricserver_performance.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2026 The KEDA Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metricscollector
+
+import (
+	"context"
+	"os"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
+	api "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/metric"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const (
+	requestResultSuccess = "success"
+	requestResultError   = "error"
+)
+
+var (
+	adapterExternalMetricRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: DefaultPromMetricsNamespace,
+			Subsystem: "external_metrics_provider",
+			Name:      "requests_total",
+			Help:      "Total number of external metric requests served to the Kubernetes HPA, labeled by outcome.",
+		},
+		[]string{"namespace", "scaled_object", "metric", "result"},
+	)
+
+	adapterExternalMetricRequestDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: DefaultPromMetricsNamespace,
+			Subsystem: "external_metrics_provider",
+			Name:      "request_duration_seconds",
+			Help:      "Duration in seconds of external metric requests served to the Kubernetes HPA.",
+			Buckets:   []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+		},
+		[]string{"result"},
+	)
+
+	adapterPerformancePromOnce sync.Once
+
+	adapterOtelMetrics               *adapterOtelPerformanceMetrics
+	adapterOtelPerformanceMetricsLog = logf.Log.WithName("adapter_otel_performance_metrics")
+)
+
+type adapterOtelPerformanceMetrics struct {
+	externalMetricRequests api.Int64Counter
+	externalMetricDuration api.Float64Histogram
+}
+
+// RegisterAdapterPerformancePromMetrics registers HPA-facing performance metrics.
+// Registration runs at most once; the first call wins and later calls are no-ops even if a different registerer is passed.
+// Intended for use by keda-metrics-apiserver which exposes metrics through legacyregistry.
+func RegisterAdapterPerformancePromMetrics(registerer prometheus.Registerer) {
+	adapterPerformancePromOnce.Do(func() {
+		registerer.MustRegister(adapterExternalMetricRequestsTotal)
+		registerer.MustRegister(adapterExternalMetricRequestDuration)
+	})
+}
+
+// InitAdapterOtelPerformanceMetrics initializes OpenTelemetry export of HPA-facing performance metrics.
+func InitAdapterOtelPerformanceMetrics() {
+	if adapterOtelMetrics != nil {
+		return
+	}
+
+	protocol := os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
+
+	var exporter metric.Exporter
+	var err error
+	switch protocol {
+	case "grpc":
+		adapterOtelPerformanceMetricsLog.V(1).Info("start OTEL grpc client for adapter performance metrics")
+		exporter, err = otlpmetricgrpc.New(context.Background())
+	default:
+		adapterOtelPerformanceMetricsLog.V(1).Info("start OTEL http client for adapter performance metrics")
+		exporter, err = otlpmetrichttp.New(context.Background())
+	}
+
+	if err != nil {
+		adapterOtelPerformanceMetricsLog.Error(err, "failed to initialize adapter OTEL performance metrics")
+		return
+	}
+
+	initAdapterOtelPerformanceMetricsWithReader(metric.NewPeriodicReader(exporter))
+}
+
+func initAdapterOtelPerformanceMetricsWithReader(reader metric.Reader) {
+	if adapterOtelMetrics != nil {
+		return
+	}
+
+	meterProvider := metric.NewMeterProvider(metric.WithReader(reader))
+	otel.SetMeterProvider(meterProvider)
+
+	meter := meterProvider.Meter("keda-adapter-performance-metrics")
+	msg := "failed to create OpenTelemetry instrument for adapter performance metrics"
+
+	externalMetricRequests, err := meter.Int64Counter(
+		"keda.external_metrics_provider.requests.count",
+		api.WithDescription("Total number of external metric requests served to the Kubernetes HPA"),
+	)
+	if err != nil {
+		adapterOtelPerformanceMetricsLog.Error(err, msg)
+		return
+	}
+
+	externalMetricDuration, err := meter.Float64Histogram(
+		"keda.external_metrics_provider.request.duration.seconds",
+		api.WithDescription("Duration in seconds of external metric requests served to the Kubernetes HPA"),
+		api.WithUnit("s"),
+	)
+	if err != nil {
+		adapterOtelPerformanceMetricsLog.Error(err, msg)
+		return
+	}
+
+	adapterOtelMetrics = &adapterOtelPerformanceMetrics{
+		externalMetricRequests: externalMetricRequests,
+		externalMetricDuration: externalMetricDuration,
+	}
+}
+
+// RecordAdapterExternalMetricRequest records an HPA external metric request handled by keda-metrics-apiserver.
+func RecordAdapterExternalMetricRequest(durationSeconds float64, err error, namespace, scaledObject, metricName string) {
+	result := requestResult(err)
+
+	adapterExternalMetricRequestsTotal.WithLabelValues(namespace, scaledObject, metricName, result).Inc()
+	adapterExternalMetricRequestDuration.WithLabelValues(result).Observe(durationSeconds)
+
+	if adapterOtelMetrics != nil {
+		counterOpt := api.WithAttributes(
+			attribute.Key("namespace").String(namespace),
+			attribute.Key("scaled_object").String(scaledObject),
+			attribute.Key("metric").String(metricName),
+			attribute.Key("result").String(result),
+		)
+		histOpt := api.WithAttributes(attribute.Key("result").String(result))
+
+		adapterOtelMetrics.externalMetricRequests.Add(context.Background(), 1, counterOpt)
+		adapterOtelMetrics.externalMetricDuration.Record(context.Background(), durationSeconds, histOpt)
+	}
+}
+
+func requestResult(err error) string {
+	if err != nil {
+		return requestResultError
+	}
+	return requestResultSuccess
+}