[RHOAIENG-31975] - Onboard metrics to the centralized metrics platform (opendatahub-io#502)

chore: 	Add the `monitoring.opendatahub.io/scrape: "true"` annotation to
	Pod and Service monitors so the metrics can be collected by the
	Centralized RHOAI Observability platform.

Signed-off-by: Spolti <fspolti@redhat.com>

Author: spolti

internal/controller/constants/constants.go

@@ -31,6 +31,7 @@ const (
 	IstioSidecarInjectAnnotationName = "sidecar.istio.io/inject"
 	KserveNetworkVisibility          = "networking.kserve.io/visibility"
 	KserveGroupAnnotation            = "serving.kserve.io/inferenceservice"
+	RhoaiObservabilityLabel          = "monitoring.opendatahub.io/scrape"
 
 	EnableAuthODHAnnotation   = "security.opendatahub.io/enable-auth"
 	LabelAuthGroup            = "security.opendatahub.io/authorization-group"

internal/controller/serving/reconcilers/kserve_istio_podmonitor_reconciler.go

@@ -19,6 +19,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/opendatahub-io/odh-model-controller/internal/controller/constants"
+
 	"github.com/go-logr/logr"
 	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
 	v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -84,6 +86,9 @@ func (r *KserveIstioPodMonitorReconciler) createDesiredResource(ctx context.Cont
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      istioPodMonitorName,
 			Namespace: isvc.Namespace,
+			Labels: map[string]string{
+				constants.RhoaiObservabilityLabel: "true",
+			},
 		},
 		Spec: v1.PodMonitorSpec{
 			Selector: metav1.LabelSelector{

internal/controller/serving/reconcilers/kserve_metrics_service_reconciler.go

@@ -22,7 +22,7 @@ import (
 	"github.com/go-logr/logr"
 	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
 	"github.com/kserve/kserve/pkg/constants"
-	constants2 "github.com/opendatahub-io/odh-model-controller/internal/controller/constants"
+	odhconstants "github.com/opendatahub-io/odh-model-controller/internal/controller/constants"
 	"github.com/opendatahub-io/odh-model-controller/internal/controller/utils"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -96,7 +96,8 @@ func (r *KserveRawMetricsServiceReconciler) createDesiredResource(ctx context.Co
 			Name:      getMetricsServiceName(isvc),
 			Namespace: isvc.Namespace,
 			Labels: map[string]string{
-				"name": getMetricsServiceName(isvc),
+				"name":                               getMetricsServiceName(isvc),
+				odhconstants.RhoaiObservabilityLabel: "true",
 			},
 		},
 		Spec: v1.ServiceSpec{
@@ -110,7 +111,7 @@ func (r *KserveRawMetricsServiceReconciler) createDesiredResource(ctx context.Co
 			},
 			Type: v1.ServiceTypeClusterIP,
 			Selector: map[string]string{
-				constants2.KserveGroupAnnotation: isvc.Name,
+				odhconstants.KserveGroupAnnotation: isvc.Name,
 			},
 		},
 	}

internal/controller/serving/reconcilers/kserve_metrics_servicemonitor_reconciler.go

@@ -18,6 +18,8 @@ package reconcilers
 import (
 	"context"
 
+	"github.com/opendatahub-io/odh-model-controller/internal/controller/constants"
+
 	"github.com/go-logr/logr"
 	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
 	"github.com/opendatahub-io/odh-model-controller/internal/controller/utils"
@@ -81,6 +83,9 @@ func (r *KserveRawMetricsServiceMonitorReconciler) createDesiredResource(ctx con
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      getMetricsServiceMonitorName(isvc),
 			Namespace: isvc.Namespace,
+			Labels: map[string]string{
+				constants.RhoaiObservabilityLabel: "true",
+			},
 		},
 		Spec: v1.ServiceMonitorSpec{
 			Endpoints: []v1.Endpoint{