Update Metrics endpoints for ODH operator (#349)

* Fix ODH and Argo monitoring

Signed-off-by: Anish Asthana <anishasthana1@gmail.com>

* Increase replica count to 2 for HA

Signed-off-by: Anish Asthana <anishasthana1@gmail.com>

* Update Prometheus name and corresponding test

Signed-off-by: Anish Asthana <anishasthana1@gmail.com>

* Restructure Service Monitors

This separate the ODH operator and ODH application monitoring into two
seperate Service Monitors.

Signed-off-by: Anish Asthana <anishasthana1@gmail.com>

Author: anishasthana

prometheus/operator/base/kustomization.yaml

@@ -4,10 +4,25 @@ resources:
 - kafka-podmonitors.yaml
 - prometheus.yaml
 - route.yaml
-- servicemonitor.yaml
+- service-monitors
+- prometheus-monitoring-role.yaml
+- prometheus-monitoring-role-binding.yaml
+
 namespace: opendatahub
 commonLabels:
   opendatahub.io/component: "true"
   component.opendatahub.io/name: prometheus
 generatorOptions:
   disableNameSuffixHash: true
+
+vars:
+  - name: namespace
+    objref:
+      kind: Prometheus
+      name: odh-monitoring
+      apiVersion: monitoring.coreos.com/v1
+    fieldref:
+      fieldpath: metadata.namespace
+
+configurations:
+  - params.yaml

prometheus/operator/base/params.yaml

@@ -0,0 +1,4 @@
+varReference:
+  - path: subjects/namespace
+    kind: ClusterRoleBinding
+    apiVersion: rbac.authorization.k8s.io/v1

prometheus/operator/base/prometheus-monitoring-role-binding.yaml

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: odh-prometheus-monitoring-rb
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: $(namespace)
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: odh-prometheus-monitoring

prometheus/operator/base/prometheus-monitoring-role.yaml

@@ -0,0 +1,23 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: odh-prometheus-monitoring
+  namespace: opendatahub
+rules:
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - ''
+    resources:
+      - services
+      - endpoints
+      - pods
+  - verbs:
+      - get
+    apiGroups:
+      - ''
+    resources:
+      - configmaps

prometheus/operator/base/prometheus.yaml

@@ -1,12 +1,12 @@
 apiVersion: monitoring.coreos.com/v1
 kind: Prometheus
 metadata:
-  name: prometheus
+  name: odh-monitoring
   labels:
-    prometheus: k8s
+    app: odh-monitoring
   namespace: prometheus
 spec:
-  replicas: 1
+  replicas: 2
   serviceAccountName: prometheus-k8s
   securityContext: {}
   serviceMonitorSelector:

…etheus/operator/base/servicemonitor.yaml …onitors/application-service-monitor.yamlprometheus/operator/base/servicemonitor.yaml renamed to prometheus/operator/base/service-monitors/application-service-monitor.yaml prometheus/operator/base/servicemonitor.yaml renamed to prometheus/operator/base/service-monitors/application-service-monitor.yaml

@@ -3,10 +3,10 @@ kind: ServiceMonitor
 metadata:
   labels:
     team: opendatahub
-  name: odhservicemonitor
+  name: odh-application-servicemonitor
 spec:
   endpoints:
-    - port: web # odh-operator, Argo
+    - port: metrics # Argo
     - bearerTokenSecret:
         key: PROMETHEUS_API_TOKEN
         name: jupyterhub

prometheus/operator/base/service-monitors/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- application-service-monitor.yaml
+- operator-service-monitor.yaml

prometheus/operator/base/service-monitors/operator-service-monitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    team: opendatahub
+  name: odh-operator-servicemonitor
+spec:
+  endpoints:
+    - port: http-metrics # Open Data Hub Operator
+    - port: cr-metrics # Open Data Hub Operator
+  selector:
+    matchLabels:
+      name: opendatahub-operator
+  namespaceSelector:
+    matchNames:
+      - openshift-operators

tests/basictests/prometheus.sh

@@ -20,9 +20,9 @@ function test_prometheus() {
     os::cmd::try_until_text "oc get pods -l k8s-app=prometheus-operator --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}'" "prometheus-operator" $odhdefaulttimeout $odhdefaultinterval
     runningbuspods=($(oc get pods -l k8s-app=prometheus-operator --field-selector="status.phase=Running" -o jsonpath="{$.items[*].metadata.name}"))
     os::cmd::expect_success_and_text "echo ${#runningbuspods[@]}" "1"
-    os::cmd::try_until_text "oc get pods -l app=prometheus --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}'" "prometheus-prometheus" $odhdefaulttimeout $odhdefaultinterval
-    runningbuspods=($(oc get pods -l app=prometheus --field-selector="status.phase=Running" -o jsonpath="{$.items[*].metadata.name}"))
-    os::cmd::expect_success_and_text "echo ${#runningbuspods[@]}" "1"
+    os::cmd::try_until_text "oc get pods -l prometheus=odh-monitoring --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}'" "prometheus-odh-monitoring" $odhdefaulttimeout $odhdefaultinterval
+    runningbuspods=($(oc get pods -l prometheus=odh-monitoring --field-selector="status.phase=Running" -o jsonpath="{$.items[*].metadata.name}"))
+    os::cmd::expect_success_and_text "echo ${#runningbuspods[@]}" "2"
     test_promportal
 }