add verify for ec2 hosted runner and kepler_model_server case

.github/workflows/test.yml

@@ -4,6 +4,8 @@ on:
   # we can run mannually to see if it can be fixed by update version of
   # - kubectl
   # - k8s cluster provider's 
+  schedule:
+    - cron: "27 4 * * *"
   workflow_dispatch:
   pull_request:
   push:
@@ -36,41 +38,43 @@ jobs:
           PROMETHEUS_ENABLE: false
           GRAFANA_ENABLE: false
           TEKTON_ENABLE: false
+          KUBEVIRT_ENABLE: false
         - cluster_provider: kind
           prometheus_operator_version: v0.12.0
           PROMETHEUS_ENABLE: true
           GRAFANA_ENABLE: true
           TEKTON_ENABLE: false
+          KUBEVIRT_ENABLE: false
         - cluster_provider: kind
           prometheus_operator_version: v0.12.0
           PROMETHEUS_ENABLE: false
           GRAFANA_ENABLE: false
           TEKTON_ENABLE: true
+          KUBEVIRT_ENABLE: false
         - cluster_provider: kind
           prometheus_operator_version: v0.12.0
           PROMETHEUS_ENABLE: true
           GRAFANA_ENABLE: true
           TEKTON_ENABLE: true
+          KUBEVIRT_ENABLE: false
+        - cluster_provider: kind
+          prometheus_operator_version: v0.12.0
+          PROMETHEUS_ENABLE: true
+          GRAFANA_ENABLE: true
+          TEKTON_ENABLE: true
+          KUBEVIRT_ENABLE: true
         - cluster_provider: microshift
           prometheus_operator_version: v0.12.0
           PROMETHEUS_ENABLE: false
           GRAFANA_ENABLE: false
           TEKTON_ENABLE: false
+          KUBEVIRT_ENABLE: false
         - cluster_provider: microshift
           prometheus_operator_version: v0.12.0
           PROMETHEUS_ENABLE: true
           GRAFANA_ENABLE: true
           TEKTON_ENABLE: false
-        #- cluster_provider: microshift
-        #  prometheus_operator_version: v0.12.0
-        #  PROMETHEUS_ENABLE: false
-        #  GRAFANA_ENABLE: false
-        #  TEKTON_ENABLE: true
-        #- cluster_provider: microshift
-        #  prometheus_operator_version: v0.12.0
-        #  PROMETHEUS_ENABLE: true
-        #  GRAFANA_ENABLE: true
-        #  TEKTON_ENABLE: true        
+          KUBEVIRT_ENABLE: false     
     steps:
     - uses: actions/checkout@v3
     - name: start from local
@@ -80,11 +84,13 @@ jobs:
         export GRAFANA_ENABLE=${{matrix.GRAFANA_ENABLE}}
         export CLUSTER_PROVIDER=${{matrix.cluster_provider}}
         export TEKTON_ENABLE=${{matrix.TEKTON_ENABLE}}
+        export KUBEVIRT_ENABLE=${{matrix.KUBEVIRT_ENABLE}}
         ./main.sh prerequisites
         ./main.sh up
     - name: verify
       run: |
         export CLUSTER_PROVIDER=${{matrix.cluster_provider}}
+        export GRAFANA_ENABLE=${{matrix.GRAFANA_ENABLE}}
         ./verify.sh
 
   containertestubuntu:

lib/prometheus.sh

@@ -44,8 +44,6 @@ deploy_prometheus_operator() {
 		mv kube-prometheus/manifests/prometheus-prometheus.yaml.tmp \
 			kube-prometheus/manifests/prometheus-prometheus.yaml
 
-		_setup_dashboard
-		_run_yq
 		_load_prometheus_operator_images_to_local_registry
 		kubectl create -f kube-prometheus/manifests/setup
 		kubectl wait \
@@ -62,6 +60,7 @@ deploy_prometheus_operator() {
 		is_set "$GRAFANA_ENABLE" && {
 			find kube-prometheus/manifests -name 'grafana-*.yaml' -type f \
 				-exec kubectl create -f {} \;
+			_setup_dashboard
 			ok "Grafana deployed"
 		}
 
@@ -118,7 +117,10 @@ _setup_dashboard(){
 	if [ -f "$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" ]; then
 		return 0
 	else
-	header "Setup Dashboard"
+	if [ -f "$KUBE_PROM_DIR/manifests/grafana-dashboardDefinitions.yaml"  ]; then
+		exit 1
+	fi
+	header "Setup Dashboard base on $DASHBOARD_DIR"
 	mkdir -p "$DASHBOARD_DIR/grafana-dashboards/"
 	cat - > "$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" << EOF
 apiVersion: v1
@@ -137,11 +139,3 @@ metadata:
 EOF
     fi
 }
-
-_run_yq(){
-	f="$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" \
-	yq -i e '.items += [load(env(f))]' "$KUBE_PROM_DIR"/manifests/grafana-dashboardDefinitions.yaml;
-	yq -i e '.spec.template.spec.containers.0.volumeMounts += [ {"mountPath": "/grafana-dashboard-definitions/0/kepler-exporter", "name": "grafana-dashboard-kepler-exporter", "readOnly": false} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml
-	yq -i e '.spec.template.spec.volumes += [ {"configMap": {"name": "grafana-dashboard-kepler-exporter"}, "name": "grafana-dashboard-kepler-exporter"} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml;
-	ok "Dashboard setup complete"
-}

main.sh

@@ -123,6 +123,7 @@ print_config() {
 		Monitoring
 		  * Install Prometheus : $prom_install_msg
 		  * Install Grafana    : $GRAFANA_ENABLE
+		if Grafana been enabled, the kepler dashboard will be installed.
 
 		Tekton
 		  * Install Tekton : $TEKTON_ENABLE

verify.sh

@@ -21,6 +21,8 @@ set -eu -o pipefail
 PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 declare -r PROJECT_ROOT
 declare -r KUBECONFIG_ROOT_DIR=${KUBECONFIG_ROOT_DIR:-$PROJECT_ROOT/.kube}
+declare -r KUBE_PROM_DIR="$PROJECT_ROOT/tmp/kube-prometheus"
+declare -r DASHBOARD_DIR="$KUBE_PROM_DIR/grafana-dashboards"
 # shellcheck source=lib/utils.sh
 source "$PROJECT_ROOT/lib/utils.sh"
 
@@ -58,6 +60,20 @@ verify_cluster() {
 		rollout_status "$res" "$NAMESPACE"
 	done
 
+	# verify kubeconf for ec2_hosted_runner usage
+	info "Verifying cluster status for CI usage"
+	cp .kube/config /tmp/kubeconfig
+	export KUBECONFIG="/tmp/kubeconfig"
+	run kubectl cluster-info || die "failed to get the cluster-info"
+
+	# check k8s system pod is there...
+	[[ $(kubectl get pods --all-namespaces | wc -l) == 0 ]] &&
+		die "it seems k8s cluster is not started"
+
+	is_set "$GRAFANA_ENABLE" && {
+		run_yq
+	}
+
 	ok "Cluster is up and running"
 }
 
@@ -67,6 +83,18 @@ containerruntime() {
 	docker info || true
 }
 
+
+run_yq(){
+	if [ -f "$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" ]; then
+		return 0
+	fi
+	f="$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" \
+	yq -i e '.items += [load(env(f))]' "$KUBE_PROM_DIR"/manifests/grafana-dashboardDefinitions.yaml;
+	yq -i e '.spec.template.spec.containers.0.volumeMounts += [ {"mountPath": "/grafana-dashboard-definitions/0/kepler-exporter", "name": "grafana-dashboard-kepler-exporter", "readOnly": false} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml
+	yq -i e '.spec.template.spec.volumes += [ {"configMap": {"name": "grafana-dashboard-kepler-exporter"}, "name": "grafana-dashboard-kepler-exporter"} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml;
+	ok "Dashboard setup complete"
+}
+
 main() {
 	# verify the deployment of cluster
 	case $1 in