chore(cherry-pick): downgrade health check error log to warn (#3108) (#3387)

Signed-off-by: adarsh0728 <gooneriitk@gmail.com>
Signed-off-by: Vigith Maurice <vigith@gmail.com>
Signed-off-by: Vaibhav Tiwari <vaibhav.tiwari33@gmail.com>
Co-authored-by: Adarsh Jain <gooneriitk@gmail.com>
Co-authored-by: Vigith Maurice <vigith@gmail.com>

Author: 3 people

pkg/mvtxdaemon/server/service/health_status.go

@@ -158,7 +158,7 @@ func (hc *HealthChecker) getDesiredReplica(mvtxMetrics *mvtxdaemon.MonoVertexMet
 	// Extract the processing rate from the metrics for the default lookback period
 	rate, existing := mvtxMetrics.ProcessingRates["default"]
 	// Rate not available
-	// send back error that we calculate the health status right now
+	// send back error that we cannot calculate the health status right now
 	if !existing || rate.GetValue() < 0 {
 		return 0, fmt.Errorf("cannot check data health, MonoVertex %s has no rate information", mvtxMetrics.MonoVertex)
 	} else {

pkg/mvtxdaemon/server/service/mvtx_service.go

@@ -154,9 +154,11 @@ func (mvs *MonoVertexService) startHealthCheck(ctx context.Context) {
 			criticality, err := mvs.healthChecker.getMonoVertexDataCriticality(ctx, mvtxMetrics.Metrics)
 			logger.Debugw("MonoVertex Health check", zap.Any("criticality", criticality))
 			if err != nil {
+				// Warn level is used to avoid flooding the logs with errors
+				// in case there are 0 pods running.
+				logger.Warnw("Failed to check MonoVertex data criticality, perhaps there are no running pods!", zap.Error(err))
 				// If there is an error, set the current health status to unknown.
-				// as we are not able to determine the health of the pipeline.
-				logger.Errorw("Failed to MonoVertex data criticality", zap.Error(err))
+				// as we are not able to determine the health of the MonoVertex.
 				mvs.healthChecker.setCurrentHealth(defaultDataHealthResponse)
 			} else {
 				// convert the MonoVertex health state to API response

test/fixtures/util.go

@@ -345,7 +345,6 @@ func WaitForVertexPodRunning(kubeClient kubernetes.Interface, vertexClient flowp
 		}
 		ok = ok && len(podList.Items) > 0 && len(podList.Items) == vertexList.Items[0].CalculateReplicas() // pod number should equal to desired replicas
 		for _, p := range podList.Items {
-			log.Println("Checking for vertex pod ready: ", p.Name)
 			ok = ok && isPodReady(p)
 		}
 		if ok {