fix: get metrics failed if values contain NaN

feat: set retention time to 2h to reduce memory usage of prometheus

Signed-off-by: Feng Kun <[email protected]>

Author: kevinfeng

pkg/monitor/storage/thanos/metric/metric.go

@@ -21,6 +21,7 @@ package metric
 import (
 	"context"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 	"tkestack.io/tke/api/monitor"
@@ -222,7 +223,9 @@ func MergeResult(results model.Matrix, timestamp string, groupByWithoutTimestamp
 				}
 			}
 			values[index].([]interface{})[0] = sp.Timestamp.Unix() * 1000
-			values[index].([]interface{})[fieldIndex+1] = float64(sp.Value)
+			if !math.IsNaN(float64(sp.Value)) {
+				values[index].([]interface{})[fieldIndex+1] = float64(sp.Value)
+			}
 			for j, tag := range tags {
 				values[index].([]interface{})[len(fieldIndexes)+1+j] = tag
 			}

pkg/platform/controller/addon/prometheus/controller.go

@@ -1166,6 +1166,7 @@ func createPrometheusCRD(components images.Components, clusterName string, remot
 			ScrapeInterval:     "60s",
 			RemoteRead:         remoteReadSpecs,
 			RemoteWrite:        remoteWriteSpecs,
+			Retention:          "2h",
 			EvaluationInterval: "1m",
 			AdditionalScrapeConfigs: &corev1.SecretKeySelector{
 				LocalObjectReference: corev1.LocalObjectReference{Name: prometheusSecret},