take zero cache hits into account.

Author: zetxqx

config/charts/inferencepool/templates/epp-config.yaml

@@ -15,8 +15,11 @@ data:
     - name: default
       plugins:
       - pluginRef: queue-scorer
+        weight: 2
       - pluginRef: kv-cache-utilization-scorer
+        weight: 2
       - pluginRef: prefix-cache-scorer
+        weight: 3
   {{- if (hasKey .Values.inferenceExtension "pluginsCustomConfig") }}
   {{- .Values.inferenceExtension.pluginsCustomConfig | toYaml | nindent 2 }}
   {{- end }}

pkg/epp/handlers/response.go

@@ -86,9 +86,11 @@ func (s *StreamingServer) HandleResponseBodyModelStreaming(ctx context.Context,
 		reqCtx.Usage = resp.Usage
 		metrics.RecordInputTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, resp.Usage.PromptTokens)
 		metrics.RecordOutputTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, resp.Usage.CompletionTokens)
+		cachedToken := 0
 		if resp.Usage.PromptTokenDetails != nil {
-			metrics.RecordPromptCachedTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, resp.Usage.PromptTokenDetails.CachedTokens)
+			cachedToken = resp.Usage.PromptTokenDetails.CachedTokens
 		}
+		metrics.RecordPromptCachedTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, cachedToken)
 		_, err := s.director.HandleResponseBodyComplete(ctx, reqCtx)
 		if err != nil {
 			logger.Error(err, "error in HandleResponseBodyComplete")

pkg/epp/handlers/server.go

@@ -316,9 +316,11 @@ func (s *StreamingServer) Process(srv extProcPb.ExternalProcessor_ProcessServer)
 						metrics.RecordResponseSizes(reqCtx.IncomingModelName, reqCtx.TargetModelName, reqCtx.ResponseSize)
 						metrics.RecordInputTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, reqCtx.Usage.PromptTokens)
 						metrics.RecordOutputTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, reqCtx.Usage.CompletionTokens)
+						cachedToken := 0
 						if reqCtx.Usage.PromptTokenDetails != nil {
-							metrics.RecordPromptCachedTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, reqCtx.Usage.PromptTokenDetails.CachedTokens)
+							cachedToken = reqCtx.Usage.PromptTokenDetails.CachedTokens
 						}
+						metrics.RecordPromptCachedTokens(reqCtx.IncomingModelName, reqCtx.TargetModelName, cachedToken)
 					}
 				}
 			}

pkg/epp/metrics/metrics.go

@@ -384,9 +384,7 @@ func RecordOutputTokens(modelName, targetModelName string, size int) {
 
 // RecordPromptCachedTokens records prompt cached tokens count.
 func RecordPromptCachedTokens(modelName, targetModelName string, size int) {
-	if size > 0 {
-		promptCachedTokens.WithLabelValues(modelName, targetModelName).Observe(float64(size))
-	}
+	promptCachedTokens.WithLabelValues(modelName, targetModelName).Observe(float64(size))
 }
 
 // RecordNormalizedTimePerOutputToken (NTPOT) records the normalized time per output token.