fix pod unknown stats

Author: andyxning

Godeps/Godeps.json

@@ -26,6 +26,7 @@ import (
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/util/node"
 )
 
 var (
@@ -352,12 +353,14 @@ func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
 	labelKeys, labelValues := kubeLabelsToPrometheusLabels(p.Labels)
 	addGauge(podLabelsDesc(labelKeys), 1, labelValues...)
 
-	if p := p.Status.Phase; p != "" {
-		addGauge(descPodStatusPhase, boolFloat64(p == v1.PodPending), string(v1.PodPending))
-		addGauge(descPodStatusPhase, boolFloat64(p == v1.PodRunning), string(v1.PodRunning))
-		addGauge(descPodStatusPhase, boolFloat64(p == v1.PodSucceeded), string(v1.PodSucceeded))
-		addGauge(descPodStatusPhase, boolFloat64(p == v1.PodFailed), string(v1.PodFailed))
-		addGauge(descPodStatusPhase, boolFloat64(p == v1.PodUnknown), string(v1.PodUnknown))
+	if phase := p.Status.Phase; phase != "" {
+		addGauge(descPodStatusPhase, boolFloat64(phase == v1.PodPending), string(v1.PodPending))
+		addGauge(descPodStatusPhase, boolFloat64(phase == v1.PodSucceeded), string(v1.PodSucceeded))
+		addGauge(descPodStatusPhase, boolFloat64(phase == v1.PodFailed), string(v1.PodFailed))
+		// This logic is directly copied from: https://github.com/kubernetes/kubernetes/blob/d39bfa0d138368bbe72b0eaf434501dcb4ec9908/pkg/printers/internalversion/printers.go#L597-L601
+		// For more info, please go to: https://github.com/kubernetes/kube-state-metrics/issues/410
+		addGauge(descPodStatusPhase, boolFloat64(phase == v1.PodRunning && !(p.DeletionTimestamp != nil && p.Status.Reason == node.NodeUnreachablePodReason)), string(v1.PodRunning))
+		addGauge(descPodStatusPhase, boolFloat64(phase == v1.PodUnknown || (p.DeletionTimestamp != nil && p.Status.Reason == node.NodeUnreachablePodReason)), string(v1.PodUnknown))
 	}
 
 	if !p.CreationTimestamp.IsZero() {

collectors/pod.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/util/node"
 )
 
 type mockPodStore struct {
@@ -481,15 +482,35 @@ func TestPodCollector(t *testing.T) {
 						Namespace: "ns1",
 					},
 					Status: v1.PodStatus{
-						Phase: "Running",
+						Phase: v1.PodRunning,
 					},
 				}, {
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "pod2",
 						Namespace: "ns2",
 					},
 					Status: v1.PodStatus{
-						Phase: "Pending",
+						Phase: v1.PodPending,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "pod3",
+						Namespace: "ns3",
+					},
+					Status: v1.PodStatus{
+						Phase: v1.PodUnknown,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:              "pod4",
+						Namespace:         "ns4",
+						DeletionTimestamp: &metav1.Time{},
+					},
+					Status: v1.PodStatus{
+						Phase:  v1.PodRunning,
+						Reason: node.NodeUnreachablePodReason,
 					},
 				},
 			},
@@ -504,6 +525,16 @@ func TestPodCollector(t *testing.T) {
 				kube_pod_status_phase{namespace="ns2",phase="Running",pod="pod2"} 0
 				kube_pod_status_phase{namespace="ns2",phase="Succeeded",pod="pod2"} 0
 				kube_pod_status_phase{namespace="ns2",phase="Unknown",pod="pod2"} 0
+				kube_pod_status_phase{namespace="ns3",phase="Failed",pod="pod3"} 0
+				kube_pod_status_phase{namespace="ns3",phase="Pending",pod="pod3"} 0
+				kube_pod_status_phase{namespace="ns3",phase="Running",pod="pod3"} 0
+				kube_pod_status_phase{namespace="ns3",phase="Succeeded",pod="pod3"} 0
+				kube_pod_status_phase{namespace="ns3",phase="Unknown",pod="pod3"} 1
+				kube_pod_status_phase{namespace="ns4",phase="Failed",pod="pod4"} 0
+				kube_pod_status_phase{namespace="ns4",phase="Pending",pod="pod4"} 0
+				kube_pod_status_phase{namespace="ns4",phase="Running",pod="pod4"} 0
+				kube_pod_status_phase{namespace="ns4",phase="Succeeded",pod="pod4"} 0
+				kube_pod_status_phase{namespace="ns4",phase="Unknown",pod="pod4"} 1
 				`,
 			metrics: []string{"kube_pod_status_phase"},
 		}, {