Skip to content

Commit 4fc0d4b

Browse files
authored
Merge pull request #24 from MbolotSuse/local-exclude
Excluding local cluster nodes from node count
2 parents e7129b2 + 4f67e69 commit 4fc0d4b

File tree

6 files changed

+270
-3
lines changed

6 files changed

+270
-3
lines changed

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ require (
3535
github.com/aws/aws-sdk-go-v2/service/licensemanager v1.15.3
3636
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3
3737
github.com/google/uuid v1.2.0
38+
github.com/prometheus/client_model v0.2.0
3839
github.com/prometheus/common v0.32.1
3940
github.com/rancher/lasso v0.0.0-20220412224715-5f3517291ad4
4041
github.com/rancher/rancher v0.0.0-20220309231411-e4af2465c5b4
@@ -78,7 +79,6 @@ require (
7879
github.com/pkg/errors v0.9.1 // indirect
7980
github.com/pmezard/go-difflib v1.0.0 // indirect
8081
github.com/prometheus/client_golang v1.12.1 // indirect
81-
github.com/prometheus/client_model v0.2.0 // indirect
8282
github.com/prometheus/procfs v0.7.3 // indirect
8383
github.com/rancher/aks-operator v1.0.5 // indirect
8484
github.com/rancher/eks-operator v1.1.3 // indirect

pkg/manager/aws.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,8 @@ func (m *AWS) runComplianceCheck(ctx context.Context) error {
127127
currentCheckoutInfo.EntitledLicenses = checkoutAmount
128128
currentCheckoutInfo.Expiry = parseExpirationTimestamp(*resp.Expiration)
129129
}
130-
} else {
130+
} else if requiredLicenses != 0 {
131+
// extend our checkout as long as we have something checked out
131132
newCheckoutInfo, err := m.extendCheckout(ctx, 5*managerInterval, currentCheckoutInfo)
132133
if err != nil {
133134
currentCheckoutInfo.EntitledLicenses = 0

pkg/manager/aws_test.go

+22
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,17 @@ func TestCheckout(t *testing.T) {
113113
cachedToken: true,
114114
},
115115
},
116+
{
117+
numRancherNodes: 0,
118+
numAWSEntitlements: 1,
119+
currentEntitlements: 0,
120+
result: testResult{
121+
errResult: false,
122+
inCompliance: true,
123+
numUsedEntitlements: 0,
124+
cachedToken: false,
125+
},
126+
},
116127
}
117128
for _, scenario := range scenarios {
118129
scenario.runScenario(t)
@@ -155,6 +166,17 @@ func TestCheckInCheckout(t *testing.T) {
155166
cachedToken: true,
156167
},
157168
},
169+
{
170+
numRancherNodes: 0,
171+
numAWSEntitlements: 1,
172+
currentEntitlements: 1,
173+
result: testResult{
174+
errResult: false,
175+
inCompliance: true,
176+
numUsedEntitlements: 0,
177+
cachedToken: false,
178+
},
179+
},
158180
}
159181
for _, scenario := range scenarios {
160182
scenario.runScenario(t)

pkg/metrics/mocks_test.go

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package metrics
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"strings"
7+
)
8+
9+
type mockPrometheusServer struct {
10+
nodesForCluster map[string]int
11+
labelSkip map[string]struct{}
12+
validTokens []string
13+
}
14+
15+
func newMockPrometheusServer() mockPrometheusServer {
16+
return mockPrometheusServer{
17+
nodesForCluster: map[string]int{},
18+
labelSkip: map[string]struct{}{},
19+
validTokens: []string{},
20+
}
21+
}
22+
23+
func (m *mockPrometheusServer) SetNodesForCluster(nodes int, cluster string, skipLabel bool) {
24+
m.nodesForCluster[cluster] = nodes
25+
if skipLabel {
26+
m.labelSkip[cluster] = struct{}{}
27+
}
28+
}
29+
30+
func (m *mockPrometheusServer) Clear() {
31+
m.nodesForCluster = map[string]int{}
32+
m.validTokens = []string{}
33+
m.labelSkip = map[string]struct{}{}
34+
}
35+
36+
func (m *mockPrometheusServer) AddAuthToken(token string) {
37+
m.validTokens = append(m.validTokens, token)
38+
}
39+
func (m *mockPrometheusServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
40+
// main serve method for the test http server
41+
if strings.HasSuffix(r.URL.String(), "/metrics") {
42+
//do auth and return the node metrics text
43+
if !m.requestAuthenticated(r) {
44+
w.WriteHeader(http.StatusUnauthorized)
45+
return
46+
}
47+
if !m.requestAuthorized(r) {
48+
w.WriteHeader(http.StatusForbidden)
49+
return
50+
}
51+
requestText := m.getNodeMetricsText()
52+
w.WriteHeader(http.StatusOK)
53+
w.Write([]byte(requestText))
54+
return
55+
}
56+
w.WriteHeader(http.StatusNotFound)
57+
return
58+
}
59+
60+
func (m *mockPrometheusServer) getNodeMetricsText() string {
61+
retText := "# TYPE cluster_manager_nodes gauge\n"
62+
for cluster, nodeCount := range m.nodesForCluster {
63+
if _, ok := m.labelSkip[cluster]; ok {
64+
retText += fmt.Sprintf("%s{} %d\n", nodeGaugeMetricName, nodeCount)
65+
} else {
66+
retText += fmt.Sprintf("%s{%s=\"%s\"} %d\n", nodeGaugeMetricName, clusterNameLabel, cluster, nodeCount)
67+
}
68+
}
69+
return retText
70+
}
71+
72+
func (m *mockPrometheusServer) requestAuthorized(req *http.Request) bool {
73+
header := req.Header.Get("Authorization")
74+
splitHeader := strings.Split(header, " ")
75+
if len(splitHeader) < 2 {
76+
return false
77+
}
78+
token := splitHeader[1]
79+
for _, validToken := range m.validTokens {
80+
if token == validToken {
81+
return true
82+
}
83+
}
84+
return false
85+
}
86+
87+
func (m *mockPrometheusServer) requestAuthenticated(req *http.Request) bool {
88+
return req.Header.Get("Authorization") != ""
89+
}

pkg/metrics/scraper.go

+24-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"net/http"
66
"strings"
77

8+
prometheusClient "github.com/prometheus/client_model/go"
89
"github.com/prometheus/common/expfmt"
10+
"github.com/sirupsen/logrus"
911
"k8s.io/client-go/rest"
1012
)
1113

@@ -30,6 +32,8 @@ func NewScraper(rancherHost string, cfg *rest.Config) Scraper {
3032

3133
const (
3234
nodeGaugeMetricName = "cluster_manager_nodes"
35+
clusterNameLabel = "cluster_id"
36+
localClusterID = "local"
3337
)
3438

3539
type NodeCounts struct {
@@ -66,10 +70,29 @@ func (s *scraper) ScrapeAndParse() (*NodeCounts, error) {
6670

6771
var nodeCount int
6872
for _, metric := range nodeMetricFamily.GetMetric() {
69-
nodeCount += int(metric.GetGauge().GetValue())
73+
isMetricForLocal, err := isMetricForLocalCluster(metric)
74+
clusterNodeCount := int(metric.GetGauge().GetValue())
75+
logrus.Debugf("scraper found nodes: %d, isMetricForLocal: %t, err: %v", clusterNodeCount, isMetricForLocal, err)
76+
if err != nil {
77+
logrus.Warnf("error when attempting to determine if count was for local cluster: %s, will not include %d nodes in total", err.Error(), clusterNodeCount)
78+
continue
79+
}
80+
if !isMetricForLocal {
81+
nodeCount += clusterNodeCount
82+
}
83+
7084
}
7185

7286
return &NodeCounts{
7387
Total: nodeCount,
7488
}, nil
7589
}
90+
91+
func isMetricForLocalCluster(metric *prometheusClient.Metric) (bool, error) {
92+
for _, label := range metric.GetLabel() {
93+
if label.Name != nil && *label.Name == clusterNameLabel {
94+
return label.Value != nil && *label.Value == localClusterID, nil
95+
}
96+
}
97+
return false, fmt.Errorf("unable to determine if metric is for local cluster due to missing label")
98+
}

pkg/metrics/scraper_test.go

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package metrics
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"net/http/httptest"
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
"k8s.io/client-go/rest"
11+
)
12+
13+
func TestScrapeAndParse(t *testing.T) {
14+
metricsServer := newMockPrometheusServer()
15+
server := httptest.NewServer(&metricsServer)
16+
defer server.Close()
17+
tests := []struct {
18+
name string
19+
localClusterNodes int
20+
skipLocalLabel bool
21+
numOtherClusters int
22+
nodesPerOtherClusters int
23+
skipOtherLabel bool
24+
authed bool
25+
expectedTotal int
26+
expectedError bool
27+
}{
28+
{
29+
name: "no downstream, local only",
30+
localClusterNodes: 2,
31+
skipLocalLabel: false,
32+
numOtherClusters: 0,
33+
nodesPerOtherClusters: 0,
34+
skipOtherLabel: false,
35+
authed: true,
36+
expectedTotal: 0,
37+
expectedError: false,
38+
},
39+
{
40+
name: "downstream and local",
41+
localClusterNodes: 2,
42+
skipLocalLabel: false,
43+
numOtherClusters: 1,
44+
nodesPerOtherClusters: 2,
45+
skipOtherLabel: false,
46+
authed: true,
47+
expectedTotal: 2,
48+
expectedError: false,
49+
},
50+
{
51+
name: "multiple downstream and local",
52+
localClusterNodes: 2,
53+
skipLocalLabel: false,
54+
numOtherClusters: 3,
55+
nodesPerOtherClusters: 4,
56+
skipOtherLabel: false,
57+
authed: true,
58+
expectedTotal: 12,
59+
expectedError: false,
60+
},
61+
{
62+
name: "local not labeled, local nodes excluded",
63+
localClusterNodes: 2,
64+
skipLocalLabel: true,
65+
numOtherClusters: 0,
66+
nodesPerOtherClusters: 0,
67+
skipOtherLabel: false,
68+
authed: true,
69+
expectedTotal: 0,
70+
expectedError: false,
71+
},
72+
{
73+
name: "local not labeled, downstream not labeled, all nodes excluded",
74+
localClusterNodes: 2,
75+
skipLocalLabel: true,
76+
numOtherClusters: 2,
77+
nodesPerOtherClusters: 3,
78+
skipOtherLabel: true,
79+
authed: true,
80+
expectedTotal: 0,
81+
expectedError: false,
82+
},
83+
{
84+
name: "local labeled, downstream not labeled, all nodes excluded",
85+
localClusterNodes: 2,
86+
skipLocalLabel: false,
87+
numOtherClusters: 2,
88+
nodesPerOtherClusters: 4,
89+
skipOtherLabel: true,
90+
authed: true,
91+
expectedTotal: 0,
92+
expectedError: false,
93+
},
94+
{
95+
name: "error, no auth",
96+
localClusterNodes: 2,
97+
skipLocalLabel: false,
98+
numOtherClusters: 0,
99+
nodesPerOtherClusters: 0,
100+
skipOtherLabel: false,
101+
authed: false,
102+
expectedTotal: 0,
103+
expectedError: true,
104+
},
105+
}
106+
for _, test := range tests {
107+
t.Run(test.name, func(t *testing.T) {
108+
metricsServer.Clear()
109+
metricsServer.SetNodesForCluster(test.localClusterNodes, localClusterID, test.skipLocalLabel)
110+
for i := 0; i < test.numOtherClusters; i++ {
111+
metricsServer.SetNodesForCluster(test.nodesPerOtherClusters, fmt.Sprintf("cluster-%d", i), test.skipOtherLabel)
112+
}
113+
config := &rest.Config{BearerToken: "abc123abc123abc123"}
114+
if test.authed {
115+
metricsServer.AddAuthToken(config.BearerToken)
116+
}
117+
metricsScraper := scraper{
118+
metricsURL: fmt.Sprintf("%s/metrics", server.URL),
119+
cli: &http.Client{},
120+
cfg: config,
121+
}
122+
res, err := metricsScraper.ScrapeAndParse()
123+
if test.expectedError {
124+
assert.Error(t, err, "expected an error but err was nil")
125+
} else {
126+
assert.NoError(t, err, "expected no error but there was an error")
127+
assert.NotNil(t, res, "expected a result but was nil")
128+
assert.Equal(t, test.expectedTotal, res.Total, "did not get expected number of nodes")
129+
}
130+
})
131+
}
132+
}

0 commit comments

Comments
 (0)