Skip to content

Commit 344e3c3

Browse files
chenchunchenchun
committed
Add metrics
1 parent 0d262c3 commit 344e3c3

File tree

8 files changed

+111
-15
lines changed

8 files changed

+111
-15
lines changed

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/kr/pretty v0.1.0 // indirect
1818
github.com/onsi/ginkgo v1.10.3
1919
github.com/onsi/gomega v1.7.1
20+
github.com/prometheus/client_golang v0.9.2
2021
github.com/spf13/pflag v1.0.5
2122
github.com/vishvananda/netlink v1.0.0
2223
github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f

‎go.sum‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:
2525
github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
2626
github.com/bazelbuild/bazel-gazelle v0.0.0-20181012220611-c728ce9f663e/go.mod h1:uHBSeeATKpVazAACZBDPL/Nk/UhQDDsJWDlqYJo8/Us=
2727
github.com/bazelbuild/buildtools v0.0.0-20180226164855-80c7f0d45d7e/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
28+
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
2829
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
2930
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
3031
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
@@ -223,6 +224,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
223224
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
224225
github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
225226
github.com/mattn/go-shellwords v0.0.0-20180605041737-f8471b0a71de/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
227+
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
226228
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
227229
github.com/mesos/mesos-go v0.0.9/go.mod h1:kPYCMQ9gsOXVAle1OsoY4I1+9kPu8GHkf88aV59fDr4=
228230
github.com/mholt/caddy v0.0.0-20180213163048-2de495001514/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
@@ -275,9 +277,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
275277
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
276278
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
277279
github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
280+
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
278281
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
282+
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
279283
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
284+
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
280285
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
286+
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
281287
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
282288
github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
283289
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=

‎pkg/ipam/floatingip/ipam.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"fmt"
2121
"net"
2222

23+
"github.com/prometheus/client_golang/prometheus"
2324
"k8s.io/apimachinery/pkg/util/sets"
2425
"tkestack.io/galaxy/pkg/api/galaxy/constant"
2526
)
@@ -63,6 +64,8 @@ type IPAM interface {
6364
NodeSubnetsByKey(key string) (sets.String, error)
6465
// Name returns IPAM's name.
6566
Name() string
67+
// implements metrics Collector interface
68+
prometheus.Collector
6669
}
6770

6871
// FloatingIPInfo is floatingIP information

‎pkg/ipam/floatingip/ipam_crd.go‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"sync"
2525
"time"
2626

27+
"github.com/prometheus/client_golang/prometheus"
2728
"k8s.io/apimachinery/pkg/util/sets"
2829
"k8s.io/client-go/tools/cache"
2930
glog "k8s.io/klog"
@@ -59,9 +60,11 @@ type crdIpam struct {
5960
ipType Type
6061
//caches for FloatingIP crd, both stores allocated FloatingIPs and unallocated FloatingIPs
6162
cacheLock *sync.RWMutex
62-
// key is FloatingIP name (ip typed as uint32)
63+
// key is ip string
6364
allocatedFIPs map[string]*FloatingIP
6465
unallocatedFIPs map[string]*FloatingIP
66+
67+
ipCounterDesc *prometheus.Desc
6568
}
6669

6770
// NewCrdIPAM init IPAM struct.
@@ -72,6 +75,8 @@ func NewCrdIPAM(fipClient crd_clientset.Interface, ipType Type, informer crdInfo
7275
cacheLock: new(sync.RWMutex),
7376
allocatedFIPs: make(map[string]*FloatingIP),
7477
unallocatedFIPs: make(map[string]*FloatingIP),
78+
ipCounterDesc: prometheus.NewDesc("galaxy_ip_counter", "Galaxy floating ip counter",
79+
[]string{"type", "subnet", "first_ip"}, nil),
7580
}
7681
// manually creating and fip to reserve it
7782
if informer != nil {
@@ -257,9 +262,13 @@ func (ci *crdIpam) Release(key string, ip net.IP) error {
257262

258263
// First returns the first matched IP by key.
259264
func (ci *crdIpam) First(key string) (*FloatingIPInfo, error) {
260-
fip, err := ci.findFloatingIPByKey(key)
261-
if err != nil {
262-
return nil, err
265+
ci.cacheLock.RLock()
266+
defer ci.cacheLock.RUnlock()
267+
var fip *FloatingIP
268+
for _, spec := range ci.allocatedFIPs {
269+
if spec.Key == key {
270+
fip = spec
271+
}
263272
}
264273
if fip == nil {
265274
return nil, nil
@@ -317,6 +326,8 @@ func (ci *crdIpam) ByPrefix(prefix string) ([]FloatingIP, error) {
317326
}
318327

319328
func (ci *crdIpam) NodeSubnet(nodeIP net.IP) *net.IPNet {
329+
ci.cacheLock.RLock()
330+
defer ci.cacheLock.RUnlock()
320331
for j := range ci.FloatingIPs {
321332
nodeSubnets := ci.FloatingIPs[j].NodeSubnets
322333
for k := range nodeSubnets {
@@ -463,17 +474,6 @@ func (ci *crdIpam) syncCacheAfterDel(released *FloatingIP) {
463474
return
464475
}
465476

466-
func (ci *crdIpam) findFloatingIPByKey(key string) (*FloatingIP, error) {
467-
ci.cacheLock.RLock()
468-
defer ci.cacheLock.RUnlock()
469-
for _, spec := range ci.allocatedFIPs {
470-
if spec.Key == key {
471-
return spec, nil
472-
}
473-
}
474-
return nil, nil
475-
}
476-
477477
func (ci *crdIpam) filterAllocatedSubnet(key string) sets.String {
478478
//key would not be empty
479479
subnetSet := sets.NewString()
@@ -548,3 +548,45 @@ func (ci *crdIpam) ReleaseIPs(ipToKey map[string]string) (map[string]string, map
548548
}
549549
return deleted, undeleted, nil
550550
}
551+
552+
// Describe sends metrics description to ch
553+
func (ci *crdIpam) Describe(ch chan<- *prometheus.Desc) {
554+
ch <- ci.ipCounterDesc
555+
}
556+
557+
// Collect sends metrics to ch
558+
func (ci *crdIpam) Collect(ch chan<- prometheus.Metric) {
559+
allocated, unallocated := map[string]*FloatingIP{}, map[string]*FloatingIP{}
560+
ci.cacheLock.RLock()
561+
pools := make([]*FloatingIPPool, len(ci.FloatingIPs))
562+
for ipStr, fip := range ci.allocatedFIPs {
563+
allocated[ipStr] = fip
564+
}
565+
for ipStr, fip := range ci.unallocatedFIPs {
566+
unallocated[ipStr] = fip
567+
}
568+
for i := range ci.FloatingIPs {
569+
pools[i] = ci.FloatingIPs[i]
570+
}
571+
ci.cacheLock.RUnlock()
572+
for _, pool := range pools {
573+
subnetStr := pool.IPNet().String()
574+
var firstIP string
575+
var allocatedNum float64
576+
for _, ipr := range pool.IPRanges {
577+
firstIP = ipr.First.String()
578+
break
579+
}
580+
for _, fip := range allocated {
581+
if !pool.Contains(fip.IP) {
582+
continue
583+
}
584+
allocatedNum += 1
585+
}
586+
// since subnetStr may be the same for different pools, add a first ip tag
587+
ch <- prometheus.MustNewConstMetric(ci.ipCounterDesc, prometheus.GaugeValue, allocatedNum,
588+
"allocated", subnetStr, firstIP)
589+
ch <- prometheus.MustNewConstMetric(ci.ipCounterDesc, prometheus.GaugeValue, float64(pool.Size()),
590+
"total", subnetStr, firstIP)
591+
}
592+
}

‎pkg/ipam/metrics/metrics.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package metrics
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
)
6+
7+
var (
8+
ScheduleLatency = prometheus.NewHistogramVec(
9+
prometheus.HistogramOpts{
10+
Name: "galaxy_schedule_latency",
11+
Help: "Galaxy schedule latency in seconds",
12+
Buckets: prometheus.ExponentialBuckets(0.1, 2, 7),
13+
}, []string{"func"})
14+
15+
CloudProviderLatency = prometheus.NewHistogramVec(
16+
prometheus.HistogramOpts{
17+
Name: "galaxy_cloud_provider_latency",
18+
Help: "Galaxy cloud provider latency in seconds",
19+
Buckets: prometheus.ExponentialBuckets(0.1, 2, 7),
20+
}, []string{"func"})
21+
)
22+
23+
// MustRegister registers all metrics
24+
func MustRegister() {
25+
prometheus.MustRegister(ScheduleLatency, CloudProviderLatency)
26+
}

‎pkg/ipam/schedulerplugin/cloudprovider.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,21 @@ package schedulerplugin
1818

1919
import (
2020
"fmt"
21+
"time"
2122

2223
glog "k8s.io/klog"
2324
"tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc"
25+
"tkestack.io/galaxy/pkg/ipam/metrics"
2426
)
2527

2628
// cloudProviderAssignIP send assign ip req to cloud provider
2729
func (p *FloatingIPPlugin) cloudProviderAssignIP(req *rpc.AssignIPRequest) error {
2830
if p.cloudProvider == nil {
2931
return nil
3032
}
33+
start := time.Now()
3134
reply, err := p.cloudProvider.AssignIP(req)
35+
metrics.CloudProviderLatency.WithLabelValues("assign").Observe(time.Since(start).Seconds())
3236
if err != nil {
3337
return fmt.Errorf("cloud provider AssignIP reply err %v", err)
3438
}
@@ -47,7 +51,9 @@ func (p *FloatingIPPlugin) cloudProviderUnAssignIP(req *rpc.UnAssignIPRequest) e
4751
if p.cloudProvider == nil {
4852
return nil
4953
}
54+
start := time.Now()
5055
reply, err := p.cloudProvider.UnAssignIP(req)
56+
metrics.CloudProviderLatency.WithLabelValues("unassign").Observe(time.Since(start).Seconds())
5157
if err != nil {
5258
return fmt.Errorf("cloud provider UnAssignIP reply err %v", err)
5359
}

‎pkg/ipam/schedulerplugin/floatingip_plugin.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
"tkestack.io/galaxy/pkg/ipam/cloudprovider"
3737
"tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc"
3838
"tkestack.io/galaxy/pkg/ipam/floatingip"
39+
"tkestack.io/galaxy/pkg/ipam/metrics"
3940
"tkestack.io/galaxy/pkg/ipam/schedulerplugin/util"
4041
)
4142

@@ -148,6 +149,7 @@ func (p *FloatingIPPlugin) updateConfigMap() (bool, error) {
148149
// If the given pod doesn't want floating IP, none failedNodes returns
149150
func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev1.Node, schedulerapi.FailedNodesMap,
150151
error) {
152+
start := time.Now()
151153
failedNodesMap := schedulerapi.FailedNodesMap{}
152154
if !p.hasResourceName(&pod.Spec) {
153155
return nodes, failedNodesMap, nil
@@ -178,6 +180,7 @@ func (p *FloatingIPPlugin) Filter(pod *corev1.Pod, nodes []corev1.Node) ([]corev
178180
}
179181
glog.V(5).Infof("filtered nodes %v failed nodes %v", nodeNames, failedNodesMap)
180182
}
183+
metrics.ScheduleLatency.WithLabelValues("filter").Observe(time.Since(start).Seconds())
181184
return filteredNodes, failedNodesMap, nil
182185
}
183186

@@ -320,6 +323,7 @@ func (p *FloatingIPPlugin) allocateIP(key string, nodeName string, pod *corev1.P
320323

321324
// Bind binds a new floatingip or reuse an old one to pod
322325
func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error {
326+
start := time.Now()
323327
pod, err := p.PluginFactoryArgs.PodLister.Pods(args.PodNamespace).Get(args.PodName)
324328
if err != nil {
325329
return fmt.Errorf("failed to find pod %s: %w", util.Join(args.PodName, args.PodNamespace), err)
@@ -375,6 +379,7 @@ func (p *FloatingIPPlugin) Bind(args *schedulerapi.ExtenderBindingArgs) error {
375379
// If fails to update, depending on resync to update
376380
return fmt.Errorf("update pod %s: %w", keyObj.KeyInDB, err1)
377381
}
382+
metrics.ScheduleLatency.WithLabelValues("bind").Observe(time.Since(start).Seconds())
378383
return nil
379384
}
380385

‎pkg/ipam/server/server.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727

2828
"github.com/emicklei/go-restful"
2929
"github.com/emicklei/go-restful-swagger12"
30+
"github.com/prometheus/client_golang/prometheus"
31+
"github.com/prometheus/client_golang/prometheus/promhttp"
3032
corev1 "k8s.io/api/core/v1"
3133
extensionClient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
3234
"k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,6 +49,7 @@ import (
4749
"tkestack.io/galaxy/pkg/ipam/client/clientset/versioned"
4850
crdInformer "tkestack.io/galaxy/pkg/ipam/client/informers/externalversions"
4951
"tkestack.io/galaxy/pkg/ipam/crd"
52+
"tkestack.io/galaxy/pkg/ipam/metrics"
5053
"tkestack.io/galaxy/pkg/ipam/schedulerplugin"
5154
"tkestack.io/galaxy/pkg/ipam/server/options"
5255
"tkestack.io/galaxy/pkg/utils/httputil"
@@ -345,6 +348,10 @@ func (s *Server) startAPIServer() {
345348
Writes(httputil.Resp{Code: http.StatusOK}))
346349

347350
restful.Add(ws)
351+
// register prometheus metrics
352+
prometheus.MustRegister(s.plugin.GetIpam())
353+
metrics.MustRegister()
354+
restful.DefaultContainer.Handle("/metrics", promhttp.Handler())
348355
addSwaggerUISupport(restful.DefaultContainer)
349356
if err := http.ListenAndServe(fmt.Sprintf("%s:%d", s.Bind, s.APIPort), nil); err != nil {
350357
glog.Fatalf("unable to listen: %v.", err)

0 commit comments

Comments
 (0)