From b6acb939004710141b08170d07dcfbe3db923347 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli Date: Tue, 8 Dec 2020 13:45:47 +0100 Subject: [PATCH] feat: exclude nodes which are not schedulable from neighbour checks --- go.mod | 4 +- go.sum | 27 +++++++-- main.go | 62 ++++++++++++++++++--- pkg/checker/checker.go | 43 +++++++++------ pkg/checker/types.go | 2 + pkg/kubediscovery/kubediscovery.go | 79 ++++++++++++++++----------- pkg/kubediscovery/nodewatcher.go | 79 +++++++++++++++++++++++++++ pkg/kubediscovery/nodewatcher_test.go | 45 +++++++++++++++ 8 files changed, 277 insertions(+), 64 deletions(-) create mode 100644 pkg/kubediscovery/nodewatcher.go create mode 100644 pkg/kubediscovery/nodewatcher_test.go diff --git a/go.mod b/go.mod index 72fe79dc..bc6e3466 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.15 require ( github.com/prometheus/client_golang v1.8.0 - k8s.io/api v0.19.4 // indirect + github.com/stretchr/testify v1.4.0 + k8s.io/api v0.19.4 k8s.io/apimachinery v0.19.4 k8s.io/client-go v0.19.4 - k8s.io/klog v1.0.0 // indirect k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect ) diff --git a/go.sum b/go.sum index 140b4511..e14d0408 100644 --- a/go.sum +++ b/go.sum @@ -84,11 +84,13 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -117,6 +119,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -140,6 +143,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -150,12 +154,12 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.3 h1:2qsuRm+bzgwSIKikigPASa2GhW8H2Dn4Qq7UxD8K/48= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -180,11 +184,13 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -211,8 +217,10 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= @@ -255,9 +263,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -277,8 +287,10 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -327,6 +339,7 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -335,6 +348,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -485,6 +499,7 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -497,6 +512,7 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -533,14 +549,17 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -563,15 +582,11 @@ k8s.io/apimachinery v0.19.4 h1:+ZoddM7nbzrDCp0T3SWnyxqf8cbWPT2fkZImoyvHUG0= k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/client-go v0.19.4 h1:85D3mDNoLF+xqpyE9Dh/OtrJDyJrSRKkHmDXIbEzer8= k8s.io/client-go v0.19.4/go.mod h1:ZrEy7+wj9PjH5VMBCuu/BDlvtUAku0oVFk4MmnW9mWA= -k8s.io/client-go v1.5.1 h1:XaX/lo2/u3/pmFau8HN+sB5C/b4dc4Dmm2eXjBH4p1E= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= diff --git a/main.go b/main.go index 74bd4682..90ad90f0 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" @@ -10,7 +11,9 @@ import ( "log" "net/http" "os" + "os/signal" "strconv" + "syscall" "time" "github.com/postfinance/kubenurse/pkg/checker" @@ -24,10 +27,39 @@ const ( ) func main() { - // Setup http transport + mux := http.NewServeMux() + server := http.Server{ + Addr: ":8080", + Handler: mux, + } + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + select { + case s := <-sig: + log.Printf("shutting down, received signal %s", s) + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + log.Fatalln(err) + } + + cancel() + case <-ctx.Done(): + } + }() + + // setup http transport transport, err := GenerateRoundTripper() if err != nil { log.Printf("using default transport: %s", err) + transport = http.DefaultTransport } @@ -36,8 +68,12 @@ func main() { Transport: transport, } - // Setup checker - chk := checker.New(client, 3*time.Second) + // setup checker + chk, err := checker.New(ctx, client, 3*time.Second) + if err != nil { + log.Fatalln(err) + } + chk.KubenurseIngressURL = os.Getenv("KUBENURSE_INGRESS_URL") chk.KubenurseServiceURL = os.Getenv("KUBENURSE_SERVICE_URL") chk.KubernetesServiceHost = os.Getenv("KUBERNETES_SERVICE_HOST") @@ -45,11 +81,11 @@ func main() { chk.KubenurseNamespace = os.Getenv("KUBENURSE_NAMESPACE") chk.NeighbourFilter = os.Getenv("KUBENURSE_NEIGHBOUR_FILTER") - // Setup http routes - http.HandleFunc("/alive", aliveHandler(chk)) - http.HandleFunc("/alwayshappy", func(http.ResponseWriter, *http.Request) {}) - http.Handle("/metrics", promhttp.Handler()) - http.Handle("/", http.RedirectHandler("/alive", http.StatusMovedPermanently)) + // setup http routes + mux.HandleFunc("/alive", aliveHandler(chk)) + mux.HandleFunc("/alwayshappy", func(http.ResponseWriter, *http.Request) {}) + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/", http.RedirectHandler("/alive", http.StatusMovedPermanently)) fmt.Println(nurse) // most important line of this project @@ -59,7 +95,15 @@ func main() { log.Fatalln("checker exited") }() - log.Fatal(http.ListenAndServe(":8080", nil)) + go func() { + if err := server.ListenAndServe(); err != nil { + if err != http.ErrServerClosed { + log.Fatalln(err) + } + } + }() + + <-ctx.Done() } func aliveHandler(chk *checker.Checker) func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/checker/checker.go b/pkg/checker/checker.go index 76da4c33..efffd3ad 100644 --- a/pkg/checker/checker.go +++ b/pkg/checker/checker.go @@ -1,6 +1,8 @@ +// Package checker implements the checks the kubenurse performs. package checker import ( + "context" "fmt" "log" "net/http" @@ -12,11 +14,17 @@ import ( // New configures the checker with a httpClient and a cache timeout for check // results. Other parameters of the Checker struct need to be configured seperately. -func New(httpClient *http.Client, cacheTtl time.Duration) *Checker { +func New(ctx context.Context, httpClient *http.Client, cacheTTL time.Duration) (*Checker, error) { + discovery, err := kubediscovery.New(ctx) + if err != nil { + return nil, fmt.Errorf("create k8s discovery client: %w", err) + } + return &Checker{ + discovery: discovery, httpClient: httpClient, - cacheTTL: cacheTtl, - } + cacheTTL: cacheTTL, + }, nil } // Run runs an check and returns the result togeter with a boolean, if it wasn't @@ -34,10 +42,10 @@ func (c *Checker) Run() (Result, bool) { // Run Checks res := Result{} - res.APIServerDirect, err = measure(c.ApiServerDirect, "api_server_direct") + res.APIServerDirect, err = measure(c.APIServerDirect, "api_server_direct") haserr = haserr || (err != nil) - res.APIServerDNS, err = measure(c.ApiServerDNS, "api_server_dns") + res.APIServerDNS, err = measure(c.APIServerDNS, "api_server_dns") haserr = haserr || (err != nil) res.MeIngress, err = measure(c.MeIngress, "me_ingress") @@ -46,7 +54,7 @@ func (c *Checker) Run() (Result, bool) { res.MeService, err = measure(c.MeService, "me_service") haserr = haserr || (err != nil) - res.Neighbourhood, err = kubediscovery.GetNeighbourhood(c.KubenurseNamespace, c.NeighbourFilter) + res.Neighbourhood, err = c.discovery.GetNeighbours(context.TODO(), c.KubenurseNamespace, c.NeighbourFilter) haserr = haserr || (err != nil) // Neighbourhood special error treating @@ -66,21 +74,21 @@ func (c *Checker) Run() (Result, bool) { } // RunScheduled runs the check run in the specified interval which can be used -// to keep the metrics up-to-date +// to keep the metrics up-to-date. func (c *Checker) RunScheduled(d time.Duration) { for range time.Tick(d) { c.Run() } } -// ApiServerDirect checks the /version endpoint of the Kubernetes API Server through the direct link -func (c *Checker) ApiServerDirect() (string, error) { +// APIServerDirect checks the /version endpoint of the Kubernetes API Server through the direct link +func (c *Checker) APIServerDirect() (string, error) { apiurl := fmt.Sprintf("https://%s:%s/version", c.KubernetesServiceHost, c.KubernetesServicePort) return c.doRequest(apiurl) } -// ApiServerDNS checks the /version endpoint of the Kubernetes API Server through the Cluster DNS URL -func (c *Checker) ApiServerDNS() (string, error) { +// APIServerDNS checks the /version endpoint of the Kubernetes API Server through the Cluster DNS URL +func (c *Checker) APIServerDNS() (string, error) { apiurl := fmt.Sprintf("https://kubernetes.default.svc.cluster.local:%s/version", c.KubernetesServicePort) return c.doRequest(apiurl) } @@ -95,14 +103,17 @@ func (c *Checker) MeService() (string, error) { return c.doRequest(c.KubenurseServiceURL + "/alwayshappy") } -// checkNeighbours checks every provided neighbour at the /alwayshappy endpoint +// checkNeighbours checks the /alwayshappy endpoint from every discovered kubenurse neighbour. Neighbour pods on nodes +// which are not schedulable are excluded from this check to avoid possible false errors. func (c *Checker) checkNeighbours(nh []kubediscovery.Neighbour) { for _, neighbour := range nh { - check := func() (string, error) { - return c.doRequest("http://" + neighbour.PodIP + ":8080/alwayshappy") - } + if neighbour.NodeSchedulable { + check := func() (string, error) { + return c.doRequest("http://" + neighbour.PodIP + ":8080/alwayshappy") + } - measure(check, "path_"+neighbour.NodeName) + measure(check, "path_"+neighbour.NodeName) + } } } diff --git a/pkg/checker/types.go b/pkg/checker/types.go index 3db21001..546ded65 100644 --- a/pkg/checker/types.go +++ b/pkg/checker/types.go @@ -21,6 +21,8 @@ type Checker struct { KubenurseNamespace string NeighbourFilter string + discovery *kubediscovery.Client + // Http Client for https requests httpClient *http.Client diff --git a/pkg/kubediscovery/kubediscovery.go b/pkg/kubediscovery/kubediscovery.go index 10d8b44f..388996ab 100644 --- a/pkg/kubediscovery/kubediscovery.go +++ b/pkg/kubediscovery/kubediscovery.go @@ -1,7 +1,9 @@ +// Package kubediscovery implements a discovery mechanism to find other k8s resources. package kubediscovery import ( "context" + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -9,55 +11,70 @@ import ( "k8s.io/client-go/rest" ) -func getClientset() (*kubernetes.Clientset, error) { - // create in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - return nil, err - } - - // create clientset - return kubernetes.NewForConfig(config) +// Client provides the kubediscovery client methods. +type Client struct { + k8s kubernetes.Interface + nodeCache *nodeCache } // Neighbour represents a kubenurse which should be reachable type Neighbour struct { - PodName string - PodIP string - HostIP string - NodeName string - Phase string // Pod Phase + PodName string + PodIP string + HostIP string + NodeName string + NodeSchedulable bool + Phase string // Pod Phase } -// GetNeighbourhood returns a slice of neighbour kubenurses for the given namespace -// and labelSelector -func GetNeighbourhood(namespace, labelSelector string) ([]Neighbour, error) { - clientset, err := getClientset() +// New creates a new kubediscovery client. The context is used to stop the k8s watchers/informers. +func New(ctx context.Context) (*Client, error) { + // create in-cluster config + config, err := rest.InClusterConfig() if err != nil { - return nil, err + return nil, fmt.Errorf("creating in-cluster configuration: %w", err) } + cliset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("creating clientset: %w", err) + } + + nodeCache, err := watchNodes(ctx, cliset) + if err != nil { + return nil, fmt.Errorf("starting node watcher: %w", err) + } + + return &Client{ + k8s: cliset, + nodeCache: nodeCache, + }, nil +} + +// GetNeighbours returns a slice of neighbour kubenurses for the given namespace and labelSelector. +func (c *Client) GetNeighbours(ctx context.Context, namespace, labelSelector string) ([]Neighbour, error) { // Get all pods - pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + pods, err := c.k8s.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ LabelSelector: labelSelector, }) - if err != nil { - return nil, err + return nil, fmt.Errorf("list pods: %w", err) } - // Process pods - res := []Neighbour{} + var neighbours []Neighbour + + // process pods for _, pod := range pods.Items { n := Neighbour{ - PodName: pod.Name, - PodIP: pod.Status.PodIP, - HostIP: pod.Status.HostIP, - Phase: string(pod.Status.Phase), - NodeName: pod.Spec.NodeName, + PodName: pod.Name, + PodIP: pod.Status.PodIP, + HostIP: pod.Status.HostIP, + Phase: string(pod.Status.Phase), + NodeName: pod.Spec.NodeName, + NodeSchedulable: c.nodeCache.isSchedulable(pod.Spec.NodeName), } - res = append(res, n) + neighbours = append(neighbours, n) } - return res, nil + return neighbours, nil } diff --git a/pkg/kubediscovery/nodewatcher.go b/pkg/kubediscovery/nodewatcher.go new file mode 100644 index 00000000..4f63a00e --- /dev/null +++ b/pkg/kubediscovery/nodewatcher.go @@ -0,0 +1,79 @@ +package kubediscovery + +import ( + "context" + "fmt" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + resyncPeriod = time.Hour * 1 +) + +type nodeCache struct { + nodes map[string]bool + mu *sync.RWMutex +} + +// watchNodes starts an informer to watch v1.Node resource, the context can be used to stop the informer +func watchNodes(ctx context.Context, client kubernetes.Interface) (*nodeCache, error) { + nc := nodeCache{ + nodes: make(map[string]bool), + mu: new(sync.RWMutex), + } + + informer := informers.NewSharedInformerFactory(client, resyncPeriod).Core().V1().Nodes().Informer() + + informer.AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: nc.add, + UpdateFunc: nc.update, + DeleteFunc: nc.delete, + }, + ) + + go informer.Run(ctx.Done()) + + if ok := cache.WaitForCacheSync(ctx.Done(), informer.HasSynced); !ok { + return nil, fmt.Errorf("watching nodes: initial cache sync not successful") + } + + return &nc, nil +} + +func (nc *nodeCache) add(obj interface{}) { + node := obj.(*corev1.Node) + + nc.mu.Lock() + nc.nodes[node.Name] = node.Spec.Unschedulable + nc.mu.Unlock() +} + +func (nc *nodeCache) delete(obj interface{}) { + node := obj.(*corev1.Node) + + nc.mu.Lock() + delete(nc.nodes, node.Name) + nc.mu.Unlock() +} + +func (nc *nodeCache) update(_ interface{}, obj interface{}) { + node := obj.(*corev1.Node) + + nc.mu.Lock() + nc.nodes[node.Name] = node.Spec.Unschedulable + nc.mu.Unlock() +} + +func (nc *nodeCache) isSchedulable(node string) bool { + nc.mu.RLock() + defer nc.mu.RUnlock() + + return !nc.nodes[node] +} diff --git a/pkg/kubediscovery/nodewatcher_test.go b/pkg/kubediscovery/nodewatcher_test.go new file mode 100644 index 00000000..58a8053e --- /dev/null +++ b/pkg/kubediscovery/nodewatcher_test.go @@ -0,0 +1,45 @@ +package kubediscovery + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestNodeWatcher(t *testing.T) { + node := &corev1.Node{} + node.Name = "testnode" + node.Spec.Unschedulable = false + + r := require.New(t) + fakeClient := fake.NewSimpleClientset() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // start the informer + nc, err := watchNodes(ctx, fakeClient) + r.NoError(err) + r.NotNil(nc) + + _, err = fakeClient.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + r.NoError(err) + r.True(nc.isSchedulable(node.Name), "node is schedulable") + + node.Spec.Unschedulable = true + + _, err = fakeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + r.NoError(err) + time.Sleep(100 * time.Millisecond) // the informer needs some time... + r.False(nc.isSchedulable(node.Name), "node is not schedulable") + + err = fakeClient.CoreV1().Nodes().Delete(ctx, node.Name, metav1.DeleteOptions{}) + r.NoError(err) + + r.True(nc.isSchedulable("unknown"), "node not in cache") +}