From 7beac30751e8fd96093486fad9ada30f314e7dc4 Mon Sep 17 00:00:00 2001 From: Thomas Gosteli <22346145+ghouscht@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:31:15 +0100 Subject: [PATCH] refactor!: rewrite and cleanup kubenurse server code (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor!: rewrite and cleanup kubenurse server code By using a package and multiple separate files the code is easier to understand and test. A new /ready handler was added so we can configure a readiness probe to allow seamless updates of kubenurse. * build: update golangci-lint version * build: update golangci-lint timeout, default is too short * build: extract lint step and use go version 1.17 * feat: configure new readinessprobe in kustomize and helm templates * fix: linter errors * chore: cleanup, remove not needed WaitGroup * refactor!: move pkg/kubediscovery to internal/kubediscovery * refactor!: move pkg/checker to internal/servicecheck * refactor!: incorporate pkg/metrics in internal/servicecheck * refactor!: more refactorings to allow easier unit testing * feat: more unit tests and coverage calculation in workflows * docs: include ci and coverage badges in readme * docs: fix coverage status URL Co-authored-by: Clément Nussbaumer * chore: embed servicecheck.Result in /alive output for simplicity Co-authored-by: Clément Nussbaumer --- .github/workflows/ci-helm-deploy.yml | 6 +- .github/workflows/ci-kustomize-deploy.yml | 6 +- .github/workflows/helm-lint.yml | 12 - .github/workflows/lint.yml | 37 +++ .github/workflows/release.yml | 4 +- .golangci.yml | 48 ++-- README.md | 3 + examples/daemonset.yaml | 9 + go.mod | 40 +++- helm/kubenurse/templates/daemonset.yaml | 9 + .../kubediscovery/kubediscovery.go | 19 +- internal/kubediscovery/kubediscovery_test.go | 55 +++++ .../kubediscovery/nodewatcher.go | 0 .../kubediscovery/nodewatcher_test.go | 0 internal/kubenurse/handler.go | 65 +++++ internal/kubenurse/handler_test.go | 62 +++++ internal/kubenurse/server.go | 202 ++++++++++++++++ internal/kubenurse/server_test.go | 39 +++ internal/kubenurse/transport.go | 55 +++++ .../servicecheck}/cache.go | 2 +- .../servicecheck/servicecheck.go | 85 +++++-- internal/servicecheck/servicecheck_test.go | 69 ++++++ .../servicecheck}/transport.go | 14 +- .../servicecheck}/types.go | 12 +- main.go | 225 +++--------------- pkg/metrics/metrics.go | 37 --- 26 files changed, 788 insertions(+), 327 deletions(-) delete mode 100644 .github/workflows/helm-lint.yml create mode 100644 .github/workflows/lint.yml rename {pkg => internal}/kubediscovery/kubediscovery.go (87%) create mode 100644 internal/kubediscovery/kubediscovery_test.go rename {pkg => internal}/kubediscovery/nodewatcher.go (100%) rename {pkg => internal}/kubediscovery/nodewatcher_test.go (100%) create mode 100644 internal/kubenurse/handler.go create mode 100644 internal/kubenurse/handler_test.go create mode 100644 internal/kubenurse/server.go create mode 100644 internal/kubenurse/server_test.go create mode 100644 internal/kubenurse/transport.go rename {pkg/checker => internal/servicecheck}/cache.go (96%) rename pkg/checker/checker.go => internal/servicecheck/servicecheck.go (58%) create mode 100644 internal/servicecheck/servicecheck_test.go rename {pkg/checker => internal/servicecheck}/transport.go (69%) rename {pkg/checker => internal/servicecheck}/types.go (83%) delete mode 100644 pkg/metrics/metrics.go diff --git a/.github/workflows/ci-helm-deploy.yml b/.github/workflows/ci-helm-deploy.yml index 7c13ea0d..252616bb 100644 --- a/.github/workflows/ci-helm-deploy.yml +++ b/.github/workflows/ci-helm-deploy.yml @@ -1,5 +1,5 @@ --- -name: ci-helm-deploy +name: deploy with helm on: push: pull_request: @@ -11,10 +11,8 @@ jobs: uses: actions/checkout@v2 - name: Setup Go uses: actions/setup-go@v2 - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 with: - version: v1.32 + go-version: 1.17 - name: GoReleaser uses: goreleaser/goreleaser-action@v2 with: diff --git a/.github/workflows/ci-kustomize-deploy.yml b/.github/workflows/ci-kustomize-deploy.yml index 86853a59..f3ac4d9a 100644 --- a/.github/workflows/ci-kustomize-deploy.yml +++ b/.github/workflows/ci-kustomize-deploy.yml @@ -1,5 +1,5 @@ --- -name: ci-kustomize-deploy +name: deploy with kustomize on: push: pull_request: @@ -11,10 +11,8 @@ jobs: uses: actions/checkout@v2 - name: Setup Go uses: actions/setup-go@v2 - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 with: - version: v1.32 + go-version: 1.17 - name: GoReleaser uses: goreleaser/goreleaser-action@v2 with: diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml deleted file mode 100644 index 099bed27..00000000 --- a/.github/workflows/helm-lint.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: Yaml lint helm chart -on: [push, pull_request] -jobs: - lintAllTheThings: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Helm lint - shell: bash - run: | - helm lint ./helm/kubenurse/ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..e28cd035 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,37 @@ +--- +name: lint and test +on: + push: + pull_request: +jobs: + lint-go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: golangci/golangci-lint-action@v2 + with: + version: v1.43 + args: --timeout 5m + lint-helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Helm lint + shell: bash + run: | + helm lint ./helm/kubenurse/ + test-go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.17 + - name: Run unit tests + run: go test -race -covermode atomic -coverprofile=profile.cov ./... + - name: Send coverage report + env: + COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + go install github.com/mattn/goveralls@v0.0.11 + goveralls -coverprofile=profile.cov -service=github diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d437da22..59e76a18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,10 +12,8 @@ jobs: uses: actions/checkout@v2 - name: Setup Go uses: actions/setup-go@v2 - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 with: - version: v1.32 + go-version: 1.17 - name: Login to DockerHub uses: docker/login-action@v1 with: diff --git a/.golangci.yml b/.golangci.yml index da47a286..ada0007f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,19 +1,20 @@ -# options for analysis running +--- run: - tests: true - timeout: 3m - - -# all available settings of specific linters + tests: false + skip-dirs: + - .github + - build + - web + - .go linters-settings: dupl: - threshold: 150 + threshold: 100 funlen: lines: 100 statements: 50 goconst: min-len: 2 - min-occurrences: 3 + min-occurrences: 2 gocritic: enabled-tags: - diagnostic @@ -23,21 +24,27 @@ linters-settings: - style disabled-checks: - whyNoLint + - hugeParam gocyclo: min-complexity: 15 - golint: + revive: min-confidence: 0.8 govet: check-shadowing: true - maligned: - suggest-new: true + lll: + line-length: 140 misspell: locale: UK + nolintlint: + allow-leading-space: false + require-explanation: true + allow-no-explanation: + - gocognit + - funlen + - gocyclo linters: - disable: - - gomnd - - lll + disable-all: true enable: - bodyclose - deadcode @@ -46,28 +53,26 @@ linters: - dupl - errcheck - funlen + - nolintlint - gochecknoglobals - gochecknoinits - gocognit - goconst - gocritic - gocyclo - - godox - gofmt - goimports - - golint + - revive - goprintffuncname - gosec - gosimple - govet - ineffassign - - interfacer - - maligned - misspell - nakedret - prealloc - rowserrcheck - - scopelint # todo + - exportloopref - staticcheck - structcheck - stylecheck @@ -78,12 +83,11 @@ linters: - varcheck - whitespace - wsl - - issues: exclude: # Very commonly not checked. - - 'Error return value of .(l.Sync|.*Close|.*Flush|os\.Remove(All)?|os\.(Un)?Setenv). is not checked' + - 'Error return value of .(l.Sync|.*Close|.*.Write|.*Flush|os\.Remove(All)?|os\.(Un)?Setenv). is not checked' + - 'G104:.*' - 'exported method (.*\.MarshalJSON|.*\.UnmarshalJSON) should have comment or be unexported' - 'shadow: declaration of "err" shadows declaration.*' max-same-issues: 0 diff --git a/README.md b/README.md index cbf9887c..a19b4d41 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![CI](https://github.com/postfinance/kubenurse/actions/workflows/release.yml/badge.svg)](https://github.com/postfinance/kubenurse/actions/workflows/release.yml) +[![Coverage Status](https://coveralls.io/repos/github/postfinance/kubenurse/badge.svg?branch=master)](https://coveralls.io/github/postfinance/kubenurse?branch=master) + # Kubenurse kubenurse is a little service that monitors all network connections in a kubernetes cluster and exports the taken metrics as prometheus endpoint. diff --git a/examples/daemonset.yaml b/examples/daemonset.yaml index bd9a7f11..61a1502e 100644 --- a/examples/daemonset.yaml +++ b/examples/daemonset.yaml @@ -36,6 +36,15 @@ spec: ports: - containerPort: 8080 protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /ready + port: 8080 + scheme: HTTP + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master diff --git a/go.mod b/go.mod index 94920339..9f973248 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/postfinance/kubenurse -go 1.15 +go 1.17 require ( github.com/prometheus/client_golang v1.11.0 @@ -9,3 +9,41 @@ require ( k8s.io/apimachinery v0.22.2 k8s.io/client-go v0.22.2 ) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/evanphx/json-patch v4.11.0+incompatible // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + google.golang.org/appengine v1.6.5 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/helm/kubenurse/templates/daemonset.yaml b/helm/kubenurse/templates/daemonset.yaml index 7aa315fe..0e689e0b 100644 --- a/helm/kubenurse/templates/daemonset.yaml +++ b/helm/kubenurse/templates/daemonset.yaml @@ -45,6 +45,15 @@ spec: ports: - containerPort: 8080 protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /ready + port: 8080 + scheme: HTTP + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master diff --git a/pkg/kubediscovery/kubediscovery.go b/internal/kubediscovery/kubediscovery.go similarity index 87% rename from pkg/kubediscovery/kubediscovery.go rename to internal/kubediscovery/kubediscovery.go index 6b73c834..fe1483da 100644 --- a/pkg/kubediscovery/kubediscovery.go +++ b/internal/kubediscovery/kubediscovery.go @@ -8,7 +8,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) // Client provides the kubediscovery client methods. @@ -42,19 +41,11 @@ type Neighbour struct { // New creates a new kubediscovery client. The context is used to stop the k8s watchers/informers. // When allowUnschedulable is true, no node watcher is created and kubenurses // on unschedulable nodes are considered as neighbours. -func New(ctx context.Context, allowUnschedulable bool) (*Client, error) { - // create in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - return nil, fmt.Errorf("creating in-cluster configuration: %w", err) - } - - cliset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("creating clientset: %w", err) - } - - var nc *nodeCache +func New(ctx context.Context, cliset kubernetes.Interface, allowUnschedulable bool) (*Client, error) { + var ( + nc *nodeCache + err error + ) // Watch nodes only if we do not consider kubenurses on unschedulable nodes if !allowUnschedulable { diff --git a/internal/kubediscovery/kubediscovery_test.go b/internal/kubediscovery/kubediscovery_test.go new file mode 100644 index 00000000..64a91b90 --- /dev/null +++ b/internal/kubediscovery/kubediscovery_test.go @@ -0,0 +1,55 @@ +package kubediscovery + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +var ( + kubenursePod = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubenurse-dummy", + Labels: map[string]string{ + "app": "kubenurse", + }, + }, + } + differentPod = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "different", + Labels: map[string]string{ + "app": "different", + }, + }, + } +) + +func TestGetNeighbours(t *testing.T) { + r := require.New(t) + fakeClient := fake.NewSimpleClientset() + + createFakePods(fakeClient) + + client, err := New(context.Background(), fakeClient, false) + r.NoError(err) + + neighbours, err := client.GetNeighbours(context.Background(), "kube-system", "app=kubenurse") + r.NoError(err) + r.Len(neighbours, 1) + r.Equal(kubenursePod.ObjectMeta.Name, neighbours[0].PodName) +} + +func createFakePods(k8s kubernetes.Interface) { + for _, pod := range []v1.Pod{kubenursePod, differentPod} { + _, err := k8s.CoreV1().Pods("kube-system").Create(context.Background(), &pod, metav1.CreateOptions{}) + if err != nil { + panic(err) + } + } +} diff --git a/pkg/kubediscovery/nodewatcher.go b/internal/kubediscovery/nodewatcher.go similarity index 100% rename from pkg/kubediscovery/nodewatcher.go rename to internal/kubediscovery/nodewatcher.go diff --git a/pkg/kubediscovery/nodewatcher_test.go b/internal/kubediscovery/nodewatcher_test.go similarity index 100% rename from pkg/kubediscovery/nodewatcher_test.go rename to internal/kubediscovery/nodewatcher_test.go diff --git a/internal/kubenurse/handler.go b/internal/kubenurse/handler.go new file mode 100644 index 00000000..9a1ef0d9 --- /dev/null +++ b/internal/kubenurse/handler.go @@ -0,0 +1,65 @@ +package kubenurse + +import ( + "encoding/json" + "net/http" + "os" + + "github.com/postfinance/kubenurse/internal/kubediscovery" + "github.com/postfinance/kubenurse/internal/servicecheck" +) + +func (s *Server) readyHandler() func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.ready { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + } +} + +func (s *Server) aliveHandler() func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + type Output struct { + Hostname string `json:"hostname"` + Headers map[string][]string `json:"headers"` + UserAgent string `json:"user_agent"` + RequestURI string `json:"request_uri"` + RemoteAddr string `json:"remote_addr"` + + // checker.Result + servicecheck.Result + + // kubediscovery + NeighbourhoodState string `json:"neighbourhood_state"` + Neighbourhood []kubediscovery.Neighbour `json:"neighbourhood"` + } + + // Run checks now + res, haserr := s.checker.Run() + if haserr { + w.WriteHeader(http.StatusInternalServerError) + } + + // Add additional data + out := Output{ + Result: res, + Headers: r.Header, + UserAgent: r.UserAgent(), + RequestURI: r.RequestURI, + RemoteAddr: r.RemoteAddr, + Neighbourhood: res.Neighbourhood, + NeighbourhoodState: res.NeighbourhoodState, + } + out.Hostname, _ = os.Hostname() + + // Generate output output + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + _ = enc.Encode(out) + } +} diff --git a/internal/kubenurse/handler_test.go b/internal/kubenurse/handler_test.go new file mode 100644 index 00000000..5bf65092 --- /dev/null +++ b/internal/kubenurse/handler_test.go @@ -0,0 +1,62 @@ +package kubenurse + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes/fake" +) + +func TestServerHandler(t *testing.T) { + r := require.New(t) + + fakeClient := fake.NewSimpleClientset() + + kubenurse, err := New(context.Background(), fakeClient) + r.NoError(err) + r.NotNil(kubenurse) + + ts := httptest.NewServer(kubenurse.http.Handler) + defer ts.Close() + + var tests = map[string]struct { + wantCode int + }{ + "/": { + wantCode: http.StatusMovedPermanently, + }, + "/ready": { + wantCode: http.StatusOK, + }, + "/alive": { + // 500 since servicechecks won't work + wantCode: http.StatusInternalServerError, + }, + "/alwayshappy": { + wantCode: http.StatusOK, + }, + // TODO: also test that metrics are present + "/metrics": { + wantCode: http.StatusOK, + }, + } + + testClient := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + for path, tc := range tests { + t.Run(path, func(t *testing.T) { + r := require.New(t) + + res, err := testClient.Get(ts.URL + path) + r.NoError(err) + r.Equal(tc.wantCode, res.StatusCode) + }) + } +} diff --git a/internal/kubenurse/server.go b/internal/kubenurse/server.go new file mode 100644 index 00000000..b337d57a --- /dev/null +++ b/internal/kubenurse/server.go @@ -0,0 +1,202 @@ +// Package kubenurse contains the server code for the kubenurse service. +package kubenurse + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "sync" + "time" + + "github.com/postfinance/kubenurse/internal/kubediscovery" + "github.com/postfinance/kubenurse/internal/servicecheck" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/kubernetes" +) + +// Server is used to build the kubenurse http/https server(s). +type Server struct { + http http.Server + https http.Server + + checker *servicecheck.Checker + + // Configuration options + useTLS bool + // If we want to consider kubenurses on unschedulable nodes + allowUnschedulable bool + extraCA string + insecure bool + + // Mutex to protect ready flag + mu *sync.Mutex + ready bool +} + +// New creates a new kubenurse server. The server can be configured with the following environment variables: +// * KUBENURSE_USE_TLS +// * KUBENURSE_ALLOW_UNSCHEDULABL +// * KUBENURSE_INGRESS_URL +// * KUBENURSE_SERVICE_URL +// * KUBERNETES_SERVICE_HOST +// * KUBERNETES_SERVICE_PORT +// * KUBENURSE_NAMESPACE +// * KUBENURSE_NEIGHBOUR_FILTER +// * KUBENURSE_EXTRA_CA +// * KUBENURSE_INSECURE +func New(ctx context.Context, k8s kubernetes.Interface) (*Server, error) { + mux := http.NewServeMux() + + server := &Server{ + http: http.Server{ + Addr: ":8080", + Handler: mux, + }, + https: http.Server{ + Addr: ":8443", + Handler: mux, + }, + + //nolint:goconst // No need to make "true" a constant in my opinion, readability is better like this. + useTLS: os.Getenv("KUBENURSE_USE_TLS") == "true", + allowUnschedulable: os.Getenv("KUBENURSE_ALLOW_UNSCHEDULABLE") == "true", + extraCA: os.Getenv("KUBENURSE_EXTRA_CA"), + insecure: os.Getenv("KUBENURSE_INSECURE") == "true", + + mu: new(sync.Mutex), + ready: true, + } + + promRegistry := prometheus.NewRegistry() + promRegistry.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + + // setup http transport + transport, err := server.generateRoundTripper() + if err != nil { + log.Printf("using default transport: %s", err) + + transport = http.DefaultTransport + } + + httpClient := &http.Client{ + Timeout: 5 * time.Second, + Transport: transport, + } + + discovery, err := kubediscovery.New(ctx, k8s, server.allowUnschedulable) + if err != nil { + return nil, fmt.Errorf("create k8s discovery client: %w", err) + } + + // setup checker + chk, err := servicecheck.New(ctx, httpClient, discovery, promRegistry, server.allowUnschedulable, 3*time.Second) + if err != nil { + return nil, err + } + + chk.KubenurseIngressURL = os.Getenv("KUBENURSE_INGRESS_URL") + chk.KubenurseServiceURL = os.Getenv("KUBENURSE_SERVICE_URL") + chk.KubernetesServiceHost = os.Getenv("KUBERNETES_SERVICE_HOST") + chk.KubernetesServicePort = os.Getenv("KUBERNETES_SERVICE_PORT") + chk.KubenurseNamespace = os.Getenv("KUBENURSE_NAMESPACE") + chk.NeighbourFilter = os.Getenv("KUBENURSE_NEIGHBOUR_FILTER") + chk.UseTLS = server.useTLS + + server.checker = chk + + // setup http routes + mux.HandleFunc("/ready", server.readyHandler()) + mux.HandleFunc("/alive", server.aliveHandler()) + mux.HandleFunc("/alwayshappy", func(http.ResponseWriter, *http.Request) {}) + mux.Handle("/metrics", promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{})) + mux.Handle("/", http.RedirectHandler("/alive", http.StatusMovedPermanently)) + + return server, nil +} + +// Run starts the periodic checker and the http/https server(s) and blocks until Shutdown was called. +func (s *Server) Run() error { + var ( + wg sync.WaitGroup + errc = make(chan error, 2) // max two errors can happen + ) + + wg.Add(1) + + go func() { + defer wg.Done() + + s.checker.RunScheduled(5 * time.Second) + log.Printf("checker exited") + }() + + wg.Add(1) + + go func() { + defer wg.Done() + + if err := s.http.ListenAndServe(); err != nil { + if err != http.ErrServerClosed { + errc <- fmt.Errorf("listen http: %w", err) + } + } + }() + + if s.useTLS { + wg.Add(1) + + go func() { + defer wg.Done() + + if err := s.https.ListenAndServeTLS( + os.Getenv("KUBENURSE_CERT_FILE"), + os.Getenv("KUBENURSE_CERT_KEY"), + ); err != nil { + if err != http.ErrServerClosed { + errc <- fmt.Errorf("listen https: %w", err) + } + } + }() + } + + wg.Wait() + close(errc) + + // return the first error if there was one + for err := range errc { + if err != nil { + return err + } + } + + return nil +} + +// Shutdown disables the readiness probe and then gracefully halts the kubenurse http/https server(s). +func (s *Server) Shutdown(ctx context.Context) error { + s.mu.Lock() + s.ready = false + s.mu.Unlock() + + // stop the scheduled checker + s.checker.StopScheduled() + + if err := s.http.Shutdown(ctx); err != nil { + return fmt.Errorf("stop http server: %w", err) + } + + if s.useTLS { + if err := s.https.Shutdown(ctx); err != nil { + return fmt.Errorf("stop https server: %w", err) + } + } + + return nil +} diff --git a/internal/kubenurse/server_test.go b/internal/kubenurse/server_test.go new file mode 100644 index 00000000..b45d47c7 --- /dev/null +++ b/internal/kubenurse/server_test.go @@ -0,0 +1,39 @@ +package kubenurse + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes/fake" +) + +func TestCombined(t *testing.T) { + r := require.New(t) + + fakeClient := fake.NewSimpleClientset() + + kubenurse, err := New(context.Background(), fakeClient) + r.NoError(err) + r.NotNil(kubenurse) + + t.Run("start/stop", func(t *testing.T) { + r := require.New(t) + errc := make(chan error, 1) + + go func() { + // blocks until shutdown is called + err := kubenurse.Run() + + errc <- err + close(errc) + }() + + // Shutdown, Run() should stop after function completes + err := kubenurse.Shutdown(context.Background()) + r.NoError(err) + + err = <-errc // blocks until kubenurse.Run() finishes and eventually returns an error + r.NoError(err) + }) +} diff --git a/internal/kubenurse/transport.go b/internal/kubenurse/transport.go new file mode 100644 index 00000000..29dda6d0 --- /dev/null +++ b/internal/kubenurse/transport.go @@ -0,0 +1,55 @@ +package kubenurse + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "os" +) + +const ( + caFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +) + +// generateRoundTripper returns a custom http.RoundTripper, including the k8s CA. +func (s *Server) generateRoundTripper() (http.RoundTripper, error) { + // Append default certpool + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + // Append ServiceAccount cacert + caCert, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("could not load certificate %s: %w", caFile, err) + } + + if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { + return nil, errors.New("could not append ca cert to system certpool") + } + + // Append extra CA, if set + if s.extraCA != "" { + caCert, err := os.ReadFile(s.extraCA) + if err != nil { + return nil, fmt.Errorf("could not load certificate %s: %w", s.extraCA, err) + } + + if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { + return nil, errors.New("could not append extra ca cert to system certpool") + } + } + + // Configure transport + tlsConfig := &tls.Config{ + InsecureSkipVerify: s.insecure, //nolint:gosec // Can be true if the user requested this. + RootCAs: rootCAs, + } + + transport := &http.Transport{TLSClientConfig: tlsConfig} + + return transport, nil +} diff --git a/pkg/checker/cache.go b/internal/servicecheck/cache.go similarity index 96% rename from pkg/checker/cache.go rename to internal/servicecheck/cache.go index 32d8fac5..72fbf22c 100644 --- a/pkg/checker/cache.go +++ b/internal/servicecheck/cache.go @@ -1,4 +1,4 @@ -package checker +package servicecheck import "time" diff --git a/pkg/checker/checker.go b/internal/servicecheck/servicecheck.go similarity index 58% rename from pkg/checker/checker.go rename to internal/servicecheck/servicecheck.go index bdaa7e15..006f5e95 100644 --- a/pkg/checker/checker.go +++ b/internal/servicecheck/servicecheck.go @@ -1,5 +1,5 @@ -// Package checker implements the checks the kubenurse performs. -package checker +// Package servicecheck implements the checks the kubenurse performs. +package servicecheck import ( "context" @@ -8,28 +8,52 @@ import ( "net/http" "time" - "github.com/postfinance/kubenurse/pkg/kubediscovery" - "github.com/postfinance/kubenurse/pkg/metrics" + "github.com/postfinance/kubenurse/internal/kubediscovery" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + okStr = "ok" + errStr = "error" ) // New configures the checker with a httpClient and a cache timeout for check // results. Other parameters of the Checker struct need to be configured separately. -func New(ctx context.Context, httpClient *http.Client, cacheTTL time.Duration, allowUnschedulable bool) (*Checker, error) { - discovery, err := kubediscovery.New(ctx, allowUnschedulable) - if err != nil { - return nil, fmt.Errorf("create k8s discovery client: %w", err) - } +func New(ctx context.Context, httpClient *http.Client, discovery *kubediscovery.Client, + promRegistry *prometheus.Registry, allowUnschedulable bool, cacheTTL time.Duration) (*Checker, error) { + errorCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "kubenurse_errors_total", + Help: "Kubenurse error counter partitioned by error type", + }, + []string{"type"}, + ) + + durationSummary := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "kubenurse_request_duration", + Help: "Kubenurse request duration partitioned by error type", + MaxAge: 1 * time.Minute, + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{"type"}, + ) + + promRegistry.MustRegister(errorCounter, durationSummary) return &Checker{ allowUnschedulable: allowUnschedulable, discovery: discovery, httpClient: httpClient, cacheTTL: cacheTTL, + errorCounter: errorCounter, + durationSummary: durationSummary, + stop: make(chan struct{}), }, nil } -// Run runs an check and returns the result togeter with a boolean, if it wasn't -// successful. It respects the cache. +// Run runs all servicechecks and returns the result togeter with a boolean which indicates success. The cache +// is respected. func (c *Checker) Run() (Result, bool) { var ( haserr bool @@ -45,16 +69,16 @@ func (c *Checker) Run() (Result, bool) { // Run Checks res := Result{} - res.APIServerDirect, err = measure(c.APIServerDirect, "api_server_direct") + res.APIServerDirect, err = c.measure(c.APIServerDirect, "api_server_direct") haserr = haserr || (err != nil) - res.APIServerDNS, err = measure(c.APIServerDNS, "api_server_dns") + res.APIServerDNS, err = c.measure(c.APIServerDNS, "api_server_dns") haserr = haserr || (err != nil) - res.MeIngress, err = measure(c.MeIngress, "me_ingress") + res.MeIngress, err = c.measure(c.MeIngress, "me_ingress") haserr = haserr || (err != nil) - res.MeService, err = measure(c.MeService, "me_service") + res.MeService, err = c.measure(c.MeService, "me_service") haserr = haserr || (err != nil) res.Neighbourhood, err = c.discovery.GetNeighbours(context.TODO(), c.KubenurseNamespace, c.NeighbourFilter) @@ -64,7 +88,7 @@ func (c *Checker) Run() (Result, bool) { if err != nil { res.NeighbourhoodState = err.Error() } else { - res.NeighbourhoodState = "ok" + res.NeighbourhoodState = okStr // Check all neighbours if the neighbourhood was discovered c.checkNeighbours(res.Neighbourhood) @@ -76,14 +100,27 @@ func (c *Checker) Run() (Result, bool) { return res, haserr } -// RunScheduled runs the check run in the specified interval which can be used -// to keep the metrics up-to-date. +// RunScheduled runs the checks in the specified interval which can be used to keep the metrics up-to-date. This +// function does not return until StopScheduled is called. func (c *Checker) RunScheduled(d time.Duration) { - for range time.Tick(d) { - c.Run() + ticker := time.NewTicker(d) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + c.Run() + case <-c.stop: + return + } } } +// StopScheduled is used to stop the scheduled run of checks. +func (c *Checker) StopScheduled() { + close(c.stop) +} + // APIServerDirect checks the /version endpoint of the Kubernetes API Server through the direct link func (c *Checker) APIServerDirect() (string, error) { apiurl := fmt.Sprintf("https://%s:%s/version", c.KubernetesServiceHost, c.KubernetesServicePort) @@ -120,24 +157,24 @@ func (c *Checker) checkNeighbours(nh []kubediscovery.Neighbour) { return c.doRequest("http://" + neighbour.PodIP + ":8080/alwayshappy") } - _, _ = measure(check, "path_"+neighbour.NodeName) + _, _ = c.measure(check, "path_"+neighbour.NodeName) } } } // measure implements metric collections for the check -func measure(check Check, label string) (string, error) { +func (c *Checker) measure(check Check, label string) (string, error) { start := time.Now() // Execute check res, err := check() // Process metrics - metrics.DurationSummary.WithLabelValues(label).Observe(time.Since(start).Seconds()) + c.durationSummary.WithLabelValues(label).Observe(time.Since(start).Seconds()) if err != nil { log.Printf("failed request for %s with %v", label, err) - metrics.ErrorCounter.WithLabelValues(label).Inc() + c.errorCounter.WithLabelValues(label).Inc() } return res, err diff --git a/internal/servicecheck/servicecheck_test.go b/internal/servicecheck/servicecheck_test.go new file mode 100644 index 00000000..4137758b --- /dev/null +++ b/internal/servicecheck/servicecheck_test.go @@ -0,0 +1,69 @@ +package servicecheck + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/postfinance/kubenurse/internal/kubediscovery" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +var fakeNeighbourPod = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubenurse-dummy", + Labels: map[string]string{ + "app": "kubenurse", + }, + }, + Spec: v1.PodSpec{ + NodeName: "dummy", + }, + Status: v1.PodStatus{ + HostIP: "127.0.0.1", + PodIP: "127.0.0.1", + }, +} + +func TestCombined(t *testing.T) { + r := require.New(t) + + // fake client, with a dummy neighbour pod + fakeClient := fake.NewSimpleClientset() + _, err := fakeClient.CoreV1().Pods("kube-system").Create(context.Background(), &fakeNeighbourPod, metav1.CreateOptions{}) + r.NoError(err) + + discovery, err := kubediscovery.New(context.Background(), fakeClient, false) + r.NoError(err) + + checker, err := New(context.Background(), http.DefaultClient, discovery, prometheus.NewRegistry(), false, 3*time.Second) + r.NoError(err) + r.NotNil(checker) + + t.Run("run", func(t *testing.T) { + r := require.New(t) + result, hadError := checker.Run() + r.True(hadError) + r.Len(result.Neighbourhood, 1) + }) + + t.Run("scheduled", func(t *testing.T) { + stopped := make(chan struct{}) + + go func() { + // blocks until StopScheduled() + checker.RunScheduled(time.Second * 5) + + close(stopped) + }() + + checker.StopScheduled() + + <-stopped + }) +} diff --git a/pkg/checker/transport.go b/internal/servicecheck/transport.go similarity index 69% rename from pkg/checker/transport.go rename to internal/servicecheck/transport.go index 83df907f..ae64de68 100644 --- a/pkg/checker/transport.go +++ b/internal/servicecheck/transport.go @@ -1,27 +1,27 @@ -package checker +package servicecheck import ( "errors" "fmt" - "io/ioutil" "net/http" + "os" "strings" ) const ( - //nolint:gosec + //nolint:gosec // This is the well-known path to Kubernetes serviceaccount tokens. tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) // doRequest does an http request only to get the http status code func (c *Checker) doRequest(url string) (string, error) { // Read Bearer Token file from ServiceAccount - token, err := ioutil.ReadFile(tokenFile) + token, err := os.ReadFile(tokenFile) if err != nil { - return "error", fmt.Errorf("could not load token %s: %s", tokenFile, err) + return errStr, fmt.Errorf("load kubernetes serviceaccount token from %s: %w", tokenFile, err) } - req, _ := http.NewRequest("GET", url, nil) + req, _ := http.NewRequest("GET", url, http.NoBody) // Only add the Bearer for API Server Requests if strings.HasSuffix(url, "/version") { @@ -37,7 +37,7 @@ func (c *Checker) doRequest(url string) (string, error) { _ = resp.Body.Close() if resp.StatusCode == http.StatusOK { - return "ok", nil + return okStr, nil } return resp.Status, errors.New(resp.Status) diff --git a/pkg/checker/types.go b/internal/servicecheck/types.go similarity index 83% rename from pkg/checker/types.go rename to internal/servicecheck/types.go index 559bd611..428806c8 100644 --- a/pkg/checker/types.go +++ b/internal/servicecheck/types.go @@ -1,10 +1,11 @@ -package checker +package servicecheck import ( "net/http" "time" - "github.com/postfinance/kubenurse/pkg/kubediscovery" + "github.com/postfinance/kubenurse/internal/kubediscovery" + "github.com/prometheus/client_golang/prometheus" ) // Checker implements the kubenurse checker @@ -27,6 +28,10 @@ type Checker struct { discovery *kubediscovery.Client + // metrics + errorCounter *prometheus.CounterVec + durationSummary *prometheus.SummaryVec + // Http Client for https requests httpClient *http.Client @@ -35,6 +40,9 @@ type Checker struct { // cacheTTL defines the TTL of how long a cached result is valid cacheTTL time.Duration + + // stop is used to cancel RunScheduled + stop chan struct{} } // Result contains the result of a performed check run diff --git a/main.go b/main.go index 05fdaef3..c2c8f809 100644 --- a/main.go +++ b/main.go @@ -2,229 +2,62 @@ package main import ( "context" - "crypto/tls" - "crypto/x509" - "encoding/json" - "errors" "fmt" - "io/ioutil" "log" - "net/http" - "os" "os/signal" - "strconv" "syscall" "time" - "github.com/postfinance/kubenurse/pkg/checker" - "github.com/postfinance/kubenurse/pkg/kubediscovery" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/postfinance/kubenurse/internal/kubenurse" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) const ( - caFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - nurse = "I'm ready to help you!" + nurse = "I'm ready to help you!" ) -//nolint:funlen func main() { - mux := http.NewServeMux() - server := http.Server{ - Addr: ":8080", - Handler: mux, - } - serverTLS := http.Server{ - Addr: ":8443", - Handler: mux, - } - useTLS := os.Getenv("KUBENURSE_USE_TLS") == "true" - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - ctx, cancel := context.WithCancel(context.Background()) - - go func() { - select { - case s := <-sig: - log.Printf("shutting down, received signal %s", s) - - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer shutdownCancel() + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil { - log.Fatalln(err) - } - - if useTLS { - if err := serverTLS.Shutdown(shutdownCtx); err != nil { - log.Fatalln(err) - } - } - - cancel() - case <-ctx.Done(): - } - }() - - // setup http transport - transport, err := GenerateRoundTripper() + // create in-cluster config + config, err := rest.InClusterConfig() if err != nil { - log.Printf("using default transport: %s", err) - - transport = http.DefaultTransport + log.Printf("creating in-cluster configuration: %s", err) + return } - client := &http.Client{ - Timeout: 5 * time.Second, - Transport: transport, + cliset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Printf("creating clientset: %s", err) + return } - // If we want to consider kubenurses on unschedulable nodes - allowUnschedulable := os.Getenv("KUBENURSE_ALLOW_UNSCHEDULABLE") == "true" - - // setup checker - chk, err := checker.New(ctx, client, 3*time.Second, allowUnschedulable) + server, err := kubenurse.New(ctx, cliset) if err != nil { - log.Fatalln(err) + log.Printf("%s", err) + return } - chk.KubenurseIngressURL = os.Getenv("KUBENURSE_INGRESS_URL") - chk.KubenurseServiceURL = os.Getenv("KUBENURSE_SERVICE_URL") - chk.KubernetesServiceHost = os.Getenv("KUBERNETES_SERVICE_HOST") - chk.KubernetesServicePort = os.Getenv("KUBERNETES_SERVICE_PORT") - chk.KubenurseNamespace = os.Getenv("KUBENURSE_NAMESPACE") - chk.NeighbourFilter = os.Getenv("KUBENURSE_NEIGHBOUR_FILTER") - chk.UseTLS = useTLS - - // setup http routes - mux.HandleFunc("/alive", aliveHandler(chk)) - mux.HandleFunc("/alwayshappy", func(http.ResponseWriter, *http.Request) {}) - mux.Handle("/metrics", promhttp.Handler()) - mux.Handle("/", http.RedirectHandler("/alive", http.StatusMovedPermanently)) - - fmt.Println(nurse) // most important line of this project - - // Start listener and checker go func() { - chk.RunScheduled(5 * time.Second) - log.Fatalln("checker exited") - }() - - go func() { - if err := server.ListenAndServe(); err != nil { - if err != http.ErrServerClosed { - log.Fatalln(err) - } - } - }() - - if useTLS { - go func() { - if err := serverTLS.ListenAndServeTLS(os.Getenv("KUBENURSE_CERT_FILE"), os.Getenv("KUBENURSE_CERT_KEY")); err != nil { - if err != http.ErrServerClosed { - log.Fatalln(err) - } - } - }() - } + <-ctx.Done() // blocks until ctx is canceled - <-ctx.Done() -} - -func aliveHandler(chk *checker.Checker) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - type Output struct { - Hostname string `json:"hostname"` - Headers map[string][]string `json:"headers"` - UserAgent string `json:"user_agent"` - RequestURI string `json:"request_uri"` - RemoteAddr string `json:"remote_addr"` - - // checker.Result - APIServerDirect string `json:"api_server_direct"` - APIServerDNS string `json:"api_server_dns"` - MeIngress string `json:"me_ingress"` - MeService string `json:"me_service"` - - // kubediscovery - NeighbourhoodState string `json:"neighbourhood_state"` - Neighbourhood []kubediscovery.Neighbour `json:"neighbourhood"` - } - - // Run checks now - res, haserr := chk.Run() - if haserr { - w.WriteHeader(http.StatusInternalServerError) - } - - // Add additional data - out := Output{ - APIServerDNS: res.APIServerDNS, - APIServerDirect: res.APIServerDirect, - MeIngress: res.MeIngress, - MeService: res.MeService, - Headers: r.Header, - UserAgent: r.UserAgent(), - RequestURI: r.RequestURI, - RemoteAddr: r.RemoteAddr, - Neighbourhood: res.Neighbourhood, - NeighbourhoodState: res.NeighbourhoodState, - } - out.Hostname, _ = os.Hostname() - - // Generate output output - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - _ = enc.Encode(out) - } -} - -// GenerateRoundTripper returns a custom http.RoundTripper, including the k8s -// CA. If env KUBENURSE_INSECURE is set to true, certificates are not validated. -func GenerateRoundTripper() (http.RoundTripper, error) { - // Parse environment variables - extraCA := os.Getenv("KUBENURSE_EXTRA_CA") - insecureEnv := os.Getenv("KUBENURSE_INSECURE") - insecure, _ := strconv.ParseBool(insecureEnv) + log.Println("shutting down, received signal to stop") - // Append default certpool - rootCAs, _ := x509.SystemCertPool() - if rootCAs == nil { - rootCAs = x509.NewCertPool() - } - - // Append ServiceAccount cacert - caCert, err := ioutil.ReadFile(caFile) - if err != nil { - return nil, fmt.Errorf("could not load certificate %s: %s", caFile, err) - } - - if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { - return nil, errors.New("could not append ca cert to system certpool") - } - - // Append extra CA, if set - if extraCA != "" { - //nolint:gosec - caCert, err := ioutil.ReadFile(extraCA) + // background ctx since, the "root" context is already canceled + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() - if err != nil { - return nil, fmt.Errorf("could not load certificate %s: %s", extraCA, err) + if err := server.Shutdown(shutdownCtx); err != nil { + log.Printf("gracefully halting kubenurse server: %s", err) } + }() - if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { - return nil, errors.New("could not append extra ca cert to system certpool") - } - } + fmt.Println(nurse) // most important line of this project - // Configure transport - tlsConfig := &tls.Config{ - InsecureSkipVerify: insecure, //nolint:gosec - RootCAs: rootCAs, + // blocks, until the server is stopped by calling Shutdown() + if err := server.Run(); err != nil { + log.Printf("running kubenurse: %s", err) } - - transport := &http.Transport{TLSClientConfig: tlsConfig} - - return transport, nil } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go deleted file mode 100644 index 4bc5bce6..00000000 --- a/pkg/metrics/metrics.go +++ /dev/null @@ -1,37 +0,0 @@ -// Package metrics sets-up the metrics which will be exported by kubenurse. TODO: rewrite this package. -package metrics - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -//nolint:gochecknoglobals -var ( - // ErrorCounter provides the kubenurse_errors_total metric - ErrorCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "kubenurse_errors_total", - Help: "Kubenurse error counter partitioned by error type", - }, - []string{"type"}, - ) - - // DurationSummary provides the kubenurse_request_duration metric - DurationSummary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Name: "kubenurse_request_duration", - Help: "Kubenurse request duration partitioned by error type", - MaxAge: 1 * time.Minute, - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, - []string{"type"}, - ) -) - -//nolint:gochecknoinits -func init() { - prometheus.MustRegister(ErrorCounter) - prometheus.MustRegister(DurationSummary) -}