diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index f0aac834b..961674082 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -21,6 +21,8 @@ import ( "github.com/stakater/Reloader/internal/pkg/options" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + + _ "net/http/pprof" ) // NewReloaderCommand starts the reloader controller diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index 7dc7664e8..3d713fdb6 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/handler" "github.com/stakater/Reloader/internal/pkg/metrics" @@ -103,7 +104,12 @@ func NewController( // Add function to add a new object to the queue in case of creating a resource func (c *Controller) Add(obj interface{}) { - + if cm, ok := obj.(*v1.ConfigMap); ok { + logrus.Infof("add: %s/%s", cm.GetNamespace(), cm.GetName()) + } else if secret, ok := obj.(*v1.Secret); ok { + logrus.Infof("add: %s/%s", secret.GetNamespace(), secret.GetName()) + } + c.collectors.QueueSize.With(prometheus.Labels{"resource": c.resource}).Set(float64(c.queue.Len())) switch object := obj.(type) { case *v1.Namespace: c.addSelectedNamespaceToCache(*object) @@ -166,6 +172,12 @@ func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { // Update function to add an old object and a new object to the queue in case of updating a resource func (c *Controller) Update(old interface{}, new interface{}) { + if cm, ok := new.(*v1.ConfigMap); ok { + logrus.Infof("update: %s/%s", cm.GetNamespace(), cm.GetName()) + } else if secret, ok := new.(*v1.Secret); ok { + logrus.Infof("update: %s/%s", secret.GetNamespace(), secret.GetName()) + } + c.collectors.QueueSize.With(prometheus.Labels{"resource": c.resource}).Set(float64(c.queue.Len())) switch new.(type) { case *v1.Namespace: return @@ -183,7 +195,12 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { - + if cm, ok := old.(*v1.ConfigMap); ok { + logrus.Infof("delete: %s/%s", cm.GetNamespace(), cm.GetName()) + } else if secret, ok := old.(*v1.Secret); ok { + logrus.Infof("delete: %s/%s", secret.GetNamespace(), secret.GetName()) + } + c.collectors.QueueSize.With(prometheus.Labels{"resource": c.resource}).Set(float64(c.queue.Len())) if options.ReloadOnDelete == "true" { if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized { c.queue.Add(handler.ResourceDeleteHandler{ @@ -239,6 +256,7 @@ func (c *Controller) runWorker() { func (c *Controller) processNextItem() bool { // Wait until there is a new item in the working queue resourceHandler, quit := c.queue.Get() + c.collectors.QueueSize.With(prometheus.Labels{"resource": c.resource}).Set(float64(c.queue.Len())) if quit { return false } @@ -264,6 +282,8 @@ func (c *Controller) handleErr(err error, key interface{}) { return } + c.collectors.Errors.With(prometheus.Labels{"error_type": "general"}).Inc() + // This controller retries 5 times if something goes wrong. After that, it stops trying. if c.queue.NumRequeues(key) < 5 { logrus.Errorf("Error syncing events: %v", err) @@ -271,10 +291,12 @@ func (c *Controller) handleErr(err error, key interface{}) { // Re-enqueue the key rate limited. Based on the rate limiter on the // queue and the re-enqueue history, the key will be processed later again. c.queue.AddRateLimited(key) + c.collectors.Requeues.With(prometheus.Labels{"resource": c.resource}).Inc() return } c.queue.Forget(key) + c.collectors.Dropped.With(prometheus.Labels{"resource": c.resource}).Inc() // Report to an external entity that, even after several retries, we could not successfully process this key runtime.HandleError(err) logrus.Errorf("Dropping key out of the queue: %v", err) diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index 94153eace..57114fb61 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -11,6 +11,10 @@ import ( type Collectors struct { Reloaded *prometheus.CounterVec ReloadedByNamespace *prometheus.CounterVec + QueueSize *prometheus.GaugeVec + Errors *prometheus.CounterVec + Requeues *prometheus.CounterVec + Dropped *prometheus.CounterVec } func NewCollectors() Collectors { @@ -40,9 +44,57 @@ func NewCollectors() Collectors { "namespace", }, ) + + queueSize := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "reloader", + Name: "queue_size", + Help: "Gauge for the size of the work queue.", + }, + []string{ + "resource", + }, + ) + + errors := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "errors_total", + Help: "Counter of errors encountered by Reloader.", + }, + []string{ + "error_type", + }, + ) + requeues := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "requeues_total", + Help: "Counter of requeues encountered by Reloader.", + }, + []string{ + "resource", + }, + ) + + dropped := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "dropped_total", + Help: "Counter of dropped events by Reloader.", + }, + []string{ + "resource", + }, + ) + return Collectors{ Reloaded: reloaded, ReloadedByNamespace: reloaded_by_namespace, + QueueSize: queueSize, + Errors: errors, + Requeues: requeues, + Dropped: dropped, } } @@ -54,6 +106,11 @@ func SetupPrometheusEndpoint() Collectors { prometheus.MustRegister(collectors.ReloadedByNamespace) } + prometheus.MustRegister(collectors.QueueSize) + prometheus.MustRegister(collectors.Errors) + prometheus.MustRegister(collectors.Requeues) + prometheus.MustRegister(collectors.Dropped) + http.Handle("/metrics", promhttp.Handler()) return collectors