Merge pull request #150 from pagbrl/improve-reliability

Make querying interval configurable and improve error handling

Author: haad

README.md

@@ -54,6 +54,7 @@ The exporter can be configured using env variables or command flags.
 | `LISTEN` |  listen on addr:port (default `:8080`), omit addr to listen on all interfaces |
 | `METRICS_PATH` |  path for metrics, default `/metrics` |
 | `SCRAPE_DELAY` | scrape delay in seconds, default `300` |
+| `SCRAPE_INTERVAL` | scrape interval in seconds (will query cloudflare every SCRAPE_INTERVAL seconds), default `60` |
 | `CF_BATCH_SIZE` | cloudflare request zones batch size (1 - 10), default `10` |
 | `METRICS_DENYLIST` | (Optional) cloudflare-exporter metrics to not export, comma delimited list of cloudflare-exporter metrics. If not set, all metrics are exported |
 | `ENABLE_PPROF` | (Optional) enable pprof profiling endpoints at `/debug/pprof/`. Accepts `true` or `false`, default `false`. **Warning**: Only enable in development/debugging environments |
@@ -70,6 +71,7 @@ Corresponding flags:
   -listen=":8080": listen on addr:port ( default :8080), omit addr to listen on all interfaces
   -metrics_path="/metrics": path for metrics, default /metrics
   -scrape_delay=300: scrape delay in seconds, defaults to 300
+  -scrape_interval=60: scrape interval in seconds, defaults to 60
   -cf_batch_size=10: cloudflare zones batch size (1-10)
   -metrics_denylist="": cloudflare-exporter metrics to not export, comma delimited list
   -enable_pprof=false: enable pprof profiling endpoints at /debug/pprof/

main.go

@@ -187,8 +187,11 @@ func runExporter() {
 	log.Debugf("Metrics set: %v", metricsSet)
 	mustRegisterMetrics(metricsSet)
 
+	scrapeInterval := time.Duration(viper.GetInt("scrape_interval")) * time.Second
+	log.Info("Scrape interval set to ", scrapeInterval)
+
 	go func() {
-		for ; true; <-time.NewTicker(60 * time.Second).C {
+		for ; true; <-time.NewTicker(scrapeInterval).C {
 			go fetchMetrics()
 		}
 	}()
@@ -255,6 +258,10 @@ func main() {
 	viper.BindEnv("scrape_delay")
 	viper.SetDefault("scrape_delay", 300)
 
+	flags.Int("scrape_interval", 60, "scrape interval in seconds, defaults to 60")
+	viper.BindEnv("scrape_interval")
+	viper.SetDefault("scrape_interval", 60)
+
 	flags.Int("cf_batch_size", 10, "cloudflare zones batch size (1-10)")
 	viper.BindEnv("cf_batch_size")
 	viper.SetDefault("cf_batch_size", 10)

prometheus.go

@@ -9,6 +9,7 @@ import (
 	"github.com/biter777/countries"
 	cloudflare "github.com/cloudflare/cloudflare-go"
 	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
@@ -448,6 +449,7 @@ func fetchWorkerAnalytics(account cloudflare.Account, wg *sync.WaitGroup) {
 
 	r, err := fetchWorkerTotals(account.ID)
 	if err != nil {
+		log.Error("failed to fetch worker analytics for account ", account.ID, ": ", err)
 		return
 	}
 
@@ -481,6 +483,7 @@ func fetchLogpushAnalyticsForAccount(account cloudflare.Account, wg *sync.WaitGr
 	r, err := fetchLogpushAccount(account.ID)
 
 	if err != nil {
+		log.Error("failed to fetch logpush analytics for account ", account.ID, ": ", err)
 		return
 	}
 
@@ -532,6 +535,7 @@ func fetchLogpushAnalyticsForZone(zones []cloudflare.Zone, wg *sync.WaitGroup) {
 	r, err := fetchLogpushZone(zoneIDs)
 
 	if err != nil {
+		log.Error("failed to fetch logpush analytics for zones: ", err)
 		return
 	}
 
@@ -560,6 +564,7 @@ func fetchZoneColocationAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) {
 
 	r, err := fetchColoTotals(zoneIDs)
 	if err != nil {
+		log.Error("failed to fetch colocation analytics for zones: ", err)
 		return
 	}
 	for _, z := range r.Viewer.Zones {
@@ -589,6 +594,7 @@ func fetchZoneAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) {
 
 	r, err := fetchZoneTotals(zoneIDs)
 	if err != nil {
+		log.Error("failed to fetch zone analytics: ", err)
 		return
 	}
 
@@ -740,6 +746,7 @@ func fetchLoadBalancerAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) {
 
 	l, err := fetchLoadBalancerTotals(zoneIDs)
 	if err != nil {
+		log.Error("failed to fetch load balancer analytics: ", err)
 		return
 	}
 	for _, lb := range l.Viewer.Zones {