DataSet api usage improvements

Author: jmakar-s1

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.0.6
+
+DataSet api usage improvements
+
 ## 3.0.5
 
 - Update yarn dependency @grafana/toolkit to 8.5.0

go.mod

@@ -5,5 +5,6 @@ go 1.16
 require (
 	github.com/grafana/grafana-plugin-sdk-go v0.126.0
 	github.com/magefile/mage v1.12.1 // indirect
+	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	moul.io/http2curl v1.0.0 // indirect
 )

go.sum

@@ -446,6 +446,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
+golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sentinelone-dataset-datasource",
-  "version": "3.0.5",
+  "version": "3.0.6",
   "description": "Scalyr Observability Platform",
   "scripts": {
     "build": "grafana-toolkit plugin:build",

pkg/plugin/client.go

@@ -2,13 +2,16 @@ package plugin
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"time"
 
+	"golang.org/x/time/rate"
+
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 )
 
@@ -20,27 +23,38 @@ type FacetRequest struct {
 }
 
 type DataSetClient struct {
-	dataSetUrl string
-	apiKey     string
-	netClient  *http.Client
+	dataSetUrl  string
+	apiKey      string
+	netClient   *http.Client
+	rateLimiter *rate.Limiter
 }
 
 func NewDataSetClient(dataSetUrl string, apiKey string) *DataSetClient {
 	// Consider using the backend.httpclient package provided by the Grafana SDK.
 	// This would allow a per-instance configurable timeout, rather than the hardcoded value here.
-	var netClient = &http.Client{
+	netClient := &http.Client{
 		Timeout: time.Second * 10,
 	}
 
+	// TODO Are there alternate approaches to implementing rate limits via the Grafana SDK?
+	//      Consult with Grafana support about this, potentially there's a simplier option.
+	rateLimiter := rate.NewLimiter(rate.Every(1 * time.Minute), 100) // 100 requests / minute
+
 	return &DataSetClient{
-		dataSetUrl: dataSetUrl,
-		apiKey:     apiKey,
-		netClient:  netClient,
+		dataSetUrl:  dataSetUrl,
+		apiKey:      apiKey,
+		netClient:   netClient,
+		rateLimiter: rateLimiter,
 	}
 }
 
 func (d *DataSetClient) newRequest(method, url string, body io.Reader) (*http.Request, error) {
-	const VERSION = "3.0.5"
+	const VERSION = "3.0.6"
+
+	if err := d.rateLimiter.Wait(context.Background()); err != nil {
+		log.DefaultLogger.Error("error applying rate limiter", "err", err)
+		return nil, err
+	}
 
 	request, err := http.NewRequest(method, url, body)
 	if err != nil {
@@ -83,6 +97,10 @@ func (d *DataSetClient) doPingRequest(req interface{}) (*LRQResult, error) {
 
 	var respBody LRQResult
 	var token string
+
+	delay := 250 * time.Millisecond
+	const maxDelay = 1 * time.Second
+
 	for i := 0; ; i++ {
 		resp, err := d.netClient.Do(request)
 		if err != nil {
@@ -115,7 +133,10 @@ func (d *DataSetClient) doPingRequest(req interface{}) (*LRQResult, error) {
 			token = resp.Header.Get(TOKEN_HEADER)
 		}
 
-		time.Sleep(100 * time.Millisecond)
+		time.Sleep(delay)
+		if delay < maxDelay {
+			delay *= 2
+		}
 
 		u := fmt.Sprintf("%s/v2/api/queries/%s?lastStepSeen=%d", d.dataSetUrl, respBody.Id, respBody.StepsCompleted)
 		request, err = d.newRequest("GET", u, nil)

src/plugin.json

@@ -43,8 +43,8 @@
         "path": "img/DatasetConfig.png"
       }
     ],
-    "version": "3.0.5",
-    "updated": "2022-08-02"
+    "version": "3.0.6",
+    "updated": "2022-10-04"
   },
   "dependencies": {
     "grafanaDependency": ">=8.2.0",