Merge pull request #114 from scalyr/DSET-2220

DSET-2220: Better handling of query options

Author: jmakar-s1

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.1.1
+
+- Bugfix around use of query options (max data points & interval)
+
 ## 3.1.0
 
 - Bumped Golang version to 1.20

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sentinelone-dataset-datasource",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "Scalyr Observability Platform",
   "scripts": {
     "build": "webpack -c ./.config/webpack/webpack.config.ts --env production",

pkg/plugin/client.go

@@ -23,14 +23,21 @@ type FacetRequest struct {
 	Field string `json:"field"`
 }
 
-type DataSetClient struct {
+type DataSetClient interface {
+	DoLRQRequest(ctx context.Context, req LRQRequest) (*LRQResult, error)
+	DoFacetValuesRequest(ctx context.Context, req FacetQuery) (*LRQResult, error)
+	DoTopFacetRequest(ctx context.Context, req TopFacetRequest) (*LRQResult, error)
+	DoFacetRequest(ctx context.Context, req FacetRequest) (int, error)
+}
+
+type dataSetClient struct {
 	dataSetUrl  string
 	apiKey      string
 	netClient   *http.Client
 	rateLimiter *rate.Limiter
 }
 
-func NewDataSetClient(dataSetUrl string, apiKey string) *DataSetClient {
+func NewDataSetClient(dataSetUrl string, apiKey string) DataSetClient {
 	// Consider using the backend.httpclient package provided by the Grafana SDK.
 	// This would allow a per-instance configurable timeout, rather than the hardcoded value here.
 	netClient := &http.Client{
@@ -41,16 +48,16 @@ func NewDataSetClient(dataSetUrl string, apiKey string) *DataSetClient {
 	//      Consult with Grafana support about this, potentially there's a simplier option.
 	rateLimiter := rate.NewLimiter(100*rate.Every(1*time.Minute), 100) // 100 requests / minute
 
-	return &DataSetClient{
+	return &dataSetClient{
 		dataSetUrl:  dataSetUrl,
 		apiKey:      apiKey,
 		netClient:   netClient,
 		rateLimiter: rateLimiter,
 	}
 }
 
-func (d *DataSetClient) newRequest(method, url string, body io.Reader) (*http.Request, error) {
-	const VERSION = "3.1.0"
+func (d *dataSetClient) newRequest(method, url string, body io.Reader) (*http.Request, error) {
+	const VERSION = "3.1.1"
 
 	if err := d.rateLimiter.Wait(context.Background()); err != nil {
 		log.DefaultLogger.Error("error applying rate limiter", "err", err)
@@ -72,7 +79,7 @@ func (d *DataSetClient) newRequest(method, url string, body io.Reader) (*http.Re
 	return request, nil
 }
 
-func (d *DataSetClient) doPingRequest(ctx context.Context, req interface{}) (*LRQResult, error) {
+func (d *dataSetClient) doPingRequest(ctx context.Context, req interface{}) (*LRQResult, error) {
 	// Long-Running Query (LRQ) api usage:
 	// - An initial POST request is made containing the standard/power query
 	// - Its response may or may not contain the results
@@ -206,19 +213,19 @@ loop:
 	return &respBody, nil
 }
 
-func (d *DataSetClient) DoLRQRequest(ctx context.Context, req LRQRequest) (*LRQResult, error) {
+func (d *dataSetClient) DoLRQRequest(ctx context.Context, req LRQRequest) (*LRQResult, error) {
 	return d.doPingRequest(ctx, req)
 }
 
-func (d *DataSetClient) DoFacetValuesRequest(ctx context.Context, req FacetQuery) (*LRQResult, error) {
+func (d *dataSetClient) DoFacetValuesRequest(ctx context.Context, req FacetQuery) (*LRQResult, error) {
 	return d.doPingRequest(ctx, req)
 }
 
-func (d *DataSetClient) DoTopFacetRequest(ctx context.Context, req TopFacetRequest) (*LRQResult, error) {
+func (d *dataSetClient) DoTopFacetRequest(ctx context.Context, req TopFacetRequest) (*LRQResult, error) {
 	return d.doPingRequest(ctx, req)
 }
 
-func (d *DataSetClient) DoFacetRequest(ctx context.Context, req FacetRequest) (int, error) {
+func (d *dataSetClient) DoFacetRequest(ctx context.Context, req FacetRequest) (int, error) {
 	body, err := json.Marshal(req)
 	if err != nil {
 		log.DefaultLogger.Error("error marshalling request to DataSet", "err", err)

pkg/plugin/plugin.go

@@ -35,7 +35,7 @@ func NewDataSetDatasource(settings backend.DataSourceInstanceSettings) (instance
 }
 
 type DataSetDatasource struct {
-	dataSetClient *DataSetClient
+	dataSetClient DataSetClient
 }
 
 // Dispose here tells plugin SDK that plugin wants to clean up resources when a new instance
@@ -96,7 +96,10 @@ func (d *DataSetDatasource) query(ctx context.Context, query backend.DataQuery)
 
 		// Setting the LRQ api's autoAlign would override the data points requested by the user (via query options).
 		// The query options support explicitly specifying data points (MaxDataPoints) or implicitly via time range and interval.
-		slices := query.MaxDataPoints
+		slices := int64(query.TimeRange.Duration() / query.Interval)
+		if slices > query.MaxDataPoints {
+			slices = query.MaxDataPoints
+		}
 		if slices > 10000 {
 			slices = 10000
 		}

pkg/plugin/plugin_test.go

@@ -99,7 +99,7 @@ func TestLiveQueryDataPQ(t *testing.T) {
 
 	dataResp := resp.Responses[refId]
 	if dataResp.Error != nil {
-		t.Error(err)
+		t.Error(dataResp.Error)
 	}
 
 	fields := dataResp.Frames[0].Fields
@@ -128,6 +128,7 @@ func TestLiveQueryDataPlot(t *testing.T) {
 						From: time.Now().Add(-4 * time.Hour),
 						To:   time.Now(),
 					},
+					Interval:      1 * time.Minute,
 					MaxDataPoints: 1000,
 					JSON:          []byte(`{"expression":"count(severity != 3)","queryType":"Standard","breakDownFacetValue":"severity"}`),
 				},
@@ -144,7 +145,7 @@ func TestLiveQueryDataPlot(t *testing.T) {
 
 	dataResp := resp.Responses[refId]
 	if dataResp.Error != nil {
-		t.Error(err)
+		t.Error(dataResp.Error)
 	}
 
 	fields := dataResp.Frames[0].Fields
@@ -158,3 +159,79 @@ func TestLiveQueryDataPlot(t *testing.T) {
 		}
 	}
 }
+
+type dataSetClientMock struct {
+	lastLRQRequest LRQRequest
+}
+
+func (d *dataSetClientMock) DoLRQRequest(ctx context.Context, req LRQRequest) (*LRQResult, error) {
+	d.lastLRQRequest = req
+	return &LRQResult{Data: []byte("{}")}, nil
+}
+
+func (d *dataSetClientMock) DoFacetValuesRequest(ctx context.Context, req FacetQuery) (*LRQResult, error) {
+	return &LRQResult{Data: []byte("{}")}, nil
+}
+
+func (d *dataSetClientMock) DoTopFacetRequest(ctx context.Context, req TopFacetRequest) (*LRQResult, error) {
+	return &LRQResult{Data: []byte("{}")}, nil
+}
+
+func (d *dataSetClientMock) DoFacetRequest(ctx context.Context, req FacetRequest) (int, error) {
+	return 0, nil
+}
+
+func TestQueryDataOptions(t *testing.T) {
+	var clientMock dataSetClientMock
+	datasource := DataSetDatasource{dataSetClient: &clientMock}
+
+	queryDataSlices := func(interval time.Duration, maxDataPoints int64) int64 {
+		refId := "A"
+		resp, err := datasource.QueryData(
+			context.Background(),
+			&backend.QueryDataRequest{
+				Queries: []backend.DataQuery{
+					{
+						RefID: refId,
+						TimeRange: backend.TimeRange{
+							From: time.Now().Add(-4 * time.Hour),
+							To:   time.Now(),
+						},
+						Interval:      interval,
+						MaxDataPoints: maxDataPoints,
+						JSON:          []byte(`{"expression":"count(severity != 3)","queryType":"Standard"}`),
+					},
+				},
+			},
+		)
+		if err != nil {
+			t.Error(err)
+		}
+
+		if len(resp.Responses) != 1 {
+			t.Fatal("QueryData must return a response")
+		}
+
+		if err := resp.Responses[refId].Error; err != nil {
+			t.Error(err)
+		}
+
+		return clientMock.lastLRQRequest.Plot.Slices
+	}
+
+	if queryDataSlices(1*time.Minute, 1000) != 240 {
+		t.Error("unexpected slice count")
+	}
+	if queryDataSlices(15*time.Second, 1000) != 960 {
+		t.Error("unexpected slice count")
+	}
+	if queryDataSlices(10*time.Second, 1000) != 1000 {
+		t.Error("unexpected slice count")
+	}
+	if queryDataSlices(10*time.Second, 2000) != 1440 {
+		t.Error("unexpected slice count")
+	}
+	if queryDataSlices(1*time.Second, 14400) != 10000 {
+		t.Error("unexpected slice count")
+	}
+}

src/plugin.json

@@ -43,8 +43,8 @@
         "path": "img/DatasetConfig.png"
       }
     ],
-    "version": "3.1.0",
-    "updated": "2023-03-22"
+    "version": "3.1.1",
+    "updated": "2023-03-31"
   },
   "dependencies": {
     "grafanaDependency": ">=8.2.0",