dev -> master (#567)

* IG-16745 - ignore grafana empty search requests (#561)

* ignore grafana empty search requests

* lint

* IG-16807: making sure key passed in inferschema is valid (#558)

* Fix error in tests. (#562)

Co-authored-by: Gal Topper <galt@iguazio.com>

* IG-16524: in case history directory was not created yet, return empty frame (#564)

* IG-16666 (#563)

* sort labels when creating tsdb frame

* rename key -> labelName

* IG-16911: returning error in case field in query is illegal (#566)

* IG-17095 check client and server version match (#556)

* check client and server version match

* python styling

* keep version in server instead of in api

* fixes

* ignore empty sessions

* add env var for py tests

* remove print

* add flag to python makefile

* remove flag from proto build

* change env var to be passed in kwargs

Co-authored-by: Dina Nimrodi <dinan@iguazio.com>

Co-authored-by: Tal Neiman <33829179+talIguaz@users.noreply.github.com>
Co-authored-by: Katya Katsenelenbogen <katyak@iguazio.com>
Co-authored-by: Gal Topper <galt@iguaz.io>
Co-authored-by: Gal Topper <galt@iguazio.com>
Co-authored-by: Dina Nimrodi <dinan@iguazio.com>

Author: 6 people

backends/kv/inferschema.go

@@ -183,6 +183,10 @@ func schemaFromKeys(keyField string, rowSet []map[string]interface{}) (v3ioutils
 			}
 			return nil, errors.Errorf("could not determine which column is the table's primary-key attribute, because %s", reason)
 		}
+	} else {
+		if val, ok := columnCanBeFullKey[keyField]; !ok || !val {
+			return nil, errors.Errorf("%s is not one of the columns", keyField)
+		}
 	}
 
 	newSchema := v3ioutils.NewSchemaWithHashingBuckets(keyField, sortingKeyField, hashingBuckets)

backends/kv/inferschema_test.go

@@ -259,6 +259,18 @@ func (suite *InferSchemaTestSuite) TestInferSchemaWithKeyLikeHashed() {
 	suite.Require().Equal(0, concreteSchema.HashingBucketNum)
 }
 
+func (suite *InferSchemaTestSuite) TestInferSchemaWithInvalidKey() {
+	keyField := "invalid_key"
+	rowSet := []map[string]interface{}{
+		{"__name": "rocky", "name": "rocky", "animal": "dog", "age": 2},
+		{"__name": "mocha", "name": "mocha", "animal": "dog", "age": 3},
+		{"__name": "scratchy", "name": "scratchy", "animal": "cat", "age": 9},
+	}
+	_, err := schemaFromKeys(keyField, rowSet)
+	suite.Require().Error(err)
+	suite.Require().Equal("invalid_key is not one of the columns", err.Error())
+}
+
 func TestInferSchemaTestSuite(t *testing.T) {
 	suite.Run(t, new(InferSchemaTestSuite))
 }

backends/tsdb/reader.go

@@ -21,6 +21,7 @@ such restriction.
 package tsdb
 
 import (
+	"sort"
 	"strings"
 	"time"
 
@@ -180,13 +181,19 @@ func (i *tsdbIterator) Next() bool {
 		columns[i], _ = frame.Column(colName) // Because we are iterating over the Names() it is safe to discard the error
 	}
 
-	for labelName, labelValue := range frame.Labels() {
+	var labelNames []string
+	labels := frame.Labels()
+	for labelName := range labels {
+		labelNames = append(labelNames, labelName)
+	}
+	sort.Strings(labelNames)
+	for _, labelName := range labelNames {
 		name := labelName
 		if name == config.PrometheusMetricNameAttribute {
 			name = "metric_name"
 		}
 
-		icol, err := frames.NewLabelColumn(name, labelValue, frame.Len())
+		icol, err := frames.NewLabelColumn(name, labels[labelName], frame.Len())
 		if err != nil {
 			i.err = err
 			return false

backends/utils/history_server.go

@@ -364,6 +364,15 @@ func (m *HistoryServer) GetLogs(request *frames.HistoryRequest, out chan frames.
 	iter, err := v3ioutils.NewFilesCursor(m.container, &v3io.GetContainerContentsInput{Path: m.LogsFolderPath + "/"})
 
 	if err != nil {
+		if errWithStatusCode, ok := err.(v3ioerrors.ErrorWithStatusCode); ok &&
+			errWithStatusCode.StatusCode() == http.StatusNotFound {
+			frame, err := frames.NewFrameWithNullValues(nil, nil, nil, nil)
+			if err != nil {
+				return err
+			}
+			out <- frame
+			return nil
+		}
 		return fmt.Errorf("Failed to list Frames History log folder %v for read, err: %v", m.LogsFolderPath, err)
 	}
 

clients/py/tests/test_client.py

@@ -32,7 +32,7 @@ def test_client(proto, cls):
         'password': 'duck season',
     }
 
-    client = v3f.Client(address, **session_params)
+    client = v3f.Client(address, should_check_version=False, **session_params)
     assert client.__class__ is cls, 'wrong class'
     for key, value in session_params.items():
         key = 'url' if key == 'data_url' else key
@@ -52,7 +52,7 @@ def test_client_wrong_params(proto, cls):
     }
 
     try:
-        v3f.Client(address, **session_params)
+        v3f.Client(address, should_check_version=False, **session_params)
         raise ValueError('expected fail but finished successfully')
     except ValueError:
         return

clients/py/tests/test_http.py

@@ -43,6 +43,8 @@ def post(self, *args, **kw):
             return self._read(*args, **kw)
         elif args[0].endswith('/write'):
             return self._write(*args, **kw)
+        elif args[0].endswith('/version'):
+            return self._version()
 
     def _read(self, *args, **kw):
         io = BytesIO()
@@ -80,6 +82,21 @@ def json():
 
         return Response
 
+    def _version(self):
+        class Response:
+            ok = True
+
+            @staticmethod
+            def json():
+                return {
+                    'version': 'test_version',
+                }
+
+        return Response
+
+    def close(self):
+        pass
+
 
 def test_read():
     address = 'https://nuclio.io'
@@ -141,4 +158,5 @@ def new_test_client(address='', session=None):
     return v3f.HTTPClient(
         address=address or 'http://example.com',
         session=session,
+        should_check_version=False,
     )

clients/py/tests/test_v3io_frames.py

@@ -33,19 +33,19 @@ def setenv(key, value):
 
 
 def test_client():
-    c = v3f.Client('localhost:8081')
+    c = v3f.Client('localhost:8081', should_check_version=False)
     assert isinstance(c, v3f.gRPCClient), 'default is not grpc'
-    c = v3f.Client('grpc://localhost:8081')
+    c = v3f.Client('grpc://localhost:8081', should_check_version=False)
     assert isinstance(c, v3f.gRPCClient), 'not gRPC'
-    c = v3f.Client('http://localhost:8081')
+    c = v3f.Client('http://localhost:8081', should_check_version=False)
     assert isinstance(c, v3f.HTTPClient), 'not HTTP'
 
 
 def test_client_env():
     url = 'localhost:8080'
     data = json.dumps({'url': url})
     with setenv(v3f.SESSION_ENV_KEY, data):
-        c = v3f.Client('localhost:8081')
+        c = v3f.Client('localhost:8081', should_check_version=False)
 
     assert c.session.url == url, 'missing URL from env'
 

clients/py/v3io_frames/__init__.py

@@ -34,7 +34,7 @@
 
 def Client(address='', data_url='', container='', user='',
            password='', token='', session_id='', frame_factory=pd.DataFrame,
-           concat=pd.concat, persist_connection=False):
+           concat=pd.concat, persist_connection=False, should_check_version=True):
     """Creates a new Frames client object
     NOTE: User authentication must be done using any of the following methods:
     setting the `token` parameter or the V3IO_ACCESS_KEY environment variable
@@ -77,6 +77,8 @@ def Client(address='', data_url='', container='', user='',
         are made often. When True is used, due to the nature of the
         underlying clients, rapid instantiation of the client
         may cause failures (e.g. HTTP NewConnectionError)
+    should_check_version (Optional) : str
+            whether client and server version should be checked; default: true
 
     Return Value
     ----------
@@ -110,7 +112,7 @@ def Client(address='', data_url='', container='', user='',
 
     cls = gRPCClient if protocol == 'grpc' else HTTPClient
     return cls(address, session, persist_connection,
-               frame_factory=frame_factory, concat=concat)
+               frame_factory=frame_factory, concat=concat, should_check_version=should_check_version)
 
 
 def session_from_env():

clients/py/v3io_frames/client.py

@@ -158,7 +158,7 @@ def is_null(self, index, column_name):
 
 class ClientBase:
     def __init__(self, address, session, persist_connection=False,
-                 frame_factory=pd.DataFrame, concat=pd.concat):
+                 frame_factory=pd.DataFrame, concat=pd.concat, should_check_version=True):
         """Creates a new Frames client object
 
         Parameters
@@ -176,6 +176,8 @@ def __init__(self, address, session, persist_connection=False,
             DataFrame factory; currently, pandas and cuDF are supported
         concat : function
             Function for concatenating DataFrames; default: pandas concat
+        should_check_version (Optional) : str
+            whether client and server version should be checked; default: true
 
         Return Value
         ----------
@@ -186,6 +188,7 @@ def __init__(self, address, session, persist_connection=False,
         self._persist_connection = persist_connection
         self.frame_factory = frame_factory
         self.concat = concat
+        self.should_check_version = should_check_version
 
     def read(self, backend, table='', query='', columns=None, filter='',
              group_by='', limit=0, data_format='', row_layout=False,

clients/py/v3io_frames/errors.py

@@ -14,7 +14,7 @@
 
 __all__ = [
     'Error', 'BadRequest', 'MessageError', 'ReadError', 'CreateError',
-    'DeleteError', 'ExecuteError', 'WriteError'
+    'DeleteError', 'ExecuteError', 'WriteError', 'HistoryError', 'VersionError'
 ]
 
 
@@ -52,3 +52,7 @@ class ExecuteError(Error):
 
 class HistoryError(Error):
     """An error in querying history logs"""
+
+
+class VersionError(Error):
+    """An error in getting server version"""