Cumulative changes for RLEC/encoding

.github/workflows/ci.yml

@@ -68,6 +68,7 @@ jobs:
         make BUILD_TLS=yes -j
         ./src/redis-server --version
         cd ..
+
     - name: Generate test certificates
       run: |
         cd redis

RLTest/__main__.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import argparse
+import io
 import os
 import cmd
 import traceback
@@ -165,6 +166,10 @@ def do_normal_conn(self, line):
     '--verbose', '-v', action='count', default=0,
     help='print more information about the test')
 
+parser.add_argument(
+    '--print-server-cmd', action='store_true',
+    help='print redis-server incovation commands')
+
 parser.add_argument(
     '--debug', action='store_const', const=True, default=False,
     help='stop before each test allow gdb attachment')
@@ -292,6 +297,7 @@ def __exit__(self, type, value, traceback):
 
 
 class RLTest:
+
     def __init__(self):
         # adding the current path to sys.path for test import puspused
         sys.path.append(os.getcwd())
@@ -376,6 +382,7 @@ def __init__(self):
         Defaults.env = self.args.env
         Defaults.binary = self.args.oss_redis_path
         Defaults.verbose = self.args.verbose
+        Defaults.print_server_cmd = self.args.print_server_cmd
         Defaults.logdir = self.args.log_dir
         Defaults.use_slaves = self.args.use_slaves
         Defaults.num_shards = self.args.shards_count
@@ -523,7 +530,8 @@ def _runTest(self, test, numberOfAssertionFailed=0, prefix='', before=None, afte
 
         if len(inspect.getargspec(test.target).args) > 0 and not test.is_method:
             try:
-                env = Env(testName=test.name)
+                # env = Env(testName=test.name)
+                env = Defaults.env_factory(testName=test.name)
             except Exception as e:
                 self.handleFailure(exception=e, prefix=msgPrefix, testname=test.name)
                 return 0
@@ -607,7 +615,8 @@ def execute(self):
         Env.RTestInstance = self
         if self.args.env_only:
             Defaults.verbose = 2
-            env = Env(testName='manual test env')
+            # env = Env(testName='manual test env')
+            env = Defaults.env_factory(testName='manual test env')
             if self.args.interactive_debugger:
                 while env.isUp():
                     time.sleep(1)
@@ -670,6 +679,9 @@ def execute(self):
 
 
 def main():
+    # Aviod "UnicodeEncodeError: 'ascii' codec can't encode character" errors
+    sys.stdout = io.open(sys.stdout.fileno(), 'w', encoding='utf8')
+    sys.stderr = io.open(sys.stderr.fileno(), 'w', encoding='utf8')
     RLTest().execute()
 
 

RLTest/env.py

@@ -28,15 +28,18 @@ def method(*argc, **nargs):
 
 
 class Query:
-    def __init__(self, env, *query):
+    def __init__(self, env, *query, **kwargs):
         self.query = query
         self.env = env
+        self.kwargs = kwargs
+        if 'conn' not in kwargs:
+            self.kwargs['conn'] = env.con
         self.errorRaised = False
         self._evaluate()
 
     def _evaluate(self):
         try:
-            self.res = self.env.cmd(*self.query)
+            self.res = self.env.cmd(*self.query, **self.kwargs)
         except Exception as e:
             self.res = str(e)
             self.errorRaised = True
@@ -58,6 +61,14 @@ def debugPrint(self):
         self.env.debugPrint('query: %s, result: %s' % (self.query, self.res), force=True)
         return self
 
+    def apply(self, fn):
+        self.res = fn(self.res)
+        return self
+
+    def map(self, fn):
+        self.res = list(map(fn, self.res))
+        return self
+
     def equal(self, expected):
         self.env.assertEqual(self.res, expected, 1)
         return self
@@ -103,6 +114,7 @@ class Defaults:
     module_args = None
 
     env = 'oss'
+    env_factory = lambda *args, **kwargs: Env(*args, **kwargs)
     binary = 'redis-server'
     proxy_binary = None
     re_binary = None
@@ -152,7 +164,7 @@ def getKwargs(self):
 class Env:
     RTestInstance = None
     EnvCompareParams = ['module', 'moduleArgs', 'env', 'useSlaves', 'shardsCount', 'useAof',
-                        'useRdbPreamble', 'forceTcp']
+                        'useRdbPreamble', 'forceTcp', 'decodeResponses']
 
     def compareEnvs(self, env):
         if env is None:
@@ -182,7 +194,7 @@ def __init__(self, testName=None, testDescription=None, module=None,
         self.env = env if env else Defaults.env
         self.useSlaves = useSlaves if useSlaves else Defaults.use_slaves
         self.shardsCount = shardsCount if shardsCount else Defaults.num_shards
-        self.decodeResponses = decodeResponses if decodeResponses else Defaults.decode_responses
+        self.decodeResponses = decodeResponses if decodeResponses is not None else Defaults.decode_responses
         self.useAof = useAof if useAof else Defaults.use_aof
         self.useRdbPreamble = useRdbPreamble if useRdbPreamble is not None else Defaults.use_rdb_preamble
         self.verbose = Defaults.verbose
@@ -289,6 +301,7 @@ def getEnvKwargs(self):
             'debugger': Defaults.debugger,
             'noCatch': Defaults.no_capture_output,
             'verbose': Defaults.verbose,
+            'printServerCmd': Defaults.print_server_cmd,
             'useTLS': self.useTLS,
             'tlsCertFile': self.tlsCertFile,
             'tlsKeyFile': self.tlsKeyFile,
@@ -336,7 +349,7 @@ def flush(self):
         self.envRunner.flush()
 
     def isCluster(self):
-        return 'cluster' in self.env
+        return 'cluster' in self.env or os.getenv("RLEC_CLUSTER") == "1"
 
     def isEnterpiseCluster(self):
         return isinstance(self.envRunner, EnterpriseRedisClusterEnv)
@@ -417,11 +430,15 @@ def assertIsInstance(self, value, instance, depth=0):
     def assertAlmostEqual(self, value1, value2, delta, depth=0):
         self._assertion('%s almost equels %s (delta %s)' % (repr(value1), repr(value2), repr(delta)), abs(value1 - value2) <= delta, depth)
 
-    def expect(self, *query):
-        return Query(self, *query)
+    def expect(self, *query, **kwargs):
+        conn = kwargs.pop('conn', None)
+        return Query(self, conn=conn, *query, **kwargs)
 
-    def cmd(self, *query):
-        res = self.con.execute_command(*query)
+    def cmd(self, *query, **kwargs):
+        conn = kwargs.pop('conn', None)
+        if conn is None:
+            conn = self.con
+        res = conn.execute_command(*query, **kwargs)
         self.debugPrint('query: %s, result: %s' % (repr(query), repr(res)))
         return res
 
@@ -436,9 +453,9 @@ def assertExists(self, val, depth=0):
         warnings.warn("AssertExists is deprecated, use cmd instead", DeprecationWarning)
         self._assertion('%s exists in db' % repr(val), self.con.exists(val), depth=0)
 
-    def executeCommand(self, *query):
+    def executeCommand(self, *query, **kwargs):
         warnings.warn("execute_command is deprecated, use cmd instead", DeprecationWarning)
-        return self.cmd(*query)
+        return self.cmd(*query, **kwargs)
 
     def reloadingIterator(self):
         yield 1
@@ -512,6 +529,10 @@ def skipOnCluster(self):
         if self.isCluster():
             self.skip()
 
+    def skipOnExistingEnv(self):
+        if self.env == 'existing-env':
+            self.skip()
+
     def isUnixSocket(self):
         return self.envRunner.isUnixSocket()
 

RLTest/exists_redis.py

@@ -1,3 +1,4 @@
+
 from __future__ import print_function
 import redis
 import subprocess
@@ -12,10 +13,12 @@
 
 
 class ExistsRedisEnv(object):
-    def __init__(self, addr='localhost:6379', password = None, **kargs):
+    def __init__(self, addr='localhost:6379', password = None, decodeResponses=False, **kwargs):
         self.host, self.port = addr.split(':')
         self.port = int(self.port)
         self.password = password
+        self.decodeResponses = decodeResponses
+        self.useTLS = kwargs['useTLS']
 
     @property
     def has_interactive_debugger(self):
@@ -36,7 +39,7 @@ def stopEnv(self, masters = True, slaves = True):
         pass
 
     def getConnection(self, shardId=1):
-        return redis.StrictRedis(self.host, self.port, password=self.password)
+        return redis.StrictRedis(self.host, self.port, password=self.password, decode_responses=self.decodeResponses)
 
     def getSlaveConnection(self):
         raise Exception('asked for slave connection but no slave exists')
@@ -63,7 +66,7 @@ def _waitForBgsaveToFinish(self):
         while True:
             if not self.getConnection().execute_command('info', 'Persistence')['rdb_bgsave_in_progress']:
                 break
-        
+
     def flush(self):
         self.getConnection().flushall()
         self._waitForBgsaveToFinish()
@@ -100,6 +103,9 @@ def checkExitCode(self):
     def isUp(self):
         return self.getConnection().ping()
 
+    def isTLS(self):
+        return self.useTLS
+
     def exists(self, val):
         return self.getConnection().exists(val)
 

RLTest/redis_std.py

@@ -21,7 +21,7 @@ class StandardEnv(object):
     def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None, outputFilesFormat=None,
                  dbDirPath=None, useSlaves=False, serverId=1, password=None, libPath=None, clusterEnabled=False, decodeResponses=False,
                  useAof=False, useRdbPreamble=True, debugger=None, noCatch=False, unix=False, verbose=False, useTLS=False, tlsCertFile=None,
-                 tlsKeyFile=None, tlsCaCertFile=None, clusterNodeTimeout = None):
+                 tlsKeyFile=None, tlsCaCertFile=None, clusterNodeTimeout = None, printServerCmd=False):
         self.uuid = uuid.uuid4().hex
         self.redisBinaryPath = os.path.expanduser(redisBinaryPath) if redisBinaryPath.startswith(
             '~/') else redisBinaryPath
@@ -47,6 +47,7 @@ def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None,
         self.slaveProcess = None
         self.slaveExitCode = None
         self.verbose = verbose
+        self.printServerCmd = printServerCmd
         self.role = MASTER
         self.useTLS = useTLS
         self.tlsCertFile = tlsCertFile
@@ -265,14 +266,14 @@ def startEnv(self, masters = True, slaves = True):
             'env': self.environ
         }
 
-        if self.verbose:
+        if self.verbose or self.printServerCmd:
             print(Colors.Green("Redis master command: " + ' '.join(self.masterCmdArgs)))
         if masters and self.masterProcess is None:
             self.masterProcess = subprocess.Popen(args=self.masterCmdArgs, **options)
             con = self.getConnection()
             self.waitForRedisToStart(con)
         if self.useSlaves and slaves and self.slaveProcess is None:
-            if self.verbose:
+            if self.verbose or self.printServerCmd:
                 print(Colors.Green("Redis slave command: " + ' '.join(self.slaveCmdArgs)))
             self.slaveProcess = subprocess.Popen(args=self.slaveCmdArgs, **options)
             con = self.getSlaveConnection()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "RLTest"
-version = "0.4.2"
+version = "0.4.2.dev1"
 description="Redis Labs Test Framework, allow to run tests on redis and modules on a variety of environments"
 authors = ["RedisLabs <[email protected]>"]
 license = "BSD-3-Clause"
@@ -25,8 +25,10 @@ classifiers = [
 [tool.poetry.dependencies]
 python = "^2.7,<2.8 || >= 3.5.0"
 distro = "^1.5.0"
-redis = "^3.5.3"
-redis-py-cluster = "*"
+# redis = "^3.5.3"
+# redis-py-cluster = "*"
+redis = { git = "https://github.com/redisfab/redis-py.git", branch = "3.5" }
+redis-py-cluster = { git = "https://github.com/redisfab/redis-py-cluster.git", branch = "2.1" }
 psutil = "^5.8.0"
 pytest-cov = "2.5"