Add repeat option to run untill tests fail

Add the `--repeat` command line option so each test should succeed
`--repeat` times to pass. One may think of the option as a `--retries`
analog, but instead of assuring that each test passes at least once, we
require each test to pass every time. Combining both non-trivial
`--retries` and `--repeat` options values require each test to pass at
least once in `--retries` runs `--repeat` times in a row.

Needed for tarantool/tarantool#6646

Author: LevKats

lib/options.py

@@ -319,6 +319,22 @@ def __init__(self):
                     Default: ${TEST_RUN_RETRIES} or 0.
                     """))
 
+        parser.add_argument(
+                "--repeat",
+                dest='repeat',
+                default=env_int('TEST_RUN_REPEAT', 1),
+                type=int,
+                help=format_help(
+                    """
+                    The number of times each test will be repeated.
+
+                    Combining both non-trivial --retries and --repeat options
+                    values require each test to pass at least once in
+                    --retries runs --repeat times in a row.
+
+                    Default: ${TEST_RUN_REPEAT} or 1.
+                    """))
+
         parser.add_argument(
                 "-p", "--pattern",
                 dest='pattern',

lib/worker.py

@@ -96,14 +96,15 @@ def get_task_groups():
     """
     suites = find_suites()
     res = collections.OrderedDict()
+    repeat = Options().args.repeat
     for suite in suites:
         key = os.path.basename(suite.suite_path)
         gen_worker = functools.partial(Worker, suite)  # get _id as an arg
         # Split stable and fragile tasks to two groups. Run stable
         # tasks in parallel with other ones. Run fragile tasks one
         # by one when all parallel tasks will be finished.
-        stable_task_ids = [task.id for task in suite.stable_tests()]
-        fragile_task_ids = [task.id for task in suite.fragile_tests()]
+        stable_task_ids = [task.id for task in suite.stable_tests()] * repeat
+        fragile_task_ids = [task.id for task in suite.fragile_tests()] * repeat
         if stable_task_ids:
             res[key] = {
                 'gen_worker': gen_worker,