Run the "from_files" recovery tests in separate runners (#7014)

Co-authored-by: Amaury Chamayou <[email protected]>

Author: eddyashton

tests/recovery.py

@@ -13,7 +13,6 @@
 import subprocess
 import json
 from infra.runner import ConcurrentRunner
-from distutils.dir_util import remove_tree, copy_tree
 from infra.consortium import slurp_file
 import infra.health_watcher
 import time
@@ -536,10 +535,23 @@ def test_recover_service_from_files(
         os.path.dirname(os.path.realpath(__file__)), "testdata", directory
     )
 
-    new_common = infra.network.get_common_folder_name(args.workspace, args.label)
-    remove_tree(new_common)
     old_common = os.path.join(service_dir, "common")
-    copy_tree(old_common, new_common)
+    LOG.info(f"Copying common folder: {old_common}")
+    new_common = infra.network.get_common_folder_name(args.workspace, args.label)
+
+    cmd = ["rm", "-rf", new_common]
+    assert (
+        infra.proc.ccall(*cmd).returncode == 0
+    ), f"Could not remove existing {new_common} directory"
+    cmd = ["mkdir", "-p", new_common]
+    assert (
+        infra.proc.ccall(*cmd).returncode == 0
+    ), f"Could not create fresh {new_common} directory"
+    for file in os.listdir(old_common):
+        cmd = ["cp", os.path.join(old_common, file), new_common]
+        assert (
+            infra.proc.ccall(*cmd).returncode == 0
+        ), f"Could not copy {file} to {new_common}"
 
     network = infra.network.Network(args.nodes, args.binary_dir)
 
@@ -1078,27 +1090,13 @@ def run(args):
                         chunk_start_seqno == seqno
                     ), f"{service_status} service at seqno {seqno} did not start a new ledger chunk (started at {chunk_start_seqno})"
 
-    test_recover_service_from_files(
-        args, directory="expired_service", expected_recovery_count=2, test_receipt=True
-    )
-    # sgx_service is historical ledger, from 1.x -> 2.x -> 3.x -> 5.x -> main.
-    # This is used to test recovery from SGX to SNP.
-    test_recover_service_from_files(
-        args, directory="sgx_service", expected_recovery_count=4, test_receipt=False
-    )
 
+def run_recovery_from_files(args):
     test_recover_service_from_files(
         args,
-        directory="double_sealed_service",
-        expected_recovery_count=2,
-        test_receipt=False,
-    )
-
-    test_recover_service_from_files(
-        args,
-        directory="cose_flipflop_service",
-        expected_recovery_count=0,
-        test_receipt=False,
+        directory=args.directory,
+        expected_recovery_count=args.expected_recovery_count,
+        test_receipt=args.test_receipt,
     )
 
 
@@ -1357,6 +1355,28 @@ def add(parser):
         snapshot_tx_interval=30,
     )
 
+    for directory, expected_recovery_count, test_receipt in (
+        ("expired_service", 2, True),
+        # sgx_service is historical ledger, from 1.x -> 2.x -> 3.x -> 5.x -> main.
+        # This is used to test recovery from SGX to SNP.
+        ("sgx_service", 4, False),
+        # double_sealed_service is a regression test for the issue described in #6906
+        ("double_sealed_service", 2, False),
+        # cose_flipflop_service is a regression test for the issue described in #7002
+        ("cose_flipflop_service", 0, False),
+    ):
+        cr.add(
+            f"recovery_from_{directory}",
+            run_recovery_from_files,
+            package="samples/apps/logging/liblogging",
+            nodes=infra.e2e_args.min_nodes(cr.args, f=1),
+            ledger_chunk_bytes="50KB",
+            snapshot_tx_interval=30,
+            directory=directory,
+            expected_recovery_count=expected_recovery_count,
+            test_receipt=test_receipt,
+        )
+
     # Note: `run_corrupted_ledger` runs with very a specific node configuration
     # so that the contents of recovered (and tampered) ledger chunks
     # can be dictated by the test. In particular, the signature interval is large