improvement(nemesis_tests): Rework nemesis tests

- Move them to separate directory
- Use custom subclass hierarchies to make them less dependent on the implementation
- Add NemesisRegistry tests
- Fixes #10592

Author: pehala

unit_tests/nemesis/__init__.py

@@ -0,0 +1,69 @@
+"""Mock classes"""
+
+from dataclasses import dataclass, field
+
+from sdcm.cluster import BaseScyllaCluster
+from unit_tests.dummy_remote import LocalLoaderSetDummy
+
+
+PARAMS = dict(nemesis_interval=1, nemesis_filter_seeds=False)
+
+
+@dataclass
+class Node:
+    running_nemesis = None
+    public_ip_address: str = '127.0.0.1'
+    name: str = 'Node1'
+
+    @property
+    def scylla_shards(self):
+        return 8
+
+    def log_message(self, *args, **kwargs):
+        pass
+
+
+@dataclass
+class Cluster:
+    nodes: list
+    params: dict = field(default_factory=lambda: PARAMS)
+
+    def check_cluster_health(self):
+        pass
+
+    @property
+    def data_nodes(self):
+        return self.nodes
+
+    @property
+    def zero_nodes(self):
+        return self.nodes
+
+    def log_message(self, *args, **kwargs):
+        pass
+
+
+@dataclass
+class FakeTester:
+    params: dict = field(default_factory=lambda: PARAMS)
+    loaders: LocalLoaderSetDummy = field(default_factory=LocalLoaderSetDummy)
+    db_cluster: Cluster | BaseScyllaCluster = field(default_factory=lambda: Cluster(nodes=[Node(), Node()]))
+    monitors: list = field(default_factory=list)
+
+    def __post_init__(self):
+        self.db_cluster.params = self.params
+
+    def create_stats(self):
+        pass
+
+    def update(self, *args, **kwargs):
+        pass
+
+    def get_scylla_versions(self):
+        pass
+
+    def get_test_details(self):
+        pass
+
+    def id(self):
+        return 0

unit_tests/nemesis/fake_cluster.py

@@ -0,0 +1,157 @@
+"""
+This module tests NemesisRegistry method on custom Subclass tree
+Should not be dependent on the implementation of Nemesis class
+"""
+
+import pytest
+from sdcm.nemesis_registry import NemesisRegistry
+
+
+class TestNemesis:
+    COMMON_STRING = "called test function "
+    flag_a = False
+    flag_b = False
+    flag_c = False
+    flag_d = False
+    flag_common = False
+
+    def disrupt_method_a(self):
+        print(self.COMMON_STRING + "a")
+
+    def disrupt_method_b(self):
+        print(self.COMMON_STRING + "b")
+
+    def disrupt_method_c(self):
+        print(self.COMMON_STRING + "c")
+
+    def disrupt_method_d(self):
+        print(self.COMMON_STRING + "d")
+
+
+class CustomNemesisA(TestNemesis):
+    flag_a = True
+    flag_common = True
+
+    def disrupt(self):
+        self.disrupt_method_a()
+
+
+class CustomNemesisB(TestNemesis):
+    flag_b = True
+    flag_common = True
+
+    def disrupt(self):
+        self.disrupt_method_b()
+
+
+class CustomNemesisC(CustomNemesisA):
+    flag_c = True
+
+    def disrupt(self):
+        self.disrupt_method_c()
+
+
+class CustomNemesisD(CustomNemesisB):
+    flag_d = True
+    flag_a = True
+    flag_common = True
+
+    def disrupt(self):
+        self.disrupt_method_d()
+
+
+@pytest.fixture
+def registry():
+    return NemesisRegistry(base_class=TestNemesis)
+
+
+@pytest.mark.parametrize(
+    "logical_phrase, expected_classes",
+    [
+        ("flag_a", {CustomNemesisA, CustomNemesisD}),
+        ("flag_b", {CustomNemesisB}),
+        ("flag_c", {CustomNemesisC}),
+        ("flag_d", {CustomNemesisD}),
+        ("flag_common", {CustomNemesisA, CustomNemesisB, CustomNemesisD}),
+    ],
+)
+def test_filter_subclasses_by_single_flag(registry, logical_phrase, expected_classes):
+    filtered = registry.filter_subclasses(registry.get_subclasses(), logical_phrase)
+    assert set(filtered) == expected_classes
+
+
+@pytest.mark.parametrize(
+    "logical_phrase, expected_classes",
+    [
+        ("flag_a and flag_d", {CustomNemesisD}),
+        ("flag_common and not flag_c", {CustomNemesisA, CustomNemesisB, CustomNemesisD}),
+        ("CustomNemesisA or flag_d", {CustomNemesisA, CustomNemesisD}),
+    ],
+)
+def test_filter_subclasses_by_combined_flags(registry, logical_phrase, expected_classes):
+    filtered = registry.filter_subclasses(registry.get_subclasses(), logical_phrase)
+    assert set(filtered) == expected_classes
+
+
+@pytest.mark.parametrize(
+    "logical_phrase, expected_classes",
+    [
+        ("(CustomNemesisA or CustomNemesisD) and not flag_d", {CustomNemesisA}),
+        ("flag_common and not flag_a", {CustomNemesisB}),
+    ],
+)
+def test_filter_subclasses_with_complex_expression(registry, logical_phrase, expected_classes):
+    filtered = registry.filter_subclasses(registry.get_subclasses(), logical_phrase)
+    assert set(filtered) == expected_classes
+
+
+def test_get_subclasses(registry):
+    subclasses = registry.get_subclasses()
+    assert set(subclasses) == {CustomNemesisA, CustomNemesisB, CustomNemesisC, CustomNemesisD}
+
+
+def test_gather_properties(registry):
+    class_properties, method_properties = registry.gather_properties()
+
+    expected_class_properties = {
+        "CustomNemesisA": {"flag_a": True, "flag_common": True},
+        "CustomNemesisB": {"flag_b": True, "flag_common": True},
+        "CustomNemesisC": {"flag_c": True},
+        "CustomNemesisD": {"flag_a": True, "flag_d": True, "flag_common": True},
+    }
+
+    expected_method_properties = {
+        "disrupt_method_a": {"flag_a": True, "flag_common": True},
+        "disrupt_method_b": {"flag_b": True, "flag_common": True},
+        "disrupt_method_c": {"flag_c": True},
+        "disrupt_method_d": {"flag_a": True, "flag_d": True, "flag_common": True},
+    }
+
+    assert class_properties == expected_class_properties
+    assert method_properties == expected_method_properties
+
+
+@pytest.mark.parametrize(
+    "logical_phrase, expected_methods",
+    [
+        ("flag_a", {CustomNemesisA.disrupt_method_a, CustomNemesisD.disrupt_method_d}),
+        ("flag_b", {CustomNemesisB.disrupt_method_b}),
+        ("flag_common and not flag_b", {CustomNemesisA.disrupt_method_a, CustomNemesisD.disrupt_method_d}),
+        ("flag_c", {CustomNemesisC.disrupt_method_c}),
+        ("flag_d", {CustomNemesisD.disrupt_method_d}),
+    ],
+)
+def test_get_disrupt_methods(registry, logical_phrase, expected_methods):
+    disrupt_methods = registry.get_disrupt_methods(logical_phrase)
+    assert set(disrupt_methods) == expected_methods
+
+
+def test_get_disrupt_method_execution(registry, capsys):
+    """Tests how you can use get_disrupt_methods to actually call returned methods"""
+    disrupt_methods = registry.get_disrupt_methods("flag_common and not flag_b")
+    for disrupt_method in disrupt_methods:
+        disrupt_method(TestNemesis())
+
+    captured = capsys.readouterr()
+    assert "called test function a" in captured.out
+    assert "called test function d" in captured.out

unit_tests/nemesis/test_registry.py

@@ -0,0 +1,141 @@
+"""
+This module tests Nemesis/SisyphusMonkey specific class feature directly, with custom Subclass tree
+Should not be dependent on the implementation of Nemesis class
+
+"""
+import pytest
+
+from sdcm.nemesis import Nemesis, SisyphusMonkey
+from sdcm.nemesis_registry import NemesisRegistry
+from unit_tests.nemesis.fake_cluster import FakeTester, PARAMS, Cluster, Node
+from unit_tests.test_tester import ClusterTesterForTests
+
+
+class TestNemesisClass(Nemesis):
+    COMMON_STRING = "called test function "
+    kubernetes = False
+    flag_a = False
+    flag_b = False
+    flag_c = False
+    flag_common = False
+
+    def __init__(self, tester_obj, termination_event, *args, nemesis_selector=None, nemesis_seed=None, **kwargs):
+        super().__init__(tester_obj, termination_event, *args, nemesis_selector=nemesis_selector,
+                         nemesis_seed=nemesis_seed, **kwargs)
+        self.nemesis_registry = NemesisRegistry(base_class=TestNemesisClass)
+
+    def disrupt_method_a(self):
+        print(self.COMMON_STRING + "a")
+
+    def disrupt_method_b(self):
+        print(self.COMMON_STRING + "b")
+
+    def disrupt_method_c(self):
+        print(self.COMMON_STRING + "c")
+
+
+class AddRemoveDCMonkey(TestNemesisClass):
+    flag_common = True
+
+    @TestNemesisClass.add_disrupt_method
+    def disrupt_rnd_method(self):
+        print("disrupt_rnd_method")
+
+    def disrupt(self):
+        self.disrupt_rnd_method()
+
+
+class DisabledMonkey(TestNemesisClass):
+    disabled = True
+    flag_b = True
+    flag_common = True
+
+    def disrupt(self):
+        self.disrupt_method_b()
+
+
+class DisruptAMonkey(TestNemesisClass):
+    flag_a = True
+    flag_common = True
+
+    def disrupt(self):
+        self.disrupt_method_a()
+
+
+class DisruptCMonkey(TestNemesisClass):
+    flag_c = True
+
+    def disrupt(self):
+        self.disrupt_method_c()
+
+
+# Use multiple inheritance to ensure we overide registry after Nemesis but before Sisyphus
+class FakeSisyphusMonkey(SisyphusMonkey, TestNemesisClass):
+    def __init__(self, tester_obj, *args, termination_event=None, nemesis_selector=None, nemesis_seed=None, **kwargs):
+        super().__init__(tester_obj, termination_event, *args, nemesis_selector=nemesis_selector,
+                         nemesis_seed=nemesis_seed, **kwargs)
+
+
+@pytest.fixture()
+def get_sisyphus():
+    def _create_sisyphus(params=PARAMS):
+        return FakeSisyphusMonkey(FakeTester(params=params))
+    return _create_sisyphus
+
+
+@pytest.mark.parametrize(
+    "params, expected",
+    [
+        pytest.param({"nemesis_exclude_disabled": True},
+                     {"disrupt_method_a", "disrupt_method_c", "disrupt_rnd_method"},
+                     id="exclude_disabled"),
+        pytest.param({"nemesis_exclude_disabled": False},
+                     {"disrupt_method_a", "disrupt_method_c", "disrupt_rnd_method", "disrupt_method_b"},
+                     id="disabled"),
+    ]
+)
+def test_disruptions_list(get_sisyphus, params, expected):
+    if params:
+        params.update(PARAMS)
+    nemesis = get_sisyphus(params=params)
+    assert expected == set(method.__name__ for method in nemesis.disruptions_list)
+
+
+def test_list_nemesis_of_added_disrupt_methods(get_sisyphus, capsys):
+    nemesis = get_sisyphus()
+    nemesis.disruptions_list = nemesis.build_disruptions_by_name(['disrupt_rnd_method'])
+    nemesis.call_next_nemesis()
+    captured = capsys.readouterr()
+    assert "disrupt_rnd_method" in captured.out
+
+
+def test_add_sisyphus_with_filter_in_parallel_nemesis_run():
+    tester = ClusterTesterForTests()
+    tester._init_params()
+    tester.db_cluster = Cluster(nodes=[Node(), Node()])
+    tester.db_cluster.params = tester.params
+    tester.params["nemesis_class_name"] = "SisyphusMonkey:1 SisyphusMonkey:2"
+    tester.params["nemesis_selector"] = ["flag_common",
+                                         "flag_common and not flag_a",
+                                         "flag_c"]
+    tester.params["nemesis_exclude_disabled"] = True
+    tester.params["nemesis_multiply_factor"] = 1
+    nemesises = tester.get_nemesis_class()
+
+    expected_selectors = ["flag_common", "flag_common and not flag_a",  "flag_c"]
+    for i, nemesis_settings in enumerate(nemesises):
+        assert nemesis_settings['nemesis'] == SisyphusMonkey, \
+            f"Wrong instance of nemesis class {nemesis_settings['nemesis']} expected SisyphusMonkey"
+        assert nemesis_settings['nemesis_selector'] == expected_selectors[i], \
+            f"Wrong nemesis filter selecters {nemesis_settings['nemesis_selector']} expected {expected_selectors[i]}"
+
+    active_nemesis = []
+    for nemesis in nemesises:
+        sisyphus = FakeSisyphusMonkey(tester, nemesis_selector=nemesis["nemesis_selector"])
+        active_nemesis.append(sisyphus)
+
+    expected_methods = [{"disrupt_method_a", "disrupt_rnd_method"},
+                        {"disrupt_rnd_method"},
+                        {"disrupt_method_c"}]
+    for i, nem in enumerate(active_nemesis):
+        assert {disrupt.__name__ for disrupt in nem.disruptions_list} == expected_methods[i]