-
Notifications
You must be signed in to change notification settings - Fork 100
/
Copy pathnode_operations.py
88 lines (74 loc) · 4.24 KB
/
node_operations.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import contextlib
import logging
from typing import Optional
from sdcm.cluster import BaseNode
LOGGER = logging.getLogger(__name__)
@contextlib.contextmanager
def block_scylla_ports(target_node: BaseNode, ports: list[int] | None = None):
ports = ports or [7001, 7000, 9042, 9142, 19042, 19142]
target_node.install_package("iptables")
target_node.start_service("iptables", ignore_status=True)
target_node.log.debug("Block connections %s", target_node.name)
for port in ports:
target_node.remoter.sudo(f"iptables -A INPUT -p tcp --dport {port} -j DROP")
target_node.remoter.sudo(f"iptables -A OUTPUT -p tcp --dport {port} -j DROP")
target_node.remoter.sudo(f"ip6tables -A INPUT -p tcp --dport {port} -j DROP")
target_node.remoter.sudo(f"ip6tables -A OUTPUT -p tcp --dport {port} -j DROP")
yield
target_node.log.debug("Remove all iptable rules %s", target_node.name)
for port in ports:
target_node.remoter.sudo(f"iptables -D INPUT -p tcp --dport {port} -j DROP")
target_node.remoter.sudo(f"iptables -D OUTPUT -p tcp --dport {port} -j DROP")
target_node.remoter.sudo(f"ip6tables -D INPUT -p tcp --dport {port} -j DROP")
target_node.remoter.sudo(f"ip6tables -D OUTPUT -p tcp --dport {port} -j DROP")
target_node.stop_service("iptables", ignore_status=True)
@contextlib.contextmanager
def pause_scylla_with_sigstop(target_node: BaseNode):
target_node.log.debug("Send signal SIGSTOP to scylla process on node %s", target_node.name)
target_node.remoter.sudo("pkill --signal SIGSTOP -e scylla", timeout=60)
yield
target_node.log.debug("Send signal SIGCONT to scylla process on node %s", target_node.name)
target_node.remoter.sudo(cmd="pkill --signal SIGCONT -e scylla", timeout=60)
@contextlib.contextmanager
def block_loaders_payload_for_scylla_node(scylla_node: BaseNode, loader_nodes: list[BaseNode]):
""" Block connections from loaders to cql ports on scylla node
Make the Scylla node inaccessible to loaders by blocking
any subsequent connections to the Scylla node.
This ensures that the stress tool can continue to operate without failure
even if the Scylla node is banned and removed from the cluster.
"""
ports = [9042, 9142, 19042, 19142]
scylla_node.install_package("iptables")
scylla_node.start_service("iptables", ignore_status=True)
loader_nodes_names = [node.name for node in loader_nodes]
blocking_ips = [node.ip_address for node in loader_nodes]
scylla_node.log.debug("Block connections on %s from loader nodes %s", scylla_node.name, loader_nodes_names)
for port in ports:
scylla_node.remoter.sudo(
f"iptables -A INPUT -s {','.join(blocking_ips)} -p tcp --dport {port} -j DROP", ignore_status=True)
scylla_node.remoter.sudo(
f"ip6tables -A INPUT -s {','.join(blocking_ips)} -p tcp --dport {port} -j DROP", ignore_status=True)
yield
# if scylla_node is alive, then delete the iptables rules
if scylla_node.remoter.is_up():
for port in ports:
scylla_node.remoter.sudo(
f"iptables -D INPUT -s {','.join(blocking_ips)} -p tcp --dport {port} -j DROP", ignore_status=True)
scylla_node.remoter.sudo(
f"ip6tables -D INPUT -s {','.join(blocking_ips)} -p tcp --dport {port} -j DROP", ignore_status=True)
scylla_node.stop_service("iptables", ignore_status=True)
def is_node_removed_from_cluster(removed_node: BaseNode, verification_node: BaseNode) -> bool:
LOGGER.debug("Verification node %s", verification_node.name)
cluster_status: Optional[dict] = removed_node.parent_cluster.get_nodetool_status(
verification_node=verification_node)
if not cluster_status:
return False
result = []
for dc in cluster_status:
result.append(removed_node.ip_address not in cluster_status[dc].keys())
return all(result)
def is_node_seen_as_down(down_node: BaseNode, verification_node: BaseNode) -> bool:
LOGGER.debug("Verification node %s", verification_node.name)
nodes_status = verification_node.parent_cluster.get_nodetool_status(verification_node, dc_aware=False)
down_node_status = nodes_status.get(down_node.ip_address)
return (not down_node_status or down_node_status["state"] == "DN")