test(nemesis.py): Add a ScyllaDiagnosisReport nemesis to dump diagnostics report and verify it.

Adds a new ScyllaDiagnosisReport nemesis to send signal to scylladb to dump diagnostics
and verify it.

Fixes: #9443

Signed-off-by: Lakshmipathi.Ganapathi <[email protected]>

Author: Lakshmipathi

Diff for: collect_diagnosis_data_test.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright (c) 2025 ScyllaDB
+
+from sdcm.tester import ClusterTester
+from sdcm import nemesis
+
+
+class ScyllaDiagnosisReport(ClusterTester):
+
+    def test_diagnosis_data(self):  # pylint: disable=invalid-name
+        current_nemesis = nemesis.ScyllaDiagnosisReport(
+            tester_obj=self, termination_event=self.db_cluster.nemesis_termination_event)
+        current_nemesis.disrupt()

Diff for: configurations/nemesis/ScyllaDiagnosisReport.yaml

@@ -0,0 +1,2 @@
+nemesis_class_name: 'ScyllaDiagnosisReport'
+user_prefix: 'ScyllaDiagnosisReport'

Diff for: data_dir/nemesis_classes.yml

@@ -65,6 +65,7 @@
 - RollingRestartConfigChangeInternodeCompression
 - ScyllaKillMonkey
 - SerialRestartOfElectedTopologyCoordinatorNemesis
+- ScyllaDiagnosisReport
 - SlaDecreaseSharesDuringLoad
 - SlaIncreaseSharesByAttachAnotherSlDuringLoad
 - SlaIncreaseSharesDuringLoad

Diff for: jenkins-pipelines/oss/features/sigquit-scylla-test.jenkinsfile

@@ -0,0 +1,13 @@
+#!groovy
+
+// trick from https://github.com/jenkinsci/workflow-cps-global-lib-plugin/pull/43
+def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
+
+longevityPipeline(
+    params: params,
+
+    backend: 'aws',
+    region: 'eu-west-1',
+    test_name: 'collect_diagnosis_data_test.ScyllaDiagnosisReport.test_diagnosis_data',
+    test_config: 'test-cases/features/diagnosis-report.yaml'
+)

Diff for: sdcm/cluster.py

@@ -988,6 +988,10 @@ def instance_name(self) -> str:
         """
         return self.name
 
+    @property
+    def is_diagnostics_logged(self):
+        return self.remoter.run("grep 'Diagnostics dump requested via SIGQUIT' /var/log/messages", ignore_status=True).exit_status == 0
+
     @property
     def is_spot(self):
         return False

Diff for: sdcm/nemesis.py

@@ -702,6 +702,18 @@ def _kill_scylla_daemon(self):
             self.log.info('Waiting JMX services to be restarted after we killed them...')
             self.target_node.wait_jmx_up()
         self.cluster.wait_for_schema_agreement()
+        
+    def _sigquit_scylla_daemon(self):
+        self.log.info('Sending SIGQUIT to scylla processes in %s',
+                      self.target_node)
+        self.target_node.remoter.sudo("pkill -3 scylla", ignore_status=True)
+        # Wait for few seconds before checking diagnosis data
+        time.sleep(30)
+        if self.target_node.is_diagnostics_logged:
+            self.log.info("Diagnosis dump report is found")
+        else:
+            self.log.error("Diagonosis dump report is missing")
+            raise Exception("Diagonosis dump report not found")
 
     @decorate_with_context(ignore_raft_topology_cmd_failing)
     @target_all_nodes
@@ -1744,6 +1756,14 @@ def wait_for_old_node_to_removed():
     def disrupt_kill_scylla(self):
         self._kill_scylla_daemon()
 
+    def disrupt_sigquit_scylla(self):
+        stress_cmd = self.tester.params.get('stress_cmd')
+        self.tester.run_stress_thread(
+            stress_cmd=stress_cmd, stress_num=1, stats_aggregate_cmds=False)
+        # Wait for 5 mins before sending signal
+        time.sleep(300)
+        self._sigquit_scylla_daemon()
+
     def disrupt_no_corrupt_repair(self):
 
         if SkipPerIssues("https://github.com/scylladb/scylladb/issues/18059", self.tester.params):
@@ -6612,6 +6632,14 @@ def disrupt(self):
         self.disrupt_nodetool_flush_and_reshard_on_kubernetes()
 
 
+class ScyllaDiagnosisReport(Nemesis):
+    disruptive = False
+    kubernetes = False
+
+    def disrupt(self):
+        self.disrupt_sigquit_scylla()
+
+
 class ScyllaKillMonkey(Nemesis):
     disruptive = True
     supports_high_disk_utilization = True

Diff for: test-cases/features/diagnosis-report.yaml

@@ -0,0 +1,13 @@
+test_duration: 30
+stress_cmd: "cassandra-stress write no-warmup cl=QUORUM duration=10m -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3)' -mode cql3 native -rate threads=60 -pop seq=1..30000000"
+n_db_nodes: 3
+n_loaders: 1
+n_monitor_nodes: 1
+
+instance_type_db: 'i4i.large'
+nemesis_class_name: 'ScyllaDiagnosisReport'
+nemesis_interval: 5
+
+email_recipients: ['[email protected]']
+stress_image:
+  cassandra-stress: 'scylladb/cassandra-stress:3.13.0'

Diff for: unit_tests/test_nemesis_sisyphus.py

@@ -80,7 +80,7 @@ def test_list_all_available_nemesis(generate_file=True):
     disruption_list, disruptions_dict, disruption_classes = sisyphus.get_list_of_disrupt_methods(
         subclasses_list=subclasses, export_properties=True)
 
-    assert len(disruption_list) == 92
+    assert len(disruption_list) == 93
 
     if generate_file:
         with open(sct_abs_path('data_dir/nemesis.yml'), 'w', encoding="utf-8") as outfile1: