13
13
import datetime
14
14
import pprint
15
15
import re
16
+ from contextlib import contextmanager
16
17
from functools import cached_property
17
18
import json
19
+ from typing import Iterator , Any
18
20
from unittest import SkipTest
19
21
20
22
import yaml
@@ -62,6 +64,13 @@ def tearDown(self) -> None:
62
64
63
65
super ().tearDown ()
64
66
67
+ @contextmanager
68
+ def logged_subtest (self , action : str ,
69
+ trace_id : str | None = None , metadata : dict [str , Any ] | None = None ) -> Iterator [None ]:
70
+ with self .actions_log .action_scope (action , self .node .name , trace_id , metadata ):
71
+ with self .subTest (msg = action ):
72
+ yield
73
+
65
74
# since this logic id depended on code run by SCT to mark uuid as test, since commit 617026aa, this code it run in the background
66
75
# and not being waited for, so we need to compensate for it here with retries
67
76
@retrying (n = 15 , sleep_time = 10 , allowed_exceptions = (AssertionError ,))
@@ -71,6 +80,7 @@ def check_scylla_version_in_housekeepingdb(self, prev_id: int, expected_status_c
71
80
Validate reported version
72
81
prev_id: check if new version is created
73
82
"""
83
+ self .actions_log .info ("Validating scylla version in housekeepingdb" , target = self .node .name )
74
84
assert self .node .uuid , "Node UUID wasn't created"
75
85
76
86
row = self .housekeeping .get_most_recent_record (query = f"SELECT id, version, ip, statuscode "
@@ -104,7 +114,7 @@ def check_scylla_version_in_housekeepingdb(self, prev_id: int, expected_status_c
104
114
assert row [0 ] > prev_id , f"New row wasn't saved in { self .CHECK_VERSION_TABLE } "
105
115
else :
106
116
assert row [0 ] == prev_id , f"New row was saved in { self .CHECK_VERSION_TABLE } unexpectedly"
107
-
117
+ self . actions_log . info ( "Scylla version in housekeepingdb is validated" , target = self . node . name )
108
118
return row [0 ] if row else 0
109
119
110
120
@property
@@ -134,10 +144,10 @@ def run_cassandra_stress(self, args: str):
134
144
transport_str = c_s_transport_str (
135
145
self .params .get ('peer_verification' ), self .params .get ('client_encrypt_mtls' ))
136
146
stress_cmd += f" -transport '{ transport_str } '"
137
-
138
- result = self .node .remoter .run (stress_cmd )
139
- assert "java.io.IOException" not in result .stdout
140
- assert "java.io.IOException" not in result .stderr
147
+ with self . actions_log . action_scope ( "running cassandra-stress" , target = self . node . name , metadata = { "stress_cmd" : stress_cmd }):
148
+ result = self .node .remoter .run (stress_cmd )
149
+ assert "java.io.IOException" not in result .stdout
150
+ assert "java.io.IOException" not in result .stderr
141
151
142
152
def check_scylla (self ):
143
153
self .node .run_nodetool ("status" )
@@ -223,6 +233,8 @@ def verify_snitch(self, backend_name: str):
223
233
"""
224
234
if not self .params .get ("use_preinstalled_scylla" ):
225
235
self .log .info ("Skipping verifying the snitch due to the 'use_preinstalled_scylla' being set to False" )
236
+ self .actions_log .info (
237
+ "Skipping verifying the snitch due to the 'use_preinstalled_scylla' being set to False" , target = self .node .name )
226
238
return
227
239
228
240
describecluster_snitch = self .get_describecluster_info ().snitch
@@ -234,13 +246,13 @@ def verify_snitch(self, backend_name: str):
234
246
snitch_matches_describecluster = [pattern .search (describecluster_snitch ) for pattern in snitch_patterns ]
235
247
snitch_matches_scylla_yaml = [pattern .search (scylla_yaml_snitch ) for pattern in snitch_patterns ]
236
248
237
- with self .subTest ('verify snitch against describecluster output' ):
249
+ with self .logged_subtest ('verify snitch against describecluster output' ):
238
250
self .assertTrue (any (snitch_matches_describecluster ),
239
251
msg = f"Expected snitch matches for describecluster to not be empty, but was. Snitch "
240
252
f"matches: { snitch_matches_describecluster } "
241
253
)
242
254
243
- with self .subTest ('verify snitch against scylla.yaml configuration' ):
255
+ with self .logged_subtest ('verify snitch against scylla.yaml configuration' ):
244
256
self .assertTrue (any (snitch_matches_scylla_yaml ),
245
257
msg = f"Expected snitch matches for scylla yaml to not be empty, but was. Snitch "
246
258
f"matches: { snitch_matches_scylla_yaml } "
@@ -263,6 +275,7 @@ def verify_docker_latest_match_release(self) -> None:
263
275
264
276
def verify_nvme_write_cache (self ) -> None :
265
277
if self .write_back_cache is None or self .node .parent_cluster .is_additional_data_volume_used ():
278
+ self .actions_log .info ("Skipped verifying NVMe write cache" , target = self .node .name )
266
279
return
267
280
expected_write_cache_value = "write back" if self .write_back_cache else "write through"
268
281
@@ -317,18 +330,18 @@ def run_pre_create_schema(self, replication_factor=1):
317
330
compaction = compaction_strategy , sstable_size = sstable_size )
318
331
319
332
# pylint: disable=too-many-statements,too-many-branches
320
- def test_scylla_service (self ):
333
+ def test_scylla_service (self ): # noqa: PLR0915
321
334
322
335
self .run_pre_create_schema ()
323
336
324
337
backend = self .params .get ("cluster_backend" )
325
338
326
339
if backend == "aws" :
327
- with self .subTest ("check ENA support" ):
340
+ with self .logged_subtest ("check ENA support" ):
328
341
assert self .node .ena_support , "ENA support is not enabled"
329
342
330
343
if backend in ["gce" , "aws" , "azure" ] and self .params .get ("use_preinstalled_scylla" ):
331
- with self .subTest ("check Scylla IO Params" ):
344
+ with self .logged_subtest ("check Scylla IO Params" ):
332
345
try :
333
346
if self .node .db_node_instance_type in ["t3.micro" ]:
334
347
self .skipTest (
@@ -343,57 +356,58 @@ def test_scylla_service(self):
343
356
)
344
357
except SkipTest as exc :
345
358
self .log .info ("Skipping IOTuneValidation due to %s" , exc .args )
359
+ self .actions_log .info ("Skipped IOTuneValidation" , target = self .node .name )
346
360
except Exception : # pylint: disable=broad-except # noqa: BLE001
347
361
self .log .error ("IOTuneValidator failed" , exc_info = True )
348
362
TestFrameworkEvent (source = {self .__class__ .__name__ },
349
363
message = "Error during IOTune params validation." ,
350
364
severity = Severity .ERROR ).publish ()
351
365
352
- with self .subTest ("verify write cache for NVMe devices" ):
366
+ with self .logged_subtest ("verify write cache for NVMe devices" ):
353
367
self .verify_nvme_write_cache ()
354
368
355
369
if (backend != "docker" and not self .params .get ("nonroot_offline_install" )
356
370
and self .node .db_node_instance_type != "t3.micro" ):
357
- with self .subTest ("verify XFS online discard enabled" ):
371
+ with self .logged_subtest ("verify XFS online discard enabled" ):
358
372
self .verify_xfs_online_discard_enabled ()
359
373
360
374
if backend == "gce" :
361
- with self .subTest ("verify users" ):
375
+ with self .logged_subtest ("verify users" ):
362
376
self .verify_users ()
363
377
364
378
expected_housekeeping_status_code = 'cr' if backend == "docker" else 'r'
365
379
366
380
if self .params .get ("use_preinstalled_scylla" ) and backend != "docker" :
367
- with self .subTest ("check the cluster name" ):
381
+ with self .logged_subtest ("check the cluster name" ):
368
382
self .check_cluster_name ()
369
383
370
- with self .subTest ('verify snitch' ):
384
+ with self .logged_subtest ('verify snitch' ):
371
385
self .verify_snitch (backend_name = backend )
372
386
373
- with self .subTest ('verify node health' ):
387
+ with self .logged_subtest ('verify node health' ):
374
388
self .verify_node_health ()
375
389
376
- with self .subTest ("check Scylla server after installation" ):
390
+ with self .logged_subtest ("check Scylla server after installation" ):
377
391
self .check_scylla ()
378
392
379
- with self .subTest ("check cqlsh installation" ):
393
+ with self .logged_subtest ("check cqlsh installation" ):
380
394
self .check_cqlsh ()
381
395
382
- with self .subTest ("check node_exporter liveness" ):
396
+ with self .logged_subtest ("check node_exporter liveness" ):
383
397
node_info_service = NodeLoadInfoServices ().get (self .node )
384
398
assert node_info_service .cpu_load_5
385
399
assert node_info_service .get_node_boot_time_seconds ()
386
400
387
- with self .subTest ( "check scylla_doctor results " ):
388
- if self .params . get ( "run_scylla_doctor " ):
401
+ if self .params . get ( "run_scylla_doctor " ):
402
+ with self .logged_subtest ( "check scylla_doctor results " ):
389
403
self .run_scylla_doctor ()
390
- else :
391
- self .log .info ("Running scylla-doctor is disabled" )
404
+ else :
405
+ self .log .info ("Running scylla-doctor is disabled" )
392
406
393
407
# We don't install any time sync service in docker, so the test is unnecessary:
394
408
# https://github.com/scylladb/scylla/tree/master/dist/docker/etc/supervisord.conf.d
395
409
if backend != "docker" :
396
- with self .subTest ("check if scylla unnecessarily installed a time synchronization service" ):
410
+ with self .logged_subtest ("check if scylla unnecessarily installed a time synchronization service" ):
397
411
# Checks https://github.com/scylladb/scylla/issues/8339
398
412
# If the instance already has systemd-timesyncd
399
413
is_timesyncd_service_installed = self .check_service_existence (service_name = "systemd-timesyncd" )
@@ -444,24 +458,24 @@ def test_scylla_service(self):
444
458
# backend=backend)
445
459
446
460
version_id_after_stop = 0
447
- with self .subTest ("check Scylla server after stop/start" ):
461
+ with self .logged_subtest ("check Scylla server after stop/start" ):
448
462
self .node .stop_scylla (verify_down = True )
449
463
self .node .start_scylla (verify_up = True )
450
-
451
464
# Scylla service has been stopped/started after installation and re-configuration.
452
465
# So we don't need to stop and to start it again
453
466
self .check_scylla ()
454
467
455
468
if not self .node .is_nonroot_install :
456
469
self .log .info ("Validate version after stop/start" )
457
- self .check_housekeeping_service_status (backend = backend )
458
- version_id_after_stop = self .check_scylla_version_in_housekeepingdb (
459
- prev_id = 0 ,
460
- expected_status_code = expected_housekeeping_status_code ,
461
- new_row_expected = False ,
462
- backend = backend )
463
-
464
- with self .subTest ("check Scylla server after restart" ):
470
+ with self .actions_log .action_scope ("Validate version after stop/start" ):
471
+ self .check_housekeeping_service_status (backend = backend )
472
+ version_id_after_stop = self .check_scylla_version_in_housekeepingdb (
473
+ prev_id = 0 ,
474
+ expected_status_code = expected_housekeeping_status_code ,
475
+ new_row_expected = False ,
476
+ backend = backend )
477
+
478
+ with self .logged_subtest ("check Scylla server after restart" ):
465
479
self .node .restart_scylla (verify_up_after = True )
466
480
self .check_scylla ()
467
481
@@ -474,35 +488,41 @@ def test_scylla_service(self):
474
488
backend = backend )
475
489
476
490
if backend != 'docker' :
477
- with self .subTest ("Check the output of perftune.py" ):
491
+ with self .logged_subtest ("Check the output of perftune.py" ):
478
492
perftune_checker = PerftuneOutputChecker (self .node )
479
493
perftune_checker .compare_perftune_results ()
480
494
481
495
if backend == 'docker' :
482
- with self .subTest ("Check docker latest tags" ):
496
+ with self .logged_subtest ("Check docker latest tags" ):
483
497
self .verify_docker_latest_match_release ()
484
498
485
499
def run_scylla_doctor (self ):
486
500
if self .params .get ('client_encrypt' ) and SkipPerIssues ("https://github.com/scylladb/field-engineering/issues/2280" , self .params ):
487
501
self .log .info ("Scylla Doctor test is skipped for encrypted environment due to issue field-engineering#2280" )
502
+ self .actions_log .info (
503
+ "Scylla Doctor test is skipped for encrypted environment due to issue field-engineering#2280" )
488
504
return
489
505
490
506
if self .db_cluster .nodes [0 ].is_nonroot_install and \
491
507
SkipPerIssues ("https://github.com/scylladb/scylla-cluster-tests/issues/10540" , self .params ):
492
508
self .log .info ("Scylla Doctor test is skipped for non-root test due to issue field-engineering#2254. " )
509
+ self .actions_log .info (
510
+ "Scylla Doctor test is skipped for non-root test due to issue field-engineering#2254." )
493
511
return
494
512
495
513
if self .node .parent_cluster .cluster_backend == "docker" :
496
- self .log .info ("Scylla Doctor check in SCT isn't yet support for docker backend" )
514
+ self .log .info ("Scylla Doctor check in SCT isn't yet supported for docker backend" )
515
+ self .actions_log .info ("Scylla Doctor check in SCT isn't yet supported for docker backend" )
497
516
return
498
517
499
518
for node in self .db_cluster .nodes :
500
- scylla_doctor = ScyllaDoctor (node , self .test_config , bool (self .params .get ('unified_package' )))
501
- scylla_doctor .install_scylla_doctor ()
502
- scylla_doctor .argus_collect_sd_package ()
503
- scylla_doctor .run_scylla_doctor_and_collect_results ()
504
- scylla_doctor .analyze_vitals ()
505
- scylla_doctor .analyze_and_verify_results ()
519
+ with self .actions_log .action_scope ("installing and running Scylla Doctor" , target = node .name ):
520
+ scylla_doctor = ScyllaDoctor (node , self .test_config , bool (self .params .get ('unified_package' )))
521
+ scylla_doctor .install_scylla_doctor ()
522
+ scylla_doctor .argus_collect_sd_package ()
523
+ scylla_doctor .run_scylla_doctor_and_collect_results ()
524
+ scylla_doctor .analyze_vitals ()
525
+ scylla_doctor .analyze_and_verify_results ()
506
526
507
527
def get_email_data (self ):
508
528
self .log .info ("Prepare data for email" )
0 commit comments