Skip to content

Commit 069b476

Browse files
d-nagyd-nagy
committed
Refactor use of 'RestoreSettings.modify_storage_dir_name'
+ Rename the setting to "use_csp_specific_archive" to be more precise + Add "csp" property in ClusterSpec to simplify use of "use_csp_specific_archive" + Also remove unused "cloud" property of CbbackupmgrSettings and subclasses Change-Id: I73122729eb4dde25e32105bd354aa6ad267162d9 Reviewed-on: https://review.couchbase.org/c/perfrunner/+/235501 Tested-by: Build Bot <build@couchbase.com> Reviewed-by: Elliot Broxton <elliot.broxton@couchbase.com>
1 parent c535e3c commit 069b476

24 files changed

Lines changed: 44 additions & 84 deletions

‎perfrunner/helpers/reporter.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,12 @@ def _post_metric(self, metric: JSON):
4747
category = metric.get('category') or self.test_config.showfast.category
4848
sub_category = metric.get('subCategory') or self.test_config.showfast.sub_category
4949

50-
if self.cluster_spec.capella_infrastructure:
51-
sub_category = sub_category.format(provider=self.cluster_spec.capella_backend.upper())
52-
if self.cluster_spec.columnar_infrastructure:
53-
metric["provider"] = "columnar"
54-
else:
55-
metric['provider'] = self.cluster_spec.cloud_provider.lower()
56-
elif self.cluster_spec.cloud_infrastructure:
57-
sub_category = sub_category.format(provider=self.cluster_spec.cloud_provider.upper())
50+
if self.cluster_spec.cloud_infrastructure:
51+
sub_category = sub_category.format(provider=self.cluster_spec.csp.upper())
52+
if self.cluster_spec.capella_infrastructure:
53+
metric["provider"] = (
54+
"columnar" if self.cluster_spec.columnar_infrastructure else "capella"
55+
)
5856

5957
metric.update({
6058
'cluster': cluster,

‎perfrunner/settings.py‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,13 @@ def dynamic_infrastructure(self) -> bool:
129129
def cloud_infrastructure(self) -> bool:
130130
return 'infrastructure' in self.config.sections()
131131

132+
@property
133+
def csp(self) -> str:
134+
"""Cloud Service Provider."""
135+
if self.cloud_infrastructure:
136+
return self.capella_backend or self.cloud_provider
137+
return ""
138+
132139
@property
133140
def cloud_provider(self) -> str:
134141
return self.config.get('infrastructure', 'provider', fallback='')
@@ -737,15 +744,11 @@ def capella_admin_credentials(self) -> list[tuple[str, str]]:
737744
def parameters(self) -> dict:
738745
from perfrunner.helpers.config_files import ClusterMetadataFile
739746

740-
csp = ""
741-
if self.cloud_infrastructure:
742-
csp = self.capella_backend or self.cloud_provider
743-
744747
overrides = self._get_options_as_dict("parameters")
745748
cluster_name = self.config.get(
746749
"metadata", "cluster", fallback=ClusterMetadataFile.DEFAULT_KEY_GROUP
747750
)
748-
with ClusterMetadataFile(csp) as metadata_file:
751+
with ClusterMetadataFile(self.csp) as metadata_file:
749752
return metadata_file.get_parameters(cluster_name, overrides)
750753

751754
@property
@@ -2578,7 +2581,6 @@ class CbbackupmgrSettings:
25782581
ENCRYPTED = False
25792582
PASSPHRASE = "couchbase"
25802583
INCLUDE_DATA = None
2581-
CLOUD = None
25822584

25832585
def __init__(self, options: dict):
25842586
self.threads = options.get("threads", self.THREADS)
@@ -2593,7 +2595,6 @@ def __init__(self, options: dict):
25932595
for kv in options.get("env_vars", "").replace(" ", "").split(",")
25942596
if kv
25952597
)
2596-
self.cloud = self.CLOUD
25972598

25982599

25992600
class BackupSettings(CbbackupmgrSettings):
@@ -2619,14 +2620,6 @@ def __init__(self, options: dict):
26192620
self.aws_credential_path = options.get("aws_credential_path", self.AWS_CREDENTIAL_PATH)
26202621
self.backup_directory = options.get("backup_directory", self.BACKUP_DIRECTORY)
26212622

2622-
if self.backup_directory:
2623-
if self.backup_directory.startswith("s3://"):
2624-
self.cloud = "aws"
2625-
elif self.backup_directory.startswith("gs://"):
2626-
self.cloud = "gcp"
2627-
elif self.backup_directory.startswith("az://"):
2628-
self.cloud = "azure"
2629-
26302623

26312624
class RestoreSettings(CbbackupmgrSettings):
26322625
DOCS_PER_COLLECTION = 0
@@ -2645,17 +2638,9 @@ def __init__(self, options):
26452638
self.backup_repo = options.get("backup_repo", self.BACKUP_REPO)
26462639
self.import_file = options.get("import_file", self.IMPORT_FILE)
26472640
self.map_data = options.get("map_data", self.MAP_DATA)
2648-
self.modify_storage_dir_name = bool(options.get("modify_storage_dir_name", 0))
2641+
self.use_csp_specific_archive = bool(options.get("use_csp_specific_archive", 0))
26492642
self.filter_keys = options.get("filter_keys", None)
26502643

2651-
if self.backup_storage:
2652-
if self.backup_storage.startswith("s3://"):
2653-
self.cloud = "aws"
2654-
elif self.backup_storage.startswith("gs://"):
2655-
self.cloud = "gcp"
2656-
elif self.backup_storage.startswith("az://"):
2657-
self.cloud = "azure"
2658-
26592644
def __str__(self) -> str:
26602645
return str(self.__dict__)
26612646

‎perfrunner/tests/analytics.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -475,11 +475,8 @@ def _restore_remote(self):
475475

476476
self.remote.client_drop_caches()
477477

478-
if self.test_config.restore_settings.modify_storage_dir_name:
479-
suffix_repo = "aws"
480-
if self.cluster_spec.capella_infrastructure:
481-
suffix_repo = self.cluster_spec.capella_backend
482-
archive += f"/{suffix_repo}"
478+
if self.test_config.restore_settings.use_csp_specific_archive:
479+
archive += f"/{self.cluster_spec.csp.lower()}"
483480

484481
self.remote.restore(
485482
cluster_spec=self.cluster_spec,

‎perfrunner/tests/fts.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -557,11 +557,8 @@ def cloud_restore(self):
557557
"{0}._default._default={0}.scope-1.collection-1"\
558558
.format(target.bucket)
559559
archive = self.test_config.restore_settings.backup_storage
560-
if self.test_config.restore_settings.modify_storage_dir_name:
561-
suffix_repo = "aws"
562-
if self.cluster_spec.capella_infrastructure:
563-
suffix_repo = self.cluster_spec.capella_backend
564-
archive = archive + "/" + suffix_repo
560+
if self.test_config.restore_settings.use_csp_specific_archive:
561+
archive += f"/{self.cluster_spec.csp.lower()}"
565562

566563
self.remote.restore(
567564
cluster_spec=self.cluster_spec,

‎perfrunner/tests/n1ql.py‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -816,11 +816,8 @@ def restore_remote(self):
816816
self.remote.client_drop_caches()
817817

818818
archive = self.test_config.restore_settings.backup_storage
819-
if self.test_config.restore_settings.modify_storage_dir_name:
820-
suffix_repo = "aws"
821-
if self.cluster_spec.capella_infrastructure:
822-
suffix_repo = self.cluster_spec.capella_backend
823-
archive += f"/{suffix_repo}"
819+
if self.test_config.restore_settings.use_csp_specific_archive:
820+
archive += f"/{self.cluster_spec.csp.lower()}"
824821

825822
self.remote.restore(
826823
cluster_spec=self.cluster_spec,
@@ -1368,11 +1365,8 @@ def cloud_restore(self):
13681365
"{0}._default._default={0}.scope-1.collection-1"\
13691366
.format(target.bucket)
13701367
archive = self.test_config.restore_settings.backup_storage
1371-
if self.test_config.restore_settings.modify_storage_dir_name:
1372-
suffix_repo = "aws"
1373-
if self.cluster_spec.capella_infrastructure:
1374-
suffix_repo = self.cluster_spec.capella_backend
1375-
archive = archive + "/" + suffix_repo
1368+
if self.test_config.restore_settings.use_csp_specific_archive:
1369+
archive += f"/{self.cluster_spec.csp.lower()}"
13761370

13771371
self.remote.restore(
13781372
cluster_spec=self.cluster_spec,

‎perfrunner/tests/secondary.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -163,11 +163,8 @@ def cloud_restore(self):
163163
"{0}._default._default={0}.scope-1.collection-1"\
164164
.format(target.bucket)
165165
archive = self.test_config.restore_settings.backup_storage
166-
if self.test_config.restore_settings.modify_storage_dir_name:
167-
suffix_repo = "aws"
168-
if self.cluster_spec.capella_infrastructure:
169-
suffix_repo = self.cluster_spec.capella_backend
170-
archive = archive + "/" + suffix_repo
166+
if self.test_config.restore_settings.use_csp_specific_archive:
167+
archive += f"/{self.cluster_spec.csp.lower()}"
171168

172169
self.remote.restore(
173170
cluster_spec=self.cluster_spec,

‎perfrunner/tests/syncgateway.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2726,11 +2726,8 @@ def cloud_restore(self):
27262726
"{0}._default._default={0}.scope-1.collection-1"\
27272727
.format(target.bucket)
27282728
archive = self.test_config.restore_settings.backup_storage
2729-
if self.test_config.restore_settings.modify_storage_dir_name:
2730-
suffix_repo = "aws"
2731-
if self.capella_infra:
2732-
suffix_repo = self.cluster_spec.capella_backend
2733-
archive = archive + "/" + suffix_repo
2729+
if self.test_config.restore_settings.use_csp_specific_archive:
2730+
archive += f"/{self.cluster_spec.csp.lower()}"
27342731

27352732
self.remote.restore(
27362733
cluster_spec=self.cluster_spec,

‎perfrunner/utils/terraform.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -156,12 +156,7 @@ def __init__(self, infra_spec: ClusterSpec, options: Namespace):
156156
self.infra_spec = infra_spec
157157
self.options = options
158158
self.os_arch = self.infra_spec.infrastructure_settings.get("os_arch", "x86_64")
159-
160-
self.csp = (
161-
self.infra_spec.capella_backend.lower()
162-
if (csp := self.infra_spec.cloud_provider.lower()) == "capella"
163-
else csp
164-
)
159+
self.csp = self.infra_spec.csp
165160

166161
self.uuid = self.infra_spec.infrastructure_settings.get("uuid", uuid4().hex[:6])
167162
self.infra_spec.config.set("infrastructure", "uuid", self.uuid)

‎tests/cloud/analytics/bigfun_20M_reb_out_1s_99c.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ aws_credential_path = /root/.ssh
3636

3737
[restore]
3838
backup_storage = s3://analytics-bigfun20m-99collections
39-
modify_storage_dir_name = True
39+
use_csp_specific_archive = True
4040
backup_repo = default
4141
threads = 8
4242
map_data = bucket-1.scope-1=bucket-1.scope-1

‎tests/cloud/analytics/capella_bigfun_20M_queries_3n_1s_99c.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ aws_credential_path = /root/.ssh
3737

3838
[restore]
3939
backup_storage = s3://analytics-bigfun20m-99collections
40-
modify_storage_dir_name = True
40+
use_csp_specific_archive = True
4141
backup_repo = default
4242
threads = 8
4343
map_data = bucket-1.scope-1=bucket-1.scope-1

0 commit comments

Comments
 (0)