CBPS-1529, CBPS-1645: Set up Enterprise Analytics cluster

+ Update "is_columnar" condition in ServerInfo to include EA
+ Add blob storage settings for EA
+ Add cloud EA test configs
+ Update blob storage configuration methods to support Azure

Change-Id: I685e0f305c942874cb9eefd6e439a85fee6e6411
Reviewed-on: https://review.couchbase.org/c/perfrunner/+/234988
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: <sean.corrigan@couchbase.com>

Author: d-nagy

perfrunner/helpers/cluster.py

@@ -267,15 +267,19 @@ def set_auth(self):
     def set_mem_quotas(self):
         for master in self.cluster_spec.masters:
             self.rest.set_kv_mem_quota(master, self.test_config.cluster.mem_quota)
+            if self.test_config.cluster.analytics_mem_quota:
+                self.rest.set_analytics_mem_quota(
+                    master, self.test_config.cluster.analytics_mem_quota
+                )
+
+            if ServerInfoManager().get_server_info_by_master_node(master).is_columnar:
+                continue
+
             self.rest.set_index_mem_quota(master, self.test_config.cluster.index_mem_quota)
             if self.test_config.cluster.fts_index_mem_quota:
                 self.rest.set_fts_index_mem_quota(
                     master, self.test_config.cluster.fts_index_mem_quota
                 )
-            if self.test_config.cluster.analytics_mem_quota:
-                self.rest.set_analytics_mem_quota(
-                    master, self.test_config.cluster.analytics_mem_quota
-                )
             if self.test_config.cluster.eventing_mem_quota:
                 self.rest.set_eventing_mem_quota(
                     master, self.test_config.cluster.eventing_mem_quota
@@ -815,22 +819,51 @@ def set_indexer_systemd_mem_limits(self):
             self.remote.set_indexer_systemd_mem_limits()
             self._restart_clusters()
 
-    def set_columnar_cloud_storage(self):
-        scheme, bucket_name = self.cluster_spec.backup.split("://")
-        region = self.cluster_spec.cloud_region
-        self.remote.configure_columnar_cloud_storage(bucket_name, scheme, region)
-
-    def add_columnar_cloud_storage_creds(self):
-        if self.cluster_spec.cloud_provider == "aws":
+    def set_columnar_blob_storage(self):
+        endpoint = self.test_config.columnar_settings.blob_storage_endpoint
+        scheme = self.test_config.columnar_settings.blob_storage_scheme
+        bucket_name = self.test_config.columnar_settings.blob_storage_bucket
+        region = self.test_config.columnar_settings.blob_storage_region
+
+        if self.cluster_spec.cloud_provider:
+            if self.cluster_spec.backup:
+                protocol, _bucket_name = self.cluster_spec.backup.split("://")
+                _scheme = protocol if protocol != "az" else "azblob"
+
+            scheme = scheme or _scheme
+            bucket_name = bucket_name or _bucket_name
+            region = self.cluster_spec.cloud_region or region
+            if scheme == "azblob":
+                storage_acc_name = self.cluster_spec.infrastructure_section("storage")[
+                    "storage_acc"
+                ]
+                endpoint = endpoint or f"https://{storage_acc_name}.blob.core.windows.net"
+
+        self.remote.configure_columnar_blob_storage(bucket_name, scheme, region, endpoint)
+
+    def add_columnar_blob_storage_creds(self):
+        csp = self.cluster_spec.cloud_provider
+        scheme = self.test_config.columnar_settings.blob_storage_scheme
+
+        if csp == "aws" or scheme == "s3":
             access_key_id, secret_access_key = local.get_aws_credential(
-                self.test_config.backup_settings.aws_credential_path, False
+                self.test_config.analytics_settings.aws_credential_path,
+                False,
             )
-            self.remote.store_analytics_aws_creds(access_key_id, secret_access_key)
-        elif self.cluster_spec.cloud_provider == "gcp":
+            self.remote.store_analytics_blob_storage_creds(access_key_id, secret_access_key)
+        elif csp == "gcp" or scheme == "gcs":
             run_local_shell_command(
                 f"gcloud storage buckets add-iam-policy-binding {self.cluster_spec.backup} "
                 "--member=allUsers --role=roles/storage.objectAdmin"
             )
+        elif csp == "azure" or scheme == "azblob":
+            storage_acc_name = self.cluster_spec.infrastructure_section("storage")["storage_acc"]
+            stdout, _, _ = run_local_shell_command(
+                f"az storage account keys list --account-name {storage_acc_name} "
+                "--query '[0].value' --output tsv"
+            )
+            storage_acc_key = stdout.strip()
+            self.remote.store_analytics_blob_storage_creds(storage_acc_name, storage_acc_key)
 
     def set_columnar_storage_partitions(self):
         storage_partitions = self.test_config.analytics_settings.columnar_storage_partitions

perfrunner/helpers/server.py

@@ -94,12 +94,13 @@ def _initialise_server_info(self, master_node: str, rest_helper: RestType) -> Se
             .replace("-enterprise", "")
             .replace("-community", "")
             .replace("-columnar", "")
+            .replace("-analytics", "")
         )
         return ServerInfo(
             build=version,
             raw_version=raw_version,
             build_tuple=create_build_tuple(version),
-            is_columnar="columnar" in raw_version,
+            is_columnar="columnar" in raw_version or "enterprise-analytics" in raw_version,
             master_node=master_node,
             is_community="community" in raw_version,
         )

perfrunner/remote/linux.py

@@ -1499,43 +1499,57 @@ def enable_resource_management_with_cgroup(self):
         run("systemctl daemon-reload")
 
     @master_server
-    def configure_columnar_cloud_storage(
-        self, bucket_name: str = "cb-perf-columnar", scheme: str = "s3", region: str = "us-east-1"
+    def configure_columnar_blob_storage(
+        self, bucket_name: str, scheme: str, region: str, endpoint: Optional[str]
     ):
         logger.info(
-            "Configuring Columnar cloud storage bucket. "
-            f"Bucket name: {bucket_name}, region: {region}, scheme: {scheme}"
+            "Configuring Columnar blob storage bucket. "
+            f"Bucket name: {bucket_name}, region: {region}, scheme: {scheme}, endpoint: {endpoint}"
         )
+
+        if not (bucket_name and scheme and region):
+            logger.interrupt("Bucket name, scheme, and region must all be provided.")
+
+        force_path_style = "true"
+        if not endpoint:
+            endpoint = ""
+            force_path_style = "false"
+
         command = (
-            "curl --max-time 3 --retry 3 --retry-connrefused "
+            "curl --max-time 3 --retry 3 --retry-connrefused -i "
             "--request POST "
             "--url http://localhost:8091/settings/analytics "
             "--header 'Content-Type: application/x-www-form-urlencoded' "
             f"--data blobStorageRegion={region} "
             "--data blobStoragePrefix= "
             f"--data blobStorageBucket={bucket_name} "
             f"--data blobStorageScheme={scheme} "
+            f"--data blobStorageEndpoint='{endpoint}' "
+            f"--data blobStorageForcePathStyle={force_path_style} "
             f"--data blobStorageAnonymousAuth=false"
         )
-        run(command)
+        output = run(command, warn_only=True)
+        http_code = output.stdout.splitlines()[0].split()[1]
+        if output.return_code != 0 or not http_code.startswith("2"):
+            logger.interrupt(
+                f"Failed to configure columnar blob storage.\n"
+                f"Stdout: {output.stdout}\n"
+                f"Stderr: {output.stderr}"
+            )
 
     @all_servers
-    def store_analytics_aws_creds(self, access_key_id: str, secret_access_key: str):
-        access_key_url = \
-            "http://localhost:8091/_metakv/cbas/debug/settings/blob_storage_access_key_id"
-        secret_access_key_url = \
-            "http://localhost:8091/_metakv/cbas/debug/settings/blob_storage_secret_access_key"
-
-        logger.info('Add AWS access key')
-        command = ("curl --max-time 3 --retry 3 --retry-connrefused -v "
-                   "-X PUT {} --data-urlencode value={}"
-                   .format(access_key_url, access_key_id))
-        run(command)
-
-        logger.info('Add AWS secret access key')
-        command = ("curl -v -X PUT {} --data-urlencode value={}"
-                   .format(secret_access_key_url, secret_access_key))
-        run(command)
+    def store_analytics_blob_storage_creds(self, access_key_id: str, secret_access_key: str):
+        base_url = "http://localhost:8091/_metakv/cbas/debug/settings"
+
+        for setting, value in [
+            ("blob_storage_access_key_id", access_key_id),
+            ("blob_storage_secret_access_key", secret_access_key),
+        ]:
+            logger.info(f"Setting {setting}")
+            run(
+                "curl --max-time 3 --retry 3 --retry-connrefused -v "
+                f"-X PUT {base_url}/{setting} --data-urlencode value={value}"
+            )
 
     @all_servers
     def set_columnar_storage_partitions(self, num_partitions: int):

perfrunner/settings.py

@@ -2780,7 +2780,7 @@ class AnalyticsSettings:
     INDEX_CONF_FILE = ""
     DROP_DATASET = ""
     ANALYTICS_LINK = "Local"
-    EXTERNAL_DATASET_TYPE = "s3"
+    EXTERNAL_DATASET_TYPE = "s3"  # alt: gcs, azblob
     EXTERNAL_DATASET_REGION = "us-east-1"
     EXTERNAL_BUCKET = None
     EXTERNAL_FILE_INCLUDE = None
@@ -2981,6 +2981,12 @@ def __init__(self, options: dict):
             if (s := dataset.split(":"))
         ]
 
+        # Enterprise Analytics settings
+        self.blob_storage_endpoint = options.get("blob_storage_endpoint")
+        self.blob_storage_bucket = options.get("blob_storage_bucket")
+        self.blob_storage_scheme = options.get("blob_storage_scheme")
+        self.blob_storage_region = options.get("blob_storage_region")
+
 
 class AuditSettings:
 

perfrunner/utils/cluster.py

@@ -108,8 +108,8 @@ def main():
     if cm.cluster_spec.columnar_infrastructure:
         if cm.cluster_spec.infrastructure_kafka_clusters:
             cm.set_kafka_links_settings()
-        cm.set_columnar_cloud_storage()
-        cm.add_columnar_cloud_storage_creds()
+        cm.add_columnar_blob_storage_creds()
+        cm.set_columnar_blob_storage()
         cm.set_columnar_storage_partitions()
 
     cm.disable_wan()

tests/cloud/analytics/bigfun_20M_ea_remote_4n_kv_config_1s_3c.test

@@ -0,0 +1,11 @@
+[cluster]
+mem_quota = 51200
+initial_nodes = 4
+num_buckets = 1
+
+[bucket]
+backend_storage = magma
+eviction_policy = fullEviction
+
+[collection]
+config = collections/1bucket_1scope_3collections_analytics.json

tests/cloud/analytics/bigfun_20M_ea_remote_sync_4n_no_index_1s_3c_incr.test

@@ -0,0 +1,40 @@
+[test_case]
+test = perfrunner.tests.analytics.BigFunIncrSyncTest
+
+[showfast]
+title = 4 nodes, BigFUN 20M users (320M docs), no indexes, s=1 c=3, Remote Link, Enterprise Analytics
+component = analyticscloud
+category = sync
+sub_category = {provider}
+orderby = _ssd_4n_3
+
+[stats]
+server_processes = java
+
+[cluster]
+mem_quota = 100
+analytics_mem_quota = 25600
+initial_nodes = 4 4
+num_buckets = 1
+
+[bucket]
+eviction_policy = fullEviction
+
+[collection]
+config = collections/1bucket_1scope_3collections_analytics.json
+
+[analytics]
+num_io_devices = 1
+dataset_conf_file = tests/analytics/config/bigfun_3_to_3_datasets.json
+analytics_link = RemoteLink
+
+[backup]
+obj_staging_dir = /stage
+obj_region = us-east-1
+aws_credential_path = /root/.ssh
+
+[restore]
+backup_storage = s3://analytics-bigfun20m-3collections
+backup_repo = default
+map_data = bucket-1.scope-1=bucket-1.scope-1
+threads = 8