Adding coexistence tests for bhive and composite vector indexes

Change-Id: I540765f6742c930be76e8f560427328a169b29de
Reviewed-on: https://review.couchbase.org/c/testrunner/+/226413
Reviewed-by: <pavan.pb@couchbase.com>
Tested-by: <yash.dodderi@couchbase.com>

Author: yash-dodderi7

conf/gsi/bhive-index.conf

@@ -50,7 +50,11 @@ gsi.composite_vector_index.CompositeVectorIndex:
     test_crash_indexer_data_nodes,nodes_init=3,services_init=kv:n1ql-index-index,default_bucket=False,skip_cleanup=True,skip_load=True,create_primary_index=False,json_template=Cars,use_magma_loader=True,vector_backup_filename=backup_zips/200k_default_scope_named_scope.zip,s3_bucket=gsi-vector-car-backups,quantization_algo_description_vector=PQ32x8,bhive_index=True,num_index_replica=1,kill_parallel=False,num_of_docs_per_collection=100000,GROUP=P1
     test_recovery_post_deleting_recovery_files,nodes_init=3,services_init=kv:n1ql-index-index,default_bucket=False,skip_cleanup=True,skip_load=True,create_primary_index=False,json_template=Cars,use_magma_loader=True,vector_backup_filename=backup_zips/200k_default_scope_named_scope.zip,s3_bucket=gsi-vector-car-backups,quantization_algo_description_vector=PQ32x8,bhive_index=True,num_index_replica=1,num_of_docs_per_collection=100000,GROUP=P1
 
-
+    # Bhive and composite_vector_index coexistence tests
+    test_batch_index_create_drop_bhive_composite_combo,init_nodes=False,nodes_init=3,services_init=kv:n1ql-index-index,default_bucket=False,skip_cleanup=True,skip_load=True,create_primary_index=False,json_template=Cars,num_of_docs_per_collection=100000,use_magma_loader=True,num_index_replica=1,index_quota_percent=75,index_category_dropped=composite_vector,num_indexes_batch=3,GROUP=P3
+    test_batch_index_create_drop_bhive_composite_combo,init_nodes=False,nodes_init=3,services_init=kv:n1ql-index-index,default_bucket=False,skip_cleanup=True,skip_load=True,create_primary_index=False,json_template=Cars,num_of_docs_per_collection=100000,use_magma_loader=True,num_index_replica=1,index_quota_percent=75,index_category_dropped=bhive_vector,num_indexes_batch=3,GROUP=P3
+    test_skewed_bhive_composite_co_existence_test,init_nodes=False,nodes_init=3,services_init=kv:n1ql-index-index,default_bucket=False,skip_cleanup=True,skip_load=True,create_primary_index=False,json_template=Cars,num_of_docs_per_collection=100000,use_magma_loader=True,num_index_replica=1,index_quota_percent=75,num_indexes_batch=3,bhive_index=True,GROUP=P3
+    test_skewed_bhive_composite_co_existence_test,init_nodes=False,nodes_init=3,services_init=kv:n1ql-index-index,default_bucket=False,skip_cleanup=True,skip_load=True,create_primary_index=False,json_template=Cars,num_of_docs_per_collection=100000,use_magma_loader=True,num_index_replica=1,index_quota_percent=75,num_indexes_batch=3,bhive_index=False,GROUP=P3
 
     #mixed dimension test
     test_mixed_dimension_data,init_nodes=False,nodes_init=2,services_init=kv:n1ql-index,default_bucket=False,skip_cleanup=True,skip_load=True,num_of_docs_per_collection=100000,json_template=Cars,bhive_index=True,GROUP=P2

pytests/gsi/base_gsi.py

@@ -3212,6 +3212,29 @@ def get_all_array_index_names(self):
                 array_indexes.append(index['indexName'])
         return array_indexes
 
+    def get_all_primary_index_names(self):
+        """Returns a list of all primary indexes in the cluster
+        """
+        primary_index = []
+        index_metadata = self.index_rest.get_indexer_metadata()['status']
+        for index in index_metadata:
+            # Check if index definition contains primary indexing syntax like PRIMARY
+            if 'definition' in index and ('isPrimary' in index):
+                primary_index.append(index['indexName'])
+        return primary_index
+
+    def get_all_bhive_index_names(self):
+        """Returns a list of all bhive indexes in the cluster
+        """
+        bhive_index = []
+        index_metadata = self.index_rest.get_indexer_metadata()['status']
+        for index in index_metadata:
+            # Check if index definition contains bhive indexing syntax like VECTOR
+            # TODO to make tweak to validation once https://jira.issues.couchbase.com/browse/MB-66285 is fixed
+            if 'definition' in index and ('VECTOR' in index['definition'][:14]):
+                bhive_index.append(index['indexName'])
+        return bhive_index
+
     def get_storage_stats_map(self, node):
         """
         Fetches index storage stats from /stats/storage endpoint
@@ -3245,6 +3268,8 @@ def backstore_mainstore_check(self):
         idx_node_list = self.get_nodes_from_services_map(service_type="index", get_all_nodes=True)
         # Exclude all array indexes
         ignore_count_index_list = self.get_all_array_index_names()
+        primary_index_list = self.get_all_primary_index_names()
+        bhive_index_list = self.get_all_bhive_index_names()
         errors = []
         for node in idx_node_list:
             self.log.info(f"Checking stats for node {node}")
@@ -3253,16 +3278,22 @@ def backstore_mainstore_check(self):
             for index in list(content.values()):
                 for index_name, stats in index.items():
                     self.log.info(f"checking for index {index_name}")
+                    if index_name in primary_index_list:
+                        continue
                     if index_name in ignore_count_index_list or "BackStore" not in stats:
                         continue
-                    if stats["MainStore"]["item_count"] != stats["BackStore"]["item_count"]:
+                    if index in bhive_index_list:
+                        key = "item_count"
+                    else:
+                        key = "items_count"
+                    if stats["MainStore"][key] != stats["BackStore"][key]:
                         self.log.info(f"Index map as seen during backstore_mainstore_check is {stats}")
                         self.log.error(f"Item count mismatch in backstore and mainstore for {index_name}")
                         errors_obj = dict()
                         errors_obj["type"] = "mismatch in backstore and mainstore"
                         errors_obj["index_name"] = index_name
-                        errors_obj["mainstore_count"] = stats["MainStore"]["item_count"]
-                        errors_obj["backstore_count"] = stats["BackStore"]["item_count"]
+                        errors_obj["mainstore_count"] = stats["MainStore"][key]
+                        errors_obj["backstore_count"] = stats["BackStore"][key]
                         errors.append(errors_obj)
         if len(errors) > 0:
             return errors