Automation for Search History, Advertising Trends and Boolean queries

Change-Id: If28544e01d22c99718217bae31c5e3c6b8f89fee
Reviewed-on: https://review.couchbase.org/c/testrunner/+/245317
Tested-by: <dananjay.s@couchbase.com>
Reviewed-by: Balakumaran G <balakumaran.gopal@couchbase.com>

Author: dananjay-s

conf/fts/py-fts-movingtopology.conf

@@ -45,23 +45,23 @@ fts.moving_topology_fts.MovingTopFTS:
     ### 2. Change in topology between indexing and querying ####
 
     # Rebalance between indexing and querying
-    rebalance_in_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
+    rebalance_in_between_indexing_and_querying,items=100000,cluster=D,D,F,F,search_history=True,GROUP=P0
     #rebalance_in_between_indexing_and_querying,items=100000,cluster=D,D,F,F,sasl_buckets=1,GROUP=P0
     rebalance_out_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,num_rebalance=2,GROUP=P0;MULTIPLE_CA
-    rebalance_out_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,GROUP=P0
-    rebalance_out_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,standard_buckets=3,GROUP=P0
+    rebalance_out_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,search_history=True,GROUP=P0
+    rebalance_out_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,standard_buckets=3,search_history=True,GROUP=P0
     rebalance_out_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,index_replicas=1,num_rebalance=2,scoring_model=bm25,GROUP=P0;MULTIPLE_CA
-    swap_rebalance_between_indexing_and_querying,items=100000,cluster=D,D,F,F,replicas=0,GROUP=P0
+    swap_rebalance_between_indexing_and_querying,items=100000,cluster=D,D,F,F,replicas=0,search_history=True,GROUP=P0
     swap_rebalance_between_indexing_and_querying,items=100000,cluster=D,D,F,F,F,replicas=0,GROUP=P0,num_rebalance=2
 
-    hard_failover_and_remove_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
+    hard_failover_and_remove_between_indexing_and_querying,items=100000,cluster=D,D,F,F,search_history=True,GROUP=P0
     hard_failover_no_rebalance_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
     failover_no_rebalance_with_replicas_between_indexing_and_querying,items=100000,index_replicas=1,cluster=D,D,F,F,compare_es=True,num_queries=20,GROUP=P1
     failover_no_rebalance_with_replicas_between_indexing_and_querying,items=100000,index_replicas=1,cluster=D,D,F,F,compare_es=True,graceful=True,num_queries=20,scoring_model=bm25,GROUP=P1
     hard_failover_master_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
     hard_failover_and_delta_recovery_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
     hard_failover_and_full_recovery_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
-    graceful_failover_and_full_recovery_between_indexing_and_querying,items=100000,cluster=D,D,F,F,GROUP=P0
+    graceful_failover_and_full_recovery_between_indexing_and_querying,items=100000,cluster=D,D,F,F,search_history=True,GROUP=P0
     graceful_failover_and_delta_recovery_between_indexing_and_querying,items=100000,cluster=D,D,F,F,scoring_model=bm25,GROUP=P0
 
     # Node crash/reboot scenarios
@@ -86,7 +86,7 @@ fts.moving_topology_fts.MovingTopFTS:
 
     hard_failover_no_rebalance_during_querying,items=100000,cluster=D,D,F,F,GROUP=P1,index_replicas=1
     hard_failover_rebalance_out_during_querying,items=100000,cluster=D,D,F,F,GROUP=P1
-    failover_and_addback_during_querying,items=100000,compare_es=False,cluster=D,D,F,F,num_queries=20,graceful=False,GROUP=P1
+    failover_and_addback_during_querying,items=100000,compare_es=False,cluster=D,D,F,F,num_queries=20,graceful=False,search_history=True,GROUP=P1
     failover_and_addback_during_querying,items=100000,compare_es=True,cluster=D,D,F,F,num_queries=20,graceful=False,index_replicas=1,GROUP=P1
     failover_and_addback_during_querying,items=100000,compare_es=False,cluster=D,D,F,F,num_queries=20,recovery=delta,graceful=False,scoring_model=bm25,GROUP=P1;MULTIPLE_CA
     failover_and_addback_during_querying,items=100000,compare_es=False,cluster=D,D,F,F,num_queries=20,recovery=full,graceful=False,GROUP=P1

conf/fts/py-fts-simpletopology.conf

@@ -66,14 +66,14 @@ fts.stable_topology_fts.StableTopFTS:
     index_wiki,items=100000,lang=FR,analyzer=fr,GROUP=P1;COLLECTIONS;MAGMA
 
     # Simple queries
-    run_default_index_query,items=100000,GROUP=PS;COLLECTIONS
-    run_default_index_query,items=100000,query="""{"match": "safiya@mcdiabetes.com", "field": "email"}""",expected_hits=1000,GROUP=PS;COLLECTIONS;MULTIPLE_CA
-    run_default_index_query,items=100000,query="""{"terms": ["Darcei", "Safiya"], "field":"manages.reports"}""",expected_hits=0,GROUP=P0;COLLECTIONS;MAGMA
-    run_default_index_query,items=100000,query="""{"match_phrase": "Darcei gel", "field": "manages.reports"}""",expected_hits=0,GROUP=P0;COLLECTIONS;MAGMA
-    run_default_index_query,items=100000,update=True,upd=100,query="""{"min": 1, "max":1, "field": "mutated", "inclusive_min": true, "inclusive_max": true}""",expected_hits=1000,GROUP=P0;COLLECTIONS
+    run_default_index_query,items=100000,search_history=True,index_insights=True,GROUP=PS;COLLECTIONS
+    run_default_index_query,items=100000,query="""{"match": "safiya@mcdiabetes.com", "field": "email"}""",expected_hits=1000,search_history=True,index_insights=True,GROUP=PS;COLLECTIONS;MULTIPLE_CA
+    run_default_index_query,items=100000,query="""{"terms": ["Darcei", "Safiya"], "field":"manages.reports"}""",expected_hits=0,search_history=True,index_insights=True,GROUP=P0;COLLECTIONS;MAGMA
+    run_default_index_query,items=100000,query="""{"match_phrase": "Darcei gel", "field": "manages.reports"}""",expected_hits=0,search_history=True,index_insights=True,GROUP=P0;COLLECTIONS;MAGMA
+    run_default_index_query,items=100000,update=True,upd=100,query="""{"min": 1, "max":1, "field": "mutated", "inclusive_min": true, "inclusive_max": true}""",expected_hits=1000,search_history=True,index_insights=True,GROUP=P0;COLLECTIONS
     run_default_index_query,items=100000,update=True,upd=100,query="""{"min": 1, "max":1, "field": "mutated", "inclusive_min": true, "inclusive_max": true}""",expected_hits=1000,restart_couchbase=True,scoring_model=bm25,GROUP=P0;COLLECTIONS
-    run_default_index_query,items=100000,delete=True,del=100,query="""{"min": 0,"max": 1,"field":"mutated","inclusive_max": true}""",expected_hits=0,GROUP=P0;COLLECTIONS
-    run_default_index_query,items=100000,standard_buckets=2,index_per_bucket=2,GROUP=P0;COLLECTIONS
+    run_default_index_query,items=100000,delete=True,del=100,query="""{"min": 0,"max": 1,"field":"mutated","inclusive_max": true}""",expected_hits=0,search_history=True,GROUP=P0;COLLECTIONS
+    run_default_index_query,items=100000,standard_buckets=2,index_per_bucket=2,search_history=True,index_insights=True,GROUP=P0;COLLECTIONS
 
     #positive testcases for size param
     run_default_index_query,items=100000,update=True,upd=100,query="""{"min": 1, "max":1, "field": "mutated", "inclusive_min": true, "inclusive_max": true}""",expected_hits=1000,GROUP=P0;COLLECTIONS,query_max_matches=10,query_limit_matches=4,expected_no_of_results=10
@@ -92,17 +92,19 @@ fts.stable_topology_fts.StableTopFTS:
     # ["match", "phrase", "bool", "match_phrase","prefix", "fuzzy", "conjunction", "disjunction"
     #  "wildcard", "regexp", "query_string", "numeric_range", "date_range", "match_all","match_none"]
     test_query_type,items=100000,cluster=D,F+Q,num_queries=2,query_type=match,hierarchical=True,hierarchical_doc_prefix=hier_,hierarchical_query=[{"name":"Alice","role":"Manager"}],GROUP=P0;COLLECTIONS
-    test_query_type,items=100000,num_queries=2,query_type=phrase,GROUP=P0;COLLECTIONS
-    test_query_type,items=100000,num_queries=2,query_type=match_phrase,scoring_model=bm25,GROUP=P0;COLLECTIONS;MAGMA
-    test_query_type,items=100000,num_queries=2,query_type=conjunction,GROUP=P0;COLLECTIONS;MULTIPLE_CA
-    test_query_type,items=100000,num_queries=2,query_type=disjunction,GROUP=P0;COLLECTIONS
-    test_query_type,items=100000,num_queries=2,query_type=prefix,GROUP=P0;COLLECTIONS;MAGMA
-    test_query_type,items=100000,num_queries=2,query_type=bool,GROUP=P0;COLLECTIONS
-    test_query_type,items=100000,num_queries=2,query_type=date_range,GROUP=P0;COLLECTIONS
-    test_query_type,items=100000,num_queries=2,query_type=numeric_range,GROUP=P0;COLLECTIONS;MAGMA
-    test_query_type,items=100000,num_queries=30,query_type=match,prefix,phrase,bool,scoring_model=bm25,GROUP=PS;COLLECTIONS
-    test_match_all,items=100000,GROUP=P0;COLLECTIONS
-    test_match_none,items=100000,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=2,query_type=phrase,search_history=True,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=2,query_type=match_phrase,scoring_model=bm25,search_history=True,GROUP=P0;COLLECTIONS;MAGMA
+    test_query_type,items=100000,num_queries=2,query_type=conjunction,search_history=True,index_insights=True,GROUP=P0;COLLECTIONS;MULTIPLE_CA
+    test_query_type,items=100000,num_queries=2,query_type=disjunction,search_history=True,index_insights=True,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=2,query_type=prefix,search_history=True,GROUP=P0;COLLECTIONS;MAGMA
+    test_query_type,items=100000,num_queries=2,query_type=bool,search_history=True,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=5,query_type=bool_filter,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=5,query_type=bool_filter,scoring_model=bm25,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=2,query_type=date_range,search_history=True,GROUP=P0;COLLECTIONS
+    test_query_type,items=100000,num_queries=2,query_type=numeric_range,search_history=True,GROUP=P0;COLLECTIONS;MAGMA
+    test_query_type,items=100000,num_queries=30,query_type=match,prefix,phrase,bool,scoring_model=bm25,search_history=True,index_insights=True,GROUP=PS;COLLECTIONS
+    test_match_all,items=100000,search_history=True,GROUP=P0;COLLECTIONS
+    test_match_none,items=100000,search_history=True,GROUP=P0;COLLECTIONS
     test_match_consistency,update=True,consistency_level=at_plus,consistency_vectors="""{"default_index_1": {"1": 8}}""",GROUP=P1;COLLECTIONS;MULTIPLE_CA
     test_match_consistency,update=True,upd_del_fields=['is_manager','dept','manages.team_size'],consistency_level=at_plus,consistency_vectors="""{"default_index_1": {"13": 10}}""",scoring_model=bm25,GROUP=P1;COLLECTIONS;MULTIPLE_CA
 

conf/fts/py-fts-vector-search.conf

@@ -2,7 +2,7 @@ fts.fts_vector_search.VectorSearch:
     #Advanced filtering
     test_docfilter,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=2000,num_queries=10,index_retry=100,skip_validation_if_no_query_hits=False,prefilter_docs=True,perform_k_validation=False,dynamic_store=False,run_n1ql_search_function=False,doc_filter=True,collection=["c1"]
 
-    test_basic_vector_search,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False
+    test_basic_vector_search,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,search_history=True,index_insights=True
     test_basic_vector_search_store_all,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=20,index_retry=100,skip_validation_if_no_query_hits=False
     test_vector_search_with_wrong_dimensions,cluster=F+Q,F,kv=["b1.s1.c1"],dimension=50,num_queries=5,index_retry=100
     create_vector_with_constant_queries_in_background,cluster=F+Q,F,kv=["b1.s1.c1","b2.s2.c2"],num_queries=25,index_retry=100
@@ -14,7 +14,7 @@ fts.fts_vector_search.VectorSearch:
     test_vector_index_update_dimensions,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,num_queries=5,index_retry=100
     test_vector_search_update_replicas,cluster=F+Q,F,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,update_replicas=2,num_queries=5,index_retry=100
     test_vector_search_update_partitions,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,update_partitions=3,num_queries=5,index_retry=100
-    test_vector_search_update_similarity,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,num_queries=5,index_retry=100
+    test_vector_search_update_similarity,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,num_queries=5,index_retry=100,search_history=True,index_insights=True
     # test_vector_search_update_doc,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,num_queries=5,index_retry=100
     test_vector_search_update_index_concurrently,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,num_queries=5,index_retry=100
     test_vector_search_update_index_concurrently,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,create_alias=True,num_queries=5,index_retry=100
@@ -28,12 +28,12 @@ fts.fts_vector_search.VectorSearch:
 
     test_vector_search_update_doc_dimension,cluster=F+Q,F,kv=["b1.s1.c1"],dimension=128,num_queries=10,index_retry=100,per_to_resize=[1],dims_to_resize=[150],faiss_indexes=['faiss_idx1'],perform_faiss_validation=True,k=100
 
-    test_nprobe_settings,cluster=F+Q,F,kv=["b1.s1.c1"],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,change_nprobe_settings=True,perform_k_validation=True
+    test_nprobe_settings,cluster=F+Q,F,kv=["b1.s1.c1"],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,change_nprobe_settings=True,perform_k_validation=True,search_history=True
     test_nprobe_settings,cluster=F+Q,F,kv=["b1.s1.c1"],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,change_nprobe_settings=True,perform_k_validation=True,num_partitions=5
     test_nprobe_settings_negative,cluster=F+Q,F,kv=["b1.s1.c1"],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,change_nprobe_settings=True,knn_params="""{"ivf_nprobe_pct": 1000, "ivf_max_codes_pct": 1000}"""
 
 
-    test_prefiltering,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,prefilter_docs=True,start_key=200000,num_partitions=3,perform_k_validation=True
+    test_prefiltering,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,prefilter_docs=True,start_key=200000,num_partitions=3,perform_k_validation=True,search_history=True
     test_prefiltering,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,prefilter_docs=True,start_key=200000,num_partitions=3,perform_k_validation=True,change_nprobe_settings=True
 
     test_prefiltering,cluster=F+Q,F,kv=["b1.s1.c1"],idx=[("i1","b1.s1.c1")],dimension=128,k=100,num_queries=100,index_retry=100,skip_validation_if_no_query_hits=False,prefilter_docs=True,start_key=200000,perform_k_validation=True,conjuction_query_with_prefilter=True

lib/membase/api/on_prem_rest_client.py

@@ -4353,6 +4353,36 @@ def set_node_setting(self, setting_name, value):
             raise Exception("Error Updating {0}: {1}".format(setting_name, content))
         return status
 
+    def get_search_history(self, limit=25, offset=0, index=None, min_duration=None):
+        api = self.fts_baseUrl + "api/searchHistory?"
+        params = "limit={0}&offset={1}".format(limit, offset)
+        if index:
+            params += "&index={0}".format(index)
+        if min_duration:
+            params += "&minDuration={0}".format(min_duration)
+        api += params
+        status, content, header = self._http_request(api)
+        if status:
+            return json.loads(content)
+        else:
+            raise Exception("Error fetching search history: {0}".format(content))
+
+    def get_index_insights(self, index_name, field, insight, limit=5, descending=True):
+        api = self.fts_baseUrl + "api/index/{0}/insights".format(index_name)
+        payload = {
+            "field": field,
+            "insight": insight,
+            "limit": limit,
+            "descending": descending
+        }
+        status, content, header = self.urllib_request(
+            api, verb='POST', params=json.dumps(payload))
+        if status:
+            return json.loads(content)
+        else:
+            raise Exception("Error fetching index insights for {0}: {1}"
+                            .format(index_name, content))
+
     def unfreeze_fts_index_partitions(self, name):
         """ method to freeze index partitions asignment"""
         api = self.fts_baseUrl+ "api/index/{0}/planFreezeControl/unfreeze".format(name)