scan plus fts automation

Change-Id: I2eb5608b8334601ad3fd5b9cccbabd6680192c93
Reviewed-on: https://review.couchbase.org/c/testrunner/+/243922
Reviewed-by: Balakumaran G <balakumaran.gopal@couchbase.com>
Tested-by: Couchbase QE

Author: nishanthvm-couchbase

conf/fts/py-fts-scan-plus.conf

@@ -0,0 +1,146 @@
+fts.stable_topology_fts.StableTopFTS:
+
+    # --- Functional tests ---
+    # Exact consistency: CRUD paused before each query, result must match
+    # HashMap perfectly (count + val).
+
+    # P0 smoke: 2-node, default params
+    test_scan_plus_functional,cluster=F,F,initial_docs=5,batch_duration=60,num_batches=3,num_crud_threads=4,GROUP=P0
+
+    # More seed docs, more batches, higher concurrency
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,GROUP=P0
+
+    # 4-partition index: KV-verified check for scan_plus partition fan-out bug
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,index_partitions=4,GROUP=P0
+
+    # --- Manager options variants ---
+    # use_bucket_seqnos=true: whole-bucket seqno path (avoids per-collection contention, different code path)
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,use_bucket_seqnos=true,GROUP=P0
+
+    # num_workers=1 (extreme low): serialised seqno retrieval; stresses multi-partition fan-out ordering
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,index_partitions=4,num_workers=1,GROUP=P0
+
+    # num_workers=100 (extreme high): max parallelism; may surface race conditions in seqno ack path
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,num_workers=100,GROUP=P0
+
+    # getseqnos_retries=1 (extreme low): one attempt only; surfaces any timing/fail-fast issues
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,getseqnos_retries=1,GROUP=P0
+
+    # getseqnos_retries=100 (extreme high): very patient seqno retrieval; baseline sanity with high retries
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,getseqnos_retries=100,GROUP=P0
+
+    # Combined: bucket seqnos + high workers (both non-defaults together)
+    test_scan_plus_functional,cluster=F,F,initial_docs=100,batch_duration=60,num_batches=5,num_crud_threads=8,use_bucket_seqnos=true,num_workers=100,GROUP=P0
+
+    # --- Stress / performance tests ---
+    # Continuous CRUD + random queries; validates result.val >= snapshot.val
+    # and flags deleted docs that still appear in results.
+
+    # 2-node, 10-minute run
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=4,GROUP=P1
+
+    # 3-node, 10-minute run, higher CRUD concurrency
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,GROUP=P1
+
+    # 2-partition index: scan_plus must gather seqno from both partitions
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,index_partitions=2,GROUP=P1
+
+    # 4-partition index: higher fan-out, more seqno coordination points
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,index_partitions=4,GROUP=P1
+
+    # 1 index replica: scan_plus querying a replica shard must also be consistent
+    test_scan_plus_stress,cluster=F,F,F,initial_docs=50,duration=600,num_crud_threads=8,num_replicas=1,GROUP=P1
+
+    # Extended 30-minute soak, larger seed set
+    test_scan_plus_stress,cluster=F,F,initial_docs=200,duration=1800,num_crud_threads=8,GROUP=P1
+
+    # --- Stress manager options variants ---
+    # num_workers=1: serialised seqno retrieval under continuous load
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,num_workers=1,GROUP=P1
+
+    # num_workers=100: max parallelism under continuous load
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,num_workers=100,GROUP=P1
+
+    # use_bucket_seqnos=true: bucket-level seqno path under continuous load
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,use_bucket_seqnos=true,GROUP=P1
+
+    # getseqnos_retries=1: aggressive fail-fast under continuous CRUD pressure
+    test_scan_plus_stress,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=8,getseqnos_retries=1,GROUP=P1
+
+    # --- Multi-collection tests ---
+    # One FTS index spans 3 collections in a custom scope.
+    # CRUD runs concurrently against all 3 collections; scan_plus queries
+    # must return a globally consistent view across collection boundaries.
+
+    # Baseline: 3 collections, default partition
+    test_scan_plus_multi_collection,cluster=F,F,initial_docs=30,duration=600,num_crud_threads=6,GROUP=P1
+
+    # 3 collections + 4 partitions: exercises per-partition seqno tracking across collections
+    test_scan_plus_multi_collection,cluster=F,F,initial_docs=50,duration=600,num_crud_threads=9,index_partitions=4,GROUP=P1
+
+    # 3 collections + 1 replica: consistency when querying replica shards in a multi-collection index
+    test_scan_plus_multi_collection,cluster=F,F,F,initial_docs=30,duration=600,num_crud_threads=6,num_replicas=1,GROUP=P1
+
+    # --- Vector tests ---
+    # KNN scan_plus: index has val + vec (1536-dim); CRUD only changes val.
+    # K=1000 always covers the entire live corpus, so validation is identical
+    # to the functional test — exact bidirectional val-consistency check.
+    test_scan_plus_vector,cluster=F,F,initial_docs=10,batch_duration=60,num_batches=3,num_crud_threads=4,vec_dims=1536,GROUP=P1
+
+fts.moving_topology_fts.MovingTopFTS:
+
+    # --- Rebalance / failover tests ---
+    # Background CRUD + scan_plus queries fire continuously during each topology change.
+    # Violations (stale/missing hits) during the disruption are logged as warnings.
+    # The test fails only if the final exact-consistency check fails after recovery.
+    # All tests require D,D,F,F initial topology (2 KV + 2 FTS nodes);
+    # rebalance-in cases additionally need spare nodes available in the .ini file.
+
+    # RF-1: scan_plus during KV node rebalance out + in
+    test_scan_plus_kv_rebalance,cluster=D,D,F,F,GROUP=moving
+
+    # RF-2: scan_plus during FTS node rebalance out + in
+    test_scan_plus_fts_rebalance,cluster=D,D,F,F,GROUP=moving
+
+    # RF-3: scan_plus during graceful KV failover + full recovery
+    test_scan_plus_kv_graceful_failover,cluster=D,D,F,F,GROUP=moving
+
+    # RF-4: scan_plus during hard KV failover (memcached kill) + full recovery
+    test_scan_plus_kv_hard_failover,cluster=D,D,F,F,GROUP=moving
+
+    # RF-5: scan_plus during FTS node hard failover + full recovery
+    test_scan_plus_fts_failover,cluster=D,D,F,F,GROUP=moving
+
+    # RF-6: scan_plus across sequential KV failover then FTS failover
+    test_scan_plus_sequential_kv_fts_failover,cluster=D,D,F,F,GROUP=moving
+
+    # RF-7: scan_plus during 2-node swap rebalance (needs 2 spare FTS nodes in .ini)
+    test_scan_plus_swap_rebalance,cluster=D,D,F,F,GROUP=moving
+
+    # RF-8: scan_plus after KV failover with delta recovery
+    test_scan_plus_delta_recovery,cluster=D,D,F,F,GROUP=moving
+
+    # --- Manager options variants (run against a representative subset) ---
+    # num_workers=1: serialised seqno path during KV rebalance
+    test_scan_plus_kv_rebalance,cluster=D,D,F,F,num_workers=1,GROUP=moving
+
+    # num_workers=100: max parallelism during KV rebalance
+    test_scan_plus_kv_rebalance,cluster=D,D,F,F,num_workers=100,GROUP=moving
+
+    # use_bucket_seqnos=true during FTS failover
+    test_scan_plus_fts_failover,cluster=D,D,F,F,use_bucket_seqnos=true,GROUP=moving
+
+    # num_workers=1 during FTS failover: worst-case seqno serialisation under disruption
+    test_scan_plus_fts_failover,cluster=D,D,F,F,num_workers=1,GROUP=moving
+
+fts.upgrade_fts.UpgradeFTS:
+
+    # --- Online upgrade test ---
+    # Validates scan_plus behavior at each upgrade stage:
+    #   Stage 0 (all pre-8.1)  : query must fail
+    #   Stage 2 (mixed)        : V-2 upgraded coordinator succeeds,
+    #                            V-3 old coordinator fails with version error
+    #   Stage 4 (all 8.1)      : exact consistency validation (V-1)
+    # Requires >= 2 FTS nodes; upgrade_to must be supplied at runtime.
+
+    test_scan_plus_online_upgrade,cluster=F,F,F,nodes_init=3,GROUP=upgrade