Add Basic Jenkins Integ Test for Demo Setup

.github/workflows/rfs_pr_e2e_test.yml → .github/workflows/full_pr_e2e_test.yml

@@ -15,7 +15,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
-  rfs-e2e-aws-test:
+  full-es68-e2e-aws-test:
     runs-on: ubuntu-latest
     steps:
       - name: Sanitize branch and repo names
@@ -34,6 +34,6 @@ jobs:
         uses: lewijacn/[email protected]
         with:
           jenkins_url: "https://migrations.ci.opensearch.org"
-          job_name: "rfs-default-e2e-test"
+          job_name: "full-es68source-e2e-test"
           api_token: "${{ secrets.JENKINS_MIGRATIONS_GENERIC_WEBHOOK_TOKEN }}"
           job_params: "GIT_REPO_URL=${{ steps.sanitize-input.outputs.pr_repo_url }},GIT_BRANCH=${{ steps.sanitize-input.outputs.branch_name }}"

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py

@@ -13,8 +13,13 @@ def pytest_configure(config):
 
 def pytest_addoption(parser):
     parser.addoption("--unique_id", action="store", default=uuid.uuid4().hex)
+    parser.addoption("--stage", action="store", default="dev")
     parser.addoption("--config_file_path", action="store", default="/etc/migration_services.yaml",
                      help="Path to config file for console library")
+    parser.addoption("--source_proxy_alb_endpoint", action="store", default=None,
+                     help="Specify the Migration ALB endpoint for the source capture proxy")
+    parser.addoption("--target_proxy_alb_endpoint", action="store", default=None,
+                     help="Specify the Migration ALB endpoint for the target proxy")
 
 
 @pytest.fixture

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/e2e_tests.py → TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py

@@ -1,8 +1,9 @@
 import logging
+import os
 import pytest
 import unittest
 from http import HTTPStatus
-from console_link.middleware.clusters import run_test_benchmarks, connection_check, clear_indices, ConnectionResult
+from console_link.middleware.clusters import connection_check, clear_indices, ConnectionResult
 from console_link.models.cluster import Cluster
 from console_link.models.backfill_base import Backfill
 from console_link.models.replayer_base import Replayer
@@ -12,8 +13,7 @@
 from console_link.middleware.kafka import delete_topic
 from console_link.models.metadata import Metadata
 from console_link.cli import Context
-from common_operations import (create_index, create_document, check_doc_counts_match, wait_for_running_replayer,
-                               EXPECTED_BENCHMARK_DOCS)
+from common_operations import (create_index, create_document, check_doc_counts_match, wait_for_running_replayer)
 logger = logging.getLogger(__name__)
 
 
@@ -25,6 +25,19 @@ def initialize(request):
     pytest.unique_id = unique_id
     source_cluster = pytest.console_env.source_cluster
     target_cluster = pytest.console_env.target_cluster
+    # If in AWS, modify source and target objects here to route requests through the created ALB to verify its operation
+    if 'AWS_EXECUTION_ENV' in os.environ:
+        logger.info("Detected an AWS environment")
+        source_proxy_alb_endpoint = request.config.getoption("--source_proxy_alb_endpoint")
+        target_proxy_alb_endpoint = request.config.getoption("--target_proxy_alb_endpoint")
+        logger.info("Checking original source and target endpoints can be reached, before using ALB endpoints for test")
+        direct_source_con_result: ConnectionResult = connection_check(source_cluster)
+        assert direct_source_con_result.connection_established is True
+        direct_target_con_result: ConnectionResult = connection_check(target_cluster)
+        assert direct_target_con_result.connection_established is True
+        source_cluster.endpoint = source_proxy_alb_endpoint
+        target_cluster.endpoint = target_proxy_alb_endpoint
+        target_cluster.allow_insecure = True
     backfill: Backfill = pytest.console_env.backfill
     assert backfill is not None
     metadata: Metadata = pytest.console_env.metadata
@@ -74,7 +87,6 @@ def test_e2e_0001_default(self):
         replayer: Replayer = pytest.console_env.replay
 
         # Load initial data
-        run_test_benchmarks(source_cluster)
         index_name = f"test_e2e_0001_{pytest.unique_id}"
         doc_id_base = "e2e_0001_doc"
         create_index(cluster=source_cluster, index_name=index_name, test_case=self)
@@ -87,24 +99,29 @@ def test_e2e_0001_default(self):
         status_result: CommandResult = snapshot.status()
         if status_result.success:
             snapshot.delete()
-        snapshot.create(wait=True)
-        metadata.migrate()
-        backfill.start()
-        backfill.scale(units=10)
+        snapshot_result: CommandResult = snapshot.create(wait=True)
+        assert snapshot_result.success
+        metadata_result: CommandResult = metadata.migrate()
+        assert metadata_result.success
+        backfill_start_result: CommandResult = backfill.start()
+        assert backfill_start_result.success
+        # small enough to allow containers to be reused, big enough to test scaling out
+        backfill_scale_result: CommandResult = backfill.scale(units=2)
+        assert backfill_scale_result.success
         # This document was created after snapshot and should not be included in Backfill but expected in Replay
         create_document(cluster=source_cluster, index_name=index_name, doc_id=doc_id_base + "_2",
                         expected_status_code=HTTPStatus.CREATED, test_case=self)
 
         ignore_list = [".", "searchguard", "sg7", "security-auditlog", "reindexed-logs"]
-        expected_docs = dict(EXPECTED_BENCHMARK_DOCS)
+        expected_docs = {}
         # Source should have both documents
-        expected_docs[index_name] = {"docs.count": "2"}
+        expected_docs[index_name] = {"count": 2}
         check_doc_counts_match(cluster=source_cluster, expected_index_details=expected_docs,
                                index_prefix_ignore_list=ignore_list, test_case=self)
         # Target should have one document from snapshot
-        expected_docs[index_name] = {"docs.count": "1"}
+        expected_docs[index_name] = {"count": 1}
         check_doc_counts_match(cluster=target_cluster, expected_index_details=expected_docs,
-                               index_prefix_ignore_list=ignore_list, max_attempts=40, delay=30.0, test_case=self)
+                               index_prefix_ignore_list=ignore_list, max_attempts=20, delay=30.0, test_case=self)
 
         backfill.stop()
 
@@ -114,7 +131,7 @@ def test_e2e_0001_default(self):
         replayer.start()
         wait_for_running_replayer(replayer=replayer)
 
-        expected_docs[index_name] = {"docs.count": "3"}
+        expected_docs[index_name] = {"count": 3}
         check_doc_counts_match(cluster=source_cluster, expected_index_details=expected_docs,
                                index_prefix_ignore_list=ignore_list, test_case=self)
 

jenkins/migrationIntegPipelines/rfsES68SourceE2ETestCover.groovy → jenkins/migrationIntegPipelines/fullES68SourceE2ETestCover.groovy

@@ -5,5 +5,5 @@ library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM(
         [$class: 'GitSCMSource',
          remote: "${gitUrl}"])
 
-// Shared library function (location from root: vars/rfsES68SourceE2ETest.groovy)
-rfsES68SourceE2ETest()
+// Shared library function (location from root: vars/fullES68SourceE2ETest.groovy)
+fullES68SourceE2ETest()

vars/defaultIntegPipeline.groovy

@@ -156,9 +156,11 @@ def call(Map config = [:]) {
                                     def time = new Date().getTime()
                                     def uniqueId = "integ_min_${time}_${currentBuild.number}"
                                     def test_result_file = "${testDir}/reports/${uniqueId}/report.xml"
+                                    def populatedIntegTestCommand = integTestCommand.replaceAll("<STAGE>", stage)
                                     def command = "pipenv run pytest --log-file=${testDir}/reports/${uniqueId}/pytest.log " +
-                                            "--junitxml=${test_result_file} ${integTestCommand} " +
+                                            "--junitxml=${test_result_file} ${populatedIntegTestCommand} " +
                                             "--unique_id ${uniqueId} " +
+                                            "--stage ${stage} " +
                                             "-s"
                                     withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) {
                                         withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", duration: 3600, roleSessionName: 'jenkins-session') {

vars/rfsES68SourceE2ETest.groovy → vars/fullES68SourceE2ETest.groovy

@@ -1,7 +1,7 @@
 
 def call(Map config = [:]) {
     def sourceContextId = 'source-single-node-ec2'
-    def migrationContextId = 'migration-rfs'
+    def migrationContextId = 'full-migration'
     def source_cdk_context = """
         {
           "source-single-node-ec2": {
@@ -19,13 +19,14 @@ def call(Map config = [:]) {
             "dataNodeCount": 1,
             "managerNodeCount": 0,
             "serverAccessType": "ipv4",
-            "restrictServerAccessTo": "0.0.0.0/0"
+            "restrictServerAccessTo": "0.0.0.0/0",
+            "requireImdsv2": false
           }
         }
     """
     def migration_cdk_context = """
         {
-          "migration-rfs": {
+          "full-migration": {
             "stage": "<STAGE>",
             "vpcId": "<VPC_ID>",
             "engineVersion": "OS_2.11",
@@ -34,9 +35,11 @@ def call(Map config = [:]) {
             "openAccessPolicyEnabled": true,
             "domainRemovalPolicy": "DESTROY",
             "artifactBucketRemovalPolicy": "DESTROY",
-            "trafficReplayerServiceEnabled": false,
+            "captureProxyServiceEnabled": true,
+            "targetClusterProxyServiceEnabled": true,
+            "trafficReplayerServiceEnabled": true,
+            "trafficReplayerExtraArgs": "--speedup-factor 10.0",
             "reindexFromSnapshotServiceEnabled": true,
-            "sourceClusterEndpoint": "<SOURCE_CLUSTER_ENDPOINT>",
             "sourceCluster": {
                 "endpoint": "<SOURCE_CLUSTER_ENDPOINT>",
                 "auth": {"type": "none"},
@@ -63,9 +66,9 @@ def call(Map config = [:]) {
             migrationContext: migration_cdk_context,
             sourceContextId: sourceContextId,
             migrationContextId: migrationContextId,
-            defaultStageId: 'rfs-integ-es68',
+            defaultStageId: 'full-es68',
             skipCaptureProxyOnNodeSetup: true,
-            jobName: 'rfs-es68source-e2e-test',
-            integTestCommand: '/root/lib/integ_test/integ_test/backfill_tests_es68.py'
+            jobName: 'full-es68source-e2e-test',
+            integTestCommand: '/root/lib/integ_test/integ_test/full_tests.py --source_proxy_alb_endpoint https://alb.migration.<STAGE>.local:9201 --target_proxy_alb_endpoint https://alb.migration.<STAGE>.local:9202'
     )
 }

vars/rfsDefaultE2ETest.groovy

@@ -1,6 +1,4 @@
-// Note:
-// 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to
-//    allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK.
+// Note: This integ test exists to verify that RFS can be ran independently of other migrations
 
 def call(Map config = [:]) {
     def sourceContextId = 'source-single-node-ec2'

vars/trafficReplayDefaultE2ETest.groovy

@@ -1,6 +1,4 @@
-// Note:
-// 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to
-//    allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK.
+// Note: This integ test exists to verify that Capture and Replay can be ran independently of other migrations
 
 def call(Map config = [:]) {
     def sourceContextId = 'source-single-node-ec2'