From f2b7f6463fbf5310d590e91946eb94db7eb2305e Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Tue, 22 Oct 2024 14:54:12 -0500 Subject: [PATCH 01/12] Initial attempt for full E2E test Signed-off-by: Tanner Lewis --- .../{e2e_tests.py => full_tests.py} | 8 +- .../fullDefaultE2ETestCover.groovy | 9 +++ vars/defaultIntegPipeline.groovy | 1 + vars/fullDefaultE2ETest.groovy | 73 +++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/{e2e_tests.py => full_tests.py} (93%) create mode 100644 jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy create mode 100644 vars/fullDefaultE2ETest.groovy diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/e2e_tests.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py similarity index 93% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/e2e_tests.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py index 475885d4de..041e5f67b9 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/e2e_tests.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py @@ -1,4 +1,5 @@ import logging +import os import pytest import unittest from http import HTTPStatus @@ -25,6 +26,11 @@ def initialize(request): pytest.unique_id = unique_id source_cluster = pytest.console_env.source_cluster target_cluster = pytest.console_env.target_cluster + # If in AWS, modify source and target objects here to route requests through the created ALB to verify its operation + if 'AWS_EXECUTION_ENV' in os.environ: + logger.info("Detected an AWS environment") + stage = request.config.getoption("--stage") + source_cluster.endpoint = f"https://alb.migration.{stage}.local:9201" backfill: Backfill = pytest.console_env.backfill assert backfill is not None metadata: Metadata = pytest.console_env.metadata @@ -90,7 +96,7 @@ def test_e2e_0001_default(self): snapshot.create(wait=True) metadata.migrate() backfill.start() - backfill.scale(units=10) + backfill.scale(units=2) # This document was created after snapshot and should not be included in Backfill but expected in Replay create_document(cluster=source_cluster, index_name=index_name, doc_id=doc_id_base + "_2", expected_status_code=HTTPStatus.CREATED, test_case=self) diff --git a/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy b/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy new file mode 100644 index 0000000000..97e10dc214 --- /dev/null +++ b/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy @@ -0,0 +1,9 @@ +def gitBranch = params.GIT_BRANCH ?: 'main' +def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git' + +library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( + [$class: 'GitSCMSource', + remote: "${gitUrl}"]) + +// Shared library function (location from root: vars/fullDefaultE2ETest.groovy) +fullDefaultE2ETest() diff --git a/vars/defaultIntegPipeline.groovy b/vars/defaultIntegPipeline.groovy index 548370d588..e5dfdf8f8e 100644 --- a/vars/defaultIntegPipeline.groovy +++ b/vars/defaultIntegPipeline.groovy @@ -159,6 +159,7 @@ def call(Map config = [:]) { def command = "pipenv run pytest --log-file=${testDir}/reports/${uniqueId}/pytest.log " + "--junitxml=${test_result_file} ${integTestCommand} " + "--unique_id ${uniqueId} " + + "--stage ${stage}" + "-s" withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", duration: 3600, roleSessionName: 'jenkins-session') { diff --git a/vars/fullDefaultE2ETest.groovy b/vars/fullDefaultE2ETest.groovy new file mode 100644 index 0000000000..342fed87e0 --- /dev/null +++ b/vars/fullDefaultE2ETest.groovy @@ -0,0 +1,73 @@ + +def call(Map config = [:]) { + def sourceContextId = 'source-single-node-ec2' + def migrationContextId = 'full-migration' + def source_cdk_context = """ + { + "source-single-node-ec2": { + "suffix": "ec2-source-", + "networkStackSuffix": "ec2-source-", + "distVersion": "6.8.23", + "distributionUrl": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-6.8.23.tar.gz", + "captureProxyEnabled": false, + "securityDisabled": true, + "minDistribution": false, + "cpuArch": "x64", + "isInternal": true, + "singleNodeCluster": true, + "networkAvailabilityZones": 2, + "dataNodeCount": 1, + "managerNodeCount": 0, + "serverAccessType": "ipv4", + "restrictServerAccessTo": "0.0.0.0/0" + } + } + """ + def migration_cdk_context = """ + { + "full-migration": { + "stage": "", + "vpcId": "", + "engineVersion": "OS_2.11", + "domainName": "os-cluster-", + "dataNodeCount": 2, + "openAccessPolicyEnabled": true, + "domainRemovalPolicy": "DESTROY", + "artifactBucketRemovalPolicy": "DESTROY", + "captureProxyServiceEnabled": true, + "targetClusterProxyServiceEnabled": true, + "trafficReplayerServiceEnabled": true, + "trafficReplayerExtraArgs": "--speedup-factor 10.0", + "reindexFromSnapshotServiceEnabled": true, + "sourceCluster": { + "endpoint": "", + "auth": {"type": "none"}, + "version": "ES_6.8.23" + }, + "tlsSecurityPolicy": "TLS_1_2", + "enforceHTTPS": true, + "nodeToNodeEncryptionEnabled": true, + "encryptionAtRestEnabled": true, + "vpcEnabled": true, + "vpcAZCount": 2, + "domainAZCount": 2, + "mskAZCount": 2, + "migrationAssistanceEnabled": true, + "replayerOutputEFSRemovalPolicy": "DESTROY", + "migrationConsoleServiceEnabled": true, + "otelCollectorEnabled": true + } + } + """ + + defaultIntegPipeline( + sourceContext: source_cdk_context, + migrationContext: migration_cdk_context, + sourceContextId: sourceContextId, + migrationContextId: migrationContextId, + defaultStageId: 'full-integ', + skipCaptureProxyOnNodeSetup: true, + jobName: 'full-default-e2e-test', + integTestCommand: '/root/lib/integ_test/integ_test/full_tests.py' + ) +} From 2ff28a885855d2f71b34d3520a86464b017825aa Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Tue, 22 Oct 2024 16:24:32 -0500 Subject: [PATCH 02/12] Add missing space Signed-off-by: Tanner Lewis --- vars/defaultIntegPipeline.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/defaultIntegPipeline.groovy b/vars/defaultIntegPipeline.groovy index e5dfdf8f8e..16fa15d30c 100644 --- a/vars/defaultIntegPipeline.groovy +++ b/vars/defaultIntegPipeline.groovy @@ -159,7 +159,7 @@ def call(Map config = [:]) { def command = "pipenv run pytest --log-file=${testDir}/reports/${uniqueId}/pytest.log " + "--junitxml=${test_result_file} ${integTestCommand} " + "--unique_id ${uniqueId} " + - "--stage ${stage}" + + "--stage ${stage} " + "-s" withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", duration: 3600, roleSessionName: 'jenkins-session') { From f1cf49e42c0abf7aa2e12b9a477d980f9de827ec Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Tue, 22 Oct 2024 21:54:31 -0500 Subject: [PATCH 03/12] Update integ test config Signed-off-by: Tanner Lewis --- .../migrationConsole/lib/integ_test/integ_test/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py index 5c899f9956..0f02426fed 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py @@ -20,3 +20,8 @@ def pytest_addoption(parser): @pytest.fixture def unique_id(pytestconfig): return pytestconfig.getoption("unique_id") + +@pytest.fixture +def stage(pytestconfig): + return pytestconfig.getoption("stage") + From 1e01deb62c4436a12c474acfc363d3e77cd0fa14 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Tue, 22 Oct 2024 22:38:52 -0500 Subject: [PATCH 04/12] Add fixes for full integ test Signed-off-by: Tanner Lewis --- .../lib/integ_test/integ_test/conftest.py | 5 +---- .../lib/integ_test/integ_test/full_tests.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py index 0f02426fed..52fbb6ae33 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py @@ -13,6 +13,7 @@ def pytest_configure(config): def pytest_addoption(parser): parser.addoption("--unique_id", action="store", default=uuid.uuid4().hex) + parser.addoption("--stage", action="store", default="dev") parser.addoption("--config_file_path", action="store", default="/etc/migration_services.yaml", help="Path to config file for console library") @@ -21,7 +22,3 @@ def pytest_addoption(parser): def unique_id(pytestconfig): return pytestconfig.getoption("unique_id") -@pytest.fixture -def stage(pytestconfig): - return pytestconfig.getoption("stage") - diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py index 041e5f67b9..8e7ec9bfd4 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py @@ -3,7 +3,7 @@ import pytest import unittest from http import HTTPStatus -from console_link.middleware.clusters import run_test_benchmarks, connection_check, clear_indices, ConnectionResult +from console_link.middleware.clusters import connection_check, clear_indices, ConnectionResult from console_link.models.cluster import Cluster from console_link.models.backfill_base import Backfill from console_link.models.replayer_base import Replayer @@ -13,8 +13,7 @@ from console_link.middleware.kafka import delete_topic from console_link.models.metadata import Metadata from console_link.cli import Context -from common_operations import (create_index, create_document, check_doc_counts_match, wait_for_running_replayer, - EXPECTED_BENCHMARK_DOCS) +from common_operations import (create_index, create_document, check_doc_counts_match, wait_for_running_replayer) logger = logging.getLogger(__name__) @@ -30,7 +29,7 @@ def initialize(request): if 'AWS_EXECUTION_ENV' in os.environ: logger.info("Detected an AWS environment") stage = request.config.getoption("--stage") - source_cluster.endpoint = f"https://alb.migration.{stage}.local:9201" + source_cluster.endpoint = f"https://alb.migration.{stage}.local:9200" backfill: Backfill = pytest.console_env.backfill assert backfill is not None metadata: Metadata = pytest.console_env.metadata @@ -80,7 +79,6 @@ def test_e2e_0001_default(self): replayer: Replayer = pytest.console_env.replay # Load initial data - run_test_benchmarks(source_cluster) index_name = f"test_e2e_0001_{pytest.unique_id}" doc_id_base = "e2e_0001_doc" create_index(cluster=source_cluster, index_name=index_name, test_case=self) @@ -102,15 +100,15 @@ def test_e2e_0001_default(self): expected_status_code=HTTPStatus.CREATED, test_case=self) ignore_list = [".", "searchguard", "sg7", "security-auditlog", "reindexed-logs"] - expected_docs = dict(EXPECTED_BENCHMARK_DOCS) + expected_docs = {} # Source should have both documents - expected_docs[index_name] = {"docs.count": "2"} + expected_docs[index_name] = {"count": 2} check_doc_counts_match(cluster=source_cluster, expected_index_details=expected_docs, index_prefix_ignore_list=ignore_list, test_case=self) # Target should have one document from snapshot - expected_docs[index_name] = {"docs.count": "1"} + expected_docs[index_name] = {"count": 1} check_doc_counts_match(cluster=target_cluster, expected_index_details=expected_docs, - index_prefix_ignore_list=ignore_list, max_attempts=40, delay=30.0, test_case=self) + index_prefix_ignore_list=ignore_list, max_attempts=20, delay=30.0, test_case=self) backfill.stop() @@ -120,7 +118,7 @@ def test_e2e_0001_default(self): replayer.start() wait_for_running_replayer(replayer=replayer) - expected_docs[index_name] = {"docs.count": "3"} + expected_docs[index_name] = {"count": 3} check_doc_counts_match(cluster=source_cluster, expected_index_details=expected_docs, index_prefix_ignore_list=ignore_list, test_case=self) From a958791e50c3f536bfeb5681e95f16a521cfdbc1 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Wed, 23 Oct 2024 09:54:17 -0500 Subject: [PATCH 05/12] Add more assertions around connection check Signed-off-by: Tanner Lewis --- .../lib/integ_test/integ_test/full_tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py index 8e7ec9bfd4..7897c59de4 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py @@ -29,7 +29,14 @@ def initialize(request): if 'AWS_EXECUTION_ENV' in os.environ: logger.info("Detected an AWS environment") stage = request.config.getoption("--stage") - source_cluster.endpoint = f"https://alb.migration.{stage}.local:9200" + logger.info("Checking original source and target endpoints can be reached, before using ALB endpoints for test") + direct_source_con_result: ConnectionResult = connection_check(source_cluster) + assert direct_source_con_result.connection_established is True + direct_target_con_result: ConnectionResult = connection_check(target_cluster) + assert direct_target_con_result.connection_established is True + source_cluster.endpoint = f"https://alb.migration.{stage}.local:9201" + target_cluster.endpoint = f"https://alb.migration.{stage}.local:9202" + target_cluster.allow_insecure = True backfill: Backfill = pytest.console_env.backfill assert backfill is not None metadata: Metadata = pytest.console_env.metadata From 0b13f00f62982598ac4c5fb815c6905a6d71c440 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Wed, 23 Oct 2024 10:06:28 -0500 Subject: [PATCH 06/12] Flake8 fix Signed-off-by: Tanner Lewis --- .../migrationConsole/lib/integ_test/integ_test/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py index 52fbb6ae33..9391807397 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py @@ -21,4 +21,3 @@ def pytest_addoption(parser): @pytest.fixture def unique_id(pytestconfig): return pytestconfig.getoption("unique_id") - From 4ef203d275869f0bb7b2de0ad4061f18822159e3 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Thu, 24 Oct 2024 13:36:37 -0500 Subject: [PATCH 07/12] Update passing in alb endpoints to integ test Signed-off-by: Tanner Lewis --- .../migrationConsole/lib/integ_test/integ_test/conftest.py | 5 ++++- .../lib/integ_test/integ_test/full_tests.py | 7 ++++--- vars/defaultIntegPipeline.groovy | 3 ++- vars/fullDefaultE2ETest.groovy | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py index 9391807397..4ec3605c9f 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py @@ -16,7 +16,10 @@ def pytest_addoption(parser): parser.addoption("--stage", action="store", default="dev") parser.addoption("--config_file_path", action="store", default="/etc/migration_services.yaml", help="Path to config file for console library") - + parser.addoption("--source_proxy_alb_endpoint", action="store", default=None, + help="Specify the Migration ALB endpoint for the source capture proxy") + parser.addoption("--target_proxy_alb_endpoint", action="store", default=None, + help="Specify the Migration ALB endpoint for the target proxy") @pytest.fixture def unique_id(pytestconfig): diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py index 7897c59de4..f7ef92c75a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py @@ -28,14 +28,15 @@ def initialize(request): # If in AWS, modify source and target objects here to route requests through the created ALB to verify its operation if 'AWS_EXECUTION_ENV' in os.environ: logger.info("Detected an AWS environment") - stage = request.config.getoption("--stage") + source_proxy_alb_endpoint = request.config.getoption("--source_proxy_alb_endpoint") + target_proxy_alb_endpoint = request.config.getoption("--target_proxy_alb_endpoint") logger.info("Checking original source and target endpoints can be reached, before using ALB endpoints for test") direct_source_con_result: ConnectionResult = connection_check(source_cluster) assert direct_source_con_result.connection_established is True direct_target_con_result: ConnectionResult = connection_check(target_cluster) assert direct_target_con_result.connection_established is True - source_cluster.endpoint = f"https://alb.migration.{stage}.local:9201" - target_cluster.endpoint = f"https://alb.migration.{stage}.local:9202" + source_cluster.endpoint = source_proxy_alb_endpoint + target_cluster.endpoint = target_proxy_alb_endpoint target_cluster.allow_insecure = True backfill: Backfill = pytest.console_env.backfill assert backfill is not None diff --git a/vars/defaultIntegPipeline.groovy b/vars/defaultIntegPipeline.groovy index 16fa15d30c..95dc4fd48f 100644 --- a/vars/defaultIntegPipeline.groovy +++ b/vars/defaultIntegPipeline.groovy @@ -156,8 +156,9 @@ def call(Map config = [:]) { def time = new Date().getTime() def uniqueId = "integ_min_${time}_${currentBuild.number}" def test_result_file = "${testDir}/reports/${uniqueId}/report.xml" + def populatedIntegTestCommand = integTestCommand.replaceAll("", stage) def command = "pipenv run pytest --log-file=${testDir}/reports/${uniqueId}/pytest.log " + - "--junitxml=${test_result_file} ${integTestCommand} " + + "--junitxml=${test_result_file} ${populatedIntegTestCommand} " + "--unique_id ${uniqueId} " + "--stage ${stage} " + "-s" diff --git a/vars/fullDefaultE2ETest.groovy b/vars/fullDefaultE2ETest.groovy index 342fed87e0..79afbd1e2d 100644 --- a/vars/fullDefaultE2ETest.groovy +++ b/vars/fullDefaultE2ETest.groovy @@ -68,6 +68,6 @@ def call(Map config = [:]) { defaultStageId: 'full-integ', skipCaptureProxyOnNodeSetup: true, jobName: 'full-default-e2e-test', - integTestCommand: '/root/lib/integ_test/integ_test/full_tests.py' + integTestCommand: '/root/lib/integ_test/integ_test/full_tests.py --source_proxy_alb_endpoint https://alb.migration..local:9201 --target_proxy_alb_endpoint https://alb.migration..local:9202' ) } From 3d71795331e917acb1d9d31b2acbceb5056973d5 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Thu, 24 Oct 2024 13:56:47 -0500 Subject: [PATCH 08/12] Update for flake8 Signed-off-by: Tanner Lewis --- .../migrationConsole/lib/integ_test/integ_test/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py index 4ec3605c9f..6c4e53ac2d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/conftest.py @@ -21,6 +21,7 @@ def pytest_addoption(parser): parser.addoption("--target_proxy_alb_endpoint", action="store", default=None, help="Specify the Migration ALB endpoint for the target proxy") + @pytest.fixture def unique_id(pytestconfig): return pytestconfig.getoption("unique_id") From 3476dab48f8745760fa24a1c143e43ac23f0b1da Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 28 Oct 2024 20:39:47 -0500 Subject: [PATCH 09/12] Address PR feedback for combining integ tests Signed-off-by: Tanner Lewis --- ...s_pr_e2e_test.yml => full_pr_e2e_test.yml} | 2 +- ...oovy => fullES68SourceE2ETestCover.groovy} | 4 +- .../rfsES68SourceE2ETestCover.groovy | 9 --- ...st.groovy => fullES68SourceE2ETest.groovy} | 4 +- vars/rfsDefaultE2ETest.groovy | 4 +- vars/rfsES68SourceE2ETest.groovy | 71 ------------------- vars/trafficReplayDefaultE2ETest.groovy | 4 +- 7 files changed, 7 insertions(+), 91 deletions(-) rename .github/workflows/{rfs_pr_e2e_test.yml => full_pr_e2e_test.yml} (96%) rename jenkins/migrationIntegPipelines/{fullDefaultE2ETestCover.groovy => fullES68SourceE2ETestCover.groovy} (72%) delete mode 100644 jenkins/migrationIntegPipelines/rfsES68SourceE2ETestCover.groovy rename vars/{fullDefaultE2ETest.groovy => fullES68SourceE2ETest.groovy} (96%) delete mode 100644 vars/rfsES68SourceE2ETest.groovy diff --git a/.github/workflows/rfs_pr_e2e_test.yml b/.github/workflows/full_pr_e2e_test.yml similarity index 96% rename from .github/workflows/rfs_pr_e2e_test.yml rename to .github/workflows/full_pr_e2e_test.yml index caafe6a8f8..aab80a6d19 100644 --- a/.github/workflows/rfs_pr_e2e_test.yml +++ b/.github/workflows/full_pr_e2e_test.yml @@ -34,6 +34,6 @@ jobs: uses: lewijacn/jenkins-trigger@1.0.4 with: jenkins_url: "https://migrations.ci.opensearch.org" - job_name: "rfs-default-e2e-test" + job_name: "full-es68source-e2e-test" api_token: "${{ secrets.JENKINS_MIGRATIONS_GENERIC_WEBHOOK_TOKEN }}" job_params: "GIT_REPO_URL=${{ steps.sanitize-input.outputs.pr_repo_url }},GIT_BRANCH=${{ steps.sanitize-input.outputs.branch_name }}" diff --git a/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy b/jenkins/migrationIntegPipelines/fullES68SourceE2ETestCover.groovy similarity index 72% rename from jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy rename to jenkins/migrationIntegPipelines/fullES68SourceE2ETestCover.groovy index 97e10dc214..26510e681b 100644 --- a/jenkins/migrationIntegPipelines/fullDefaultE2ETestCover.groovy +++ b/jenkins/migrationIntegPipelines/fullES68SourceE2ETestCover.groovy @@ -5,5 +5,5 @@ library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( [$class: 'GitSCMSource', remote: "${gitUrl}"]) -// Shared library function (location from root: vars/fullDefaultE2ETest.groovy) -fullDefaultE2ETest() +// Shared library function (location from root: vars/fullES68SourceE2ETest.groovy) +fullES68SourceE2ETest() diff --git a/jenkins/migrationIntegPipelines/rfsES68SourceE2ETestCover.groovy b/jenkins/migrationIntegPipelines/rfsES68SourceE2ETestCover.groovy deleted file mode 100644 index b0801a90dd..0000000000 --- a/jenkins/migrationIntegPipelines/rfsES68SourceE2ETestCover.groovy +++ /dev/null @@ -1,9 +0,0 @@ -def gitBranch = params.GIT_BRANCH ?: 'main' -def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git' - -library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( - [$class: 'GitSCMSource', - remote: "${gitUrl}"]) - -// Shared library function (location from root: vars/rfsES68SourceE2ETest.groovy) -rfsES68SourceE2ETest() diff --git a/vars/fullDefaultE2ETest.groovy b/vars/fullES68SourceE2ETest.groovy similarity index 96% rename from vars/fullDefaultE2ETest.groovy rename to vars/fullES68SourceE2ETest.groovy index 79afbd1e2d..73bec0c204 100644 --- a/vars/fullDefaultE2ETest.groovy +++ b/vars/fullES68SourceE2ETest.groovy @@ -65,9 +65,9 @@ def call(Map config = [:]) { migrationContext: migration_cdk_context, sourceContextId: sourceContextId, migrationContextId: migrationContextId, - defaultStageId: 'full-integ', + defaultStageId: 'full-es68', skipCaptureProxyOnNodeSetup: true, - jobName: 'full-default-e2e-test', + jobName: 'full-es68source-e2e-test', integTestCommand: '/root/lib/integ_test/integ_test/full_tests.py --source_proxy_alb_endpoint https://alb.migration..local:9201 --target_proxy_alb_endpoint https://alb.migration..local:9202' ) } diff --git a/vars/rfsDefaultE2ETest.groovy b/vars/rfsDefaultE2ETest.groovy index 5a5115e557..f0aea12ab4 100644 --- a/vars/rfsDefaultE2ETest.groovy +++ b/vars/rfsDefaultE2ETest.groovy @@ -1,6 +1,4 @@ -// Note: -// 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to -// allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK. +// Note: This integ test exists to verify that RFS can be ran independently of other migrations def call(Map config = [:]) { def sourceContextId = 'source-single-node-ec2' diff --git a/vars/rfsES68SourceE2ETest.groovy b/vars/rfsES68SourceE2ETest.groovy deleted file mode 100644 index cb047acbd3..0000000000 --- a/vars/rfsES68SourceE2ETest.groovy +++ /dev/null @@ -1,71 +0,0 @@ - -def call(Map config = [:]) { - def sourceContextId = 'source-single-node-ec2' - def migrationContextId = 'migration-rfs' - def source_cdk_context = """ - { - "source-single-node-ec2": { - "suffix": "ec2-source-", - "networkStackSuffix": "ec2-source-", - "distVersion": "6.8.23", - "distributionUrl": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-6.8.23.tar.gz", - "captureProxyEnabled": false, - "securityDisabled": true, - "minDistribution": false, - "cpuArch": "x64", - "isInternal": true, - "singleNodeCluster": true, - "networkAvailabilityZones": 2, - "dataNodeCount": 1, - "managerNodeCount": 0, - "serverAccessType": "ipv4", - "restrictServerAccessTo": "0.0.0.0/0" - } - } - """ - def migration_cdk_context = """ - { - "migration-rfs": { - "stage": "", - "vpcId": "", - "engineVersion": "OS_2.11", - "domainName": "os-cluster-", - "dataNodeCount": 2, - "openAccessPolicyEnabled": true, - "domainRemovalPolicy": "DESTROY", - "artifactBucketRemovalPolicy": "DESTROY", - "trafficReplayerServiceEnabled": false, - "reindexFromSnapshotServiceEnabled": true, - "sourceClusterEndpoint": "", - "sourceCluster": { - "endpoint": "", - "auth": {"type": "none"}, - "version": "ES_6.8.23" - }, - "tlsSecurityPolicy": "TLS_1_2", - "enforceHTTPS": true, - "nodeToNodeEncryptionEnabled": true, - "encryptionAtRestEnabled": true, - "vpcEnabled": true, - "vpcAZCount": 2, - "domainAZCount": 2, - "mskAZCount": 2, - "migrationAssistanceEnabled": true, - "replayerOutputEFSRemovalPolicy": "DESTROY", - "migrationConsoleServiceEnabled": true, - "otelCollectorEnabled": true - } - } - """ - - defaultIntegPipeline( - sourceContext: source_cdk_context, - migrationContext: migration_cdk_context, - sourceContextId: sourceContextId, - migrationContextId: migrationContextId, - defaultStageId: 'rfs-integ-es68', - skipCaptureProxyOnNodeSetup: true, - jobName: 'rfs-es68source-e2e-test', - integTestCommand: '/root/lib/integ_test/integ_test/backfill_tests_es68.py' - ) -} diff --git a/vars/trafficReplayDefaultE2ETest.groovy b/vars/trafficReplayDefaultE2ETest.groovy index e2822bbd78..174b57d37b 100644 --- a/vars/trafficReplayDefaultE2ETest.groovy +++ b/vars/trafficReplayDefaultE2ETest.groovy @@ -1,6 +1,4 @@ -// Note: -// 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to -// allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK. +// Note: This integ test exists to verify that Capture and Replay can be ran independently of other migrations def call(Map config = [:]) { def sourceContextId = 'source-single-node-ec2' From 2c38b641723a0ad389c742765c6db4ba8d44da65 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 28 Oct 2024 20:43:02 -0500 Subject: [PATCH 10/12] Remove unused integ test file Signed-off-by: Tanner Lewis --- .../integ_test/backfill_tests_es68.py | 92 ------------------- 1 file changed, 92 deletions(-) delete mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/backfill_tests_es68.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/backfill_tests_es68.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/backfill_tests_es68.py deleted file mode 100644 index 51a55db663..0000000000 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/backfill_tests_es68.py +++ /dev/null @@ -1,92 +0,0 @@ -import logging -import pytest -import unittest -from http import HTTPStatus -from console_link.middleware.clusters import connection_check, clear_indices, ConnectionResult -from console_link.models.cluster import Cluster -from console_link.models.backfill_base import Backfill -from console_link.models.command_result import CommandResult -from console_link.models.snapshot import Snapshot -from console_link.models.metadata import Metadata -from console_link.cli import Context -from common_operations import (get_document, create_document, create_index) - -logger = logging.getLogger(__name__) - - -def preload_data(source_cluster: Cluster, target_cluster: Cluster): - # Confirm source and target connection - source_con_result: ConnectionResult = connection_check(source_cluster) - assert source_con_result.connection_established is True - target_con_result: ConnectionResult = connection_check(target_cluster) - assert target_con_result.connection_established is True - - # Clear any existing non-system indices - clear_indices(source_cluster) - clear_indices(target_cluster) - - # Preload data that test cases will verify is migrated - # test_backfill_0001 - index_name = f"test_backfill_0001_{pytest.unique_id}" - doc_id = "backfill_0001_doc" - create_index(cluster=source_cluster, index_name=index_name) - create_document(cluster=source_cluster, index_name=index_name, doc_id=doc_id, - expected_status_code=HTTPStatus.CREATED) - - -@pytest.fixture(scope="class") -def setup_backfill(request): - config_path = request.config.getoption("--config_file_path") - unique_id = request.config.getoption("--unique_id") - pytest.console_env = Context(config_path).env - pytest.unique_id = unique_id - preload_data(source_cluster=pytest.console_env.source_cluster, - target_cluster=pytest.console_env.target_cluster) - backfill: Backfill = pytest.console_env.backfill - assert backfill is not None - metadata: Metadata = pytest.console_env.metadata - assert metadata is not None - - backfill.create() - snapshot: Snapshot = pytest.console_env.snapshot - status_result: CommandResult = snapshot.status() - if status_result.success: - snapshot.delete() - snapshot_result: CommandResult = snapshot.create(wait=True) - assert snapshot_result.success - metadata_result: CommandResult = metadata.migrate() - assert metadata_result.success - backfill_start_result: CommandResult = backfill.start() - assert backfill_start_result.success - # small enough to allow containers to be reused, big enough to test scaling out - backfill_scale_result: CommandResult = backfill.scale(units=2) - assert backfill_scale_result.success - - -@pytest.fixture(scope="session", autouse=True) -def cleanup_after_tests(): - # Setup code - logger.info("Starting backfill tests...") - - yield - - # Teardown code - logger.info("Stopping backfill...") - backfill: Backfill = pytest.console_env.backfill - backfill.stop() - - -@pytest.mark.usefixtures("setup_backfill") -class BackfillTests(unittest.TestCase): - - def test_backfill_0001_single_document(self): - index_name = f"test_backfill_0001_{pytest.unique_id}" - doc_id = "backfill_0001_doc" - source_cluster: Cluster = pytest.console_env.source_cluster - target_cluster: Cluster = pytest.console_env.target_cluster - - # Assert preloaded document exists - get_document(cluster=source_cluster, index_name=index_name, doc_id=doc_id, test_case=self) - - get_document(cluster=target_cluster, index_name=index_name, doc_id=doc_id, max_attempts=30, delay=30.0, - test_case=self) From 32126b90fbac0ef8289ac4d8eee3a3513ba6851d Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Wed, 30 Oct 2024 09:26:36 -0500 Subject: [PATCH 11/12] Minor fixes Signed-off-by: Tanner Lewis --- .github/workflows/full_pr_e2e_test.yml | 2 +- .../lib/integ_test/integ_test/full_tests.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/full_pr_e2e_test.yml b/.github/workflows/full_pr_e2e_test.yml index aab80a6d19..d7fea7a5d8 100644 --- a/.github/workflows/full_pr_e2e_test.yml +++ b/.github/workflows/full_pr_e2e_test.yml @@ -15,7 +15,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - rfs-e2e-aws-test: + full-es68-e2e-aws-test: runs-on: ubuntu-latest steps: - name: Sanitize branch and repo names diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py index f7ef92c75a..fa591c2979 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/full_tests.py @@ -99,10 +99,15 @@ def test_e2e_0001_default(self): status_result: CommandResult = snapshot.status() if status_result.success: snapshot.delete() - snapshot.create(wait=True) - metadata.migrate() - backfill.start() - backfill.scale(units=2) + snapshot_result: CommandResult = snapshot.create(wait=True) + assert snapshot_result.success + metadata_result: CommandResult = metadata.migrate() + assert metadata_result.success + backfill_start_result: CommandResult = backfill.start() + assert backfill_start_result.success + # small enough to allow containers to be reused, big enough to test scaling out + backfill_scale_result: CommandResult = backfill.scale(units=2) + assert backfill_scale_result.success # This document was created after snapshot and should not be included in Backfill but expected in Replay create_document(cluster=source_cluster, index_name=index_name, doc_id=doc_id_base + "_2", expected_status_code=HTTPStatus.CREATED, test_case=self) From e0b241d8590bd486b0e711d30894117cbd0ac621 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Wed, 30 Oct 2024 12:46:56 -0500 Subject: [PATCH 12/12] Provide setting to not enforce Imdsv2 Signed-off-by: Tanner Lewis --- vars/fullES68SourceE2ETest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vars/fullES68SourceE2ETest.groovy b/vars/fullES68SourceE2ETest.groovy index 73bec0c204..d04651379d 100644 --- a/vars/fullES68SourceE2ETest.groovy +++ b/vars/fullES68SourceE2ETest.groovy @@ -19,7 +19,8 @@ def call(Map config = [:]) { "dataNodeCount": 1, "managerNodeCount": 0, "serverAccessType": "ipv4", - "restrictServerAccessTo": "0.0.0.0/0" + "restrictServerAccessTo": "0.0.0.0/0", + "requireImdsv2": false } } """