integ-test.jenkinsfile

lib = library(identifier: 'jenkins@4.2.2', retriever: modernSCM([
    $class: 'GitSCMSource',
    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def docker_images = [
    "tar": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
    "rpm": "opensearchstaging/ci-runner:ci-runner-rockylinux8-systemd-base-integtest-v2",
    "deb": "opensearchstaging/ci-runner:ci-runner-ubuntu2004-systemd-base-integtest-v2",
]

def docker_args = [
    "tar": "-u 1000",
    "rpm": "--entrypoint=/usr/sbin/init -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro",
    "deb": "--entrypoint=/usr/sbin/init -u root --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro",
]

def agent_nodes = [
    "x64": "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host",
    "arm64": "Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host",
]

pipeline {
    options {
        timeout(time: 3, unit: 'HOURS')
    }
    agent none
    environment {
        BUILD_MANIFEST = "build-manifest.yml"
        BUILD_JOB_NAME = "distribution-build-opensearch"
        ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name')
    }
    triggers {
        parameterizedCron '''
            H 14 * * * %TEST_MANIFEST=2.8.0/opensearch-2.8.0-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/tar/builds/opensearch/manifest.yml
            H 14 * * * %TEST_MANIFEST=2.8.0/opensearch-2.8.0-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/tar/builds/opensearch/manifest.yml
            H 16 * * * %TEST_MANIFEST=2.8.0/opensearch-2.8.0-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/rpm/builds/opensearch/manifest.yml
            H 16 * * * %TEST_MANIFEST=2.8.0/opensearch-2.8.0-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/rpm/builds/opensearch/manifest.yml
            H 17 * * * %TEST_MANIFEST=2.8.0/opensearch-2.8.0-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/deb/builds/opensearch/manifest.yml
            H 17 * * * %TEST_MANIFEST=2.8.0/opensearch-2.8.0-test.yml;BUILD_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/deb/builds/opensearch/manifest.yml
            '''
    }
    parameters {
        string(
            name: 'COMPONENT_NAME',
            description: 'If this field contains one or more component names (e.g. index-management geospatial ...) separated by space, will test with "--component ...", else test everything in the TEST_MANIFEST..',
            trim: true
        )
        string(
            name: 'TEST_MANIFEST',
            description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.',
            trim: true
        )
        string(
            name: 'BUILD_MANIFEST_URL',
            description: 'The build manifest URL for OpenSearch, e.g. "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.5.0/6976/linux/x64/tar/builds/opensearch/manifest.yml".',
            trim: true
        )
    }
    stages {
        stage('verify-parameters') {
            agent { label agent_nodes["x64"]}
            steps {
                script {
                    if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) {
                        currentBuild.result = 'ABORTED'
                        error("Integration Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.")
                    }

                    if (BUILD_MANIFEST_URL == '') {
                        currentBuild.result = 'ABORTED'
                        error("Integration Tests failed to start. Build manifest url was not provided.")
                    }

                    downloadBuildManifest(
                        url: BUILD_MANIFEST_URL,
                        path: BUILD_MANIFEST
                    )

                    def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
                    def componentList = COMPONENT_NAME ? COMPONENT_NAME.trim().split(" ") as List : buildManifestObj.getNames()
                    env.architecture = buildManifestObj.getArtifactArchitecture()
                    env.buildId = buildManifestObj.getArtifactBuildId()
                    env.distribution = buildManifestObj.getDistribution()
                    env.version = buildManifestObj.build.version
                    env.platform = buildManifestObj.build.platform
                    env.artifactPath = buildManifestObj.getArtifactRoot(BUILD_JOB_NAME, buildId)
                    env.AGENT_LABEL = agent_nodes["$architecture"]

                    echo "Version: ${version}, Agent: ${AGENT_LABEL}, BuildId: ${buildId}, Distribution: ${distribution}, Components: ${componentList}"
                    currentBuild.description = "$TEST_MANIFEST, $version, $architecture, $buildId, $distribution, $componentList"
                }
            }
            post {
                always {
                    postCleanup()
                }
            }
        }
        stage('integ-test') {
            // Need to run this directly on agent node here in order to trigger stages with docker container and avoid docker within docker situation
            // Can only be run in runner that is at least 50GB per container
            agent { label AGENT_LABEL }
            steps {
                script {

                    downloadBuildManifest(
                        url: BUILD_MANIFEST_URL,
                        path: BUILD_MANIFEST
                    )

                    def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
                    def componentDefaultList = buildManifestObj.getNames()
                    def componentList = COMPONENT_NAME ? COMPONENT_NAME.trim().split(" ") as List : componentDefaultList
                    String switch_user_non_root = (distribution.equals('rpm') || distribution.equals('deb')) ? 'true' : 'false'
                    echo "switch_user_non_root: ${switch_user_non_root}"
                    echo "componentList: ${componentList}"

                    for (component_check in componentList) {
                        if (!componentDefaultList.contains(component_check)) {
                            error("${component_check} is not in build manifest: ${componentDefaultList}, exit 1")
                        }
                    }

                    echo "Downloading from S3: ${artifactPath}"
                    downloadFromS3(
                        assumedRoleName: 'opensearch-bundle',
                        roleAccountNumberCred: 'jenkins-aws-account-public',
                        downloadPath: "${artifactPath}/",
                        bucketName: "${ARTIFACT_BUCKET_NAME}",
                        localPath: "${WORKSPACE}/artifacts",
                        force: true
                      )
                    sh("mv -v $WORKSPACE/artifacts/${artifactPath} $WORKSPACE")

                    // Stash the current working directory files, aka opensearch-build repo
                    // Unstash later in each triggered stage to run integTest
                    stash includes: "**", name: "integtest-opensearch-$BUILD_NUMBER"

                    componentTests = [:]

                    for (component in componentList) {
                        // Must use local variable due to groovy for loop and closure scope
                        // Or the stage will be fixed to the last item in return when new stages are triggered here
                        // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/
                        def local_component = component.trim()
                        def local_component_index = componentList.indexOf(local_component)
                        def wait_seconds = local_component_index * 10

                        echo "Add Component: ${local_component}"
                        componentTests["Run Integtest ${local_component}"] = {
                            // Using scripted pipelines to trigger dynamic parallel stages
                            timeout(time: 2, unit: 'HOURS') {
                                node(AGENT_LABEL) {
                                    docker.withRegistry('https://public.ecr.aws/') {
                                        docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
                                            try {
                                                stage("${local_component}") {
                                                    // Jenkins tend to not clean up workspace at times even though ws clean is called
                                                    // Docker is mounting the agent directory so it can communicate with the agent
                                                    // this sometimes causes the workspace to retain last run test-results and ends with build failures
                                                    // https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
                                                    sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}")
                                                    unstash "integtest-opensearch-$BUILD_NUMBER"
                                                    sh('rm -rf test-results')
                                                    runIntegTestScript(
                                                        jobName: "$BUILD_JOB_NAME",
                                                        componentName: "${local_component}",
                                                        buildManifest: "$BUILD_MANIFEST",
                                                        testManifest: "manifests/${TEST_MANIFEST}",
                                                        localPath: "${WORKSPACE}/${distribution}",
                                                        switchUserNonRoot: "${switch_user_non_root}"
                                                    )
                                                }
                                            } catch (e) {
                                                echo "Error running integtest for component ${local_component}, creating Github issue"
                                                String issueBodyMessage = "The integration test failed at distribution level for component ${local_component}<br>Version: ${version}<br>Distribution: ${distribution}<br>Architecture: ${architecture}<br>Platform: ${platform}<br><br>Please check the logs: ${RUN_DISPLAY_URL}<br><br> * Steps to reproduce: See https://github.com/opensearch-project/opensearch-build/tree/main/src/test_workflow#integration-tests<br>* Access components yml file:<br> - [With security](https://ci.opensearch.org/ci/dbc/${JOB_NAME}/${version}/${buildId}/${platform}/${architecture}/${distribution}/test-results/${BUILD_NUMBER}/integ-test/${local_component}/with-security/${local_component}.yml) (if applicable)<br> - [Without security](https://ci.opensearch.org/ci/dbc/${JOB_NAME}/${version}/${buildId}/${platform}/${architecture}/${distribution}/test-results/${BUILD_NUMBER}/integ-test/${local_component}/without-security/${local_component}.yml) (if applicable)<br><br> _Note: All in one test report manifest with all the details coming soon. See https://github.com/opensearch-project/opensearch-build/issues/1274_"
                                                createGithubIssue(
                                                    repoUrl: buildManifestObj.getRepo("${local_component}"),
                                                    issueTitle: "[AUTOCUT] Integration Test failed for ${local_component}: ${version} ${distribution} distribution",
                                                    issueBody: issueBodyMessage,
                                                    label: "autocut,v${version},integ-test-failure"
                                                )
                                                throw new Exception("Error running integtest for component ${local_component}", e)
                                            } finally {
                                                echo "Completed running integtest for component ${local_component}"
                                                uploadTestResults(
                                                    buildManifestFileName: BUILD_MANIFEST,
                                                    jobName: JOB_NAME
                                                )
                                                postCleanup()
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                    parallel componentTests
                }
            }
            post {
                always {
                    postCleanup()
                }
            }
        }
    }

    post {
        success {
            node(AGENT_LABEL) {
                script {
                    def stashed = lib.jenkins.Messages.new(this).get(['integ-test'])
                    publishNotification(
                        icon: ':white_check_mark:',
                        message: 'Integration Tests Successful',
                        extra: stashed,
                        credentialsId: 'jenkins-integ-test-webhook',
                        manifest: TEST_MANIFEST,
                    )

                    postCleanup()
                }
            }
        }
        failure {
            node(AGENT_LABEL) {
                script  {
                    def stashed = lib.jenkins.Messages.new(this).get(['integ-test'])
                    publishNotification(
                        icon: ':warning:',
                        message: 'Failed Integration Tests',
                        extra: stashed,
                        credentialsId: 'jenkins-integ-test-webhook',
                        manifest: TEST_MANIFEST,
                    )

                    postCleanup()
                }
            }
        }
    }
}