[Extensions] Enable Integration Tests for AD Extension

build.gradle

@@ -11,6 +11,7 @@
 
 import java.util.concurrent.Callable
 import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.testclusters.ExtensionsProperties
 import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 buildscript {
@@ -34,6 +35,8 @@ buildscript {
         js_resource_folder = "src/test/resources/job-scheduler"
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
         job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build)
+        job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
+                '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + plugin_no_snapshot + '.zip'
         bwcVersionShort = "2.6.0"
         bwcVersion = bwcVersionShort + ".0"
         bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' +
@@ -92,17 +95,15 @@ repositories {
 }
 
 apply plugin: 'java'
-apply plugin: 'application'
 apply plugin: 'idea'
+apply plugin: 'opensearch.opensearchplugin'
 apply plugin: 'opensearch.testclusters'
 apply plugin: 'base'
 apply plugin: 'jacoco'
 apply plugin: 'eclipse'
 //apply plugin: 'opensearch.pluginzip'
 apply plugin: 'maven-publish'
 
-mainClassName = 'org.opensearch.ad.AnomalyDetectorExtension'
-
 ext {
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
     buildVersionQualifier = System.getProperty("build.version_qualifier")
@@ -118,6 +119,13 @@ allprojects {
     }
 }
 
+opensearchplugin {
+    name 'opensearch-anomaly-detection'
+    description 'OpenSearch anomaly detector plugin'
+    classname 'org.opensearch.ad.AnomalyDetectorExtension'
+    extendedPlugins = ['lang-painless', 'opensearch-job-scheduler']
+}
+
 ext {
     projectSubstitutions = [:]
     licenseFile = rootProject.file('LICENSE.txt')
@@ -139,10 +147,35 @@ configurations.all {
         force "net.bytebuddy:byte-buddy-agent:1.9.15"
         force "com.google.code.gson:gson:2.8.9"
         force "junit:junit:4.13.2"
+
+        force "jakarta.json:jakarta.json-api:2.1.1"
+        force "com.google.guava:guava:31.1-jre"
+        force "org.apache.logging.log4j:log4j-api:2.20.0"
+        force "org.apache.logging.log4j:log4j-core:2.20.0"
+        force "org.apache.logging.log4j:log4j-jul:2.20.0"
+        force "com.fasterxml.jackson:jackson-bom:2.15.1"
+        force "com.fasterxml.jackson.core:jackson-databind:2.15.0"
+        force "com.fasterxml.jackson.core:jackson-core:2.15.0"
+        force "com.fasterxml.jackson.core:jackson-annotations:2.15.1"
+        force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2"
+        force "com.fasterxml.jackson.datatype:jackson-datatype-guava:2.15.2"
+        force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2"
+        force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.15.2"
+        force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.2"
+        force "io.netty:netty-buffer:4.1.92.Final"
+        force "io.netty:netty-codec:4.1.92.Final"
+        force "io.netty:netty-codec-http:4.1.92.Final"
+        force "io.netty:netty-codec-http2:4.1.92.Final"
+        force "io.netty:netty-common:4.1.92.Final"
+        force "io.netty:netty-handler:4.1.92.Final"
+        force "io.netty:netty-transport-native-unix-common:4.1.92.Final"
+        force "io.netty:netty-resolver:4.1.92.Final"
+        force "io.netty:netty-transport:4.1.92.Final" 
     }
 }
 
 configurations {
+    zipArchive
     testImplementation {
         exclude group: 'org.hamcrest', module: 'hamcrest-core'
     }
@@ -170,7 +203,6 @@ publishing {
         }
     }
 }
-
 // Commented this code until https://github.com/opensearch-project/opensearch-sdk-java/issues/144
 /*tasks.named('forbiddenApisMain').configure {
     // Only enable limited check because AD code has too many violations.
@@ -185,12 +217,18 @@ publishing {
 
 // Allow test cases to be named Tests without having to be inherited from LuceneTestCase.
 // see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
-//testingConventions.enabled = false
-
-//licenseHeaders.enabled = true
-//dependencyLicenses.enabled = false
-//thirdPartyAudit.enabled = false
-//loggerUsageCheck.enabled = false
+testingConventions.enabled = false
+
+licenseHeaders.enabled = false
+dependencyLicenses.enabled = false
+thirdPartyAudit.enabled = false
+loggerUsageCheck.enabled = false
+validatePluginZipPom.enabled = false
+validateNebulaPom.enabled = false
+forbiddenApisMain.enabled = false
+forbiddenApisTest.enabled = false
+generatePomFileForPluginZipPublication.enabled = false
+publishPluginZipPublicationToMavenLocal.enabled = false
 
 // See package README.md for details on using these tasks.
 def _numNodes = findProperty('numNodes') as Integer ?: 1
@@ -213,6 +251,13 @@ test {
     systemProperty 'tests.security.manager', 'false'
 }
 
+task anomalyDetection(type: JavaExec) {
+    group = 'Execution'
+    description = 'Run AnomalyDetection Extension.'
+    mainClass = 'org.opensearch.ad.AnomalyDetectorExtension'
+    classpath = sourceSets.main.runtimeClasspath
+}
+
 task integTest(type: RestIntegTestTask) {
     description = "Run tests against a cluster"
     testClassesDirs = sourceSets.test.output.classesDirs
@@ -222,6 +267,11 @@ task integTest(type: RestIntegTestTask) {
 // Commented this code until https://github.com/opensearch-project/opensearch-sdk-java/issues/144
 /*
 tasks.named("check").configure { dependsOn(integTest) }
+*/
+
+task startTestExtension(type: Exec) {
+    commandLine 'bash', '-c', "./gradlew anomalyDetection &"
+}
 
 integTest {
     retry {
@@ -231,7 +281,7 @@ integTest {
             maxFailures = 10
         }
     }
-//    dependsOn "bundlePlugin"
+    dependsOn "startTestExtension"
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
 
@@ -286,9 +336,17 @@ integTest {
         }
     }
 }
-*/
 
-/*testClusters.integTest {
+def integADExtensionYml = new org.yaml.snakeyaml.Yaml().load(new File("src/test/resources/org/opensearch/ad/integ/anomaly-detection-extension.yml").newInputStream())
+
+task closeTestExtension (type: Exec) {
+    commandLine 'bash', '-c', "kill \$(lsof -i:${integADExtensionYml.port})"
+}
+
+tasks.named("integTest").configure { finalizedBy(closeTestExtension) }
+
+testClusters.integTest {
+    extension(new ExtensionsProperties("${integADExtensionYml.name}", "${integADExtensionYml.uniqueId}", "${integADExtensionYml.hostAddress}", "${integADExtensionYml.port}", "${integADExtensionYml.version}", "${integADExtensionYml.opensearchVersion}", "${integADExtensionYml.minimumCompatibleVersion}"))
     testDistribution = "ARCHIVE"
     // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
     if (_numNodes > 1) numberOfNodes = _numNodes
@@ -302,53 +360,19 @@ integTest {
             debugPort += 1
         }
     }
-    plugin(project.tasks.bundlePlugin.archiveFile)*/
-
-
-//    Commented this code until we have support of Job Scheduler for extensibility
-//    plugin(provider(new Callable<RegularFile>(){
-//        @Override
-//        RegularFile call() throws Exception {
-//            return new RegularFile() {
-//                @Override
-//                File getAsFile() {
-//                    if (new File("$project.rootDir/$js_resource_folder").exists()) {
-//                        project.delete(files("$project.rootDir/$js_resource_folder"))
-//                    }
-//                    project.mkdir js_resource_folder
-//                    ant.get(src: job_scheduler_build_download,
-//                            dest: js_resource_folder,
-//                            httpusecaches: false)
-//                    return fileTree(js_resource_folder).getSingleFile()
-//                }
-//            }
-//        }
-//    }))
-//
-//    // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before
-//    // the opensearch-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug.
-//    //
-//    // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opensearch-job-scheduler], dependency of [opendistro-anomaly-detection]
-//    //       at org.opensearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452)
-//    //
-//    // One explanation is that ES build script sort plugins according to the natural ordering of their names.
-//    // opendistro-anomaly-detection comes before opensearch-job-scheduler.
-//    //
-//    // The following is a comparison of different plugin installation order:
-//    // Before 7.7:
-//    // ./bin/elasticsearch-plugin install --batch file:opendistro-anomaly-detection.zip file:opensearch-job-scheduler.zip
-//    //
-//    // After 7.7:
-//    // ./bin/elasticsearch-plugin install --batch file:opensearch-job-scheduler.zip file:opendistro-anomaly-detection.zip
-//    //
-//    // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed.
-/*    nodes.each { node ->
-        def plugins = node.plugins
-        def firstPlugin = plugins.get(0)
-        plugins.remove(0)
-        plugins.add(firstPlugin)
-    }
-}*/
+
+    plugin(provider(new Callable<RegularFile>(){
+        @Override
+        RegularFile call() throws Exception {
+            return new RegularFile() {
+                @Override
+                File getAsFile() {
+                    return configurations.zipArchive.asFileTree.getSingleFile()
+                }
+            }
+        }
+    }))
+}
 
 task integTestRemote(type: RestIntegTestTask) {
     testClassesDirs = sourceSets.test.output.classesDirs
@@ -792,18 +816,19 @@ check.dependsOn jacocoTestCoverageVerification
 jacocoTestCoverageVerification.dependsOn jacocoTestReport
 
 dependencies {
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
     compileOnly "org.opensearch:opensearch:${opensearch_version}"
     compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${opensearch_version}"
     implementation "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}"
 //  Removed Common Utils dependency from AD
 //    implementation "org.opensearch:common-utils:${common_utils_version}"
     implementation "org.opensearch:opensearch-job-scheduler:${job_scheduler_version}"
     implementation "org.opensearch.sdk:opensearch-sdk-java:2.0.0-SNAPSHOT"
-    implementation "com.google.inject:guice:5.1.0"
+    implementation "com.google.inject:guice:7.0.0"
     implementation "org.opensearch.client:opensearch-java:${opensearch_version}"
     implementation "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
-    implementation group: 'com.google.guava', name: 'guava', version:'31.0.1-jre'
+    implementation group: 'com.google.guava', name: 'guava', version:'31.1-jre'
     implementation group: 'com.google.guava', name: 'failureaccess', version:'1.0.1'
     implementation group: 'org.javassist', name: 'javassist', version:'3.28.0-GA'
     implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'

src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java

@@ -190,7 +190,6 @@ public void runJob(ScheduledJobParameter scheduledJobParameter, JobExecutionCont
      * Get anomaly result, index result or handle exception if failed.
      *
      * @param jobParameter       scheduled job parameter
-     * @param lockService        lock service
      * @param lock               lock to run job
      * @param detectionStartTime detection start time
      * @param executionStartTime detection end time