Make optional by using Gradle JDK auto-detection and clean up Spark 4.0 CI workflow

Signed-off-by: Sotaro Hikita <bering1814@gmail.com>

Author: lawofcycles

.github/workflows/build_spark_40.yml

@@ -14,26 +14,6 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v3
 
-      - name: Set up JDK 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '8'
-          cache: 'gradle'
-
-      - name: 'Setup: Java 8 env'
-        run: echo "JAVA8_HOME=$JAVA_HOME" >> $GITHUB_ENV
-
-      - name: Set up JDK 11
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '11'
-          cache: 'gradle'
-
-      - name: 'Setup: Java 11 env'
-        run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV
-
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:

CHANGELOG.md

@@ -8,6 +8,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Changed
 - Switched to more reliable OpenSearch Lucene snapshot location ([#597](https://github.com/opensearch-project/opensearch-hadoop/pull/597))
+- Make $JAVA8_HOME optional by using Gradle auto-detection and remove unnecessary JDK 8/11 setup from Spark 4.0 CI workflow ([#XXX](https://github.com/opensearch-project/opensearch-hadoop/pull/XXX))
+- Make $JAVA8_HOME environment variable optional by using Gradle auto-detection to find JDK 8 ([#XXX](https://github.com/opensearch-project/opensearch-hadoop/pull/XXX))
 
 ### Deprecated
 

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/BaseBuildPlugin.groovy

@@ -77,13 +77,27 @@ class BaseBuildPlugin implements Plugin<Project> {
 
             // We snap the runtime to java 8 since Hadoop needs to see some significant
             // upgrades to support any runtime higher than that
+            // Try to find JDK 8 from BuildParams (populated by JAVA8_HOME env var).
+            // If not found, use Gradle's JavaInstallationRegistry to auto-detect installed JDKs.
             JavaHome opensearchHadoopRuntimeJava = BuildParams.javaVersions.find { it.version == 8 }
-            if (opensearchHadoopRuntimeJava == null) {
-                throw new GradleException(
-                        '$JAVA8_HOME must be set to build OpenSearch-Hadoop. ' +
-                                "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
-                                "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
-                )
+            if (opensearchHadoopRuntimeJava != null) {
+                project.rootProject.ext.runtimeJavaHome = opensearchHadoopRuntimeJava.javaHome.get()
+            } else {
+                def javaInstallationRegistry = project.rootProject.services.get(org.gradle.internal.jvm.inspection.JavaInstallationRegistry)
+                def jvmMetadataDetector = project.rootProject.services.get(org.gradle.internal.jvm.inspection.JvmMetadataDetector)
+                def jdk8Install = javaInstallationRegistry.toolchains().find { toolchain ->
+                    def metadata = jvmMetadataDetector.getMetadata(toolchain.location)
+                    metadata.languageVersion.majorVersion == '8'
+                }
+                if (jdk8Install != null) {
+                    project.rootProject.ext.runtimeJavaHome = jdk8Install.location.location
+                } else {
+                    throw new GradleException(
+                            'JDK 8 is required to build OpenSearch-Hadoop but was not found. ' +
+                                    'Either set $JAVA8_HOME or install JDK 8 via a toolchain manager (e.g. sdkman) ' +
+                                    'so that Gradle auto-detection can find it.'
+                    )
+                }
             }
 
             // Set on global build info
@@ -92,7 +106,6 @@ class BaseBuildPlugin implements Plugin<Project> {
             }
 
             // Set on build settings
-            project.rootProject.ext.runtimeJavaHome = opensearchHadoopRuntimeJava.javaHome.get()
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
 
             project.rootProject.ext.buildInfoConfigured = true

gradle.properties

@@ -5,7 +5,7 @@ systemProp.org.gradle.warning.mode=fail
 org.gradle.daemon=false
 
 # java homes resolved by environment variables
-org.gradle.java.installations.auto-detect=false
+org.gradle.java.installations.auto-detect=true
 org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME
 
 ## Dependecies Version

spark/core/build.gradle

@@ -129,9 +129,9 @@ sparkVariants {
                 source(generatedJavaDirectory)
                 if (variant.name.contains('spark40')) {
                     executable = new File(System.getenv('SPARK_TEST_JAVA_HOME') ?: System.getenv('JAVA17_HOME') ?: project.ext.runtimeJavaHome.toString(), 'bin/javadoc').absolutePath
-                    options.addStringOption('Xdoclint:none', '-quiet')
-                    failOnError = false
                 }
+                options.addStringOption('Xdoclint:none', '-quiet')
+                failOnError = false
             }
         }
 
@@ -187,8 +187,11 @@ tasks.withType(ScalaCompile) { ScalaCompile task ->
     if (task.name.contains("spark40") || task.name.contains("Spark40")) {
         return
     }
-    task.scalaCompileOptions.additionalParameters = ["-javabootclasspath", new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar').absolutePath]
-    task.options.bootstrapClasspath = layout.files(new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar'))
+    File rtJar = new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar')
+    if (rtJar.exists()) {
+        task.scalaCompileOptions.additionalParameters = ["-javabootclasspath", rtJar.absolutePath]
+        task.options.bootstrapClasspath = layout.files(rtJar)
+    }
     task.sourceCompatibility = project.ext.minimumRuntimeVersion
     task.targetCompatibility = project.ext.minimumRuntimeVersion
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath

spark/sql-30/build.gradle

@@ -150,6 +150,8 @@ sparkVariants {
         tasks.getByName(variant.taskName('javadoc')) {
             dependsOn compileScala
             source(generatedJavaDirectory)
+            options.addStringOption('Xdoclint:none', '-quiet')
+            failOnError = false
         }
 
         scaladoc {
@@ -159,8 +161,11 @@ sparkVariants {
 }
 
 tasks.withType(ScalaCompile) { ScalaCompile task ->
-    task.scalaCompileOptions.additionalParameters = ["-javabootclasspath", new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar').absolutePath]
-    task.options.bootstrapClasspath = layout.files(new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar'))
+    File rtJar = new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar')
+    if (rtJar.exists()) {
+        task.scalaCompileOptions.additionalParameters = ["-javabootclasspath", rtJar.absolutePath]
+        task.options.bootstrapClasspath = layout.files(rtJar)
+    }
     task.sourceCompatibility = project.ext.minimumRuntimeVersion
     task.targetCompatibility = project.ext.minimumRuntimeVersion
     task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath