build.gradle

import org.opensearch.hadoop.gradle.scala.SparkVariantPlugin

description = "OpenSearch Spark Core"

apply plugin: 'java-library'
apply plugin: 'scala'
apply plugin: 'opensearch.hadoop.build'
apply plugin: 'spark.variants'

sparkVariants {
    capabilityGroup 'org.opensearch.spark.variant'
    setCoreDefaultVariant "spark30scala212", spark30Version, scala212Version
    addCoreFeatureVariant "spark30scala213", spark30Version, scala213Version
    addCoreFeatureVariant "spark35scala212", spark35Version, scala212Version
    addCoreFeatureVariant "spark35scala213", spark35Version, scala213Version
    addCoreFeatureVariant "spark40scala213", spark40Version, scala213Version

    all { SparkVariantPlugin.SparkVariant variant ->

        String scalaCompileTaskName = project.sourceSets
                .getByName(variant.getSourceSetName("main"))
                .getCompileTaskName("scala")

        // Configure main compile task
        project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
            configure(compileScala.scalaCompileOptions.forkOptions) {
                memoryMaximumSize = '1g'
            }
            compileScala.scalaCompileOptions.additionalParameters = [
                    "-feature",
                    "-unchecked",
                    "-deprecation",
                    "-Xfuture",
                    "-Yno-adapted-args",
                    "-Ywarn-dead-code",
                    "-Ywarn-numeric-widen",
                    "-Xfatal-warnings"
            ]
        }

        dependencies {
            add(variant.configuration('api'), "org.scala-lang:scala-library:${variant.scalaVersion}")
            add(variant.configuration('api'), "org.scala-lang:scala-reflect:${variant.scalaVersion}")
            add(variant.configuration('api'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:${variant.sparkVersion}") {
                exclude group: 'javax.servlet'
                exclude group: 'org.apache.hadoop'
            }

            // Spark 3.5+ moved org.apache.spark.internal.Logging to spark-common-utils
            // Spark 4.1+ further split LogKey into spark-common-utils-java
            if (variant.sparkVersion.startsWith("4.")) {
                add(variant.configuration('compileOnly'), "org.apache.spark:spark-common-utils_${variant.scalaMajorVersion}:${variant.sparkVersion}")
                add(variant.configuration('compileOnly'), "org.apache.spark:spark-common-utils-java_${variant.scalaMajorVersion}:${variant.sparkVersion}")
            } else if (variant.sparkVersion.startsWith("3.5")) {
                add(variant.configuration('compileOnly'), "org.apache.spark:spark-common-utils_${variant.scalaMajorVersion}:${variant.sparkVersion}")
            }


            add(variant.configuration('implementation'), project(":opensearch-hadoop-mr"))
            add(variant.configuration('implementation'), "commons-logging:commons-logging:1.3.6")

            add(variant.configuration('compileOnly'), "com.fasterxml.jackson.module:jackson-module-scala_${variant.scalaMajorVersion}:2.9.10")
            add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.21")
            add(variant.configuration('compileOnly'), "com.google.guava:guava:23.0")
            add(variant.configuration('compileOnly'), "com.google.protobuf:protobuf-java:4.34.1")
            add(variant.configuration('compileOnly'), "org.slf4j:slf4j-api:2.0.18")

            add(variant.configuration('test', 'implementation'), project(":test:shared"))
            add(variant.configuration('test', 'implementation'), "com.esotericsoftware.kryo:kryo:2.24.0")
            add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:${variant.sparkVersion}") {
                exclude group: 'javax.servlet'
                exclude group: 'org.apache.hadoop'
            }

            add(variant.configuration('itest', 'implementation'), project(":test:shared"))
            add(variant.configuration('test', 'implementation'), "org.elasticsearch:securemock:1.2")

            add(variant.configuration('additionalSources'), project(":opensearch-hadoop-mr"))
            add(variant.configuration('javadocSources'), project(":opensearch-hadoop-mr"))
        }

        def javaFilesOnly = { FileTreeElement spec ->
            spec.file.name.endsWith('.java') || spec.isDirectory()
        }

        // Add java files from scala source set to javadocSourceElements.
        project.fileTree("src/main/scala").include(javaFilesOnly).each {
            project.artifacts.add(variant.configuration('javadocSourceElements'), it)
        }

        // Configure java source generation for javadoc purposes
        String generatedJavaDirectory = "$buildDir/generated/java/${variant.name}"
        Configuration scalaCompilerPlugin = project.configurations.maybeCreate(variant.configuration('scalaCompilerPlugin'))
        scalaCompilerPlugin.defaultDependencies { dependencies ->
            dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:0.19"))
        }

            ScalaCompile compileScala = tasks.getByName(scalaCompileTaskName) as ScalaCompile
            compileScala.scalaCompileOptions.with {
                additionalParameters = [
                        "-Xplugin:" + configurations.getByName(variant.configuration('scalaCompilerPlugin')).asPath,
                        "-P:genjavadoc:out=$generatedJavaDirectory".toString()
                ]
            }
            // Export generated Java code from the genjavadoc compiler plugin
            artifacts {
                add(variant.configuration('javadocSourceElements'), project.file(generatedJavaDirectory)) {
                    builtBy compileScala
                }
            }
            tasks.getByName(variant.taskName('javadoc')) {
                dependsOn compileScala
                source(generatedJavaDirectory)
                options.addStringOption('Xdoclint:none', '-quiet')
                failOnError = false
            }

        scaladoc {
            title = "${rootProject.description} ${version} API"
        }
    }
}

// Configure all Spark test tasks to use testJavaHome (JDK 17) with required module opens
tasks.withType(Test) { Test task ->
    task.executable = new File(project.ext.testJavaHome, 'bin/java').absolutePath
    task.jvmArgs "--add-opens=java.base/java.io=ALL-UNNAMED"
    task.jvmArgs "--add-opens=java.base/java.nio=ALL-UNNAMED"
    task.jvmArgs "--add-opens=java.base/java.lang=ALL-UNNAMED"
    task.jvmArgs "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
    task.jvmArgs "--add-opens=java.base/java.util=ALL-UNNAMED"
    task.jvmArgs "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
    task.jvmArgs "--add-opens=java.base/java.net=ALL-UNNAMED"
}

// Set minimum compatibility and java home for compiler task
tasks.withType(ScalaCompile) { ScalaCompile task ->
    // Spark 4.0 variants require JDK 17+ which does not have rt.jar
    if (task.name.contains("spark40") || task.name.contains("Spark40")) {
        return
    }
    File rtJar = new File(project.ext.runtimeJavaHome, 'jre/lib/rt.jar')
    if (rtJar.exists()) {
        task.scalaCompileOptions.additionalParameters = ["-javabootclasspath", rtJar.absolutePath]
        task.options.bootstrapClasspath = layout.files(rtJar)
    }
    task.sourceCompatibility = project.ext.minimumRuntimeVersion
    task.targetCompatibility = project.ext.minimumRuntimeVersion
    task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
}