build.gradle

import org.opensearch.gradle.ConcatFilesTask
import org.opensearch.gradle.DependenciesInfoTask
import org.opensearch.gradle.precommit.DependencyLicensesTask
import org.opensearch.hadoop.gradle.BuildPlugin

apply plugin: 'opensearch.hadoop.build'

description = "OpenSearch for Apache Hadoop"

base {
  archivesName = 'opensearch-hadoop'
}

def sparkVariantIncluded = 'spark35scala212'

configurations {
    embedded {
        canBeResolved = true
        canBeConsumed = false
        transitive = false
    }
    javadocDependencies {
        canBeResolved = true
        canBeConsumed = false
    }
    compileClasspath {
        extendsFrom javadocDependencies
    }
    dist {
        canBeResolved = true
        canBeConsumed = false
        attributes {
            attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, 'packaging'))
        }
    }
    sparkExtraJars {
        canBeResolved = true
        canBeConsumed = false
        transitive = false
    }
    thirdPartyShaded {
        canBeResolved = true
        canBeConsumed = false
        transitive = false
    }
    licenseChecks {
        extendsFrom thirdPartyShaded
        extendsFrom runtimeClasspath
    }
}

BuildPlugin.disableTransitiveDependencies(project, project.configurations.thirdPartyShaded)

def distProjects = [":opensearch-hadoop-mr", ":opensearch-hadoop-hive",
                    ":opensearch-spark-35"]

distProjects.each { distProject ->
    def configureDistDependency = { Dependency dependency ->
        if (distProject == ":opensearch-spark-35") {
            dependency.capabilities {
                requireCapability("org.opensearch.spark.sql.variant:$sparkVariantIncluded:$project.version")
            }
        }
    }
    dependencies {
        // This is only going to pull in each project's regular jar to create the project-wide uberjar.
        add('embedded', project(distProject), configureDistDependency)
        // To squash Javadoc warnings.
        add('javadocDependencies', project(distProject), configureDistDependency)
        // This will pull all java sources (including generated) for the project-wide javadoc.
        add('javadocSources', project(distProject), configureDistDependency)
        // This will pull all non-generated sources for the project-wide source jar.
        add('additionalSources', project(distProject), configureDistDependency)
        // This will pull in the regular jar, javadoc jar, and source jar to be packaged in the distribution.
        add('dist', project(distProject), configureDistDependency)
    }
}

// Additional Spark variant jars to include in the distribution zip (not in the uber jar)
['spark30scala212', 'spark30scala213'].each { variant ->
    dependencies {
        add('sparkExtraJars', project(":opensearch-spark-30")) {
            capabilities {
                requireCapability("org.opensearch.spark.sql.variant:$variant:$project.version")
            }
        }
    }
}
['spark35scala213'].each { variant ->
    dependencies {
        add('sparkExtraJars', project(":opensearch-spark-35")) {
            capabilities {
                requireCapability("org.opensearch.spark.sql.variant:$variant:$project.version")
            }
        }
    }
}
dependencies {
    sparkExtraJars(project(":opensearch-spark-40")) {
        capabilities {
            requireCapability("org.opensearch.spark.sql.variant:spark40scala213:$project.version")
        }
    }
}

dependencies {
    // For Uber pom (and Javadoc to a lesser extent)
    thirdPartyShaded("commons-httpclient:commons-httpclient:3.1")
    thirdPartyShaded("org.codehaus.jackson:jackson-mapper-asl:${project.ext.jacksonVersion}")
    thirdPartyShaded("org.codehaus.jackson:jackson-core-asl:${project.ext.jacksonVersion}")

    implementation("commons-logging:commons-logging:1.4.0")
    implementation("commons-codec:commons-codec:1.22.0")
    implementation("javax.xml.bind:jaxb-api:2.3.1")
    implementation("org.apache.hive:hive-service:$hiveVersion") {
        exclude module: "log4j-slf4j-impl"
    }
    implementation("org.apache.hive:hive-exec:$hiveVersion")
    implementation("org.apache.hive:hive-metastore:$hiveVersion")
    implementation("org.apache.spark:spark-core_${project.ext.scala212MajorVersion}:$spark35Version") {
        exclude group: 'javax.servlet'
        exclude group: 'org.apache.hadoop'
    }
    implementation("org.apache.spark:spark-yarn_${project.ext.scala212MajorVersion}:$spark35Version") {
        exclude group: 'org.apache.hadoop'
    }
    implementation("org.apache.spark:spark-sql_${project.ext.scala212MajorVersion}:$spark35Version") {
        exclude group: 'org.apache.hadoop'
    }
    implementation("org.apache.spark:spark-streaming_${project.ext.scala212MajorVersion}:$spark35Version") {
        exclude group: 'org.apache.hadoop'
    }
    implementation("org.scala-lang:scala-library:$scala212Version")
    implementation("org.scala-lang:scala-reflect:$scala212Version")
    implementation(project.ext.hadoopClient)
    implementation("org.apache.hadoop:hadoop-common:${project.ext.hadoopVersion}")
    implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${project.ext.hadoopVersion}")
    implementation("joda-time:joda-time:$jodaVersion")
    compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala212MajorVersion}:$spark35Version")
}

// Configure uber jar
jar {
    dependsOn(project.configurations.embedded)

    manifest {
        attributes['Implementation-Title'] = 'opensearch-hadoop'
    }

    from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
        include "org/opensearch/**"
        include "opensearch-hadoop-build.properties"
        include "META-INF/services/*"
    }

    // Each integration will be copying it's entire jar contents into this master jar.
    // There will be lots of duplicates since they all package up the core code inside of them.
    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}

javadoc {
    options {
        header = "opensearch-hadoop"
    }
}

publishing {
    publications {
        main {
            getPom().withXml { XmlProvider xml ->
                Node root = xml.asNode()

                // add clojars repo to pom
                Node repositories = root.appendNode('repositories')
                Node repository = repositories.appendNode('repository')
                repository.appendNode('id', 'clojars.org')
                repository.appendNode('url', 'https://clojars.org/repo')

                // Correct the artifact Id, otherwise it is listed as 'dist'
                root.get('artifactId').get(0).setValue(project.base.archivesName.get())
            }
        }
    }
}

// Name of the directory under the root of the zip file that will contain the zip contents
String zipContentDir = "opensearch-hadoop-${project.version}"

// Create a zip task for creating the distribution
task('distZip', type: Zip) {
    group = 'Distribution'
    description = "Builds zip archive, containing all jars and docs, suitable for download page."

    dependsOn(tasks.pack)

    from(project.rootDir) {
        include('README.md')
        include('LICENSE.txt')
        include('NOTICE.txt')
        into(zipContentDir)
    }

    into("$zipContentDir/dist") {
        from(project.configurations.dist)
        from(project.configurations.sparkExtraJars)
        from(tasks.jar)
        from(tasks.javadocJar)
        from(tasks.sourcesJar)
    }
}

distribution {
    dependsOn(distZip)
}

// Add a task in the root project that collects all the dependencyReport data for each project
// Concatenates the dependencies CSV files into a single file
task generateDependenciesReport(type: ConcatFilesTask) { concatDepsTask ->
    dependsOn rootProject.allprojects.collect { it.tasks.withType(DependenciesInfoTask) }
    rootProject.allprojects.collect { 
    files = fileTree(dir: project.rootDir,  include: '**/dependencies.csv' )
        it.tasks.withType(DependenciesInfoTask) { depTask ->
            concatDepsTask.dependsOn depTask
            concatDepsTask.getFiles().from(depTask.outputFile)
        }
    }
    headerLine = "name,version,url,license"
    target = new File(System.getProperty('csv')?: "${project.buildDir}/reports/dependencies/opensearch-hadoop-dependencies.csv")
}

project.tasks.named('dependencyLicenses', DependencyLicensesTask) {
    it.dependencies = project.configurations.licenseChecks.incoming.artifactView {
        componentFilter {
            !(it instanceof ProjectComponentIdentifier)
        }
    }.files
}