Update gradle build script to support configuration cache (deepjavalibrary#3782)

* Update gradle build script to support configuration cache

* Fix fasttext continuous build

Author: xyang16

.github/workflows/continuous.yml

@@ -68,7 +68,9 @@ jobs:
               - 'extensions/fasttext/**'
       - name: Compile fastText JNI macOs
         if: ${{ steps.fasttext_changes.outputs.src == 'true' && runner.os == 'macOs' }}
-        run: ./gradlew :extensions:fasttext:compileJNI
+        run: |
+          export CMAKE_POLICY_VERSION_MINIMUM=3.5
+          ./gradlew :extensions:fasttext:compileJNI
       - name: Compile fastText JNI linux
         if: ${{ steps.fasttext_changes.outputs.src == 'true' && runner.os == 'Linux' }}
         # fasttext on linux requires gcc12, but ubuntu-latest uses at least gcc13

api/build.gradle.kts

@@ -22,6 +22,7 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
+        val version = project.version
         inputs.properties(mapOf("version" to version))
         filesMatching("**/api.properties") {
             expand(mapOf("version" to version))

buildSrc/src/main/kotlin/ai/djl/cppFormatter.gradle.kts

@@ -5,33 +5,28 @@ import os
 import text
 import url
 
-fun checkClang(clang: File) {
-    val url = "https://djl-ai.s3.amazonaws.com/build-tools/osx/clang-format".url
-    if (!clang.exists()) {
-        // create the folder and download the executable
-        clang.parentFile.mkdirs()
-        // TODO original method was appending
-        clang.writeBytes(url.openStream().use { it.readAllBytes() })
-        clang.setExecutable(true)
+tasks {
+    fun checkClang(clang: File) {
+        val url = "https://djl-ai.s3.amazonaws.com/build-tools/osx/clang-format".url
+        if (!clang.exists()) {
+            // create the folder and download the executable
+            clang.parentFile.mkdirs()
+            // TODO original method was appending
+            clang.writeBytes(url.openStream().use { it.readAllBytes() })
+            clang.setExecutable(true)
+        }
     }
-}
-
-fun formatCpp(f: File, clang: File): String = when {
-    !f.name.endsWith(".cc") && !f.name.endsWith(".cpp") && !f.name.endsWith(".h") -> ""
-    else -> ProcessBuilder(
-        clang.absolutePath,
-        "-style={BasedOnStyle: Google, IndentWidth: 2, ColumnLimit: 120, AlignAfterOpenBracket: DontAlign, SpaceAfterCStyleCast: true}",
-        f.absolutePath
-    )
-        .start().inputStream.use { it.readAllBytes().decodeToString() }
-}
 
-interface FormatterConfig {
-    val exclusions: ListProperty<String>
-}
-project.extensions.create<FormatterConfig>("formatCpp")
+    fun formatCpp(f: File, clang: File): String = when {
+        !f.name.endsWith(".cc") && !f.name.endsWith(".cpp") && !f.name.endsWith(".h") -> ""
+        else -> ProcessBuilder(
+            clang.absolutePath,
+            "-style={BasedOnStyle: Google, IndentWidth: 2, ColumnLimit: 120, AlignAfterOpenBracket: DontAlign, SpaceAfterCStyleCast: true}",
+            f.absolutePath
+        )
+            .start().inputStream.use { it.readAllBytes().decodeToString() }
+    }
 
-project.tasks {
     register("formatCpp") {
         val rootProject = project.rootProject
         val clang = rootProject.projectDir / ".clang/clang-format"
@@ -78,3 +73,9 @@ project.tasks {
         }
     }
 }
+
+interface FormatterConfig {
+    val exclusions: ListProperty<String>
+}
+project.extensions.create<FormatterConfig>("formatCpp")
+

buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts

@@ -7,42 +7,42 @@ import java.io.PrintWriter
 
 tasks {
     register("formatJava") {
-        val sourceSets = project.sourceSets
+        val resultFilePath = "build/formatJava-result.txt"
+        var files = project.sourceSets.flatMap { it.allSource }
+        files = files.filter { it.name.endsWith(".java") && "generated-src" !in it.absolutePath }
+        inputs.files(files)
+        outputs.file(project.file(resultFilePath))
         doLast {
             val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`)
-            for (item in sourceSets)
-                for (file in item.allSource) {
-                    if (!file.name.endsWith(".java") || "generated-src" in file.absolutePath)
-                        continue
-                    if (formatter.format("-a", "-i", file.absolutePath) != 0)
-                        throw GradleException("Format java failed: " + file.absolutePath)
+            for (f in files) {
+                if (formatter.format("-a", "-i", f.absolutePath) != 0) {
+                    throw GradleException("Format java failed: " + f.absolutePath)
                 }
+            }
+            File(resultFilePath).writeText("Success")
         }
     }
 
     val verifyJava by registering {
         val resultFilePath = "build/verifyJava-result.txt"
-        inputs.files(project.sourceSets.flatMap { it.allSource })
-        inputs.files(project.fileTree("generated-src"))
+        var files = project.sourceSets.flatMap { it.allSource }
+        files = files.filter { it.name.endsWith(".java") && "generated-src" !in it.absolutePath }
+        inputs.files(files)
         outputs.file(project.file(resultFilePath))
-
-        val proj = project
         doLast {
             val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`)
-            for (item in proj.sourceSets)
-                for (file in item.allSource) {
-                    if (!file.name.endsWith(".java") || "generated-src" in file.absolutePath)
-                        continue
-                    if (formatter.format("-a", "-n", "--set-exit-if-changed", file.absolutePath) != 0)
-                        throw GradleException(
-                            "File not formatted: " + file.absolutePath
-                                    + System.lineSeparator()
-                                    + "In order to reformat your code, run './gradlew formatJava' (or './gradlew fJ' for short)"
-                                    + System.lineSeparator()
-                                    + "See https://github.com/deepjavalibrary/djl/blob/master/docs/development/development_guideline.md#coding-conventions for more details"
-                        )
+            for (f in files) {
+                if (formatter.format("-a", "-n", "--set-exit-if-changed", f.absolutePath) != 0) {
+                    throw GradleException(
+                        "File not formatted: " + f.absolutePath
+                                + System.lineSeparator()
+                                + "In order to reformat your code, run './gradlew formatJava' (or './gradlew fJ' for short)"
+                                + System.lineSeparator()
+                                + "See https://github.com/deepjavalibrary/djl/blob/master/docs/development/development_guideline.md#coding-conventions for more details"
+                    )
                 }
-            proj.file(resultFilePath).writeText("Success")
+            }
+            File(resultFilePath).writeText("Success")
         }
     }
 

buildSrc/src/main/kotlin/ai/djl/stats.gradle.kts

@@ -1,6 +1,5 @@
 package ai.djl
 
-import org.gradle.api.plugins.ExtraPropertiesExtension
 import org.gradle.kotlin.dsl.support.serviceOf
 import org.gradle.tooling.events.FinishEvent
 import org.gradle.tooling.events.OperationCompletionListener
@@ -46,21 +45,20 @@ abstract class StatisticsService : BuildService<StatisticsService.Parameters>,
 }
 
 tasks.test {
+    @OptIn(ExperimentalTime::class)
+    val timeSource = timeSource
+    val projectName = project.name
+    val demoListener = demoListener
+    val ext = project.ext
     doFirst {
         @OptIn(ExperimentalTime::class)
-        startTime = timeSource.markNow()
+        ext.set("startTime", timeSource.markNow())
     }
     doLast {
         @OptIn(ExperimentalTime::class)
-        if (state.didWork)
-            demoListener.get().parameters.testsResults[timeSource.markNow() - startTime] = project.name
+        if (state.didWork) {
+            val t = ext.get("startTime") as TimeSource.Monotonic.ValueTimeMark
+            demoListener.get().parameters.testsResults[timeSource.markNow() - t] = projectName
+        }
     }
 }
-
-@ExperimentalTime
-var Task.startTime: TimeSource.Monotonic.ValueTimeMark
-    get() = ext.get("startTime") as TimeSource.Monotonic.ValueTimeMark
-    set(value) = ext.set("startTime", value)
-
-val Task.ext: ExtraPropertiesExtension
-    get() = (this as ExtensionAware).extensions.getByName<ExtraPropertiesExtension>("ext")

engines/ml/xgboost/build.gradle.kts

@@ -57,13 +57,18 @@ tasks {
     }
 
     publishing {
+        val rapisVersion = libs.versions.rapis.get()
+        val projectName = project.name
+        val xgbFlavor = xgbFlavor
+        val isGpu = isGpu
+
         publications {
             named<MavenPublication>("maven") {
-                artifactId = "${project.name}$xgbFlavor"
+                artifactId = "${projectName}$xgbFlavor"
                 pom {
                     name = "DJL Engine Adapter for XGBoost"
                     description = "Deep Java Library (DJL) Engine Adapter for XGBoost"
-                    url = "https://djl.ai/engines/ml/${project.name}"
+                    url = "https://djl.ai/engines/ml/${projectName}"
 
                     withXml {
                         val pomNode = asElement()
@@ -85,7 +90,7 @@ tasks {
                                     }
                                     "groupId" addWith "ai.rapids"
                                     "artifactId" addWith "cudf"
-                                    "version" addWith libs.versions.rapis.get()
+                                    "version" addWith rapisVersion
                                     "classifier" addWith "cuda11"
                                     "scope" addWith "compile"
                                 }

engines/mxnet/mxnet-engine/build.gradle.kts

@@ -25,9 +25,11 @@ sourceSets.main {
 
 tasks {
     processResources {
-        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get()))
+        val djlVersion = libs.versions.djl.get()
+        val mxnetVersion = libs.versions.mxnet.get()
+        inputs.properties(mapOf("djlVersion" to djlVersion, "mxnetVersion" to mxnetVersion))
         filesMatching("**/mxnet-engine.properties") {
-            expand(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get()))
+            expand(mapOf("djlVersion" to djlVersion, "mxnetVersion" to mxnetVersion))
         }
     }
 
@@ -42,11 +44,13 @@ tasks {
             .withPropertyName("jna/mapping.properties file")
         outputs.dir(buildDirectory / "generated-src")
         outputs.cacheIf { true }
+        val jnaGenerator = jnaratorJar.get().outputs.files.singleFile
+        val buildDir = buildDirectory
+        val dir = projectDir
+        val prov = providers
 
-        val dir = project.projectDir
         doLast {
-            val jnaGenerator = jnaratorJar.get().outputs.files.singleFile
-            providers.javaexec {
+            prov.javaexec {
                 workingDir = dir
                 mainClass = "-jar"
                 args(
@@ -56,7 +60,7 @@ tasks {
                     "-p",
                     "ai.djl.mxnet.jna",
                     "-o",
-                    "$buildDirectory/generated-src",
+                    "$buildDir/generated-src",
                     "-m",
                     "${dir}/src/main/jna/mapping.properties",
                     "-f",

engines/mxnet/native/build.gradle.kts

@@ -24,9 +24,10 @@ tasks {
         outputs.dir("build/libs")
 
         var versionName = project.version.toString()
+        if (!isRelease)
+            versionName += "-$nowFormatted"
+
         doFirst {
-            if (!isRelease)
-                versionName += "-$nowFormatted"
             val dir = placeholder / "native/lib"
             dir.mkdirs()
             val propFile = placeholder / "native/lib/mxnet.properties"
@@ -153,7 +154,7 @@ tasks {
     withType<PublishToMavenRepository> {
         for (flavor in flavorNames) {
             if (requireSigning) {
-                dependsOn("sign${flavor.substring(0, 1).uppercase() + flavor.substring(1)}Publication")
+                dependsOn("sign${flavor.take(1).uppercase() + flavor.substring(1)}Publication")
             }
 
             val platformNames = (binaryRoot / flavor).list() ?: emptyArray()
@@ -245,7 +246,7 @@ if (project.hasProperty("staging")) {
         doLast {
             val conn = URL(url).openConnection() as HttpURLConnection
             conn.requestMethod = "POST"
-            conn.setRequestProperty("Authorization", "Bearer ${token}")
+            conn.setRequestProperty("Authorization", "Bearer $token")
             val status = conn.responseCode
             if (status != HttpURLConnection.HTTP_OK) {
                 project.logger.error("Failed to POST '${url}'. Received status code ${status}: ${conn.responseMessage}")

engines/pytorch/pytorch-engine/build.gradle.kts

@@ -19,9 +19,11 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get()))
+        val djlVersion = libs.versions.djl.get()
+        val pytorchVersion = libs.versions.pytorch.get()
+        inputs.properties(mapOf("djlVersion" to djlVersion, "pytorchVersion" to pytorchVersion))
         filesMatching("**/pytorch-engine.properties") {
-            expand(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get()))
+            expand(mapOf("djlVersion" to djlVersion, "pytorchVersion" to pytorchVersion))
         }
     }