Skip to content

Replace AutoTuner Maven lookup with offline plugin staleness detection #2127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
parthosa merged 4 commits into from
Aug 26, 2026
Merged

Replace AutoTuner Maven lookup with offline plugin staleness detection #2127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9e760e3
Remove AutoTuner Maven release lookup
parthosa Aug 24, 2026
4378e07
Recommend checking for outdated cuDF Spark plugin
parthosa Aug 24, 2026
fa4dbcd
Merge latest dev into offline plugin staleness branch
parthosa Aug 25, 2026
2f3fc13
Fix plugin staleness release boundary
parthosa Aug 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 36 additions & 31 deletions core/src/main/scala/com/nvidia/spark/rapids/tool/tuning/AutoTuner.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,11 @@

package com.nvidia.spark.rapids.tool.tuning

import java.time.YearMonth

import scala.beans.BeanProperty
import scala.collection.mutable
import scala.util.Try
import scala.util.control.NonFatal
import scala.util.matching.Regex

Expand All @@ -32,7 +35,7 @@ import org.yaml.snakeyaml.constructor.ConstructorException
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.ByteUnit
import org.apache.spark.sql.rapids.tool.ToolUtils
import org.apache.spark.sql.rapids.tool.util.{StringUtils, ValidatableProperties, WebCrawlerUtil}
import org.apache.spark.sql.rapids.tool.util.{StringUtils, ValidatableProperties}

/**
* A wrapper class that stores all the GPU properties.
Expand Down Expand Up @@ -1870,11 +1873,13 @@ abstract class AutoTuner(
* 2- If there are more than 1 entry for ".*rapids-4-spark.*jar", then add a comment that
* there should be only 1 jar in the class path.
* 3- If there are cudf jars, ignore that for now.
* 4- If there is a new release recommend that to the user
* 4- If the plugin jar's release month is at least two months old, recommend checking the
* latest release.
*/
private def recommendClassPathEntries(): Unit = {
val missingRapidsJarsEntry = classPathComments("rapids.jars.missing")
val multipleRapidsJarsEntry = classPathComments("rapids.jars.multiple")
val outdatedRapidsJarsEntry = classPathComments("rapids.jars.outdated")

appInfoProvider.getRapidsJars match {
case Seq() =>
Expand All @@ -1887,27 +1892,9 @@ abstract class AutoTuner(
case v: Seq[String] if v.length > 1 =>
val comment = s"$multipleRapidsJarsEntry [${v.mkString(", ")}]"
appendComment(comment)
case Seq(jarVer) =>
// compare jarVersion to the latest release
val latestPluginVersion = WebCrawlerUtil.getLatestPluginRelease
latestPluginVersion match {
case Some(ver) =>
if (ToolUtils.compareVersions(jarVer, ver).exists(_ < 0)) {
val jarURL = WebCrawlerUtil.getPluginMvnDownloadLink(ver)
appendComment(
"A newer NVIDIA cuDF plugin for Apache Spark release is available:\n" +
s" $jarURL\n" +
s" Version used in application is $jarVer.")
}
case None =>
logError("Could not pull the latest cuDF plugin jar release.")
val pluginRepoUrl = WebCrawlerUtil.getMVNArtifactURL("rapids.plugin")
appendComment(
"Failed to validate the latest cuDF plugin release.\n" +
s" Verify that the version used in application ($jarVer) is the latest on:\n" +
s" $pluginRepoUrl")

}
case Seq(jarVer) if autoTunerHelper.isPluginJarProbablyOutdated(jarVer) =>
appendComment(outdatedRapidsJarsEntry)
case Seq(_) => () // One recent plugin JAR needs no classpath recommendation.
}
}
}
Expand Down Expand Up @@ -2612,13 +2599,35 @@ class ProfilingAutoTuner(
* Helper trait for the AutoTuner
*/
trait AutoTunerHelper extends Logging {
private val pluginReleaseIntervalMonths = 2L

/**
* Strategy for cluster shape recommendation.
* See [[com.nvidia.spark.rapids.tool.ClusterSizingStrategy]] for different strategies.
*/
def recommendedClusterSizingStrategy(platform: Platform): ClusterSizingStrategy
// the plugin jar is in the form of rapids-4-spark_scala_binary-(version)-*.jar
lazy val pluginJarRegEx: Regex = "rapids-4-spark_\\d\\.\\d+-(\\d{2}\\.\\d{2}\\.\\d+).*\\.jar".r

/**
* Returns whether a plugin version's release month is at least two months before the current
* month, based on the expected release cadence.
* Patch releases within the same release month are intentionally ignored.
*/
def isPluginJarProbablyOutdated(
pluginVersion: String,
currentYearMonth: YearMonth = YearMonth.now()): Boolean = {
val versionParts = pluginVersion.split("\\.")
if (versionParts.length < 2) {
false
} else {
Try(YearMonth.of(2000 + versionParts(0).toInt, versionParts(1).toInt)).toOption
.exists { releaseYearMonth =>
!releaseYearMonth.isAfter(currentYearMonth.minusMonths(pluginReleaseIntervalMonths))
}
}
}

// Starting with this plugin version, the cuDF plugin auto-tunes the number of
// concurrent GPU tasks based on memory usage (see spark-rapids#12374), so the
// AutoTuner should no longer recommend `spark.rapids.sql.concurrentGpuTasks`.
Expand Down Expand Up @@ -2756,6 +2765,7 @@ object ProfilingAutoTunerHelper extends AutoTunerHelper {
trait AutoTunerStaticComments {
// scalastyle:off line.size.limit
private lazy val advancedConfigDocUrl = "https://nvidia.github.io/spark-rapids/docs/additional-functionality/advanced_configs.html#advanced-configuration"
private lazy val cudfSparkDownloadUrl = "https://nvidia.github.io/cudf-spark/docs/download.html"
private lazy val shuffleManagerDocUrl = "https://docs.nvidia.com/spark-rapids/user-guide/latest/additional-functionality/rapids-shuffle.html#rapids-shuffle-manager"

val classPathComments: Map[String, String] = Map(
Expand All @@ -2768,6 +2778,9 @@ trait AutoTunerStaticComments {
("Multiple cuDF plugin jar\n" +
" exist on the classpath.\n" +
" Make sure to keep only a single jar."),
"rapids.jars.outdated" ->
("The NVIDIA cuDF plugin for Apache Spark used by this application may be outdated.\n" +
s" Check the latest release: $cudfSparkDownloadUrl"),
"rapids.shuffle.jars" ->
("The RAPIDS Shuffle Manager requires spark.driver.extraClassPath\n" +
" and spark.executor.extraClassPath settings to include the\n" +
Expand Down Expand Up @@ -2825,14 +2838,6 @@ trait AutoTunerStaticComments {
|""".stripMargin.trim.replaceAll("\n", "\n ")
}

def latestPluginJarComment(latestJarMvnUrl: String, currentJarVer: String): String = {
s"""
|A newer NVIDIA cuDF plugin for Apache Spark release is available:
|$latestJarMvnUrl
|Version used in application is $currentJarVer.
|""".stripMargin.trim.replaceAll("\n", "\n ")
}

def notEnoughMemComment(minSizeInMB: Long): String = {
s"""
|This node/worker configuration is not ideal for using the cuDF plugin
Expand Down
6 changes: 2 additions & 4 deletions core/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ClusterRecommendationSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,6 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase
val profileLogContent = FSUtils.readFileContentAsUTF8(logFile)
val actualResults = extractAutoTunerResults(profileLogContent)

val testAppJarVer = "25.02.0"
// scalastyle:off line.size.limit
val expectedResults =
s"""|
Expand Down Expand Up @@ -370,8 +369,8 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase
|- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set.
|- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set.
|- ${getEnforcedPropertyComment("spark.sql.shuffle.partitions")}
|- ${latestPluginJarComment(latestPluginJarUrl, testAppJarVer)}
|- $shufflePartitionsCommentForSpilling
|- ${classPathComments("rapids.jars.outdated")}
|- ${classPathComments("rapids.shuffle.jars")}
|""".stripMargin.trim
// scalastyle:on line.size.limit
Expand Down Expand Up @@ -450,7 +449,6 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase
val profileLogContent = FSUtils.readFileContentAsUTF8(logFile)
val actualResults = extractAutoTunerResults(profileLogContent)

val testAppJarVer = "25.02.0"
// scalastyle:off line.size.limit
val expectedResults =
s"""|
Expand Down Expand Up @@ -490,8 +488,8 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase
|- 'spark.rapids.sql.multiThreadedRead.numThreads' was not set.
|- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set.
|- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set.
|- ${latestPluginJarComment(latestPluginJarUrl, testAppJarVer)}
|- $shufflePartitionsCommentForSpilling
|- ${classPathComments("rapids.jars.outdated")}
|- ${classPathComments("rapids.shuffle.jars")}
|""".stripMargin.trim
// scalastyle:on line.size.limit
Expand Down
145 changes: 24 additions & 121 deletions core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.nvidia.spark.rapids.tool.tuning

import java.io.{File, FileNotFoundException}
import java.time.YearMonth

import scala.collection.mutable

Expand All @@ -29,7 +30,6 @@ import org.scalatest.prop.TableFor4

import org.apache.spark.sql.TrampolineUtil
import org.apache.spark.sql.rapids.tool.util.FSUtils
import org.apache.spark.sql.rapids.tool.util.WebCrawlerUtil

/**
* Base class for Profiling AutoTuner test suites.
Expand Down Expand Up @@ -113,16 +113,6 @@ abstract class ProfilingAutoTunerSuiteBase extends BaseAutoTunerSuite {
buildAutoTunerForTests(infoProvider, platform)
}

/**
* Helper method to return the latest cuDF plugin jar URL.
*/
protected lazy val latestPluginJarUrl: String = {
val latestRelease = WebCrawlerUtil.getLatestPluginRelease match {
case Some(v) => v
case None => fail("Could not find pull the latest release successfully")
}
ToolTestUtils.pluginMvnPrefix(latestRelease) + ".jar"
}
}

/**
Expand Down Expand Up @@ -1546,58 +1536,28 @@ class ProfilingAutoTunerSuite extends ProfilingAutoTunerSuiteBase {
compareOutput(expectedResults, autoTunerOutput)
}

test("Recommend upgrading to the latest plugin release") {
// 1. Pull the latest release from mvn.
// 2. The Autotuner should warn the users that they are using an older release
// 3. Compare the output
val testAppJarVer = "23.02.0"
// scalastyle:off line.size.limit
val expectedResults =
s"""|
|Spark Properties:
|--conf spark.dataproc.enhanced.execution.enabled=false
|--conf spark.dataproc.enhanced.optimizer.enabled=false
|--conf spark.executor.instances=8
|--conf spark.executor.memory=32g
|--conf spark.executor.memoryOverhead=19660m
|--conf spark.locality.wait=0
|--conf spark.plugins=com.nvidia.spark.SQLPlugin
|--conf spark.rapids.memory.pinnedPool.size=6g
|--conf spark.rapids.shuffle.multiThreaded.maxBytesInFlight=4g
|--conf spark.rapids.shuffle.multiThreaded.reader.threads=28
|--conf spark.rapids.shuffle.multiThreaded.writer.threads=28
|--conf spark.rapids.sql.batchSizeBytes=2147483647b
|--conf spark.rapids.sql.enabled=true
|--conf spark.rapids.sql.format.parquet.multithreaded.combine.waitTime=1000
|--conf spark.rapids.sql.multiThreadedRead.numThreads=80
|--conf spark.rapids.sql.reader.multithreaded.combine.sizeBytes=10m
|--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=128m
|--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE]
|--conf spark.sql.adaptive.coalescePartitions.initialPartitionNum=200
|--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m
|
|Comments:
|- 'spark.dataproc.enhanced.execution.enabled' should be disabled. WARN: Turning this property on might case the GPU accelerated Dataproc cluster to hang.
|- 'spark.dataproc.enhanced.execution.enabled' was not set.
|- 'spark.dataproc.enhanced.optimizer.enabled' should be disabled. WARN: Turning this property on might case the GPU accelerated Dataproc cluster to hang.
|- 'spark.dataproc.enhanced.optimizer.enabled' was not set.
|- 'spark.plugins' should be set to the class name required for the cuDF plugin.
| Refer to: https://docs.nvidia.com/spark-rapids/user-guide/latest/getting-started/overview.html
|- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set.
|- 'spark.rapids.sql.batchSizeBytes' was not set.
|- 'spark.rapids.sql.enabled' was not set.
|- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set.
|- 'spark.rapids.sql.reader.multithreaded.combine.sizeBytes' was not set.
|- 'spark.sql.adaptive.advisoryPartitionSizeInBytes' was not set.
|- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set.
|- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set.
|- ${latestPluginJarComment(latestPluginJarUrl, testAppJarVer)}
|- ${classPathComments("rapids.shuffle.jars")}
|""".stripMargin
// scalastyle:on line.size.limit
val rapidsJarsArr = Seq(s"rapids-4-spark_2.12-$testAppJarVer.jar")
val autoTunerOutput = generateRecommendationsForRapidsJars(rapidsJarsArr)
compareOutput(expectedResults, autoTunerOutput)
test("Outdated cuDF plugin jar triggers a release comment") {
val output = generateRecommendationsForRapidsJars(
Seq("rapids-4-spark_2.12-23.02.0.jar"))

assert(output.contains(
"The NVIDIA cuDF plugin for Apache Spark used by this application may be outdated."))
assert(output.contains(
"Check the latest release: https://nvidia.github.io/cudf-spark/docs/download.html"))
}

test("cuDF plugin jar staleness follows the two-month release cadence") {
val testCases = Seq(
("26.06.1", YearMonth.of(2026, 7), false),
("26.06.1", YearMonth.of(2026, 8), true),
("26.06.1", YearMonth.of(2026, 9), true),
("26.08.0", YearMonth.of(2026, 9), false),
("invalid", YearMonth.of(2026, 9), false))

testCases.foreach { case (pluginVersion, currentYearMonth, expected) =>
assert(autoTunerHelper.isPluginJarProbablyOutdated(pluginVersion, currentYearMonth) ===
expected)
}
}

// Helper that runs the AutoTuner without pre-setting `spark.rapids.sql.concurrentGpuTasks`
Expand Down Expand Up @@ -1677,62 +1637,6 @@ class ProfilingAutoTunerSuite extends ProfilingAutoTunerSuiteBase {
s"Expected preserved concurrentGpuTasks to be present, got:\n$output")
}

test("No recommendation when the jar pluginJar is up-to-date") {
// 1. Pull the latest release from mvn.
// 2. The Autotuner finds tha the jar version is latest. No comments should be added
// 3. Compare the output
val latestRelease = WebCrawlerUtil.getLatestPluginRelease match {
case Some(v) => v
case None => fail("Could not find pull the latest release successfully")
}
// scalastyle:off line.size.limit
val expectedResults =
s"""|
|Spark Properties:
|--conf spark.dataproc.enhanced.execution.enabled=false
|--conf spark.dataproc.enhanced.optimizer.enabled=false
|--conf spark.executor.instances=8
|--conf spark.executor.memory=32g
|--conf spark.executor.memoryOverhead=19660m
|--conf spark.locality.wait=0
|--conf spark.plugins=com.nvidia.spark.SQLPlugin
|--conf spark.rapids.memory.pinnedPool.size=6g
|--conf spark.rapids.shuffle.multiThreaded.maxBytesInFlight=4g
|--conf spark.rapids.shuffle.multiThreaded.reader.threads=28
|--conf spark.rapids.shuffle.multiThreaded.writer.threads=28
|--conf spark.rapids.sql.batchSizeBytes=2147483647b
|--conf spark.rapids.sql.enabled=true
|--conf spark.rapids.sql.format.parquet.multithreaded.combine.waitTime=1000
|--conf spark.rapids.sql.multiThreadedRead.numThreads=80
|--conf spark.rapids.sql.reader.multithreaded.combine.sizeBytes=10m
|--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=128m
|--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE]
|--conf spark.sql.adaptive.coalescePartitions.initialPartitionNum=200
|--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m
|
|Comments:
|- 'spark.dataproc.enhanced.execution.enabled' should be disabled. WARN: Turning this property on might case the GPU accelerated Dataproc cluster to hang.
|- 'spark.dataproc.enhanced.execution.enabled' was not set.
|- 'spark.dataproc.enhanced.optimizer.enabled' should be disabled. WARN: Turning this property on might case the GPU accelerated Dataproc cluster to hang.
|- 'spark.dataproc.enhanced.optimizer.enabled' was not set.
|- 'spark.plugins' should be set to the class name required for the cuDF plugin.
| Refer to: https://docs.nvidia.com/spark-rapids/user-guide/latest/getting-started/overview.html
|- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set.
|- 'spark.rapids.sql.batchSizeBytes' was not set.
|- 'spark.rapids.sql.enabled' was not set.
|- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set.
|- 'spark.rapids.sql.reader.multithreaded.combine.sizeBytes' was not set.
|- 'spark.sql.adaptive.advisoryPartitionSizeInBytes' was not set.
|- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set.
|- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set.
|- ${classPathComments("rapids.shuffle.jars")}
|""".stripMargin
// scalastyle:on line.size.limit
val rapidsJarsArr = Seq(s"rapids-4-spark_2.12-$latestRelease.jar")
val autoTunerOutput = generateRecommendationsForRapidsJars(rapidsJarsArr)
compareOutput(expectedResults, autoTunerOutput)
}

// Note: This test verifies that the AutoTuner comments about enabling the file cache
// but does not actually enable since this requires knowledge of the disk bandwidth
// and available disk space.
Expand Down Expand Up @@ -3307,7 +3211,6 @@ class ProfilingAutoTunerSuite extends ProfilingAutoTunerSuiteBase {
val profileLogContent = FSUtils.readFileContentAsUTF8(logFile)
val actualResults = extractAutoTunerResults(profileLogContent)

val testAppJarVer = "25.02.0"
// scalastyle:off line.size.limit
val expectedResults =
s"""|
Expand Down Expand Up @@ -3337,10 +3240,10 @@ class ProfilingAutoTunerSuite extends ProfilingAutoTunerSuiteBase {
|- 'spark.rapids.sql.multiThreadedRead.numThreads' was not set.
|- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set.
|- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set.
|- ${latestPluginJarComment(latestPluginJarUrl, testAppJarVer)}
|- ${notEnoughMemCommentForKey("spark.executor.memory")}
|- ${notEnoughMemCommentForKey("spark.rapids.memory.pinnedPool.size")}
|- $shufflePartitionsCommentForSpilling
|- ${classPathComments("rapids.jars.outdated")}
|- ${classPathComments("rapids.shuffle.jars")}
|- ${notEnoughMemComment(40140)}
|- $missingGpuDiscoveryScriptComment
Expand Down
Loading
Loading