feat(tests): Test against a matrix of Iceberg and Spark/Flink versions (#106)

This change introduces a "simple" way to play Spark and Flink tests against a matrix of Iceberg and Spark/Flink versions.

It is "simple" because it may not work if different engine versions become so incompatible, that the test classes themselves need to be changed. Once that happens (and it may soon, with Spark 4.0) then we'll need to look for a more complex setup.

But for now, this change at least allows testing against Iceberg 1.9.0 and 1.9.1, and against the two last versions of each engine.

The versions to use are controlled by the following properties:

* `authmgr.test.iceberg.versions`: 1.9.1, 1.9.0
* `authmgr.test.spark.versions`: 3.5.6, 3.5.5
* `authmgr.test.flink.versions`: 1.20.2, 1.20.1

Summary of tasks:

* The `intTest` task runs the tests with the latest versions.
* The `intTestMatrix` task runs the entire matrix.
* The `intTest_iceberg190_spark355` task runs the tests with Iceberg 1.9.0 and Spark 3.5.5. There are similar tasks for each combination of iceberg/engine versions available.
* The `printTestMatrix` task prints the different tasks available.

Author: adutra

.github/workflows/ci.yml

@@ -118,10 +118,10 @@ jobs:
           path: |
             **/build/test-results/**
 
-  runtime-tests:
+  spark-tests:
 
     runs-on: ubuntu-latest
-    name: Spark & Flink Tests
+    name: Spark Tests
     permissions:
       contents: read
 
@@ -143,16 +143,51 @@ jobs:
 
       - name: Gradle Run Spark Tests
         run: |
-          ./gradlew --continue --scan :authmgr-oauth2-runtime-spark-tests:intTest
+          ./gradlew --continue --scan \
+            :authmgr-oauth2-runtime-spark-tests:printTestMatrix \
+            :authmgr-oauth2-runtime-spark-tests:intTestMatrix
+
+      - name: Archive Test Results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: spark-tests-results
+          path: |
+            **/build/test-results/**
+
+  flink-tests:
+
+    runs-on: ubuntu-latest
+    name: Flink Tests
+    permissions:
+      contents: read
+
+    steps:
+
+      - name: Checkout Project
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          validate-wrappers: false
 
       - name: Gradle Run Flink Tests
         run: |
-          ./gradlew --continue --scan :authmgr-oauth2-runtime-flink-tests:intTest
+          ./gradlew --continue --scan \
+            :authmgr-oauth2-runtime-flink-tests:printTestMatrix \
+            :authmgr-oauth2-runtime-flink-tests:intTestMatrix
 
       - name: Archive Test Results
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: runtime-tests-results
+          name: flink-tests-results
           path: |
             **/build/test-results/**

gradle.properties

@@ -28,3 +28,10 @@ org.gradle.configuration-cache=false
 #org.gradle.configuration-cache-problems=warn
 # bump the Gradle daemon heap size (you can set bigger heap sizes as well)
 org.gradle.jvmargs=-Xms2g -Xmx4g -XX:MaxMetaspaceSize=768m
+
+# Matrix testing configuration for integration tests
+# Comma-separated list of versions to test against
+# The last version in the list is the default and will be used for the main intTest task
+authmgr.test.iceberg.versions=1.9.0,1.9.2
+authmgr.test.spark.versions=3.5.5,3.5.6
+authmgr.test.flink.versions=1.20.1,1.20.2

gradle/libs.versions.toml

@@ -19,13 +19,11 @@
 
 [versions]
 errorprone = "2.36.0"
-flink = "1.20.1"
 hadoop = "3.4.1"
-iceberg = "1.9.1"
-immutables = "2.10.1"
+iceberg = "1.9.1" # When updating iceberg version, also update the iceberg test versions in gradle.properties
+immutables = "2.11.1"
 mockserver = "5.15.0"
-slf4j = "2.0.16"
-spark = "3.5.5"
+slf4j = "2.0.17"
 
 [bundles]
 
@@ -41,12 +39,6 @@ bouncycastle-bcpkix = { module = "org.bouncycastle:bcpkix-jdk15on", version = "1
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.0" }
 errorprone = { module = "com.google.errorprone:error_prone_core", version.ref = "errorprone"  }
 errorprone-annotations = { module = "com.google.errorprone:error_prone_annotations", version.ref = "errorprone" }
-flink-table-api-java = { module = "org.apache.flink:flink-table-api-java", version.ref = "flink" }
-flink-table-runtime = { module = "org.apache.flink:flink-table-runtime", version.ref = "flink" }
-flink-table-planner-loader = { module = "org.apache.flink:flink-table-planner-loader", version.ref = "flink" }
-flink-clients = { module = "org.apache.flink:flink-clients", version.ref = "flink" }
-flink-connector-base = { module = "org.apache.flink:flink-connector-base", version.ref = "flink" }
-flink-connector-files = { module = "org.apache.flink:flink-connector-files", version.ref = "flink" }
 hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" }
 hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client", version.ref = "hadoop" }
 hadoop-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-client-core", version.ref = "hadoop" }
@@ -65,7 +57,6 @@ mockserver-client-java = { module = "org.mock-server:mockserver-client-java", ve
 mockserver-netty = { module = "org.mock-server:mockserver-netty", version.ref = "mockserver" }
 s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version = "4.3.0" }
 slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
-spark-sql = { module = "org.apache.spark:spark-sql_2.12", version.ref = "spark" }
 spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.9.0" }
 testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.20.4" }
 testcontainers-keycloak = { module = "com.github.dasniko:testcontainers-keycloak", version = "3.6.0" }

oauth2/runtime-flink-tests/build.gradle.kts

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+import org.gradle.api.plugins.jvm.JvmTestSuite
+import org.gradle.kotlin.dsl.register
+
 plugins {
   id("authmgr-java")
   id("authmgr-java-testing")
@@ -23,72 +26,179 @@ description = "Flink tests for Dremio AuthManager for Apache Iceberg"
 
 ext { set("mavenName", "Auth Manager for Apache Iceberg - OAuth2 - Flink Tests") }
 
+// Matrix testing configuration
+val icebergVersions = project.findProperty("authmgr.test.iceberg.versions").toString().split(",")
+val flinkVersions = project.findProperty("authmgr.test.flink.versions").toString().split(",")
+
+// Use the last combination as default for regular intTest
+val defaultIcebergVersion = icebergVersions.last()
+val defaultFlinkVersion = flinkVersions.last()
+
+val intTestBase =
+  configurations.create("intTestBase") {
+    description = "Base configuration holding common dependencies for Flink integration tests"
+    isCanBeResolved = false
+    isCanBeConsumed = false
+  }
+
+// Make intTestImplementation extend from intTestBase
+configurations.intTestImplementation.get().extendsFrom(intTestBase)
+
 dependencies {
 
   // Note: iceberg-core will be provided by the iceberg-flink-runtime jar,
   // with shaded dependencies; it should not leak into this project unshaded.
 
-  intTestImplementation(project(":authmgr-oauth2-runtime"))
+  intTestBase(project(":authmgr-oauth2-runtime"))
 
-  intTestImplementation(testFixtures(project(":authmgr-oauth2-core")) as ModuleDependency) {
+  intTestBase(testFixtures(project(":authmgr-oauth2-core")) as ModuleDependency) {
     exclude(group = "org.apache.iceberg")
   }
 
+  intTestBase(libs.hadoop.common)
+  intTestBase(libs.hadoop.hdfs.client)
+  intTestBase(libs.hadoop.mapreduce.client.core)
+
+  intTestBase(platform(libs.testcontainers.bom))
+  intTestBase("org.testcontainers:testcontainers")
+  intTestBase(libs.s3mock.testcontainers)
+
+  intTestBase(platform(libs.junit.bom))
+  intTestBase("org.junit.jupiter:junit-jupiter")
+  intTestBase("org.junit.jupiter:junit-jupiter-api")
+  intTestBase("org.junit.platform:junit-platform-launcher")
+
+  intTestBase(libs.assertj.core)
+  intTestBase(libs.mockito.core)
+  intTestBase(libs.logback.classic)
+
+  // Add to intTestImplementation all Iceberg/Flink dependencies (with default versions)
+  // that are required for compilation of test classes
   intTestImplementation(platform(libs.iceberg.bom))
   intTestImplementation("org.apache.iceberg:iceberg-flink-runtime-1.20")
+  intTestImplementation("org.apache.flink:flink-table-api-java:$defaultFlinkVersion")
+}
+
+// Create matrix test tasks for each version combination
+val matrixTestTasks = mutableListOf<TaskProvider<Test>>()
+
+icebergVersions.forEach { icebergVersion ->
+  flinkVersions.forEach { flinkVersion ->
+    val suiteName =
+      "intTest_iceberg${icebergVersion.replace(".", "_")}_flink${flinkVersion.replace(".", "_")}"
 
-  intTestImplementation(libs.flink.table.api.java)
-  intTestImplementation(libs.flink.table.runtime)
-  intTestImplementation(libs.flink.table.planner.loader)
-  intTestImplementation(libs.flink.clients)
-  intTestImplementation(libs.flink.connector.base)
-  intTestImplementation(libs.flink.connector.files)
+    val runtimeConfig =
+      configurations.create(suiteName) {
+        extendsFrom(intTestBase)
+        isCanBeResolved = true
+        isCanBeConsumed = false
+      }
 
-  intTestImplementation(libs.hadoop.common)
-  intTestImplementation(libs.hadoop.hdfs.client)
-  intTestImplementation(libs.hadoop.mapreduce.client.core)
+    // Add version-specific dependencies
+    dependencies {
+      runtimeConfig(platform("org.apache.iceberg:iceberg-bom:$icebergVersion"))
+      runtimeConfig("org.apache.iceberg:iceberg-flink-runtime-1.20:$icebergVersion")
+      runtimeConfig("org.apache.flink:flink-table-api-java:$flinkVersion")
+      runtimeConfig("org.apache.flink:flink-table-runtime:$flinkVersion")
+      runtimeConfig("org.apache.flink:flink-table-planner-loader:$flinkVersion")
+      runtimeConfig("org.apache.flink:flink-clients:$flinkVersion")
+      runtimeConfig("org.apache.flink:flink-connector-base:$flinkVersion")
+      runtimeConfig("org.apache.flink:flink-connector-files:$flinkVersion")
+      runtimeConfig("org.apache.iceberg:iceberg-aws-bundle:$icebergVersion")
+    }
 
-  intTestRuntimeOnly("org.apache.iceberg:iceberg-aws")
-  intTestRuntimeOnly("org.apache.iceberg:iceberg-aws-bundle")
+    testing {
+      suites {
+        register<JvmTestSuite>(suiteName) {
+          targets.all {
+            testTask.configure {
+              shouldRunAfter("test")
 
-  intTestImplementation(platform(libs.testcontainers.bom))
-  intTestImplementation("org.testcontainers:testcontainers")
-  intTestImplementation(libs.s3mock.testcontainers)
+              if (System.getenv("CI") == null) {
+                maxParallelForks = 2
+              }
 
-  intTestImplementation(platform(libs.junit.bom))
-  intTestImplementation("org.junit.jupiter:junit-jupiter")
-  intTestImplementation("org.junit.jupiter:junit-jupiter-api")
+              description =
+                "Runs Flink integration tests with Iceberg $icebergVersion and Flink $flinkVersion"
 
-  intTestImplementation(libs.assertj.core)
-  intTestImplementation(libs.mockito.core)
+              // Use shared test classes from src/intTest
+              testClassesDirs = sourceSets.intTest.get().output.classesDirs
+              classpath = runtimeConfig + sourceSets.intTest.get().output
+
+              dependsOn(":authmgr-oauth2-runtime:shadowJar")
+
+              environment("AWS_REGION", "us-west-2")
+              environment("AWS_ACCESS_KEY_ID", "fake")
+              environment("AWS_SECRET_ACCESS_KEY", "fake")
+
+              jvmArgs(
+                "--add-exports",
+                "java.base/sun.nio.ch=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.util=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.lang=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.lang.reflect=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.io=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.net=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.nio=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.util.concurrent=ALL-UNNAMED",
+                "--add-opens",
+                "java.base/java.security=ALL-UNNAMED",
+              )
+
+              // Set system properties to identify the versions being tested
+              systemProperty("authmgr.test.iceberg.version", icebergVersion)
+              systemProperty("authmgr.test.flink.version", flinkVersion)
+
+              inputs.property("icebergVersion", icebergVersion)
+              inputs.property("flinkVersion", flinkVersion)
+            }
+            matrixTestTasks.add(testTask)
+          }
+        }
+      }
+    }
+  }
 }
 
 tasks.named<Test>("intTest").configure {
-  if (System.getenv("CI") == null) {
-    maxParallelForks = 2
-  }
-  dependsOn(":authmgr-oauth2-runtime:shadowJar")
-  environment("AWS_REGION", "us-west-2")
-  environment("AWS_ACCESS_KEY_ID", "fake")
-  environment("AWS_SECRET_ACCESS_KEY", "fake")
-  jvmArgs(
-    "--add-exports",
-    "java.base/sun.nio.ch=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.util=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.lang=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.lang.reflect=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.io=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.net=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.nio=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.util.concurrent=ALL-UNNAMED",
-    "--add-opens",
-    "java.base/java.security=ALL-UNNAMED",
+  dependsOn(
+    tasks.named(
+      "intTest_iceberg${defaultIcebergVersion.replace(".", "_")}_flink${defaultFlinkVersion.replace(".", "_")}"
+    )
   )
+  // the task itself should not run any tests
+  enabled = false
+  description =
+    "Runs Flink integration tests with the default Iceberg version ($defaultIcebergVersion) and default Flink version ($defaultFlinkVersion)."
+}
+
+// Create a task to run all matrix tests
+tasks.register("intTestMatrix") {
+  group = "verification"
+  description = "Runs all integration test matrix combinations."
+  dependsOn(matrixTestTasks)
+}
+
+// Helper task to print matrix configuration
+tasks.register("printTestMatrix") {
+  group = "help"
+  description = "Prints the test matrix configuration."
+  doLast {
+    println("Flink Integration Test Matrix:")
+    println("Iceberg versions: ${icebergVersions.joinToString(", ")}")
+    println("Flink versions: ${flinkVersions.joinToString(", ")}")
+    println("Available tasks:")
+    matrixTestTasks.forEach { task ->
+      val icebergVersion = task.get().inputs.properties["icebergVersion"]
+      val flinkVersion = task.get().inputs.properties["flinkVersion"]
+      println("  - ${task.name} uses: Iceberg $icebergVersion, Flink $flinkVersion")
+    }
+  }
 }

oauth2/runtime-flink-tests/src/intTest/java/com/dremio/iceberg/authmgr/oauth2/test/flink/FlinkPolarisS3ITBase.java

@@ -31,6 +31,7 @@
 import java.util.concurrent.ExecutionException;
 import org.apache.flink.table.api.EnvironmentSettings;
 import org.apache.flink.table.api.TableEnvironment;
+import org.apache.iceberg.IcebergBuild;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -45,6 +46,20 @@ public abstract class FlinkPolarisS3ITBase {
   protected volatile PolarisContainer polaris;
   protected TableEnvironment flink;
 
+  @BeforeAll
+  public void recordExpectedVersions() {
+    var expectedIcebergVersion = System.getProperty("authmgr.test.iceberg.version");
+    var actualIcebergVersion = IcebergBuild.version();
+    if (actualIcebergVersion.equals("unspecified")) {
+      // Iceberg 1.9.0 returns "unspecified" :shrug:
+      var icebergTag = IcebergBuild.gitTags().get(0);
+      assertThat(icebergTag).startsWith("apache-iceberg-" + expectedIcebergVersion);
+    } else {
+      assertThat(actualIcebergVersion).startsWith(expectedIcebergVersion);
+    }
+    // TODO how to check Flink version? GlobalConfiguration.loadConfiguration() doesn't work
+  }
+
   @BeforeAll
   public void setup() throws ExecutionException, InterruptedException {
     var network = Network.newNetwork();