[ZEPPELIN-6195] Fix Selenium tests by switching from Chrome to Firefox and upgrading outdated Spark

.github/workflows/frontend.yml

@@ -86,32 +86,37 @@ jobs:
       - name: Run headless test
         run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw package -pl zeppelin-web-angular ${MAVEN_ARGS}
 
-  test-selenium-with-spark-module-for-spark-3-4:
+  test-selenium-with-spark-module-for-spark-3-5:
     runs-on: ubuntu-24.04
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Tune Runner VM
-        uses: ./.github/actions/tune-runner-vm
-      - name: Set up JDK 11
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: 11
-      - name: Cache local Maven repository
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.m2/repository
-            !~/.m2/repository/org/apache/zeppelin/
-            ~/.spark-dist
-            ~/.cache
-          key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-zeppelin-
+
+      - name: Install Firefox (non-snap) and geckodriver
+        run: |
+          sudo snap remove firefox || true
+
+          sudo add-apt-repository -y ppa:mozillateam/ppa
+          sudo tee -a /etc/apt/preferences.d/mozilla-firefox <<EOF
+          Package: *
+          Pin: release o=LP-PPA-mozillateam
+          Pin-Priority: 1001
+          EOF
+
+          sudo apt-get update
+          sudo apt-get install -y firefox
+
+          GECKODRIVER_VERSION=$(curl -s https://api.github.com/repos/mozilla/geckodriver/releases/latest | grep tag_name | cut -d '"' -f 4)
+          wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
+          tar -xzf geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
+          chmod +x geckodriver
+          sudo mv geckodriver /usr/local/bin/
+          geckodriver --version
+          firefox --version
+
       - name: Setup conda environment with python 3.9 and R
         uses: conda-incubator/setup-miniconda@v3
         with:
@@ -122,15 +127,22 @@ jobs:
           channel-priority: strict
           auto-activate-base: false
           use-mamba: true
-      - name: Make IRkernel available to Jupyter
-        run: |
-          R -e "IRkernel::installspec()"
+
       - name: Install Environment
         run: |
-          ./mvnw clean install -DskipTests -am -pl zeppelin-integration -Pweb-classic -Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Pweb-dist ${MAVEN_ARGS}
-      - name: run tests
+          ./mvnw clean install -DskipTests -am -pl zeppelin-integration -Pweb-classic -Pintegration -Pspark-scala-2.12 -Pspark-3.5 -Pweb-dist ${MAVEN_ARGS}
+
+      - name: Print browser version
         run: |
-          xvfb-run --auto-servernum --server-args="-screen 0 1600x1024x16" ./mvnw verify -DfailIfNoTests=false -pl zeppelin-integration -Pweb-classic -Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Pweb-dist -Pusing-source-tree ${MAVEN_ARGS}
-      - name: Print zeppelin logs
-        if: always()
-        run: if [ -d "logs" ]; then cat logs/*; fi
+          echo "Firefox version:"
+          firefox --version
+          echo "GeckoDriver version:"
+          geckodriver --version
+
+      - name: Run integration tests with Firefox
+        run: |
+          xvfb-run --auto-servernum --server-args="-screen 0 1600x1024x16" \
+            ./mvnw verify -DfailIfNoTests=false \
+            -pl zeppelin-integration \
+            -Pweb-classic -Pintegration -Pspark-scala-2.12 -Pspark-3.5 -Pweb-dist -Pusing-source-tree \
+            ${MAVEN_ARGS}

zeppelin-integration/pom.xml

@@ -38,7 +38,7 @@
 
   <properties>
     <!--test library versions-->
-    <selenium.java.version>4.11.0</selenium.java.version>
+    <selenium.java.version>4.20.0</selenium.java.version>
   </properties>
 
   <dependencies>

zeppelin-integration/src/test/java/org/apache/zeppelin/WebDriverManager.java

@@ -80,83 +80,44 @@ public WebDriver getWebDriver() {
   }
 
   private WebDriver constructWebDriver(int port) {
-    Supplier<WebDriver> chromeDriverSupplier = () -> {
-      try {
-        ChromeOptions options = new ChromeOptions();
-        return new ChromeDriver(options);
-      } catch (Exception e) {
-        LOG.error("Exception in WebDriverManager while ChromeDriver ", e);
-        return null;
-      }
-    };
-    Supplier<WebDriver> firefoxDriverSupplier = () -> {
-      try {
-        return getFirefoxDriver();
-      } catch (Exception e) {
-        LOG.error("Exception in WebDriverManager while FireFox Driver ", e);
-        return null;
-      }
-    };
-    Supplier<WebDriver> safariDriverSupplier = () -> {
-      try {
-        return new SafariDriver();
-      } catch (Exception e) {
-        LOG.error("Exception in WebDriverManager while SafariDriver ", e);
-        return null;
-      }
-    };
+    WebDriver driver = getFirefoxDriver();
 
-    WebDriver driver;
-    switch (SystemUtils.getEnvironmentVariable("ZEPPELIN_SELENIUM_BROWSER", "").toLowerCase(Locale.ROOT)) {
-      case "chrome":
-        driver = chromeDriverSupplier.get();
-        break;
-      case "firefox":
-        driver = firefoxDriverSupplier.get();
-        break;
-      case "safari":
-        driver = safariDriverSupplier.get();
-        break;
-      default:
-        driver = Stream.of(chromeDriverSupplier, firefoxDriverSupplier, safariDriverSupplier)
-            .map(Supplier::get)
-            .filter(Objects::nonNull)
-            .findFirst()
-            .orElse(null);
-    }
     if (driver == null) {
-      throw new RuntimeException("No available WebDriver");
+      throw new RuntimeException("Failed to initialize Firefox WebDriver");
     }
 
     String url = "http://localhost:" + port + "/classic";
 
     long start = System.currentTimeMillis();
     boolean loaded = false;
     driver.manage().timeouts()
-      .implicitlyWait(Duration.ofSeconds(AbstractZeppelinIT.MAX_IMPLICIT_WAIT));
+            .implicitlyWait(Duration.ofSeconds(AbstractZeppelinIT.MAX_IMPLICIT_WAIT));
     driver.get(url);
 
     while (System.currentTimeMillis() - start < 60 * 1000) {
-      // wait for page load
       try {
         (new WebDriverWait(driver, Duration.ofSeconds(30))).until(new ExpectedCondition<Boolean>() {
           @Override
           public Boolean apply(WebDriver d) {
-            return d.findElement(By.xpath("//i[@uib-tooltip='WebSocket Connected']"))
-                .isDisplayed();
+            return d.findElement(By.xpath("//i[@uib-tooltip='WebSocket Connected']")).isDisplayed();
           }
         });
         loaded = true;
         break;
       } catch (TimeoutException e) {
-        LOG.info("Exception in WebDriverManager while WebDriverWait ", e);
+        LOG.info("Retry loading the page due to timeout.", e);
         driver.navigate().to(url);
       }
     }
 
     assertTrue(loaded);
 
-    driver.manage().window().maximize();
+    try {
+      driver.manage().window().maximize();
+    } catch (Exception e) {
+      LOG.warn("Failed to maximize Firefox window. Consider using setSize instead.", e);
+    }
+
     return driver;
   }
 

zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java

@@ -65,7 +65,7 @@ public class DownloadUtils {
   private static final int PROGRESS_BAR_UPDATE_INTERVAL;
 
   private static String downloadFolder = System.getProperty("user.home") + "/.cache";
-  public static final String DEFAULT_SPARK_VERSION = "3.4.2";
+  public static final String DEFAULT_SPARK_VERSION = "3.5.6";
   public static final String DEFAULT_SPARK_HADOOP_VERSION = "3";