Run the UI tests as groups to prevent timeout error. (#1456)

* test breakdown

* clearer names

* Update build.yaml

* Update run.sh

Update run.sh

Author: anusreelakshmi934

.github/workflows/build.yaml

@@ -93,25 +93,61 @@ jobs:
   build:
     needs: fetch_merge_commit_sha_from_lsp4ij_PR
     runs-on: ${{ matrix.os }}
+    name: ${{ matrix.runtime }} - ${{ matrix.test-group }}
     strategy:
       fail-fast: false
       matrix:
         runtime: [ linux, mac, windows ]
+        test-group: [ LSP4Jakarta-Unit, Maven-MicroProfile, Gradle-MicroProfile, Maven-Custom-Liberty-Install, Gradle-Custom-Liberty-Install, Maven-MP-Config, Gradle-MP-Config, Maven-MP-SID, Gradle-MP-SID, Maven-NLT-REST, Gradle-NLT-REST, Gradle-Language-Server, Gradle-MP-Language-Server, Gradle-Jakarta-Language-Server, Maven-Multi-Module-MP ]
+        exclude:
+          # Exclude LSP4Jakarta-Unit from mac and windows - only run on linux
+          - runtime: mac
+            test-group: LSP4Jakarta-Unit
+          - runtime: windows
+            test-group: LSP4Jakarta-Unit
         include:
           - runtime: linux
             os: ubuntu-latest
-            reportName: linux-test-report
           - runtime: mac
             os: macOS-14
-            reportName: mac-test-report
           - runtime: windows
             os: windows-latest
-            reportName: windows-test-report
+          - test-group: Maven-MicroProfile
+            test-class: io.openliberty.tools.intellij.it.MavenSingleModMPProjectTest
+          - test-group: Gradle-MicroProfile
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModMPProjectTest
+          - test-group: Maven-Custom-Liberty-Install
+            test-class: io.openliberty.tools.intellij.it.MavenSingleModCustomWLPInstallProjectTest
+          - test-group: Gradle-Custom-Liberty-Install
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModCustomWLPInstallProjectTest
+          - test-group: Maven-MP-Config
+            test-class: io.openliberty.tools.intellij.it.MavenSingleModMPCfgProjectTest
+          - test-group: Gradle-MP-Config
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModMPCfgProjectTest
+          - test-group: Maven-MP-SID
+            test-class: io.openliberty.tools.intellij.it.MavenSingleModMPSIDProjectTest
+          - test-group: Gradle-MP-SID
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModMPSIDProjectTest
+          - test-group: Maven-NLT-REST
+            test-class: io.openliberty.tools.intellij.it.MavenSingleModNLTRestProjectTest
+          - test-group: Gradle-NLT-REST
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModNLTRestProjectTest
+          - test-group: Gradle-Language-Server
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModLSTest
+          - test-group: Gradle-MP-Language-Server
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModMPLSTest
+          - test-group: Gradle-Jakarta-Language-Server
+            test-class: io.openliberty.tools.intellij.it.GradleSingleModJakartaLSTest
+          - test-group: Maven-Multi-Module-MP
+            test-class: io.openliberty.tools.intellij.it.MavenMPMultipleProjectTest
+          - test-group: LSP4Jakarta-Unit
+            test-class: "io.openliberty.tools.intellij.lsp4jakarta.it.*"
     env:
       USE_LOCAL_PLUGIN: ${{ inputs.useLocalPlugin || false }}
       REF_LSP4IJ: ${{ needs.fetch_merge_commit_sha_from_lsp4ij_PR.outputs.pr_details }}
       LSP4IJ_BRANCH: ${{ needs.fetch_merge_commit_sha_from_lsp4ij_PR.outputs.checkout_name }}
       REF_LTI_TAG: ${{ inputs.refLTITag }}
+      TEST_CLASS: ${{ matrix.test-class }}
     steps:
       - name: Configure pagefile
         if: contains(matrix.os, 'windows')
@@ -152,7 +188,7 @@ jobs:
         working-directory: ./liberty-tools-intellij
         run: bash ./gradlew buildPlugin -PuseLocal=${{ env.USE_LOCAL_PLUGIN }}
       - name: 'Archive artifacts'
-        if: ${{ runner.os == 'Linux' && !failure() }}
+        if: ${{ runner.os == 'Linux' && matrix.test-group == 'LSP4Jakarta-Unit' && !failure() }}
         uses: actions/[email protected]
         with:
           name: liberty-tools-intellij-LTI-${{ env.REF_LTI_TAG || 'default' }}-LSP4IJ-${{ env.LSP4IJ_BRANCH || 'default' }}
@@ -161,14 +197,14 @@ jobs:
             ./**/libs/*liberty-tools-intellij*.jar
           if-no-files-found: warn
           retention-days: 7
-      - name: 'Run UI integration tests'
+      - name: 'Run tests'
         id: run_tests
         working-directory: ./liberty-tools-intellij
         run: bash ./src/test/resources/ci/scripts/run.sh
       - name: 'Archive Test logs and reports'
         if: ${{ failure() && steps.run_tests.conclusion == 'failure' }}
         uses: actions/[email protected]
         with:
-          name: ${{ matrix.reportName }}-LTI-${{ env.REF_LTI_TAG || 'default' }}-LSP4IJ-${{ env.LSP4IJ_BRANCH || 'default' }}
+          name: ${{ matrix.runtime }}-${{ matrix.test-group }}-test-report-LTI-${{ env.REF_LTI_TAG || 'default' }}-LSP4IJ-${{ env.LSP4IJ_BRANCH || 'default' }}
           path: |
             liberty-tools-intellij/build/reports/

src/test/resources/ci/scripts/run.sh

@@ -240,9 +240,15 @@ main() {
     for restartCount in {1..5}; do
         startIDE
         # Run the tests
-        echo -e "\n$(${currentTime[@]}): INFO: Running tests..."
         set -o pipefail # using tee requires we use this setting to gather the rc of gradlew
-        ./gradlew test -PuseLocal=$USE_LOCAL_PLUGIN | tee "$JUNIT_OUTPUT_TXT"
+        if [ -n "$TEST_CLASS" ]; then
+            echo -e "\n$(${currentTime[@]}): INFO: Running specific test class: $TEST_CLASS"
+            test_args=(--tests "$TEST_CLASS")
+        else
+            echo -e "\n$(${currentTime[@]}): INFO: Running all tests..."
+            test_args=()
+        fi
+        ./gradlew test "${test_args[@]}" -PuseLocal="$USE_LOCAL_PLUGIN" | tee "$JUNIT_OUTPUT_TXT"
         testRC=$? # gradlew test only returns 0 or 1, not the return code from JUnit
         set +o pipefail # reset this option
         grep -i "SocketTimeoutException" "$JUNIT_OUTPUT_TXT" && testRC=23