Fixes integ test CI #694
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | ||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| - 1.* | ||
| - 2.* | ||
| pull_request: | ||
| env: | ||
| GRADLE_OPTS: -Dhttp.keepAlive=false | ||
| CI_ENVIRONMENT: normal | ||
| jobs: | ||
| Get-CI-Image-Tag: | ||
| uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main | ||
| with: | ||
| product: opensearch | ||
| generate-test-list: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }} | ||
| steps: | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: 21 | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Generate list of tasks | ||
| id: set-matrix | ||
| run: | | ||
| echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT | ||
| publish-components-to-maven-local: | ||
| strategy: | ||
| matrix: | ||
| os: [ ubuntu-latest, windows-latest ] | ||
| runs-on: ${{ matrix.os }} | ||
| steps: | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin | ||
| java-version: 21 | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Publish components to Maven Local (Linux) | ||
| if: matrix.os == 'ubuntu-latest' | ||
| run: | | ||
| ./gradlew clean \ | ||
| :opensearch-resource-sharing-spi:publishToMavenLocal \ | ||
| :opensearch-security-client:publishToMavenLocal \ | ||
| -Dbuild.snapshot=false | ||
| - name: Publish components to Maven Local (Windows) | ||
| if: matrix.os == 'windows-latest' | ||
| run: | | ||
| gradlew.bat clean ^ | ||
| :opensearch-resource-sharing-spi:publishToMavenLocal ^ | ||
| :opensearch-security-client:publishToMavenLocal ^ | ||
| -Dbuild.snapshot=false | ||
| shell: cmd | ||
| - name: Cache artifacts for dependent jobs | ||
| if: ${{ always() }} | ||
| uses: actions/[email protected] | ||
| with: | ||
| # Use the appropriate Maven local repository path based on OS. | ||
| path: ${{ matrix.os == 'ubuntu-latest' | ||
| ? '~/.m2/repository/org/opensearch/' | ||
| : format('{0}\\.m2\\repository\\org\\opensearch', env.HOME) }} | ||
| key: maven-local-${{ github.run_id }} | ||
| test-windows: | ||
| name: test | ||
| needs: [generate-test-list] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }} | ||
| platform: [windows-latest] | ||
| jdk: [21] | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Build and Test | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| ${{ matrix.gradle_task }} -Dbuild.snapshot=false | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: ${{ matrix.platform }}-JDK${{ matrix.jdk }}-${{ matrix.gradle_task }}-reports | ||
| path: | | ||
| ./build/reports/ | ||
| test-linux: | ||
| name: test | ||
| needs: ["generate-test-list", "Get-CI-Image-Tag"] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }} | ||
| platform: [ubuntu-latest] | ||
| jdk: [21] | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| # using the same image which is used by opensearch-build to build the OpenSearch Distribution | ||
| # this image tag is subject to change as more dependencies and updates will arrive over time | ||
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | ||
| # need to switch to root so that github actions can install runner binary on container without permission issues. | ||
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | ||
| steps: | ||
| - name: Run start commands | ||
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Build and Test | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| ${{ matrix.gradle_task }} -Dbuild.snapshot=false | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: ${{ matrix.platform }}-JDK${{ matrix.jdk }}-${{ matrix.gradle_task }}-reports | ||
| path: | | ||
| ./build/reports/ | ||
| report-coverage: | ||
| needs: ["test-windows", "test-linux", "integration-tests-windows", "integration-tests-linux", "spi-tests-linux", "spi-tests-windows"] | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/download-artifact@v4 | ||
| with: | ||
| path: downloaded-artifacts | ||
| - name: Display structure of downloaded files | ||
| run: ls -R | ||
| working-directory: downloaded-artifacts | ||
| - name: Upload Coverage with retry | ||
| uses: Wandalen/[email protected] | ||
| with: | ||
| attempt_limit: 5 | ||
| attempt_delay: 2000 | ||
| action: codecov/codecov-action@v4 | ||
| with: | | ||
| token: ${{ secrets.CODECOV_TOKEN }} | ||
| fail_ci_if_error: true | ||
| verbose: true | ||
| integration-tests-windows: | ||
| name: integration-tests | ||
| needs: ["publish-components-to-maven-local"] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| jdk: [21] | ||
| platform: [windows-latest] | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Restore Maven Local Cache | ||
| uses: actions/[email protected] | ||
| with: | ||
| path: ${{ format('{0}\\.m2\\repository\\org\\opensearch', env.HOME) }} | ||
| key: maven-local-${{ github.run_id }} | ||
| - name: Run Integration Tests | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| :integrationTest -Dbuild.snapshot=false | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: integration-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports | ||
| path: | | ||
| ./build/reports/ | ||
| integration-tests-linux: | ||
| name: integration-tests | ||
| needs: ["Get-CI-Image-Tag"] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| jdk: [21] | ||
| platform: [ubuntu-latest] | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| # using the same image which is used by opensearch-build team to build the OpenSearch Distribution | ||
| # this image tag is subject to change as more dependencies and updates will arrive over time | ||
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | ||
| # need to switch to root so that github actions can install runner binary on container without permission issues. | ||
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | ||
| steps: | ||
| - name: Run start commands | ||
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Publish components to Maven Local | ||
| # Since this step runs inside a docker container, publish-to-maven-local job cannot be re-used here | ||
| run: | | ||
| ./gradlew clean \ | ||
| :opensearch-resource-sharing-spi:publishToMavenLocal \ | ||
| :opensearch-security-client:publishToMavenLocal \ | ||
| -Dbuild.snapshot=false | ||
| - name: Build and Test | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| :integrationTest -Dbuild.snapshot=false | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: integration-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports | ||
| path: | | ||
| ./build/reports/ | ||
| spi-tests-linux: | ||
| name: spi-tests | ||
| needs: ["Get-CI-Image-Tag"] | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| jdk: [21] | ||
| platform: [ubuntu-latest] | ||
| runs-on: ${{ matrix.platform }} | ||
| container: | ||
| # using the same image which is used by opensearch-build to build the OpenSearch Distribution | ||
| # this image tag is subject to change as more dependencies and updates will arrive over time | ||
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | ||
| # need to switch to root so that github actions can install runner binary on container without permission issues. | ||
| options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} | ||
| steps: | ||
| - name: Run start commands | ||
| run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Run SPI Tests | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| :opensearch-resource-sharing-spi:test -Dbuild.snapshot=false | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: spi-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports | ||
| path: | | ||
| ./build/reports/ | ||
| spi-tests-windows: | ||
| name: spi-tests | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| jdk: [21] | ||
| platform: [windows-latest] | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Run SPI Tests | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| :opensearch-resource-sharing-spi:test -Dbuild.snapshot=false | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: spi-${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports | ||
| path: | | ||
| ./build/reports/ | ||
| resource-tests: | ||
| env: | ||
| CI_ENVIRONMENT: resource-test | ||
| needs: publish-components-to-maven-local | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| jdk: [21] | ||
| platform: [ubuntu-latest] | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - name: Set up JDK for build and test | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout security | ||
| uses: actions/checkout@v4 | ||
| - name: Restore Maven Local Cache | ||
| uses: actions/[email protected] | ||
| with: | ||
| path: ~/.m2/repository/org/opensearch/ | ||
| key: maven-local-${{ github.run_id }} | ||
| - name: Run Resource Tests | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| :integrationTest -Dbuild.snapshot=false --tests org.opensearch.security.ResourceFocusedTests | ||
| backward-compatibility-build: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: 21 | ||
| - name: Checkout Security Repo | ||
| uses: actions/checkout@v4 | ||
| - name: Build BWC tests | ||
| uses: gradle/gradle-build-action@v3 | ||
| with: | ||
| cache-disabled: true | ||
| arguments: | | ||
| -p bwc-test build -x test -x integTest | ||
| backward-compatibility: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| jdk: [11, 17] | ||
| platform: [ubuntu-latest, windows-latest] | ||
| runs-on: ${{ matrix.platform }} | ||
| steps: | ||
| - uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: ${{ matrix.jdk }} | ||
| - name: Checkout Security Repo | ||
| uses: actions/checkout@v4 | ||
| - id: build-previous | ||
| uses: ./.github/actions/run-bwc-suite | ||
| with: | ||
| plugin-previous-branch: "2.x" | ||
| plugin-next-branch: "current_branch" | ||
| report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }} | ||
| username: admin | ||
| password: admin | ||
| code-ql: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin # Temurin is a distribution of adoptium | ||
| java-version: 21 | ||
| - uses: github/codeql-action/init@v3 | ||
| with: | ||
| languages: java | ||
| - run: ./gradlew clean assemble | ||
| - uses: github/codeql-action/analyze@v3 | ||
| build-artifact-names: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Setup Environment | ||
| uses: actions/checkout@v4 | ||
| - name: Configure Java | ||
| uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: temurin | ||
| java-version: 21 | ||
| - name: Build and Test Artifacts | ||
| run: | | ||
| # Set version variables | ||
| security_plugin_version=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}') | ||
| security_plugin_version_no_snapshot=$(echo $security_plugin_version | sed 's/-SNAPSHOT//g') | ||
| security_plugin_version_only_number=$(echo $security_plugin_version_no_snapshot | cut -d- -f1) | ||
| test_qualifier=alpha2 | ||
| # Debug print versions | ||
| echo "Versions:" | ||
| echo $security_plugin_version | ||
| echo $security_plugin_version_no_snapshot | ||
| echo $security_plugin_version_only_number | ||
| echo $test_qualifier | ||
| # Publish SPI | ||
| ./gradlew clean :opensearch-resource-sharing-spi:publishToMavenLocal && test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version.jar | ||
| ./gradlew clean :opensearch-resource-sharing-spi:publishToMavenLocal -Dbuild.snapshot=false && test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version_no_snapshot.jar | ||
| ./gradlew clean :opensearch-resource-sharing-spi:publishToMavenLocal -Dbuild.snapshot=false -Dbuild.version_qualifier=$test_qualifier && test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version_only_number-$test_qualifier.jar | ||
| ./gradlew clean :opensearch-resource-sharing-spi:publishToMavenLocal -Dbuild.version_qualifier=$test_qualifier && test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version_only_number-$test_qualifier-SNAPSHOT.jar | ||
| # Publish Client | ||
| ./gradlew clean :opensearch-security-client:publishToMavenLocal && test -s ./client/build/libs/opensearch-security-client-$security_plugin_version.jar | ||
| ./gradlew clean :opensearch-security-client:publishToMavenLocal -Dbuild.snapshot=false && test -s ./client/build/libs/opensearch-security-client-$security_plugin_version_no_snapshot.jar | ||
| ./gradlew clean :opensearch-security-client:publishToMavenLocal -Dbuild.snapshot=false -Dbuild.version_qualifier=$test_qualifier && test -s ./client/build/libs/opensearch-security-client-$security_plugin_version_only_number-$test_qualifier.jar | ||
| ./gradlew clean :opensearch-security-client:publishToMavenLocal -Dbuild.version_qualifier=$test_qualifier && test -s ./client/build/libs/opensearch-security-client-$security_plugin_version_only_number-$test_qualifier-SNAPSHOT.jar | ||
| # Build artifacts | ||
| ./gradlew clean assemble && \ | ||
| test -s ./build/distributions/opensearch-security-$security_plugin_version.zip && \ | ||
| test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version.jar && \ | ||
| test -s ./client/build/libs/opensearch-security-client-$security_plugin_version.jar | ||
| ./gradlew clean assemble -Dbuild.snapshot=false && \ | ||
| test -s ./build/distributions/opensearch-security-$security_plugin_version_no_snapshot.zip && \ | ||
| test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version_no_snapshot.jar && \ | ||
| test -s ./client/build/libs/opensearch-security-client-$security_plugin_version_no_snapshot.jar | ||
| ./gradlew clean assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=$test_qualifier && \ | ||
| test -s ./build/distributions/opensearch-security-$security_plugin_version_only_number-$test_qualifier.zip && \ | ||
| test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version_only_number-$test_qualifier.jar && \ | ||
| test -s ./client/build/libs/opensearch-security-client-$security_plugin_version_only_number-$test_qualifier.jar | ||
| ./gradlew clean assemble -Dbuild.version_qualifier=$test_qualifier && \ | ||
| test -s ./build/distributions/opensearch-security-$security_plugin_version_only_number-$test_qualifier-SNAPSHOT.zip && \ | ||
| test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version_only_number-$test_qualifier-SNAPSHOT.jar && \ | ||
| test -s ./client/build/libs/opensearch-security-client-$security_plugin_version_only_number-$test_qualifier-SNAPSHOT.jar | ||
| ./gradlew clean publishPluginZipPublicationToZipStagingRepository && \ | ||
| test -s ./build/distributions/opensearch-security-$security_plugin_version.zip && \ | ||
| test -s ./build/distributions/opensearch-security-$security_plugin_version.pom && \ | ||
| test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version.jar | ||
| ./gradlew clean publishShadowPublicationToMavenLocal && \ | ||
| test -s ./spi/build/libs/opensearch-resource-sharing-spi-$security_plugin_version.jar && \ | ||
| test -s ./client/build/libs/opensearch-security-client-$security_plugin_version.jar | ||
| - name: List files in build directory on failure | ||
| if: failure() | ||
| run: ls -al ./*/build/libs/ ./build/distributions/ | ||
