diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml index e2fd768220a..3d9ecc28b2e 100644 --- a/.github/actions/gradle-command-self-hosted-action/action.yml +++ b/.github/actions/gradle-command-self-hosted-action/action.yml @@ -42,5 +42,9 @@ runs: rm ~/.m2/settings.xml fi ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} --continue \ + -PjavaVersion=11 \ + -Porg.gradle.java.installations.auto-detect=false \ + -Porg.gradle.java.installations.auto-download=false \ + -Porg.gradle.java.installations.fromEnv=JAVA_HOME_11_ARM64,JAVA_HOME_17_ARM64,JAVA_HOME_21_ARM64,JAVA_HOME_8_X64,JAVA_HOME_11_X64,JAVA_HOME_17_X64,JAVA_HOME_21_X64 \ -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses \ ${{ inputs.arguments }} diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index d5f1f879a07..81847d7f26e 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -25,7 +25,7 @@ inputs: java-version: required: false description: 'Install Java version' - default: '' + default: 'default' go-version: required: false description: 'Install Go version' @@ -70,11 +70,16 @@ runs: tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}- - name: Install Java - if: ${{ inputs.java-version != '' }} + if: ${{ inputs.java-version != 'default' && '11' }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + - name: Install Java 11 as default for Gradle and build targets uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }} + java-version: '11' - name: Setup Gradle uses: gradle/gradle-build-action@v2 with: diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml index aeb940e3b3d..f6bae1f01e2 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml @@ -95,8 +95,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_1' \ @@ -105,8 +104,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_2' \ @@ -115,8 +113,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_3' \ @@ -125,8 +122,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_CoGBK_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml index 367a1d2853f..19848425dec 100644 --- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml @@ -95,8 +95,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_1' \ @@ -105,8 +104,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_2' \ @@ -115,8 +113,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_3' \ @@ -125,8 +122,7 @@ jobs: with: gradle-command: :sdks:java:testing:load-tests:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CoGroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_CoGBK_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml index 2dd206c5bae..8d29478999e 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml @@ -72,9 +72,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 8 + java-version: 17 - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: @@ -97,8 +95,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records uses: ./.github/actions/gradle-command-self-hosted-action @@ -108,8 +105,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records uses: ./.github/actions/gradle-command-self-hosted-action @@ -119,8 +115,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action @@ -130,8 +125,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action @@ -141,8 +135,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -152,8 +145,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -163,6 +155,5 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml index eb405be32dd..467efd0ef90 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml @@ -72,9 +72,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 8 + java-version: 17 - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: @@ -97,8 +95,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records uses: ./.github/actions/gradle-command-self-hosted-action @@ -108,8 +105,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records uses: ./.github/actions/gradle-command-self-hosted-action @@ -119,8 +115,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action @@ -130,8 +125,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action @@ -141,8 +135,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -152,8 +145,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action @@ -163,6 +155,5 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml index 78846e35ada..09d9dd584ae 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml @@ -98,8 +98,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_1' \ - name: run ParDo Dataflow V2 Batch Java Load Test 2 (200 times) uses: ./.github/actions/gradle-command-self-hosted-action @@ -109,8 +108,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_2' \ - name: run ParDo Dataflow V2 Batch Java Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -120,8 +118,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_3' \ - name: run ParDo Dataflow V2 Batch Java Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -131,6 +128,5 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_batch_ParDo_4' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml index 8ea2adc8e03..675dd5a7441 100644 --- a/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml +++ b/.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml @@ -98,8 +98,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_1 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_1' \ - name: run ParDo Dataflow V2 Streaming Java Load Test 2 (200 times) uses: ./.github/actions/gradle-command-self-hosted-action @@ -109,8 +108,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_2 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_2' \ - name: run ParDo Dataflow V2 Streaming Java Load Test 3 (10 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -120,8 +118,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_3 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_3' \ - name: run ParDo Dataflow V2 Streaming Java Load Test 4 (100 counters) uses: ./.github/actions/gradle-command-self-hosted-action @@ -131,6 +128,5 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.ParDoLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ '-PloadTest.args=${{ env.beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions_test_arguments_4 }} --appName=load_tests_Java${{ matrix.java_version }}_Dataflow_V2_streaming_ParDo_4' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 39eab26dfcf..c1ee5f08d03 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -83,6 +83,8 @@ jobs: run: | gcloud auth configure-docker us.gcr.io - name: run goPostCommitDataflowARM script - run: ./gradlew :goPostCommitDataflowARM + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :goPostCommitDataflowARM env: USER: github-actions diff --git a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml index 29a4f941adb..aa84f36725c 100644 --- a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml +++ b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml @@ -80,4 +80,5 @@ jobs: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:go:test:samzaValidatesRunner \ No newline at end of file + gradle-command: :sdks:go:test:samzaValidatesRunner + arguments: -PjavaVersion=8 diff --git a/.github/workflows/beam_PostCommit_Java_DataflowV1.yml b/.github/workflows/beam_PostCommit_Java_DataflowV1.yml index dbd23d94367..648edd19041 100644 --- a/.github/workflows/beam_PostCommit_Java_DataflowV1.yml +++ b/.github/workflows/beam_PostCommit_Java_DataflowV1.yml @@ -75,9 +75,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 11 - 8 + java-version: 11 - name: run PostCommit Java Dataflow V1 script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml index 0c7da0f60fe..7b7f8910be7 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml @@ -80,9 +80,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - ${{ matrix.java_version != '11' && matrix.java_version || '' }} - 11 + java-version: ${{ matrix.java_version }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: GCloud Docker credential helper @@ -97,8 +95,7 @@ jobs: gradle-command: :runners:google-cloud-dataflow-java:arm:examplesJavaRunnerV2IntegrationTestARM max-workers: 12 arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -Pcontainer-architecture-list=arm64,amd64 \ -Ppush-containers \ -Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \ diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml index 29b5624e73d..3d69e4e38c6 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml @@ -77,13 +77,12 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - ${{ matrix.java_version }} - 11 + java-version: ${{ matrix.java_version }} - name: run java${{ matrix.java_version }}PostCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:examples:java${{ matrix.java_version }}PostCommit + arguments: -PjavaVersion=${{ matrix.java_version }} max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml index 894fe7d1b51..c3390f90b5d 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml @@ -87,6 +87,7 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:examplesJavaDistrolessRunnerV2PreCommit + arguments: -PjavaVersion=17 max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml index b4a76ad09f4..09aea1c48f8 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml @@ -79,9 +79,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - ${{ matrix.java_version }} - 11 + java-version: ${{ matrix.java_version }} - name: run PostCommit Java Examples Dataflow V2 Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -90,8 +88,7 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -PskipCheckerFramework \ - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml index e68498be3a5..fe4a5742cf2 100644 --- a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml +++ b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml @@ -73,21 +73,13 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 8 - 11 + # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 + # Also change sdks/java/io/hcatalog/build.gradle + java-version: 8 - name: run javaHadoopVersionsTest script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :javaHadoopVersionsTest - # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 - - name: run java8HadoopVersionsTest script - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:java:io:hcatalog:hadoopVersionsTest - arguments: | - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml index 647605844f4..b4e976ce49c 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml @@ -78,7 +78,7 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest - arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + arguments: -PjavaVersion=11 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml index cea8af594bf..51710fe2aae 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml @@ -73,17 +73,14 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 8 + java-version: 17 - name: run PostCommit Java Jpms Dataflow Java17 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest arguments: -PskipCheckerFramework - -PtestJavaVersion=17 - -Pjava17Home=$JAVA_HOME_17_X64 + -PjavaVersion=17 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml index ff174b5f43b..4111e049b46 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml @@ -78,7 +78,7 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest - arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + arguments: -PjavaVersion=11 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml index e149ff76eee..85f0bf55a95 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml @@ -73,17 +73,14 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 8 + java-version: 17 - name: run PostCommit Java Jpms Direct Java17 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest arguments: -PskipCheckerFramework - -PtestJavaVersion=17 - -Pjava17Home=$JAVA_HOME_17_X64 + -PjavaVersion=17 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml index 68a47727f6f..d869e65fe71 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml @@ -74,17 +74,14 @@ jobs: uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: | - 21 - 8 + java-version: 21 - name: run PostCommit Java Jpms Direct Java21 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest arguments: -PskipCheckerFramework - -PtestJavaVersion=21 - -Pjava21Home=$JAVA_HOME_21_X64 + -PjavaVersion=21 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml index 4b3f696622b..5ef7cf9144f 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml @@ -78,7 +78,7 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:flinkRunnerIntegrationTest - arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + arguments: -PjavaVersion=11 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml index d7e5975bbb0..b91f9e8d9df 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml @@ -72,13 +72,11 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action - with: - java-version: 11 - name: run PostCommit Java Jpms Spark Java11 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:jpms-tests:sparkRunnerIntegrationTest - arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64 + arguments: -PjavaVersion=11 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml index b81bad11550..ce63e68fb59 100644 --- a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml @@ -103,16 +103,13 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 8 - 11 + java-version: 8 - name: run PostCommit Java Nexmark Dataflow V2 (streaming = ${{ matrix.streaming }}) script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:testing:nexmark:run arguments: | - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ + -PjavaVersion=8 \ -Pnexmark.runner.version=V2 \ -Pnexmark.runner=:runners:google-cloud-dataflow-java \ '${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml index 84198f24646..b5adc626df4 100644 --- a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml @@ -111,8 +111,7 @@ jobs: with: gradle-command: :sdks:java:testing:nexmark:run arguments: | - -PtestJavaVersion=${{ matrix.java_version }} \ - -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ -Pnexmark.runner.version=V2 \ -Pnexmark.runner=:runners:google-cloud-dataflow-java \ '${{ env.GRADLE_COMMAND_ARGUMENTS }}--influxTags={"runnerVersion":"V2","javaVersion":"${{ matrix.java_version }}"}--streaming=${{ matrix.streaming }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml b/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml index e98ab64bb12..4aed7b8ddb3 100644 --- a/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml +++ b/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml @@ -75,19 +75,16 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action # TODO(https://github.com/apache/beam/issues/32208) move to Java11 after bump to Samza 1.8 + # Also change runners/samza/job-server/build.gradle with: - java-version: | - 8 - 11 + java-version: 8 - name: run PostCommit Java Samza script env: CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}} uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:samza:job-server:validatesPortableRunner - arguments: | - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ + arguments: -PjavaVersion=8 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml index a84485c717f..4fe5a9d60c9 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -77,13 +77,12 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - ${{ matrix.java_version }} - 11 + java-version: ${{ matrix.java_version }} - name: run jar Java${{ matrix.java_version }} script - run: | - ./gradlew runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar + arguments: -PjavaVersion=${{ matrix.java_version }} - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -97,7 +96,7 @@ jobs: -x testJar \ -x classes \ -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml index c869c58f377..2b28a61fc43 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml @@ -77,7 +77,9 @@ jobs: with: java-version: default - name: run validatesRunner script - run: ./gradlew :runners:direct-java:validatesRunner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:direct-java:validatesRunner - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml index 11b18548249..80a175137a2 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml @@ -77,13 +77,12 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - ${{ matrix.java_version }} - 11 + java-version: ${{ matrix.java_version }} - name: run jar Java${{ matrix.java_version }} script - run: | - ./gradlew :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar + arguments: -PjavaVersion=${{ matrix.java_version }} - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -93,7 +92,7 @@ jobs: -x shadowTestJar \ -x compileJava \ -x compileTestJava \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PjavaVersion=${{ matrix.java_version }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml index 7f7b9c9270d..f60ad9fdbf2 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml @@ -75,12 +75,11 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 8 - 11 + java-version: 8 - name: run jar Java8 script - run: | - ./gradlew :runners:flink:1.19:jar :runners:flink:1.19:testJar + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:flink:1.19:jar :runners:flink:1.19:testJar - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -94,7 +93,7 @@ jobs: -x testJar \ -x classes \ -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + -PjavaVersion=8 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml index 9455717df7b..faecac41a0f 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml @@ -74,16 +74,12 @@ jobs: uses: ./.github/actions/setup-environment-action # TODO(https://github.com/apache/beam/issues/32208) move to Java11 after bump to Samza 1.8 with: - java-version: | - 8 - 11 + java-version: 8 - name: run validatesRunner script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:samza:validatesRunner - arguments: | - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ + arguments: -PjavaVersion=8 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml index 0852fcdf4af..a357669013b 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml @@ -75,12 +75,11 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 8 - 11 + java-version: 8 - name: run jar Java8 script - run: | - ./gradlew :runners:spark:3:jar :runners:spark:3:testJar + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:spark:3:jar :runners:spark:3:testJar - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -94,7 +93,7 @@ jobs: -x testJar \ -x classes \ -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + -PjavaVersion=8 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml index 1bb4e55ed79..856a691b57e 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml @@ -76,7 +76,9 @@ jobs: java-version: default python-version: default - name: run ulrLoopbackValidatesRunner script - run: ./gradlew :runners:portability:java:ulrLoopbackValidatesRunner + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:portability:java:ulrLoopbackValidatesRunner - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml index 9077803aeb0..7436f8ffe3c 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml @@ -77,9 +77,6 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 11 - 8 python-version: ${{ matrix.python_version }} - name: Set PY_VER_CLEAN id: set_py_ver_clean diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml index 985c2a705fb..f940defa498 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml @@ -77,9 +77,6 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 11 - 8 python-version: ${{ matrix.python_version }} - name: Set PY_VER_CLEAN id: set_py_ver_clean diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml index 300becc9ddb..e075c70b478 100644 --- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml +++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml @@ -89,6 +89,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:samzaValidatesRunner arguments: | + -PjavaVersion=8 \ -PpythonVersion=${{ matrix.python_version }} \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index e2d3220ae6a..e10177d5ae0 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -83,8 +83,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava arguments: | - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ + -PjavaVersion=11 \ -PpythonVersion=${{ matrix.python_version }} \ - name: Archive Python Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml index e75eb1e3c2d..7f346dbbe19 100644 --- a/.github/workflows/beam_PostCommit_XVR_Samza.yml +++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml @@ -87,6 +87,7 @@ jobs: with: gradle-command: :runners:samza:job-server:validatesCrossLanguageRunner arguments: | + -PjavaVersion=8 \ -PpythonVersion=${{ matrix.python_version }} \ -PskipNonPythonTask=true \ - name: run PostCommit XVR Samza script @@ -97,6 +98,7 @@ jobs: with: gradle-command: :runners:samza:job-server:validatesCrossLanguageRunner arguments: | + -PjavaVersion=8 \ -PpythonVersion=${{ matrix.python_version }} \ -PskipNonPythonTask=false \ - name: Archive JUnit Test Results diff --git a/.github/workflows/beam_PreCommit_ItFramework.yml b/.github/workflows/beam_PreCommit_ItFramework.yml index c110c209475..b134221cc1d 100644 --- a/.github/workflows/beam_PreCommit_ItFramework.yml +++ b/.github/workflows/beam_PreCommit_ItFramework.yml @@ -88,7 +88,10 @@ jobs: with: java-version: default - name: run ItFrameworkPrecommit script - run: ./gradlew -p it build + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: build + arguments: -p it - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml index d3f4afc8a40..7519434e8d7 100644 --- a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml @@ -86,9 +86,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: 17 - name: run Debezium IO build task uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -96,8 +94,7 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ - name: run Debezium IO additional tasks uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -107,8 +104,7 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PjavaVersion=17 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml index e7dc3620df2..6d611e892e1 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml @@ -93,14 +93,10 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - # The test requires Java 21 and Java 8 versions. - # Java 8 is installed second because JAVA_HOME needs to point to Java 8. - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 21 - 8 + java-version: 21 - name: Clean uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -116,9 +112,8 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=21 \ + -PjavaVersion=21 \ -PskipCheckerFramework \ - -Pjava21Home=$JAVA_HOME_21_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml index 5c3cf29419c..49819de8ad1 100644 --- a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml @@ -88,9 +88,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 8 - 11 + # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 + # Also change sdks/java/io/hcatalog/build.gradle + java-version: 8 - name: run HCatalog IO build script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -98,8 +98,7 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 - - name: Test HCatalog IO on Java8 + - name: Test HCatalog IO on Java uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:io:hcatalog:test @@ -107,8 +106,6 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -Dfile.encoding=UTF-8 \ - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml index 03ff102861c..98b9247f664 100644 --- a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml @@ -86,10 +86,6 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action - with: - java-version: | - 8 - 11 - name: run Java IOs PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -98,17 +94,6 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -Dfile.encoding=UTF-8 \ - # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 - - name: run Java8 IOs PreCommit script - uses: ./.github/actions/gradle-command-self-hosted-action - with: - gradle-command: :sdks:java:io:hcatalog:build - arguments: | - -PdisableSpotlessCheck=true \ - -PdisableCheckStyle=true \ - -Dfile.encoding=UTF-8 \ - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml b/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml index 9211203f4fe..f08d1bb3843 100644 --- a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml +++ b/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml @@ -87,19 +87,12 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action - with: - java-version: | - 8 - 11 - # TODO(https://github.com/apache/beam/issues/32207) Run test with Java11 - name: run sparkVersionsTest script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:spark:3:sparkVersionsTest arguments: | -PdisableSpotlessCheck=true \ - -PtestJavaVersion=8 \ - -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_SQL_Java17.yml b/.github/workflows/beam_PreCommit_SQL_Java17.yml index 8deefa08fa4..0507c17b0d6 100644 --- a/.github/workflows/beam_PreCommit_SQL_Java17.yml +++ b/.github/workflows/beam_PreCommit_SQL_Java17.yml @@ -78,14 +78,10 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) -# The test requires Java 17 and Java 8 versions. -# Java 8 is installed second because JAVA_HOME needs to point to Java 8. - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 8 + java-version: 17 python-version: default - name: Build and Test uses: ./.github/actions/gradle-command-self-hosted-action @@ -94,9 +90,8 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=17 \ + -PjavaVersion=17 \ -PskipCheckerFramework \ - -Pjava17Home=$JAVA_HOME_17_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_SQL_Java8.yml b/.github/workflows/beam_PreCommit_SQL_Java8.yml index 68b0669ce19..b8a64dafd1e 100644 --- a/.github/workflows/beam_PreCommit_SQL_Java8.yml +++ b/.github/workflows/beam_PreCommit_SQL_Java8.yml @@ -81,9 +81,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 8 - 11 + java-version: 8 python-version: default go-version: default - name: Install Flutter @@ -98,9 +96,8 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=8 \ + -PjavaVersion=8 \ -PskipCheckerFramework \ - -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 8aead7937cf..f1df00a4f15 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -119,7 +119,6 @@ jobs: with: gradle-command: :sdks:${{ matrix.container_task }}:docker arguments: | - -Pjava11Home=$JAVA_HOME_11_X64 \ -Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \ -Pdocker-tag-list=${{ github.sha }},${BEAM_VERSION},latest \ -Pcontainer-architecture-list=arm64,amd64 \ diff --git a/.github/workflows/local_env_tests.yml b/.github/workflows/local_env_tests.yml index 3983bfe7e7b..33795acd7ad 100644 --- a/.github/workflows/local_env_tests.yml +++ b/.github/workflows/local_env_tests.yml @@ -55,8 +55,10 @@ jobs: run: "sudo ./local-env-setup.sh" id: local_env_install_ubuntu - name: "Gradle check" - run: "./gradlew checkSetup" id: local_env_install_gradle_check_ubuntu + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: checkSetup run_local_env_install_mac: timeout-minutes: 50 name: "Mac run local environment shell script" @@ -72,5 +74,7 @@ jobs: run: "./local-env-setup.sh" id: local_env_install_mac - name: "Gradle check" - run: "./gradlew checkSetup" id: local_env_install_gradle_check_mac + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: checkSetup diff --git a/.test-infra/jenkins/CommonJobProperties.groovy b/.test-infra/jenkins/CommonJobProperties.groovy index 0f63b5de49f..4af1edcba5f 100644 --- a/.test-infra/jenkins/CommonJobProperties.groovy +++ b/.test-infra/jenkins/CommonJobProperties.groovy @@ -27,9 +27,6 @@ import PythonTestProperties as pythonTestProperties class CommonJobProperties { static String checkoutDir = 'src' - final static String JAVA_8_HOME = '/usr/lib/jvm/java-8-openjdk-amd64' - final static String JAVA_11_HOME = '/usr/lib/jvm/java-11-openjdk-amd64' - final static String JAVA_17_HOME = '/usr/lib/jvm/java-17-openjdk-amd64' final static String PYTHON = pythonTestProperties.DEFAULT_INTERPRETER // Sets common top-level job properties for main repository jobs. diff --git a/.test-infra/jenkins/NexmarkBuilder.groovy b/.test-infra/jenkins/NexmarkBuilder.groovy index 69fa3dcc427..25532e6a73c 100644 --- a/.test-infra/jenkins/NexmarkBuilder.groovy +++ b/.test-infra/jenkins/NexmarkBuilder.groovy @@ -124,6 +124,7 @@ class NexmarkBuilder { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) + switches("-PjavaVersion=8") switches("-Pnexmark.runner=${runner.getDependencyBySDK(sdk)}") switches("-Pnexmark.args=\"${parseOptions(options)}\"") if (jobSpecificSwitches != null) { @@ -145,8 +146,7 @@ class NexmarkBuilder { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches("-PtestJavaVersion=11") - switches("-Pjava11Home=${commonJobProperties.JAVA_11_HOME}") + switches("-PjavaVersion=11") switches("-Pnexmark.runner=${runner.getDependencyBySDK(sdk)}") switches("-Pnexmark.args=\"${parseOptions(options)}\"") if (jobSpecificSwitches != null) { @@ -168,8 +168,29 @@ class NexmarkBuilder { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:java:testing:nexmark:run') commonJobProperties.setGradleSwitches(delegate) - switches("-PtestJavaVersion=17") - switches("-Pjava17Home=${commonJobProperties.JAVA_17_HOME}") + switches("-PjavaVersion=17") + switches("-Pnexmark.runner=${runner.getDependencyBySDK(sdk)}") + switches("-Pnexmark.args=\"${parseOptions(options)}\"") + if (jobSpecificSwitches != null) { + jobSpecificSwitches.each { + switches(it) + } + } + } + } + } + + static void java21Suite(context, String title, Runner runner, SDK sdk, Map options, List jobSpecificSwitches) { + InfluxDBCredentialsHelper.useCredentials(context) + context.steps { + shell("echo \"*** RUN ${title} with Java 21***\"") + + // Run with Java 17 + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':sdks:java:testing:nexmark:run') + commonJobProperties.setGradleSwitches(delegate) + switches("-PjavaVersion=21") switches("-Pnexmark.runner=${runner.getDependencyBySDK(sdk)}") switches("-Pnexmark.args=\"${parseOptions(options)}\"") if (jobSpecificSwitches != null) { diff --git a/build.gradle.kts b/build.gradle.kts index 78536326817..2c1096f76cc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -390,6 +390,7 @@ tasks.register("sqlPreCommit") { tasks.register("javaPreCommitPortabilityApi") { dependsOn(":runners:google-cloud-dataflow-java:worker:build") + dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion${project.ext.get("javaVersion")}") } tasks.register("javaPostCommit") { @@ -572,6 +573,7 @@ tasks.register("javaExamplesDataflowPrecommit") { dependsOn(":runners:google-cloud-dataflow-java:examples:preCommit") dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit") dependsOn(":runners:google-cloud-dataflow-java:examplesJavaRunnerV2PreCommit") + dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion${project.ext.get("javaVersion")}") } tasks.register("whitespacePreCommit") { @@ -762,15 +764,3 @@ if (project.hasProperty("javaLinkageArtifactIds")) { } } } -if (project.hasProperty("testJavaVersion")) { - var testVer = project.property("testJavaVersion") - - tasks.getByName("javaPreCommitPortabilityApi").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") - tasks.getByName("javaExamplesDataflowPrecommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") -} else { - allprojects { - tasks.withType(Test::class).configureEach { - exclude("**/JvmVerification.class") - } - } -} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index a09ce2b3ec1..c93707f0af9 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -44,6 +44,7 @@ import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.javadoc.Javadoc import org.gradle.api.tasks.testing.Test import org.gradle.api.tasks.PathSensitivity +import org.gradle.jvm.toolchain.JavaLanguageVersion import org.gradle.testing.jacoco.tasks.JacocoReport @@ -181,13 +182,11 @@ class BeamModulePlugin implements Plugin { List mavenRepositories = [] /** - * Set minimal Java version needed to compile the module. - * - *

Valid values are LTS versions greater than the lowest supported - * Java version. Used when newer Java byte code version required than Beam's - * byte code compatibility version. + * The minimum supported language version to target for compilation. Only override when an + * optional project requires language features that are not present for the current default + * language version targeted by Beam since this setting also affects the class file version. */ - JavaVersion requireJavaVersion = null + int minimumLanguageVersion = 8 } /** A class defining the set of configurable properties accepted by applyPortabilityNature. */ @@ -454,53 +453,6 @@ class BeamModulePlugin implements Plugin { return 'beam' + p.path.replace(':', '-') } - static def getSupportedJavaVersion() { - if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - return 'java8' - } else if (JavaVersion.current() == JavaVersion.VERSION_11) { - return 'java11' - } else if (JavaVersion.current() == JavaVersion.VERSION_17) { - return 'java17' - } else if (JavaVersion.current() == JavaVersion.VERSION_21) { - return 'java21' - } else { - String exceptionMessage = "Your Java version is unsupported. You need Java version of 8, 11, 17 or 21 to get started, but your Java version is: " + JavaVersion.current(); - throw new GradleException(exceptionMessage) - } - } - - /* - * Set compile args for compiling and running in different java version by modifying the compiler args in place. - * - * Replace `-source X` and `-target X` or `--release X` options if already existed in compilerArgs. - */ - static def setCompileAndRuntimeJavaVersion(List compilerArgs, String ver) { - boolean foundS = false, foundT = false - int foundR = -1 - logger.fine("set java ver ${ver} to compiler args") - for (int i = 0; i < compilerArgs.size()-1; ++i) { - if (compilerArgs.get(i) == '-source') { - foundS = true - compilerArgs.set(i+1, ver) - } else if (compilerArgs.get(i) == '-target') { - foundT = true - compilerArgs.set(i+1, ver) - } else if (compilerArgs.get(i) == '--release') { - foundR = i - } - } - if (foundR != -1) { - compilerArgs.removeAt(foundR + 1) - compilerArgs.removeAt(foundR) - } - if (!foundS) { - compilerArgs.addAll('-source', ver) - } - if (!foundT) { - compilerArgs.addAll('-target', ver) - } - } - void apply(Project project) { /** ***********************************************************************************************/ @@ -552,8 +504,6 @@ class BeamModulePlugin implements Plugin { project.apply plugin: "com.dorongold.task-tree" project.taskTree { noRepeat = true } - project.ext.currentJavaVersion = getSupportedJavaVersion() - project.ext.allFlinkVersions = project.flink_versions.split(',') project.ext.latestFlinkVersion = project.ext.allFlinkVersions.last() @@ -947,64 +897,6 @@ class BeamModulePlugin implements Plugin { + suffix) } - def errorProneAddModuleOpts = [ - "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] - - // set compiler options for java version overrides to compile with a different java version - project.ext.setJavaVerOptions = { CompileOptions options, String ver -> - if (ver == '8') { - def java8Home = project.findProperty("java8Home") - options.fork = true - options.forkOptions.javaHome = java8Home as File - options.compilerArgs += ['-Xlint:-path'] - } else if (ver == '11') { - def java11Home = project.findProperty("java11Home") - options.fork = true - options.forkOptions.javaHome = java11Home as File - options.compilerArgs += ['-Xlint:-path'] - } else if (ver == '17') { - def java17Home = project.findProperty("java17Home") - options.fork = true - options.forkOptions.javaHome = java17Home as File - options.compilerArgs += ['-Xlint:-path'] - // Error prone requires some packages to be exported/opened for Java 17 - // Disabling checks since this property is only used for tests - // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support - options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - // The -J prefix is needed to workaround https://github.com/gradle/gradle/issues/22747 - options.forkOptions.jvmArgs += errorProneAddModuleOpts.collect { '-J' + it } - } else if (ver == '21') { - def java21Home = project.findProperty("java21Home") - options.fork = true - options.forkOptions.javaHome = java21Home as File - options.compilerArgs += [ - '-Xlint:-path', - '-Xlint:-this-escape' - ] - // Error prone requires some packages to be exported/opened for Java 17+ - // Disabling checks since this property is only used for tests - options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - options.forkOptions.jvmArgs += errorProneAddModuleOpts.collect { '-J' + it } - // TODO(https://github.com/apache/beam/issues/28963) - // upgrade checkerFramework to enable it in Java 21 - project.checkerFramework { - skipCheckerFramework = true - } - } else { - throw new GradleException("Unknown Java Version ${ver} for setting additional java options") - } - } - project.ext.repositories = { maven { name "testPublicationLocal" @@ -1096,8 +988,6 @@ class BeamModulePlugin implements Plugin { project.archivesBaseName = configuration.archivesBaseName } - project.apply plugin: "java" - // We create a testRuntimeMigration configuration here to extend // testImplementation, testRuntimeOnly, and default (similar to what // testRuntime did). @@ -1114,48 +1004,6 @@ class BeamModulePlugin implements Plugin { runtimeOnly.extendsFrom(provided) } - // Configure the Java compiler source language and target compatibility levels. Also ensure that - def requireJavaVersion = JavaVersion.toVersion(project.javaVersion) - if (configuration.requireJavaVersion != null) { - // Overwrite project.javaVersion if requested. - if (JavaVersion.VERSION_11.equals(configuration.requireJavaVersion)) { - project.javaVersion = '11' - } else if (JavaVersion.VERSION_17.equals(configuration.requireJavaVersion)) { - project.javaVersion = '17' - } else if (JavaVersion.VERSION_21.equals(configuration.requireJavaVersion)) { - project.javaVersion = '21' - } else { - throw new GradleException( - "requireJavaVersion has to be supported LTS version greater than the default Java version. Actual: " + - configuration.requireJavaVersion - ) - } - requireJavaVersion = configuration.requireJavaVersion - } - - String forkJavaVersion = null - if (requireJavaVersion.compareTo(JavaVersion.current()) > 0) { - // If compiled on older SDK, compile with JDK configured with compatible javaXXHome - // The order is intended here - if (requireJavaVersion.compareTo(JavaVersion.VERSION_11) <= 0 && - project.hasProperty('java11Home')) { - forkJavaVersion = '11' - } else if (requireJavaVersion.compareTo(JavaVersion.VERSION_17) <= 0 && - project.hasProperty('java17Home')) { - forkJavaVersion = '17' - } else if (requireJavaVersion.compareTo(JavaVersion.VERSION_21) <= 0 && - project.hasProperty('java21Home')) { - forkJavaVersion = '21' - } else { - logger.config("Module ${project.name} disabled. To enable, either " + - "compile on newer Java version or pass java${project.javaVersion}Home project property") - forkJavaVersion = '' - } - } - - project.sourceCompatibility = project.javaVersion - project.targetCompatibility = project.javaVersion - def defaultLintSuppressions = [ 'options', 'cast', @@ -1169,10 +1017,136 @@ class BeamModulePlugin implements Plugin { 'unchecked', 'varargs', ] - // Java21 introduced new lint "this-escape", violated by generated srcs - // TODO(yathu) remove this once generated code (antlr) no longer trigger this warning - if (JavaVersion.current().compareTo(JavaVersion.VERSION_21) >= 0) { - defaultLintSuppressions += ['this-escape'] + + // Set the toolchain language version + def toolchainLanguageVersion = JavaLanguageVersion.of(project.javaVersion) + if (!toolchainLanguageVersion.canCompileOrRun(configuration.minimumLanguageVersion)) { + def incompatibleLanguageVersion = toolchainLanguageVersion + toolchainLanguageVersion = JavaLanguageVersion.of(configuration.minimumLanguageVersion) + + logger.warning('Requested a toolchain for language version ' + toolchainLanguageVersion + ' because ' + + 'a toolchain for language version ' + incompatibleLanguageVersion + ' is incompatible with ' + + 'the minimum supported language version.') + toolchainLanguageVersion = JavaLanguageVersion.of(configuration.minimumLanguageVersion) + } + + project.apply plugin: "java" + + project.java { + toolchain { + languageVersion = toolchainLanguageVersion + } + } + + // Enable errorprone static analysis + project.apply plugin: 'net.ltgt.errorprone' + + project.dependencies { + errorprone "com.google.errorprone:error_prone_core:$errorprone_version:with-dependencies" + errorprone "jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.2" + } + + project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version:with-dependencies" } + + project.tasks.withType(JavaCompile).configureEach { + // Ensure that we configure the Java compiler to use UTF-8. + options.encoding = "UTF-8" + // Configure the Java compiler source language and target compatibility levels. + sourceCompatibility = targetCompatibility = JavaVersion.toVersion(configuration.minimumLanguageVersion) + + // As we want to add '-Xlint:-deprecation' we intentionally remove '-Xlint:deprecation' from compilerArgs here, + // as intellij is adding this, see https://youtrack.jetbrains.com/issue/IDEA-196615 + options.compilerArgs -= [ + "-Xlint:deprecation", + ] + options.compilerArgs += [ + '-parameters', + '-Xlint:all', + '-Werror' + ] + options.compilerArgs += defaultLintSuppressions.collect { "-Xlint:-${it}" } + options.compilerArgs += configuration.disableLintWarnings.collect { "-Xlint:-${it}" } + + options.errorprone { + disableWarningsInGeneratedCode = true + excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' + + // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks + disable 'AutoValueImmutableFields', + 'AutoValueSubclassLeaked', + 'BadImport', + 'BadInstanceof', + 'BigDecimalEquals', + 'ComparableType', + 'DoNotMockAutoValue', + 'EmptyBlockTag', + 'EmptyCatch', + 'EqualsGetClass', + 'EqualsUnsafeCast', + 'EscapedEntity', + 'ExtendsAutoValue', + 'InlineFormatString', + 'InlineMeSuggester', + 'InvalidBlockTag', + 'InvalidInlineTag', + 'InvalidLink', + 'InvalidParam', + 'InvalidThrows', + 'JavaTimeDefaultTimeZone', + 'JavaUtilDate', + 'JodaConstructors', + 'MalformedInlineTag', + 'MissingSummary', + 'MixedMutabilityReturnType', + 'PreferJavaTimeOverload', + 'MutablePublicArray', + 'NonCanonicalType', + 'ProtectedMembersInFinalClass', + 'Slf4jFormatShouldBeConst', + 'Slf4jSignOnlyFormat', + 'StaticAssignmentInConstructor', + 'ThreadPriorityCheck', + 'TimeUnitConversionChecker', + 'UndefinedEquals', + 'UnescapedEntity', + 'UnnecessaryLambda', + 'UnnecessaryMethodReference', + 'UnnecessaryParentheses', + 'UnrecognisedJavadocTag', + 'UnsafeReflectiveConstructionCast', + 'UseCorrectAssertInTests', + // Sometimes a static logger is preferred, which is the convention + // currently used in beam. See docs: + // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static + 'Slf4jLoggerShouldBeNonStatic' + } + } + + if (toolchainLanguageVersion.canCompileOrRun(9)) { + // Use -release 8 when targeting Java 8 and running on JDK > 8 + // + // Consider migrating compilation and testing to use JDK 9+ and setting '--release 8' as + // the default allowing 'applyJavaNature' to override it for the few modules that need JDK 9+ + // artifacts. See https://stackoverflow.com/a/43103038/4368200 for additional details. + project.tasks.withType(JavaCompile).configureEach { + // TODO(https://github.com/apache/beam/issues/23901): Fix + // optimizerOuterThis breakage + options.compilerArgs << '-XDoptimizeOuterThis=false' + // Takes precedence over sourceCompatibility and targetCompatibility + options.release = configuration.minimumLanguageVersion + } + } + + if (toolchainLanguageVersion.canCompileOrRun(21)) { + project.tasks.withType(JavaCompile).configureEach { + // Java21 introduced new lint "this-escape", violated by generated srcs + // TODO(yathu) remove this once generated code (antlr) no longer trigger this warning + options.compilerArgs << '-Xlint:-this-escape' + } + } + + project.tasks.withType(Jar).configureEach { + preserveFileTimestamps(false) } // Configure the default test tasks set of tests executed @@ -1195,6 +1169,16 @@ class BeamModulePlugin implements Plugin { maxHeapSize = '2g' } + if (toolchainLanguageVersion.canCompileOrRun(17)) { + project.tasks.withType(Test).configureEach { + jvmArgs += [ + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + ] + } + } + List skipDefRegexes = [] skipDefRegexes << "AutoValue_.*" skipDefRegexes << "AutoOneOf_.*" @@ -1217,8 +1201,7 @@ class BeamModulePlugin implements Plugin { ] // Only skip checkerframework if explicitly requested - skipCheckerFramework = project.hasProperty('enableCheckerFramework') && - !parseBooleanProperty(project, 'enableCheckerFramework') + skipCheckerFramework = !parseBooleanProperty(project, 'enableCheckerFramework') // Always exclude checkerframework on tests. It's slow, and it often // raises erroneous error because we don't have checker annotations for @@ -1490,148 +1473,6 @@ class BeamModulePlugin implements Plugin { project.tasks.analyzeDependencies.enabled = false } - // errorprone requires java9+ compiler. It can be used with Java8 but then sets a java9+ errorproneJavac. - // However, the redirect ignores any task that forks and defines either a javaHome or an executable, - // see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support - // which means errorprone cannot run when gradle runs on Java11+ but serve `-testJavaVersion=8 -Pjava8Home` options - if (!(project.findProperty('testJavaVersion') == '8')) { - // Enable errorprone static analysis - project.apply plugin: 'net.ltgt.errorprone' - - project.dependencies { - errorprone("com.google.errorprone:error_prone_core:$errorprone_version") - errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.2") - // At least JDk 9 compiler is required, however JDK 8 still can be used but with additional errorproneJavac - // configuration. For more details please see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) == 0) { - errorproneJavac("com.google.errorprone:javac:9+181-r4173-1") - } - } - - project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } - - project.tasks.withType(JavaCompile) { - options.errorprone.disableWarningsInGeneratedCode = true - options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' - - // Error Prone requires some packages to be exported/opened on Java versions that support modules, - // i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403. - // The -J prefix is not needed if forkOptions.javaHome is unset, - // see http://github.com/gradle/gradle/issues/22747 - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 - && options.forkOptions.javaHome == null) { - options.fork = true - options.forkOptions.jvmArgs += errorProneAddModuleOpts - } - - // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks - options.errorprone.errorproneArgs.add("-Xep:AutoValueImmutableFields:OFF") - options.errorprone.errorproneArgs.add("-Xep:AutoValueSubclassLeaked:OFF") - options.errorprone.errorproneArgs.add("-Xep:BadImport:OFF") - options.errorprone.errorproneArgs.add("-Xep:BadInstanceof:OFF") - options.errorprone.errorproneArgs.add("-Xep:BigDecimalEquals:OFF") - options.errorprone.errorproneArgs.add("-Xep:ComparableType:OFF") - options.errorprone.errorproneArgs.add("-Xep:DoNotMockAutoValue:OFF") - options.errorprone.errorproneArgs.add("-Xep:EmptyBlockTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:EmptyCatch:OFF") - options.errorprone.errorproneArgs.add("-Xep:EqualsGetClass:OFF") - options.errorprone.errorproneArgs.add("-Xep:EqualsUnsafeCast:OFF") - options.errorprone.errorproneArgs.add("-Xep:EscapedEntity:OFF") - options.errorprone.errorproneArgs.add("-Xep:ExtendsAutoValue:OFF") - options.errorprone.errorproneArgs.add("-Xep:InlineFormatString:OFF") - options.errorprone.errorproneArgs.add("-Xep:InlineMeSuggester:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidBlockTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidInlineTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidLink:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidParam:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidThrows:OFF") - options.errorprone.errorproneArgs.add("-Xep:JavaTimeDefaultTimeZone:OFF") - options.errorprone.errorproneArgs.add("-Xep:JavaUtilDate:OFF") - options.errorprone.errorproneArgs.add("-Xep:JodaConstructors:OFF") - options.errorprone.errorproneArgs.add("-Xep:MalformedInlineTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:MissingSummary:OFF") - options.errorprone.errorproneArgs.add("-Xep:MixedMutabilityReturnType:OFF") - options.errorprone.errorproneArgs.add("-Xep:PreferJavaTimeOverload:OFF") - options.errorprone.errorproneArgs.add("-Xep:MutablePublicArray:OFF") - options.errorprone.errorproneArgs.add("-Xep:NonCanonicalType:OFF") - options.errorprone.errorproneArgs.add("-Xep:ProtectedMembersInFinalClass:OFF") - options.errorprone.errorproneArgs.add("-Xep:Slf4jFormatShouldBeConst:OFF") - options.errorprone.errorproneArgs.add("-Xep:Slf4jSignOnlyFormat:OFF") - options.errorprone.errorproneArgs.add("-Xep:StaticAssignmentInConstructor:OFF") - options.errorprone.errorproneArgs.add("-Xep:ThreadPriorityCheck:OFF") - options.errorprone.errorproneArgs.add("-Xep:TimeUnitConversionChecker:OFF") - options.errorprone.errorproneArgs.add("-Xep:UndefinedEquals:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnescapedEntity:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnnecessaryLambda:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnnecessaryMethodReference:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnnecessaryParentheses:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnrecognisedJavadocTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnsafeReflectiveConstructionCast:OFF") - options.errorprone.errorproneArgs.add("-Xep:UseCorrectAssertInTests:OFF") - - // Sometimes a static logger is preferred, which is the convention - // currently used in beam. See docs: - // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static - options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") - } - } - - // Handle compile Java versions - project.tasks.withType(JavaCompile).configureEach { - // we configure the Java compiler to use UTF-8. - options.encoding = "UTF-8" - // If compiled on newer JDK, set byte code compatibility - if (requireJavaVersion.compareTo(JavaVersion.current()) < 0) { - def compatVersion = project.javaVersion == '1.8' ? '8' : project.javaVersion - options.compilerArgs += ['--release', compatVersion] - // TODO(https://github.com/apache/beam/issues/23901): Fix - // optimizerOuterThis breakage - options.compilerArgs += ['-XDoptimizeOuterThis=false'] - } else if (forkJavaVersion) { - // If compiled on older SDK, compile with JDK configured with compatible javaXXHome - setCompileAndRuntimeJavaVersion(options.compilerArgs, requireJavaVersion as String) - project.ext.setJavaVerOptions(options, forkJavaVersion) - } - // As we want to add '-Xlint:-deprecation' we intentionally remove '-Xlint:deprecation' from compilerArgs here, - // as intellij is adding this, see https://youtrack.jetbrains.com/issue/IDEA-196615 - options.compilerArgs -= [ - "-Xlint:deprecation", - ] - options.compilerArgs += ([ - '-parameters', - '-Xlint:all', - '-Werror' - ] - + (defaultLintSuppressions + configuration.disableLintWarnings).collect { "-Xlint:-${it}" }) - } - - if (forkJavaVersion) { - project.tasks.withType(Javadoc) { - executable = project.findProperty('java' + forkJavaVersion + 'Home') + '/bin/javadoc' - } - } - - project.tasks.withType(Jar).configureEach { - preserveFileTimestamps(false) - } - - // if specified test java version, modify the compile and runtime versions accordingly - if (['8', '11', '17', '21'].contains(project.findProperty('testJavaVersion'))) { - String ver = project.getProperty('testJavaVersion') - def testJavaHome = project.getProperty("java${ver}Home") - - // redirect java compiler to specified version for compileTestJava only - project.tasks.compileTestJava { - setCompileAndRuntimeJavaVersion(options.compilerArgs, ver) - project.ext.setJavaVerOptions(options, ver) - } - // redirect java runtime to specified version for running tests - project.tasks.withType(Test).configureEach { - useJUnit() - executable = "${testJavaHome}/bin/java" - } - } - if (configuration.shadowClosure) { // Enables a plugin which can perform shading of classes. See the general comments // above about dependency management for Java projects and how the shadow plugin @@ -1845,21 +1686,7 @@ class BeamModulePlugin implements Plugin { project.ext.includeInJavaBom = configuration.publish project.ext.exportJavadoc = configuration.exportJavadoc - boolean publishEnabledByCommand = isRelease(project) || project.hasProperty('publishing') - if (forkJavaVersion == '') { - // project needs newer version and not served. - // If not publishing ,disable the project. Otherwise, fail the build - def msg = "project ${project.name} needs newer Java version to compile. Consider set -Pjava${project.javaVersion}Home" - if (publishEnabledByCommand) { - throw new GradleException("Publish enabled but " + msg + ".") - } else { - logger.config(msg + " if needed.") - project.tasks.each { - it.enabled = false - } - } - } - if (publishEnabledByCommand && configuration.publish) { + if ((isRelease(project) || project.hasProperty('publishing')) && configuration.publish) { project.apply plugin: "maven-publish" // plugin to support repository authentication via ~/.m2/settings.xml @@ -2652,9 +2479,7 @@ class BeamModulePlugin implements Plugin { // see https://issues.apache.org/jira/browse/BEAM-6698 maxHeapSize = '4g' if (config.environment == PortableValidatesRunnerConfiguration.Environment.DOCKER) { - def ver = project.findProperty('testJavaVersion') - def javaContainerSuffix = ver ? "java$ver" : getSupportedJavaVersion() - dependsOn ":sdks:java:container:${javaContainerSuffix}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" } } } @@ -2679,7 +2504,6 @@ class BeamModulePlugin implements Plugin { def pythonDir = project.project(":sdks:python").projectDir def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner") - def javaContainerSuffix = getSupportedJavaVersion() // Sets up, collects, and runs Python pipeline tests project.tasks.register(config.name+"PythonUsingJava") { @@ -2690,7 +2514,7 @@ class BeamModulePlugin implements Plugin { for (path in config.expansionProjectPaths) { dependsOn project.project(path).shadowJar.getPath() } - dependsOn ":sdks:java:container:$javaContainerSuffix:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn "installGcpTest" if (usesDataflowRunner) { dependsOn ":sdks:python:test-suites:dataflow:py${project.ext.pythonVersion.replace('.', '')}:initializeForDataflowJob" @@ -2759,9 +2583,8 @@ class BeamModulePlugin implements Plugin { ] def serviceArgs = project.project(':sdks:python').mapToArgString(expansionServiceOpts) def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '') - def javaContainerSuffix = getSupportedJavaVersion() def setupTask = project.tasks.register(config.name+"Setup", Exec) { - dependsOn ':sdks:java:container:'+javaContainerSuffix+':docker' + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn ':sdks:python:container:py'+pythonContainerSuffix+':docker' dependsOn ':sdks:java:testing:expansion-service:buildTestExpansionServiceJar' dependsOn ":sdks:python:installGcpTest" @@ -2932,7 +2755,6 @@ class BeamModulePlugin implements Plugin { ] def serviceArgs = project.project(':sdks:python').mapToArgString(transformServiceOpts) def pythonContainerSuffix = project.project(':sdks:python').pythonVersion.replace('.', '') - def javaContainerSuffix = getSupportedJavaVersion() // Transform service delivers transforms that refer to SDK harness containers with following sufixes. def transformServiceJavaContainerSuffix = 'java11' @@ -2940,7 +2762,7 @@ class BeamModulePlugin implements Plugin { def setupTask = project.tasks.register(config.name+"Setup", Exec) { // Containers for main SDKs when running tests. - dependsOn ':sdks:java:container:'+javaContainerSuffix+':docker' + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn ':sdks:python:container:py'+pythonContainerSuffix+':docker' // Containers for external SDKs used through the transform service. dependsOn ':sdks:java:container:'+transformServiceJavaContainerSuffix+':docker' diff --git a/examples/java/build.gradle b/examples/java/build.gradle index e10d4fd6b1d..b2444b10cc1 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -54,7 +54,7 @@ configurations.sparkRunnerPreCommit { dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation library.java.vendored_guava_32_1_2_jre - if (project.findProperty('testJavaVersion') == '21' || JavaVersion.current().compareTo(JavaVersion.VERSION_21) >= 0) { + if (JavaLanguageVersion.of(project.javaVersion).canCompileOrRun(21)) { // this dependency is a provided dependency for kafka-avro-serializer. It is not needed to compile with Java<=17 // but needed for compile only under Java21, specifically, required for extending from AbstractKafkaAvroDeserializer compileOnly library.java.kafka diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 829aefd4474..44ca5391e8e 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -17,6 +17,7 @@ */ import groovy.json.JsonOutput +import org.gradle.jvm.toolchain.JavaLanguageVersion plugins { id 'org.apache.beam.module' id 'org.jetbrains.kotlin.jvm' version '1.6.10' @@ -123,16 +124,12 @@ task preCommit() { } } -compileKotlin { - kotlinOptions { - jvmTarget = project.javaVersion - } -} -compileTestKotlin { - kotlinOptions { - jvmTarget = project.javaVersion - } +kotlin { + jvmToolchain { + languageVersion = JavaLanguageVersion.of(project.javaVersion) + } } + repositories { mavenCentral() } diff --git a/examples/multi-language/README.md b/examples/multi-language/README.md index f9905ca310e..9028ae9efbe 100644 --- a/examples/multi-language/README.md +++ b/examples/multi-language/README.md @@ -156,7 +156,9 @@ export DOCKER_ROOT= docker push $DOCKER_ROOT/beam_python3.9_sdk:latest -./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=$DOCKER_ROOT -Pdocker-tag=latest -Pjava11Home=$JAVA_HOME +./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=$DOCKER_ROOT -Pdocker-tag=latest \ +-Porg.gradle.java.installations.auto-detect=false \ +-Porg.gradle.java.installations.paths=$JAVA_HOME docker push $DOCKER_ROOT/beam_java11_sdk:latest ``` diff --git a/gradle.properties b/gradle.properties index 5aa95bcf437..319bb6acd55 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,6 +19,8 @@ org.gradle.caching=true org.gradle.parallel=true org.gradle.daemon=false org.gradle.configureondemand=true +org.gradle.java.installations.auto-detect=true +org.gradle.java.installations.auto-download=false org.gradle.jvmargs=-Xss10240k org.gradle.vfs.watch=true offlineRepositoryRoot=offline-repository @@ -33,7 +35,7 @@ signing.gnupg.useLegacyGpg=true version=2.66.0-SNAPSHOT sdk_version=2.66.0.dev -javaVersion=1.8 +javaVersion=8 docker_image_default_repo_root=apache docker_image_default_repo_prefix=beam_ diff --git a/runners/google-cloud-dataflow-java/arm/build.gradle b/runners/google-cloud-dataflow-java/arm/build.gradle index ae63cdf8bdb..dbb8ec3e043 100644 --- a/runners/google-cloud-dataflow-java/arm/build.gradle +++ b/runners/google-cloud-dataflow-java/arm/build.gradle @@ -72,7 +72,7 @@ configurations { examplesJavaIntegrationTest } dependencies { examplesJavaIntegrationTest project(project.path) // TODO(yathu) Include full test classpath once gradle shadow plugin support Java21 - if (project.findProperty('testJavaVersion') == '21' || JavaVersion.current().equals(JavaVersion.VERSION_21)) { + if (JavaLanguageVersion.of(project.javaVersion).canCompileOrRun(21)) { examplesJavaIntegrationTest project(path: ":runners:google-cloud-dataflow-java") } else { examplesJavaIntegrationTest project(path: ":runners:google-cloud-dataflow-java", configuration: "testRuntimeMigration") @@ -80,16 +80,12 @@ dependencies { examplesJavaIntegrationTest project(path: ":examples:java", configuration: "testRuntimeMigration") } -def javaVer = "java8" -if (project.hasProperty('testJavaVersion')) { - javaVer = "java${project.getProperty('testJavaVersion')}" -} def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' def gcpRegion = project.findProperty('gcpRegion') ?: 'us-central1' def dataflowValidatesTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests' def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${gcpProject}/java-postcommit-it" -def DockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}${javaVer}_sdk" +def DockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${project.javaVersion}_sdk" def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') ext.DockerJavaMultiarchImageName = "${DockerJavaMultiarchImageContainer}:${dockerTag}" as String @@ -113,7 +109,7 @@ def runnerV2PipelineOptionsARM = [ // than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs // daily. def buildAndPushDockerJavaMultiarchContainer = tasks.register("buildAndPushDockerJavaMultiarchContainer") { - dependsOn ":sdks:java:container:${javaVer}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" } task printrunnerV2PipelineOptionsARM { diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index d4ce1aedf31..674e427c041 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -16,7 +16,6 @@ * limitations under the License. */ -import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion import groovy.json.JsonOutput @@ -282,10 +281,9 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> tasks.register('examplesJavaRunnerV2IntegrationTestDistroless', Test.class) { group = "verification" dependsOn 'buildAndPushDistrolessContainerImage' - def javaVer = project.findProperty('testJavaVersion') def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}" def tag = project.findProperty('dockerTag') - def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}" + def imageURL = "${repository}/beam_java${project.javaVersion}_sdk_distroless:${tag}" def pipelineOptions = [ "--runner=TestDataflowRunner", "--project=${gcpProject}", @@ -306,20 +304,19 @@ tasks.register('examplesJavaRunnerV2IntegrationTestDistroless', Test.class) { } tasks.register('buildAndPushDistrolessContainerImage', Task.class) { - // Only Java 17 and 21 are supported. - // See https://github.com/GoogleContainerTools/distroless/tree/main/java#image-contents. - def allowed = ["java17", "java21"] + onlyIf('javaVersion is a supported LTS release for distroless') { + // Only Java 17 and 21 are supported. + // See https://github.com/GoogleContainerTools/distroless/tree/main/java#image-contents. + JavaLanguageVersion.of(project.javaVersion) in [17, 21] + } + doLast { - def javaVer = project.findProperty('testJavaVersion') - if (!allowed.contains(javaVer)) { - throw new GradleException("testJavaVersion must be one of ${allowed}, got: ${javaVer}") - } if (!project.hasProperty('dockerTag')) { throw new GradleException("dockerTag is missing but required") } def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}" def tag = project.findProperty('dockerTag') - def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}" + def imageURL = "${repository}/beam_java${project.javaVersion}_sdk_distroless:${tag}" exec { executable 'docker' workingDir rootDir @@ -330,8 +327,8 @@ tasks.register('buildAndPushDistrolessContainerImage', Task.class) { imageURL, '-f', 'sdks/java/container/Dockerfile-distroless', - "--build-arg=BEAM_BASE=gcr.io/apache-beam-testing/beam-sdk/beam_${javaVer}_sdk", - "--build-arg=DISTROLESS_BASE=gcr.io/distroless/${javaVer}-debian12", + "--build-arg=BEAM_BASE=gcr.io/apache-beam-testing/beam-sdk/beam_java${project.javaVersion}_sdk", + "--build-arg=DISTROLESS_BASE=gcr.io/distroless/java${project.javaVersion}-debian12", '.' ] } @@ -347,13 +344,9 @@ tasks.register('buildAndPushDistrolessContainerImage', Task.class) { // task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { - def javaVer = getSupportedJavaVersion() - if(project.hasProperty('testJavaVersion')) { - javaVer = "java${project.getProperty('testJavaVersion')}" - } - dependsOn ":sdks:java:container:${javaVer}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" def defaultDockerImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk", + name: "${project.docker_image_default_repo_prefix}java${project.javaVersion}_sdk", root: "apache", tag: project.sdk_version) doLast { @@ -728,9 +721,7 @@ task examplesJavaRunnerV2PreCommit(type: Test) { task examplesJavaRunnerV2IntegrationTest(type: Test) { group = "Verification" dependsOn buildAndPushDockerJavaContainer - if (project.hasProperty("testJavaVersion")) { - dependsOn ":sdks:java:testing:test-utils:verifyJavaVersion${project.property("testJavaVersion")}" - } + dependsOn ":sdks:java:testing:test-utils:verifyJavaVersion${project.javaVersion}" systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions) @@ -840,13 +831,9 @@ task GCSUpload(type: JavaExec) { } def buildAndPushDistrolessDockerJavaContainer = tasks.register("buildAndPushDistrolessDockerJavaContainer") { - def javaVer = getSupportedJavaVersion() - if(project.hasProperty('testJavaVersion')) { - javaVer = "java${project.getProperty('testJavaVersion')}" - } - dependsOn ":sdks:java:container:distroless:${javaVer}:docker" + dependsOn ":sdks:java:container:distroless:java${project.javaVersion}:docker" def defaultDockerImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk_distroless", + name: "${project.docker_image_default_repo_prefix}java${project.javaVersion}_sdk_distroless", root: "apache", tag: project.sdk_version) doLast { diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle index f0898fefc88..90b2014d078 100644 --- a/runners/google-cloud-dataflow-java/examples/build.gradle +++ b/runners/google-cloud-dataflow-java/examples/build.gradle @@ -148,9 +148,12 @@ task javaPostCommit() { } task postCommitLegacyWorkerJava8(type: Test) { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(8) + } + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - systemProperty "java.specification.version", "8" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } @@ -158,10 +161,27 @@ task java8PostCommit() { dependsOn postCommitLegacyWorkerJava8 } +task postCommitLegacyWorkerJava11(type: Test) { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(11) + } + + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') +} + +task java11PostCommit() { + dependsOn postCommitLegacyWorkerJava11 +} + task postCommitLegacyWorkerJava17(type: Test) { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(17) + } + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - systemProperty "java.specification.version", "17" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } @@ -170,9 +190,12 @@ task java17PostCommit() { } task postCommitLegacyWorkerJava21(type: Test) { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(21) + } + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - systemProperty "java.specification.version", "21" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'exclude') } @@ -183,9 +206,7 @@ task java21PostCommit() { task preCommit() { dependsOn preCommitLegacyWorker dependsOn preCommitLegacyWorkerImpersonate - if (project.hasProperty("testJavaVersion")) { - dependsOn ":sdks:java:testing:test-utils:verifyJavaVersion${project.property("testJavaVersion")}" - } + dependsOn ":sdks:java:testing:test-utils:verifyJavaVersion${project.javaVersion}" } task verifyPortabilityApi() { diff --git a/runners/portability/java/build.gradle b/runners/portability/java/build.gradle index 6e3b431e802..01a18501943 100644 --- a/runners/portability/java/build.gradle +++ b/runners/portability/java/build.gradle @@ -254,7 +254,7 @@ tasks.register("validatesRunnerSickbay", Test) { } task ulrDockerValidatesRunner { - dependsOn createUlrValidatesRunnerTask("ulrDockerValidatesRunnerTests", "DOCKER", ":sdks:java:container:${project.ext.currentJavaVersion}:docker") + dependsOn createUlrValidatesRunnerTask("ulrDockerValidatesRunnerTests", "DOCKER", ":sdks:java:container:java${project.javaVersion}:docker") } task ulrLoopbackValidatesRunner { diff --git a/runners/prism/java/build.gradle b/runners/prism/java/build.gradle index e75fda999e1..d3e8d4251f1 100644 --- a/runners/prism/java/build.gradle +++ b/runners/prism/java/build.gradle @@ -235,7 +235,7 @@ tasks.register("validatesRunnerSickbay", Test) { task prismDockerValidatesRunner { Task vrTask = createPrismValidatesRunnerTask("prismDockerValidatesRunnerTests", "DOCKER") - vrTask.dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker" + vrTask.dependsOn ":sdks:java:container:java${project.javaVersion}:docker" } task prismLoopbackValidatesRunner { diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index 037d46a31ed..2b04c7afb3c 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -65,9 +65,8 @@ def sparkTestProperties(overrides = [:]) { def sparkTestJvmArgs() { - // run tests with Java 17 using -PtestJavaVersion=17 -Pjava17Home=??? - if (project.hasProperty('testJavaVersion') && - project.getProperty('testJavaVersion') in ['17', '21']) { + // run tests with Java 17 using -PjavaVersion=17 + if (JavaLanguageVersion.of(project.javaVersion).canCompileOrRun(17)) { return [ "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", // add-opens below required for Kryo FieldSerializer / SparkRunnerKryoRegistratorTest diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle index 7a4c0a63aa4..47ead458079 100644 --- a/sdks/go/test/build.gradle +++ b/sdks/go/test/build.gradle @@ -69,7 +69,7 @@ task flinkValidatesRunner { dependsOn ":sdks:go:test:goBuild" dependsOn ":sdks:go:container:docker" - dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar" dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { @@ -93,7 +93,7 @@ task flinkValidatesRunner { task samzaValidatesRunner { dependsOn ":sdks:go:test:goBuild" dependsOn ":sdks:go:container:docker" - dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn ":runners:samza:job-server:shadowJar" dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { @@ -118,7 +118,7 @@ task sparkValidatesRunner { group = "Verification" dependsOn ":sdks:go:test:goBuild" - dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn ":runners:spark:3:job-server:shadowJar" dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { @@ -149,7 +149,7 @@ tasks.register("ulrValidatesRunner") { dependsOn ":sdks:go:test:goBuild" dependsOn ":sdks:go:container:docker" - dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn "setupVirtualenv" dependsOn ":sdks:python:buildPython" dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" @@ -180,7 +180,7 @@ task prismValidatesRunner { dependsOn ":sdks:go:test:goBuild" dependsOn ":sdks:go:container:docker" - dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" doLast { def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 60dd0cd97f1..c9474097d40 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -389,7 +389,11 @@ if [[ "$RUNNER" == "dataflow" ]]; then JAVA_TAG=$(date +%Y%m%d-%H%M%S) JAVA_CONTAINER=us.gcr.io/$PROJECT/$USER/beam_java11_sdk echo "Using container $JAVA_CONTAINER for cross-language java transforms" - ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG -Pjava11Home=$JAVA11_HOME + ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG \ + -PjavaVersion=11 \ + -Porg.gradle.java.installations.auto-detect=false \ + -Porg.gradle.java.installations.auto-download=false \ + -Porg.gradle.java.installations.fromEnv=JAVA11_HOME # Verify it exists docker images | grep $JAVA_TAG diff --git a/sdks/java/container/agent/build.gradle b/sdks/java/container/agent/build.gradle index dff97980675..8d11ed7e8d7 100644 --- a/sdks/java/container/agent/build.gradle +++ b/sdks/java/container/agent/build.gradle @@ -23,7 +23,7 @@ plugins { applyJavaNature( exportJavadoc: false, publish: false, - requireJavaVersion: JavaVersion.VERSION_11 + minimumLanguageVersion: 11 ) description = "Apache Beam :: SDKs :: Java :: Container :: Agent" diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index acb6b79b346..7d7c9762cde 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -97,33 +97,6 @@ task skipPullLicenses(type: Exec) { args "-c", "mkdir -p build/target/go-licenses build/target/options build/target/third_party_licenses && touch build/target/go-licenses/skip && touch build/target/third_party_licenses/skip" } -// Java11+ container depends on the java agent project. To compile it, need a compatible JDK version: -// lower bound 11 and upper bound imageJavaVersion -task validateJavaHome { - def requiredForVer = ["11", "17", "21"] - if (requiredForVer.contains(imageJavaVersion)) { - doFirst { - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0) { - return - } - boolean propertyFound = false - // enable to build agent with compatible java versions (11-requiredForVer) - for (def checkVer : requiredForVer) { - if (project.hasProperty("java${checkVer}Home")) { - propertyFound = true - } - if (checkVer == imageJavaVersion) { - // cannot build agent with a higher version than the docker java ver - break - } - } - if (!propertyFound) { - throw new GradleException("System Java needs to have version 11+ or java${imageJavaVersion}Home required for imageJavaVersion=${imageJavaVersion}. Re-run with -Pjava${imageJavaVersion}Home") - } - } - } -} - def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") docker { @@ -162,4 +135,3 @@ if (project.rootProject.hasProperty("docker-pull-licenses") || dockerPrepare.dependsOn copySdkHarnessLauncher dockerPrepare.dependsOn copyDockerfileDependencies dockerPrepare.dependsOn copyJdkOptions -dockerPrepare.dependsOn validateJavaHome diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 6f34891c2d3..ca67573b3fd 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -244,11 +244,7 @@ task preCommit { dependsOn build dependsOn runBasicExample dependsOn runPojoExample - - if (project.hasProperty("testJavaVersion")) { - var testVer = project.property("testJavaVersion") - dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") - } + dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion${project.javaVersion}") } task postCommit { diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index d07904f3465..0037000e2f2 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -106,13 +106,10 @@ hadoopVersions.each { kv -> } } -project.tasks.withType(Test).configureEach { - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 && project.findProperty('testJavaVersion') != '8') { - useJUnit { - filter { - excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogIOTest" - excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogBeamSchemaTest" - } +tasks.withType(Test).configureEach { + // TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 + // Also change .github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(8) } - } } diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index e4d148cf556..6579d2642ae 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -1,4 +1,5 @@ import groovy.json.JsonOutput +import org.gradle.jvm.toolchain.JavaLanguageVersion import java.util.stream.Collectors @@ -130,9 +131,12 @@ hadoopVersions.each { kv -> def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' def gcpTempLocation = project.findProperty('gcpTempLocation') ?: 'gs://managed-iceberg-integration-tests/temp' -def usingJava8 = (project.findProperty('testJavaVersion') == '8' || JavaVersion.current().equals(JavaVersion.VERSION_1_8)) task integrationTest(type: Test) { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(11) + } + group = "Verification" systemProperty "beamTestPipelineOptions", JsonOutput.toJson([ "--project=${gcpProject}", @@ -143,10 +147,6 @@ task integrationTest(type: Test) { outputs.upToDateWhen { false } include '**/*IT.class' - // BQ metastore catalog doesn't support java 8 - if (usingJava8) { - exclude '**/BigQueryMetastoreCatalogIT.class' - } maxParallelForks 1 classpath = sourceSets.test.runtimeClasspath @@ -154,6 +154,10 @@ task integrationTest(type: Test) { } task dataflowIntegrationTest(type: Test) { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(11) + } + group = "Verification" def args = [ "--runner=DataflowRunner", @@ -180,12 +184,6 @@ task dataflowIntegrationTest(type: Test) { includeTestsMatching 'org.apache.beam.sdk.io.iceberg.catalog.BigQueryMetastoreCatalogIT.testStreamToPartitionedDynamicDestinations' } - doLast { - if (usingJava8) { - throw new StopExecutionException("BigQueryMetastoreCatalog doesn't support Java 8"); - } - } - maxParallelForks 4 classpath = sourceSets.test.runtimeClasspath testClassesDirs = sourceSets.test.output.classesDirs diff --git a/sdks/java/testing/jpms-tests/build.gradle b/sdks/java/testing/jpms-tests/build.gradle index ac2050df7d9..01dd9f061de 100644 --- a/sdks/java/testing/jpms-tests/build.gradle +++ b/sdks/java/testing/jpms-tests/build.gradle @@ -22,17 +22,11 @@ plugins { id 'org.apache.beam.module' } -// overwrite javaVersion before applyJavaNature -if (project.hasProperty("testJavaVersion")) { - javaVersion = "1.${project.getProperty('testJavaVersion')}" as String -} else { - javaVersion = "1.11" -} - applyJavaNature( exportJavadoc: false, publish: false, - disableLintWarnings: ['requires-transitive-automatic', 'requires-automatic'] + disableLintWarnings: ['requires-transitive-automatic', 'requires-automatic'], + minimumLanguageVersion: 11 ) provideIntegrationTestingDependencies() enableJavaPerformanceTesting() @@ -40,19 +34,6 @@ enableJavaPerformanceTesting() description = "Apache Beam :: SDKs :: Java :: Testing :: JPMS Tests" ext.summary = "E2E test for Java 9 modules" -// direct compileJava to use specified java version. -project.tasks.compileJava { - if (project.hasProperty('testJavaVersion')) { - options.fork = true - options.forkOptions.javaHome = project.findProperty("java${project.getProperty('testJavaVersion')}Home") as File - if (project.getProperty('testJavaVersion') == '17') { - setJavaVerOptions(options, '17') - } else if (project.getProperty('testJavaVersion') == '21') { - setJavaVerOptions(options, '21') - } - } -} - /* * List of runners to run integration tests on. */ @@ -118,13 +99,3 @@ plugins.withType(JavaPlugin).configureEach{ modularity.inferModulePath = true } } - -// JPMS requires JDK > 8. Test tasks enabled when either -// (i) testJavaVersion property specified (assumed to be >8) or; -// (ii) current Java version is greater than 8 -project.tasks.each { - it.onlyIf { - project.hasProperty('testJavaVersion') - || JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 - } -} diff --git a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java index 9616918eca1..66f67ae94be 100644 --- a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java +++ b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java @@ -49,26 +49,6 @@ public void verifyCodeIsCompiledWithJava8() throws IOException { assertEquals(v1_8, getByteCodeVersion(DoFn.class)); } - @Test - public void verifyTestCodeIsCompiledWithJava8() throws IOException { - assertEquals(v1_8, getByteCodeVersion(JvmVerification.class)); - } - - @Test - public void verifyTestCodeIsCompiledWithJava11() throws IOException { - assertEquals(v11, getByteCodeVersion(JvmVerification.class)); - } - - @Test - public void verifyTestCodeIsCompiledWithJava17() throws IOException { - assertEquals(v17, getByteCodeVersion(JvmVerification.class)); - } - - @Test - public void verifyTestCodeIsCompiledWithJava21() throws IOException { - assertEquals(v21, getByteCodeVersion(JvmVerification.class)); - } - // jvm @Test public void verifyRunningJVMVersionIs8() { diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle index 4211cd00947..860e7be459e 100644 --- a/sdks/python/test-suites/portable/common.gradle +++ b/sdks/python/test-suites/portable/common.gradle @@ -1,5 +1,3 @@ -import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion - import org.apache.tools.ant.taskdefs.condition.Os /* @@ -23,7 +21,6 @@ import org.apache.tools.ant.taskdefs.condition.Os def pythonRootDir = "${rootDir}/sdks/python" def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') def latestFlinkVersion = project.ext.latestFlinkVersion -def currentJavaVersion = project.ext.currentJavaVersion ext { pythonContainerTask = ":sdks:python:container:py${pythonVersionSuffix}:docker" @@ -42,8 +39,7 @@ def createFlinkRunnerTestTask(String workerType) { task.configure { dependsOn ":runners:flink:${latestFlinkVersion}:job-server:shadowJar" // The Java SDK worker is required to execute external transforms. - def suffix = getSupportedJavaVersion() - dependsOn ":sdks:java:container:${suffix}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" if (workerType == 'DOCKER') { dependsOn pythonContainerTask } else if (workerType == 'PROCESS') { @@ -214,8 +210,7 @@ def createPrismRunnerTestTask(String workerType) { task.configure { dependsOn ":runners:prism:build" // The Java SDK worker is required to execute external transforms. - def suffix = getSupportedJavaVersion() - dependsOn ":sdks:java:container:${suffix}:docker" + dependsOn ":sdks:java:container:java${project.javaVersion}:docker" if (workerType == 'DOCKER') { dependsOn pythonContainerTask } else if (workerType == 'PROCESS') { @@ -373,7 +368,7 @@ project.tasks.register("postCommitPy${pythonVersionSuffix}IT") { 'setupVirtualenv', 'installGcpTest', ":runners:flink:${latestFlinkVersion}:job-server:shadowJar", - ":sdks:java:container:${currentJavaVersion}:docker", + ":sdks:java:container:java${project.javaVersion}:docker", ':sdks:java:testing:kafka-service:buildTestKafkaServiceJar', ':sdks:java:io:expansion-service:shadowJar', ':sdks:java:io:google-cloud-platform:expansion-service:shadowJar', @@ -425,7 +420,7 @@ project.tasks.register("xlangSpannerIOIT") { 'setupVirtualenv', 'installGcpTest', ":runners:flink:${latestFlinkVersion}:job-server:shadowJar", - ":sdks:java:container:${currentJavaVersion}:docker", + ":sdks:java:container:java${project.javaVersion}:docker", ':sdks:java:io:expansion-service:shadowJar', ':sdks:java:io:google-cloud-platform:expansion-service:shadowJar', ':sdks:java:io:amazon-web-services2:expansion-service:shadowJar', diff --git a/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md b/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md index 1ce3f60060b..c3c4eb5e4d2 100644 --- a/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md +++ b/website/www/site/content/en/documentation/sdks/java-multi-language-pipelines.md @@ -221,7 +221,9 @@ python -m apache_beam.runners.portability.local_job_service_main -p $JOB_SERVER_ (this guide requires that your JAVA_HOME is set to Java 11). ``` -./gradlew :sdks:java:container:java11:docker -Pjava11Home=$JAVA_HOME +./gradlew :sdks:java:container:java11:docker \ +-Porg.gradle.java.installations.auto-detect=false \ +-Porg.gradle.java.installations.paths=$JAVA_HOME ``` 5. Run the pipeline.