Activate service account for Publish Beam SDK Snapshots workflow

.github/workflows/beam_CleanUpPrebuiltSDKImages.yml

@@ -52,7 +52,7 @@ jobs:
   beam_CleanUpPrebuiltSDKImages:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 180
+    timeout-minutes: 360
     strategy:
       matrix: 
         job_name: [beam_CleanUpPrebuiltSDKImages]

.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml

@@ -84,9 +84,21 @@ jobs:
             ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Base_Uncased.txt
             ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Language_Modeling_Bert_Large_Uncased.txt
             ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Imagenet_Classification_Resnet_152_Tesla_T4_GPU.txt
+            ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Pytorch_Sentiment_Streaming_DistilBert_Base_Uncased.txt
       # The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
       - name: get current time
         run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV
+      - name: run Pytorch Sentiment using Hugging Face distilbert-base-uncased model
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        timeout-minutes: 180
+        with:
+          gradle-command: :sdks:python:apache_beam:testing:load_tests:run
+          arguments: |
+            -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_streaming_benchmarks \
+            -Prunner=DataflowRunner \
+            -PpythonVersion=3.10 \
+            -PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
+            '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_6 }} --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \
       - name: run Pytorch Vision Classification with Resnet 101
         uses: ./.github/actions/gradle-command-self-hosted-action
         timeout-minutes: 180

.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml

@@ -52,7 +52,7 @@ env:
   GCS_BUCKET: gs://beam-flink-cluster
   FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.17.0/flink-1.17.0-bin-scala_2.12.tgz
   HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
-  FLINK_TASKMANAGER_SLOTS: 5
+  FLINK_TASKMANAGER_SLOTS: 1
   DETACHED_MODE: true
   HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
   JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest
@@ -64,7 +64,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event.comment.body == 'Run Load Tests Go CoGBK Flink Batch'
-    runs-on: [self-hosted, ubuntu-20.04, main]
+    runs-on: [self-hosted, ubuntu-20.04, highmem]
     timeout-minutes: 720
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     strategy:

.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml

@@ -64,7 +64,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event.comment.body == 'Run Load Tests Go Combine Flink Batch'
-    runs-on: [self-hosted, ubuntu-20.04, main]
+    runs-on: [self-hosted, ubuntu-20.04, highmem]
     timeout-minutes: 720
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     strategy:

.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml

@@ -52,7 +52,7 @@ env:
   GCS_BUCKET: gs://beam-flink-cluster
   FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.17.0/flink-1.17.0-bin-scala_2.12.tgz
   HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
-  FLINK_TASKMANAGER_SLOTS: 5
+  FLINK_TASKMANAGER_SLOTS: 1
   DETACHED_MODE: true
   HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
   JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest
@@ -64,7 +64,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event.comment.body == 'Run Load Tests Go GBK Flink Batch'
-    runs-on: [self-hosted, ubuntu-20.04, main]
+    runs-on: [self-hosted, ubuntu-20.04, highmem]
     timeout-minutes: 720
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     strategy:

.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml

@@ -45,8 +45,8 @@ concurrency:
 
 env:
   DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
-  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+#  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+#  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Jpms_Dataflow_Java17:
@@ -79,7 +79,7 @@ jobs:
       - name: run PostCommit Java Jpms Dataflow Java17 script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest
+          gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest --no-build-cache --rerun-tasks
           arguments:
             -PskipCheckerFramework
             -PtestJavaVersion=17

.github/workflows/beam_PostCommit_Python_Arm.yml

@@ -91,6 +91,9 @@ jobs:
           credentials_json: ${{ secrets.GCP_SA_KEY }}
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
+      - name: Docker auth
+        run: |
+          gcloud auth configure-docker ${{ env.docker_registry }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: GCloud Docker credential helper

.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml

@@ -56,7 +56,7 @@ jobs:
       github.event_name == 'pull_request_target' ||
       startsWith(github.event.comment.body, 'Run Python RC Dataflow ValidatesContainer')
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 100
+    timeout-minutes: 300
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
     strategy:
       fail-fast: false

.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml

@@ -16,7 +16,7 @@
 # TODO(https://github.com/apache/beam/issues/32492): re-enable the suite
 # on cron and add release/trigger_all_tests.json to trigger path once fixed.
 
-name: PostCommit XVR GoUsingJava Dataflow (DISABLED)
+name: PostCommit XVR GoUsingJava Dataflow
 
 on:
   # schedule:
@@ -59,7 +59,7 @@ jobs:
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event.comment.body == 'Run XVR_GoUsingJava_Dataflow PostCommit'
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 100
+    timeout-minutes: 300
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     strategy:
       matrix:
@@ -77,15 +77,33 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           python-version: default
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+#      - name: Java container
+#        uses: ./.github/actions/gradle-command-self-hosted-action
+#        with:
+#          gradle-command: :sdks:java:container:java11:docker
+#      - name: Java expansion service
+#        uses: ./.github/actions/gradle-command-self-hosted-action
+#        with:
+#          gradle-command: :sdks:java:testing:expansion-service:buildTestExpansionServiceJar
+#      - name: Python container
+#        uses: ./.github/actions/gradle-command-self-hosted-action
+#        with:
+#          gradle-command: :sdks:python:container:py39:docker
+#      - name: Go container
+#        uses: ./.github/actions/gradle-command-self-hosted-action
+#        with:
+#          gradle-command: :sdks:go:container:docker
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v3
+#        with:
+#          install: true
+#          driver: 'docker-container'  # Required for multi-platform builds
       - name: GCloud Docker credential helper
         run: |
           gcloud auth configure-docker us.gcr.io
-      - name: run XVR GoUsingJava Dataflow script
+      - name: run PostCommit XVR GoUsingJava Dataflow script
         env:
-          USER: github-actions
-          CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
+          USER: jenkins
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava
@@ -102,4 +120,4 @@ jobs:
           commit: '${{ env.prsha || env.GITHUB_SHA }}'
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
-          large_files: true
+          large_files: true

.github/workflows/beam_PostRelease_NightlySnapshot.yml

@@ -20,11 +20,11 @@ on:
     inputs:
       RELEASE:
         description: Beam version of current release (e.g. 2.XX.0)
-        required: true
-        default: '2.XX.0'
+        required: false
+        default: ''
       SNAPSHOT_URL:
         description: Location of the staged artifacts in Maven central (https://repository.apache.org/content/repositories/orgapachebeam-NNNN/).
-        required: true
+        required: false
   schedule:
     - cron: '15 16 * * *'
 

.github/workflows/beam_PreCommit_Flink_Container.yml

@@ -16,28 +16,25 @@
 name: PreCommit Flink Container
 
 on:
-  pull_request_target:
-    paths:
-      - 'model/**'
-      - 'sdks/python/apache_beam/runners/portability/**'
-      - 'release/**'
-      - 'runners/core-java/**'
-      - 'runners/flink/**'
-      - 'runners/java-fn-execution/**'
-      - '.github/trigger_files/beam_PreCommit_Flink_Container.json'
-      - 'release/trigger_all_tests.json'
-  push:
-      tags: [ 'v*' ]
-      branches: [ 'master', 'release-*' ]
-      paths:
-      - 'model/**'
-      - 'sdks/python/apache_beam/runners/portability/**'
-      - 'release/**'
-      - 'runners/core-java/**'
-      - 'runners/flink/**'
-      - 'runners/java-fn-execution/**'
-  schedule:
-    - cron: '0 */6 * * *'
+#  pull_request_target:
+#    paths:
+#      - 'model/**'
+#      - 'sdks/python/**'
+#      - 'release/**'
+#      - 'sdks/java/io/kafka/**'
+#      - 'runners/core-construction-java/**'
+#      - 'runners/core-java/**'
+#      - 'runners/extensions-java/**'
+#      - 'runners/flink/**'
+#      - 'runners/java-fn-execution/**'
+#      - 'runners/reference/**'
+#      - '.github/trigger_files/beam_PreCommit_Flink_Container.json'
+#      - 'release/trigger_all_tests.json'
+#  push:
+#    branches: ['master', 'release-*']
+#    tags: 'v*'
+#  schedule:
+#    - cron: '0 */6 * * *'
   workflow_dispatch:
 
 # Setting explicit permissions for the action to avoid the default permissions which are `write-all`

.github/workflows/beam_PreCommit_GHA.yml

@@ -16,17 +16,17 @@
 name: PreCommit GHA
 
 on:
-  push:
-    tags: ['v*']
-    branches: ['master', 'release-*']
-    paths: ['.github/**/*.yml']
-  pull_request_target:
-    branches: ['master', 'release-*' ]
-    paths: ['.github/**/*.yml', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GHA.json']
-  issue_comment:
-    types: [created]
-  schedule:
-  - cron: '0 */6 * * *'
+#  push:
+#    tags: ['v*']
+#    branches: ['master', 'release-*']
+#    paths: ['.github/**/*.yml']
+#  pull_request_target:
+#    branches: ['master', 'release-*' ]
+#    paths: ['.github/**/*.yml', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GHA.json']
+#  issue_comment:
+#    types: [created]
+#  schedule:
+#  - cron: '0 */6 * * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

.github/workflows/beam_PreCommit_Go.yml

@@ -16,17 +16,17 @@
 name: PreCommit Go
 
 on:
-  push:
-    tags: ['v*']
-    branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_Go.yml']
-  pull_request_target:
-    branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Go.json']
-  issue_comment:
-    types: [created]
-  schedule:
-    - cron: '0 1/6 * * *'
+#  push:
+#    tags: ['v*']
+#    branches: ['master', 'release-*']
+#    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_Go.yml']
+#  pull_request_target:
+#    branches: ['master', 'release-*']
+#    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Go.json']
+#  issue_comment:
+#    types: [created]
+#  schedule:
+#    - cron: '0 1/6 * * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

.github/workflows/beam_PreCommit_GoPortable.yml

@@ -16,17 +16,17 @@
 name: PreCommit GoPortable
 
 on:
-  push:
-    tags: ['v*']
-    branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_GoPortable.yml']
-  pull_request_target:
-    branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GoPortable.json']
-  issue_comment:
-    types: [created]
-  schedule:
-    - cron: '0 1/6 * * *'
+#  push:
+#    tags: ['v*']
+#    branches: ['master', 'release-*']
+#    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_GoPortable.yml']
+#  pull_request_target:
+#    branches: ['master', 'release-*']
+#    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GoPortable.json']
+#  issue_comment:
+#    types: [created]
+#  schedule:
+#    - cron: '0 1/6 * * *'
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs

.github/workflows/beam_PreCommit_GoPrism.yml

@@ -16,17 +16,17 @@
 name: PreCommit GoPrism
 
 on:
-  push:
-    tags: ['v*']
-    branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_GoPrism.yml']
-  pull_request_target:
-    branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GoPrism.json']
-  issue_comment:
-    types: [created]
-  schedule:
-    - cron: '0 1/6 * * *'
+#  push:
+#    tags: ['v*']
+#    branches: ['master', 'release-*']
+#    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_GoPrism.yml']
+#  pull_request_target:
+#    branches: ['master', 'release-*']
+#    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GoPrism.json']
+#  issue_comment:
+#    types: [created]
+#  schedule:
+#    - cron: '0 1/6 * * *'
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs