run_rc_validation.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# To learn more about GitHub Actions in Apache Beam check the CI.m
# DO NOT USE THIS WORKFLOW AND WILL BE REMOVED SOON

name: Run RC Validation
on:
  workflow_dispatch:
    inputs:
      RELEASE_VER:
        description: Beam current Release Version
        required: true
        default: 2.42.0
      USER_GCS_BUCKET:
        description: Bucket to upload results
        required: true
        default: gs://rc-validation-migration-tests
      RC_NUM:
        description: RC number
        required: true
        default: "1"
      APACHE_CONTENTS_REPO:
        description: Apache Contents Repository
        required: true
        default: https://repository.apache.org/content/repositories/orgapachebeam-1285
      RUN_SQL_TAXI_WITH_DATAFLOW:
        description: Include SQL Taxi with Dataflow?
        required: true
        type: boolean
        default: true
      RUN_PYTHON_CROSS_VALIDATION:
        description: Include Python Cross Validation?
        required: true
        type: boolean
        default: true
      RUN_DIRECT_RUNNER_TESTS:
        description: Include Direct Runner Leaderboard & Gamestates?
        required: true
        type: boolean
        default: true
      RUN_DATAFLOW_RUNNER_TESTS:
        description: Include Dataflow Runner Leaderboard & Gamestates?
        type: boolean
        required: true
        default: true
env:
  RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}"
  RELEASE_VER: ${{github.event.inputs.RELEASE_VER}}
  USER_GCP_PROJECT: apache-beam-testing
  PYTHON_RC_DOWNLOAD_URL: https://dist.apache.org/repos/dist/dev/beam
  USER_GCP_REGION: us-central1
  USER_GCP_ZONE: us-central1-a
  APACHE_CONTENTS_REPO: ${{github.event.inputs.APACHE_CONTENTS_REPO}}
  FIXED_WINDOW_DURATION: 20
jobs:
  python_release_candidate:
    runs-on: [self-hosted, ubuntu-20.04]
    permissions:
      pull-requests: write
      contents: write
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      RAW_RELEASE_VER: ${{ github.event.inputs.RELEASE_VER }}
      WORKING_BRANCH: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}_validations"
    steps:
      - name: Set Release Branch Name (Major.Minor)
        id: set_branch_name # Optional: Give the step an ID
        run: |
          VERSION="${{ env.RAW_RELEASE_VER }}" # Use the full version from input/env
          # Use shell parameter expansion to remove the last dot and everything after it
          MAJOR_MINOR_VERSION="${VERSION%.*}"
          # Construct the branch name
          BRANCH_NAME="release-$MAJOR_MINOR_VERSION"
          echo "Original Version: $VERSION"
          echo "Calculated Major.Minor Version: $MAJOR_MINOR_VERSION"
          echo "Setting RELEASE_BRANCH Env Var to: $BRANCH_NAME"
          # Export the calculated name to the environment for subsequent steps
          echo "RELEASE_BRANCH=$BRANCH_NAME" >> $GITHUB_ENV
      - name: Check out code
        uses: actions/checkout@v4
        with:
          ref: ${{ env.RC_TAG }}
      - name: Setup GitHub CLI
        uses: ./.github/actions/setup-gh-cli-linux
      - name: Set git config
        run: |
          git config user.name $GITHUB_ACTOR
          git config user.email actions@"$RUNNER_NAME".local
      - name: Verify working branch name
        run: ./scripts/ci/ci_check_git_branch.sh $WORKING_BRANCH
      - name: Create Pull Request
        run: |
          git checkout -b ${{env.WORKING_BRANCH}} ${{ env.RC_TAG }} --quiet
          touch empty_file.json
          git add empty_file.json
          git commit -m "Add empty file in order to create PR" --quiet
          git push origin ${{env.WORKING_BRANCH}}  --quiet
          GITHUB_PR_URL=$(gh pr create -B ${{env.RELEASE_BRANCH}} -H ${{env.WORKING_BRANCH}} -t"[DO NOT MERGE] Run Python RC Validation Tests" -b "PR to run Python ReleaseCandidate Jenkins Job.")
      - name: Comment on PR to Trigger Python ReleaseCandidate Test
        run: |
          gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate"

  sql_taxi_with_dataflow:
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_SQL_TAXI_WITH_DATAFLOW == 'true'}}
    strategy:
      matrix:
        py_version: [3.9]
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
       ref: ${{env.RC_TAG}}

    - name: Install Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{matrix.py_version}}


    - name: Setup Java JDK
      uses: actions/setup-java@v4
      with:
        distribution: 'temurin'
        java-version: 11

    - name: Setting python env
      uses: ./.github/actions/common-rc-validation
      with:
        RELEASE_VER: ${{env.RELEASE_VER}}
        PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}



    - name: Setting Taxi Variables
      run: |
        echo "SQL_TAXI_TOPIC=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
        echo "SQL_TAXI_SUBSCRIPTION=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM"  >> $GITHUB_ENV
        echo "SQL_EXPANSION_SERVICE_JAR=${APACHE_CONTENTS_REPO}/org/apache/beam/beam-sdks-java-extensions-sql-expansion-service/${RELEASE_VER}/beam-sdks-java-extensions-sql-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV
    - name: Creating Pub/Sub Topics
      run: |
        gcloud pubsub topics create --project=${USER_GCP_PROJECT} ${SQL_TAXI_TOPIC}
        gcloud pubsub subscriptions create --project=${USER_GCP_PROJECT} --topic=${SQL_TAXI_TOPIC} ${SQL_TAXI_SUBSCRIPTION}
    - name: XLang SQL Taxi with DataflowRunner
      run: |
        echo '***************************************************';
        echo '* Running Python XLang SQL Taxi with DataflowRunner';
        echo '***************************************************';
        timeout --preserve-status 20m python -m apache_beam.examples.sql_taxi \
        --project=${USER_GCP_PROJECT} \
        --region=${USER_GCP_REGION} \
        --runner DataflowRunner \
        --num_workers 5 \
        --output_topic projects/${USER_GCP_PROJECT}/topics/${SQL_TAXI_TOPIC} \
        --beam_services="{\":sdks:java:extensions:sql:expansion-service:shadowJar\": \"${SQL_EXPANSION_SERVICE_JAR}\"}" \
        --sdk_location apache_beam-${RELEASE_VER}.tar.gz || true
    - name: Checking Results
      run: |
        gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION}
        gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION}
    - name: Removing Pub Sub Topic
      if: always()
      run: |
        gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_TOPIC}
        gcloud pubsub subscriptions delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_SUBSCRIPTION}


  python_cross_validation:
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}}
    strategy:
      matrix:
        py_version: [3.9]
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{env.RC_TAG}}
    - name: Verify ENV values
      run: |
        echo ""
        echo "====================Checking Environment & Variables================="
        echo ""
        echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}."
    - name: Install Kubectl
      uses: azure/setup-kubectl@v3

    - name: Setup Java JDK
      uses: actions/setup-java@v4
      with:
        distribution: 'temurin'
        java-version: 11

    - name: Install Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{matrix.py_version}}


    - name: Setting python env
      uses: ./.github/actions/common-rc-validation
      with:
        RELEASE_VER: ${{env.RELEASE_VER}}
        PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}

    - name: Installing gcloud-auth-plugin
      run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin
    - name: Setting Kafka Cluster Name
      run: |
        echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM">> $GITHUB_ENV


    - name: Creating Kafka Cluster
      run: |
        gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME
        kubectl apply -R -f .test-infra/kubernetes/kafka-cluster

    - name: Waiting for Kafka cluster to be ready
      run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=1200s
    - name: Start xlang Kafka Taxi with Dataflow Runner
      run: |
        echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV
        echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
        echo "KAFKA_EXPANSION_SERVICE_JAR=${APACHE_CONTENTS_REPO}/org/apache/beam/beam-sdks-java-io-expansion-service/${RELEASE_VER}/beam-sdks-java-io-expansion-service-${RELEASE_VER}.jar" >> $GITHUB_ENV

    - name: Creating BigQuery Dataset
      run: bq mk --project_id=${USER_GCP_PROJECT} ${KAFKA_TAXI_DF_DATASET}
    - name: Running Xlang Kafka Taxi with Dataflow Runner
      run: |
        echo '*****************************************************';
        echo '* Running Python XLang Kafka Taxi with DataflowRunner';
        echo '*****************************************************';
        echo "***************************************************************";
        timeout --preserve-status 20m python -m apache_beam.examples.kafkataxi.kafka_taxi \
        --project=${USER_GCP_PROJECT} \
        --region=${USER_GCP_REGION} \
        --topic beam-runnerv2 \
        --bootstrap_servers ${BOOTSTRAP_SERVERS} \
        --bq_dataset ${KAFKA_TAXI_DF_DATASET} \
        --runner DataflowRunner \
        --num_workers 5 \
        --temp_location=${USER_GCS_BUCKET}/temp/ \
        --with_metadata \
        --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \
        --sdk_location apache_beam-${RELEASE_VER}.tar.gz || true
    - name: Checking executions results
      run: |
        bq head -n 10 ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
    - name: Remove BigQuery Dataset
      if: always()
      run: |
        bq rm -f ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
        bq rm -f ${KAFKA_TAXI_DF_DATASET}

    - name: Delete Kafka Cluster
      if: always()
      run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME


  generate_shared_pubsub:
    outputs:
      name: ${{ steps.generate_pubsub_name.outputs.pubsub }}
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' || github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS== 'true'}}
    steps:
    - name: Sending PubSub name to env
      run: |
        echo "SHARED_PUBSUB_TOPIC=leader_board-${GITHUB_ACTOR}-python-topic-$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
    - id: generate_pubsub_name
      run: |
        echo "::set-output name=pubsub::$SHARED_PUBSUB_TOPIC"
    - name: Creating Pub Sub Topics
      run: gcloud pubsub topics create --project=${USER_GCP_PROJECT} ${SHARED_PUBSUB_TOPIC}


  java_injector:
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' || github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS== 'true'}}
    needs: generate_shared_pubsub
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{env.RC_TAG}}
    - name: Install Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.9'

    - name: Setting python env
      uses: ./.github/actions/common-rc-validation
      with:
        RELEASE_VER: ${{env.RELEASE_VER}}
        PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}

    - name: Setup Java and Maven Action
      uses: actions/setup-java@v4
      with:
        java-version: 11
    - name: Updating Settings
      run: |
        cd ~
        if [[ ! -d .m2 ]]; then
          mkdir .m2
        fi
        cd .m2
        if [[ -f ~/.m2/settings.xml ]]; then
          mv settings.xml /tmp/
        fi
        touch settings.xml
        echo "<settings>" >> settings.xml
        echo "  <profiles>" >> settings.xml
        echo "    <profile>" >> settings.xml
        echo "      <id>release-repo</id>" >> settings.xml
        echo "      <activation>" >> settings.xml
        echo "        <activeByDefault>true</activeByDefault>" >> settings.xml
        echo "      </activation>" >> settings.xml
        echo "      <repositories>" >> settings.xml
        echo "        <repository>" >> settings.xml
        echo "          <id>Release ${RELEASE_VER} RC${{github.event.inputs.RC_NUM}}</id>" >> settings.xml
        echo "          <name>Release ${RELEASE_VER} RC${{github.event.inputs.RC_NUM}}</name>" >> settings.xml
        echo "          <url>${APACHE_CONTENTS_REPO}</url>" >> settings.xml
        echo "        </repository>" >> settings.xml
        echo "      </repositories>" >> settings.xml
        echo "    </profile>" >> settings.xml
        echo "  </profiles>" >> settings.xml
        echo "</settings>" >> settings.xml
        cat settings.xml
    - name: Generate Maven archetype
      run: |
        mvn archetype:generate \
        -DarchetypeGroupId=org.apache.beam \
        -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
        -DarchetypeVersion=2.41.0 \
        -DgroupId=org.example \
        -DartifactId=word-count-beam \
        -Dversion="0.1" \
        -Dpackage=org.apache.beam.examples \
        -DinteractiveMode=false \
        -DarchetypeCatalog=internal
    - name: Running Pub/Sub Java injector
      run: |
        ls
        cd word-count-beam
        timeout --preserve-status 50m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true


  direct_runner_leaderboard:
    runs-on: [self-hosted, ubuntu-20.04]
    if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }}
    strategy:
      matrix:
        py_version: [3.9]
    needs: generate_shared_pubsub
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{env.RC_TAG}}

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_version}}

      - name: Setting python env
        uses: ./.github/actions/common-rc-validation
        with:
          RELEASE_VER: ${{env.RELEASE_VER}}
          PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}

      - name: Exporting leaderboard Dataset Name
        run: echo "LEADERBOARD_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
      - name: Creating Dataset
        run: bq mk --project_id=${USER_GCP_PROJECT} ${LEADERBOARD_DIRECT_DATASET}
      - name: Starting Leaderboard with DirectRunner
        run: |
          echo '*****************************************************';
          echo '* Running Python Leaderboard with DirectRunner';
          echo '*****************************************************';
          timeout --preserve-status 5m python -m apache_beam.examples.complete.game.leader_board \
          --project=${USER_GCP_PROJECT} \
          --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \
          --dataset ${LEADERBOARD_DIRECT_DATASET} || true
      - name: Checking results
        run: |
          bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_users
          bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
      - name: Removing BigQuery Dataset
        if: always()
        run: |
          bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_users
          bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
          bq rm -f $LEADERBOARD_DIRECT_DATASET

  dataflow_runner_leaderboard:
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}}
    strategy:
      matrix:
        py_version: [3.9]
    needs: [generate_shared_pubsub]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{env.RC_TAG}}

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_version}}

      - name: Setting python env
        uses: ./.github/actions/common-rc-validation
        with:
          RELEASE_VER: ${{env.RELEASE_VER}}
          PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}

      - name: Exporting Dataflow Dataset Name
        run: echo "LEADERBOARD_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
      - name: Creating Dataset
        run: bq mk --project_id=${USER_GCP_PROJECT} ${LEADERBOARD_DF_DATASET}
      - name: Starting Leaderboard with Dataflow
        run: |
          echo '*****************************************************';
          echo '* Running Python Leaderboard with DataflowRunner';
          echo '*****************************************************';
          timeout --preserve-status 20m python -m apache_beam.examples.complete.game.leader_board \
          --project=${USER_GCP_PROJECT} \
          --region=${USER_GCP_REGION} \
          --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \
          --dataset ${LEADERBOARD_DF_DATASET} \
          --runner DataflowRunner \
          --temp_location=${USER_GCS_BUCKET}/temp/ \
          --sdk_location apache_beam-${RELEASE_VER}.tar.gz || true
      - name: Checking results
        run: |
          bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_users
          bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_teams
      - name: Removing BigQuery Dataset
        if: always()
        run: |
          bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_users
          bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_teams
          bq rm -f $LEADERBOARD_DF_DATASET


  direct_runner_gamestats:
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }}
    strategy:
      matrix:
        py_version: [3.9]
    needs: [generate_shared_pubsub]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{env.RC_TAG}}
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_version}}

      - name: Setting python env
        uses: ./.github/actions/common-rc-validation
        with:
          RELEASE_VER: ${{env.RELEASE_VER}}
          PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}

      - name: Exporting Gamestates Direct Dataset Name
        run: echo "GAMESTATS_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
      - name: Creating Dataset
        run: bq mk --project_id=${USER_GCP_PROJECT} ${GAMESTATS_DIRECT_DATASET}
      - name: Starting Gamestats with DirectRunner
        run: |
          echo '*****************************************************';
          echo '* Running GameStats with DirectRunner';
          echo '*****************************************************';
          timeout --preserve-status 40m python -m apache_beam.examples.complete.game.game_stats \
          --project=${USER_GCP_PROJECT} \
          --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \
          --dataset ${GAMESTATS_DIRECT_DATASET} \
          --fixed_window_duration ${FIXED_WINDOW_DURATION} || true
      - name: Checking Results
        run: |
          bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_teams
          bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
      - name: Removing BigQuery Dataset
        if: always()
        run: |
          bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
          bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_teams
          bq rm -f $GAMESTATS_DIRECT_DATASET


  dataflow_runner_gamestats:
    runs-on: [self-hosted,ubuntu-20.04]
    if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}}
    strategy:
      matrix:
        py_version: [3.9]
    needs: [generate_shared_pubsub]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{env.RC_TAG}}
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_version}}

      - name: Setting python env
        uses: ./.github/actions/common-rc-validation
        with:
          RELEASE_VER: ${{env.RELEASE_VER}}
          PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}

      - name: Exporting Gamestates Direct Dataset Name
        run: echo "GAMESTATS_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
      - name: Creating Dataset
        run: bq mk --project_id=${USER_GCP_PROJECT} ${GAMESTATS_DF_DATASET}
      - name: Starting Gamestats with DirectRunner
        run: |
          echo '*****************************************************';
          echo '* Running GameStats with DataflowRunner';
          echo '*****************************************************';
          timeout --preserve-status 45m python -m apache_beam.examples.complete.game.game_stats \
          --project=${USER_GCP_PROJECT} \
          --region=${USER_GCP_REGION} \
          --topic projects/${USER_GCP_PROJECT}/topics/${{needs.generate_shared_pubsub.outputs.name}} \
          --dataset ${GAMESTATS_DF_DATASET} \
          --runner DataflowRunner \
          --temp_location=${USER_GCS_BUCKET}/temp/ \
          --sdk_location apache_beam-${RELEASE_VER}.tar.gz \
          --fixed_window_duration ${FIXED_WINDOW_DURATION} || true
      - name: Checking Results
        run: |
          bq head -n 10 ${GAMESTATS_DF_DATASET}.game_stats_teams
          bq head -n 10 ${GAMESTATS_DF_DATASET}.game_stats_sessions
      - name: Removing BigQuery Dataset
        if: always()
        run: |
          bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_teams
          bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions
          bq rm -f $GAMESTATS_DF_DATASET

  remove_shared_pubsub:
    runs-on: [self-hosted,ubuntu-20.04]
    needs: [java_injector, generate_shared_pubsub]
    if: always()
    steps:
      - name: Deleting Shared Pub Sub
        run: gcloud pubsub topics delete --project=${USER_GCP_PROJECT}  ${{needs.generate_shared_pubsub.outputs.name}}