upload nightly prom metrics to s3

Author: THardy98

.github/workflows/nightly-throughput-stress.yml

@@ -6,7 +6,7 @@ on:
     - cron: '00 11 * * *'
   push:
     branches:
-      - add-nightly-throughput-stress-workflow
+      - upload-nightly-metrics-to-s3
   workflow_dispatch:
     inputs:
       duration:
@@ -35,6 +35,9 @@ env:
   # Logging and artifacts
   WORKER_LOG_DIR: /tmp/throughput-stress-logs
 
+  # AWS S3 metrics upload ARN
+  AWS_S3_METRICS_UPLOAD_ROLE_ARN: ${{ vars.AWS_S3_METRICS_UPLOAD_ROLE_ARN }}
+
   # Omes configuration
   OMES_REPO: temporalio/omes
   OMES_REF: main
@@ -44,6 +47,10 @@ jobs:
   throughput-stress:
     runs-on: ubuntu-latest-4-cores
     timeout-minutes: ${{ fromJSON(inputs.job_timeout_minutes || vars.NIGHTLY_JOB_TIMEOUT_MINUTES || 360) }}
+    permissions:
+      contents: read
+      actions: write
+      id-token: write
 
     steps:
       - name: Print test configuration
@@ -88,6 +95,14 @@ jobs:
       - name: Install Temporal CLI
         uses: temporalio/setup-temporal@v0
 
+      - name: Install Prometheus
+        run: |
+          PROM_VERSION="3.8.0"
+          wget -q https://github.com/prometheus/prometheus/releases/download/v${PROM_VERSION}/prometheus-${PROM_VERSION}.linux-amd64.tar.gz
+          tar xzf prometheus-${PROM_VERSION}.linux-amd64.tar.gz
+          sudo mv prometheus-${PROM_VERSION}.linux-amd64/prometheus /usr/local/bin/
+          prometheus --version
+
       - name: Setup log directory
         run: mkdir -p $WORKER_LOG_DIR
 
@@ -120,13 +135,37 @@ jobs:
             --duration $TEST_DURATION \
             --timeout $TEST_TIMEOUT \
             --max-concurrent 10 \
+            --prom-listen-address 127.0.0.1:9091 \
+            --worker-prom-listen-address 127.0.0.1:9092 \
+            --prom-instance-addr 127.0.0.1:9090 \
+            --prom-instance-config \
+            --prom-export-worker-metrics $RUN_ID.parquet \
             --option internal-iterations=10 \
             --option continue-as-new-after-iterations=3 \
             --option sleep-time=1s \
             --option visibility-count-timeout=5m \
             --option min-throughput-per-hour=1000 \
             2>&1 | tee $WORKER_LOG_DIR/scenario.log
 
+      - name: Configure AWS credentials
+        if: always()
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.AWS_S3_METRICS_UPLOAD_ROLE_ARN }}
+          aws-region: us-west-2
+    
+      - name: Upload metrics to S3
+        if: always()
+        run: |
+          DATE=$(date +%Y-%m-%d)
+          # Use test/ prefix on non-main branches
+          PREFIX="language=java/date=$DATE"
+          if [[ "${{ github.ref }}" != "refs/heads/main" ]]; then
+            PREFIX="test/$PREFIX"
+          fi
+          aws s3 cp omes/$RUN_ID.parquet \
+            "s3://cloud-data-ingest-prod/github/sdk_load_test/$PREFIX/$RUN_ID.parquet" 
+
       - name: Upload logs on failure
         if: failure() || cancelled()
         uses: actions/upload-artifact@v4