Regression Parallel Jobs (OPS & UAT) #1516
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regression Parallel Jobs (OPS & UAT) | |
| on: | |
| schedule: | |
| # Run OPS every 3 days at midnight PST (8 AM UTC) | |
| - cron: '0 8 */3 * *' | |
| # Run UAT every 3 days at 4 AM PST (12 PM UTC) | |
| - cron: '0 12 */3 * *' | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Environment to test' | |
| required: true | |
| type: choice | |
| options: | |
| - ops | |
| - uat | |
| env: | |
| # Edit this list for ad hoc testing. Leave blank to run the full set. | |
| TEST_CONCEPT_IDS: | |
| jobs: | |
| setup-and-chunk: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| environment: ${{ steps.prepare.outputs.environment }} | |
| environment_upper: ${{ steps.prepare.outputs.environment_upper }} | |
| harmony_status: ${{ steps.check-harmony.outputs.harmony_status }} | |
| edl_status: ${{ steps.record-edl-status.outputs.edl_status }} | |
| matrix_chunks: ${{ steps.generate_chunks.outputs.matrix_chunks }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Determine Environment | |
| id: prepare | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| ENV="${{ inputs.environment }}" | |
| elif [ "${{ github.event.schedule }}" = "0 8 */3 * *" ]; then | |
| ENV="ops" | |
| else | |
| ENV="uat" | |
| fi | |
| echo "environment=$ENV" >> "$GITHUB_OUTPUT" | |
| echo "environment_upper=${ENV^^}" >> "$GITHUB_OUTPUT" | |
| - name: Check Harmony Health | |
| id: check-harmony | |
| env: | |
| TARGET_ENV: ${{ steps.prepare.outputs.environment }} | |
| run: | | |
| if [ "$TARGET_ENV" = "uat" ]; then | |
| HEALTH_URL="https://harmony.uat.earthdata.nasa.gov/health" | |
| else | |
| HEALTH_URL="https://harmony.earthdata.nasa.gov/health" | |
| fi | |
| HEALTH_RESPONSE="$(curl -fsS --max-time 20 "$HEALTH_URL" 2>/dev/null || true)" | |
| HEALTH_STATUS="$(printf '%s' "$HEALTH_RESPONSE" | jq -r '.status // "down"' 2>/dev/null || echo down)" | |
| if [ "$HEALTH_STATUS" != "up" ]; then | |
| HEALTH_STATUS="down" | |
| fi | |
| echo "Harmony health check for $TARGET_ENV: $HEALTH_STATUS" | |
| echo "harmony_status=$HEALTH_STATUS" >> "$GITHUB_OUTPUT" | |
| - name: Set EDL token for validation | |
| id: set-edl-token | |
| env: | |
| TARGET_ENV: ${{ steps.prepare.outputs.environment }} | |
| run: | | |
| case "$TARGET_ENV" in | |
| uat) | |
| echo "edl_token=${{ secrets.UAT_URS_TOKEN }}" >> "$GITHUB_OUTPUT" | |
| ;; | |
| ops) | |
| echo "edl_token=${{ secrets.OPS_URS_TOKEN }}" >> "$GITHUB_OUTPUT" | |
| ;; | |
| *) | |
| echo "❌ Unknown environment: $TARGET_ENV" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| - name: Validate EDL token | |
| id: validate-edl-token | |
| continue-on-error: true | |
| uses: ./.github/actions/check-edl-token | |
| with: | |
| environment: ${{ steps.prepare.outputs.environment }} | |
| edl-token: ${{ steps.set-edl-token.outputs.edl_token }} | |
| - name: Record EDL validation status | |
| id: record-edl-status | |
| if: always() | |
| run: | | |
| if [ "${{ steps.validate-edl-token.outcome }}" = "success" ]; then | |
| echo "edl_status=up" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "edl_status=down" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Generate Chunks | |
| if: ${{ steps.check-harmony.outputs.harmony_status == 'up' }} | |
| id: generate_chunks | |
| env: | |
| ENV: ${{ steps.prepare.outputs.environment }} | |
| CONCEPT_IDS: ${{ env.TEST_CONCEPT_IDS }} | |
| run: | | |
| shopt -s nullglob | |
| files=(tests/cmr/l2ss-py/${ENV}/*) | |
| if [ -n "${CONCEPT_IDS:-}" ]; then | |
| mapfile -t requested_ids < <( | |
| printf '%s\n' "$CONCEPT_IDS" \ | |
| | tr ',\r' '\n\n' \ | |
| | sed '/^[[:space:]]*$/d; s/^[[:space:]]*//; s/[[:space:]]*$//' | |
| ) | |
| if [ "${#requested_ids[@]}" -eq 0 ]; then | |
| echo "No valid concept IDs were provided in concept_ids." | |
| echo "matrix_chunks=[]" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| declare -A requested_map=() | |
| for concept_id in "${requested_ids[@]}"; do | |
| requested_map["$concept_id"]=1 | |
| done | |
| filtered_files=() | |
| declare -A found_map=() | |
| for file in "${files[@]}"; do | |
| concept_id="$(basename "$file")" | |
| if [ -n "${requested_map[$concept_id]+x}" ]; then | |
| filtered_files+=("$file") | |
| found_map["$concept_id"]=1 | |
| fi | |
| done | |
| missing_ids=() | |
| for concept_id in "${requested_ids[@]}"; do | |
| if [ -z "${found_map[$concept_id]+x}" ]; then | |
| missing_ids+=("$concept_id") | |
| fi | |
| done | |
| if [ "${#missing_ids[@]}" -gt 0 ]; then | |
| echo "The following concept IDs were not found under tests/cmr/l2ss-py/${ENV}: ${missing_ids[*]}" | |
| exit 1 | |
| fi | |
| files=("${filtered_files[@]}") | |
| fi | |
| total=${#files[@]} | |
| if [ "$total" -eq 0 ]; then | |
| echo "No files found for environment $ENV!" | |
| echo "matrix_chunks=[]" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| max_chunk_size=250 | |
| num_chunks=10 | |
| chunk_size=$(( (total + num_chunks - 1) / num_chunks )) | |
| if (( chunk_size > max_chunk_size )); then | |
| chunk_size=$max_chunk_size | |
| num_chunks=$(( (total + chunk_size - 1) / chunk_size )) | |
| fi | |
| echo "Total files: $total" | |
| echo "Chunk size: $chunk_size" | |
| echo "Chunks: $num_chunks" | |
| chunks_json=$( | |
| for i in $(seq 0 $((num_chunks - 1))); do | |
| start=$((i * chunk_size)) | |
| chunk=("${files[@]:start:chunk_size}") | |
| [ ${#chunk[@]} -eq 0 ] && continue | |
| chunk_id=$((i + 1)) | |
| printf '%s\n' "${chunk[@]}" \ | |
| | jq -R 'split("/") | {env: .[-2], file: .[-1]}' \ | |
| | jq -s --arg id "$chunk_id" '{id: ($id|tonumber), include: .}' | |
| done | jq -s -c '.' | |
| ) | |
| echo "matrix_chunks=$chunks_json" >> "$GITHUB_OUTPUT" | |
| process-chunks: | |
| needs: setup-and-chunk | |
| # Only run if there are actual chunks to process | |
| if: ${{ needs.setup-and-chunk.outputs.harmony_status == 'up' && needs.setup-and-chunk.outputs.edl_status == 'up' && needs.setup-and-chunk.outputs.matrix_chunks != '[]' && needs.setup-and-chunk.outputs.matrix_chunks != '' }} | |
| name: process-chunk-${{ matrix.chunk.id }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Dynamically spawn exactly as many jobs as we have chunks | |
| chunk: ${{ fromJson(needs.setup-and-chunk.outputs.matrix_chunks) }} | |
| uses: ./.github/workflows/process_chunk.yml | |
| secrets: inherit | |
| with: | |
| # Pass the specific matrix payload as a JSON string | |
| # Only pass the include list to the child workflow; it will wrap it | |
| # into a matrix.include internally. | |
| chunk-data: ${{ toJSON(matrix.chunk.include) }} | |
| environment: ${{ needs.setup-and-chunk.outputs.environment_upper }} | |
| aggregate-failures: | |
| needs: [setup-and-chunk, process-chunks] | |
| runs-on: ubuntu-latest | |
| if: ${{ always() && needs.setup-and-chunk.outputs.harmony_status == 'up' && needs.setup-and-chunk.outputs.edl_status == 'up' && needs.setup-and-chunk.outputs.matrix_chunks != '[]' && needs.setup-and-chunk.outputs.matrix_chunks != '' }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install Poetry | |
| uses: abatilo/actions-poetry@v4 | |
| with: | |
| poetry-version: 2.3.2 | |
| - name: Poetry Install | |
| run: poetry install | |
| - name: Download all job status artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: job-status-* | |
| path: job-status | |
| - name: Set CMR bearer token dynamically | |
| id: set-cmr-token | |
| run: | | |
| case "${{ needs.setup-and-chunk.outputs.environment_upper }}" in | |
| UAT) | |
| echo "CMR_BEARER_TOKEN=${{ secrets.UAT_URS_TOKEN }}" >> $GITHUB_ENV | |
| ;; | |
| OPS) | |
| echo "CMR_BEARER_TOKEN=${{ secrets.OPS_URS_TOKEN }}" >> $GITHUB_ENV | |
| ;; | |
| *) | |
| echo "❌ Unknown environment: ${{ inputs.environment }}" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| - name: Configure AWS credentials for UAT | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CUMULUS_UAT }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CUMULUS_UAT }} | |
| aws-region: us-west-2 | |
| - name: Aggregate failed jobs and print URLs and reasons | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| REGRESSION_ENV: ${{ needs.setup-and-chunk.outputs.environment_upper }} | |
| CMR_TOKEN_LAMBDA_FUNCTION_NAME: ${{ needs.setup-and-chunk.outputs.environment }}-launchpad_token_dispenser | |
| CMR_TOKEN_LAMBDA_REGION: us-west-2 | |
| CMR_USER: ${{ secrets.CMR_USER }} | |
| CMR_PASS: ${{ secrets.CMR_PASS }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| run: | | |
| poetry run python tests/aggregate_results.py | |
| update-alert-issue: | |
| needs: setup-and-chunk | |
| runs-on: ubuntu-latest | |
| if: ${{ always() && (needs.setup-and-chunk.outputs.harmony_status != 'up' || needs.setup-and-chunk.outputs.edl_status != 'up') }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install Poetry | |
| uses: abatilo/actions-poetry@v4 | |
| with: | |
| poetry-version: 2.3.2 | |
| - name: Poetry Install | |
| run: poetry install | |
| - name: Update alert issue | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| run: | | |
| args=(--env "${{ needs.setup-and-chunk.outputs.environment }}") | |
| if [ "${{ needs.setup-and-chunk.outputs.harmony_status }}" != "up" ]; then | |
| args+=(--alert "Harmony was unavailable, so regression aggregation was skipped for this run.") | |
| fi | |
| if [ "${{ needs.setup-and-chunk.outputs.edl_status }}" != "up" ]; then | |
| args+=(--alert "EDL token validation failed, so regression jobs could not use Earthdata access.") | |
| fi | |
| poetry run python scripts/update_issue_alert.py "${args[@]}" |