[Fix #119] Implement feature to allow multiple versions of Challenges via git branches #5
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate and Process EvalAI Challenge | |
| on: | |
| push: | |
| branches: | |
| - 'challenge' | |
| - 'challenge-*-*' | |
| pull_request: | |
| types: [opened, synchronize, reopened, edited] | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| detect-and-validate-challenge-branch: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| is_challenge_branch: ${{ steps.branch-check.outputs.is_challenge_branch }} | |
| challenge_branch: ${{ steps.branch-check.outputs.challenge_branch }} | |
| steps: | |
| - name: Detect and validate challenge branch | |
| id: branch-check | |
| run: | | |
| # Get the actual branch name | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| BRANCH="${{ github.head_ref }}" | |
| else | |
| BRANCH="${{ github.ref_name }}" | |
| fi | |
| echo "🔍 Analyzing branch: $BRANCH" | |
| echo "challenge_branch=$BRANCH" >> $GITHUB_OUTPUT | |
| # Check if this is a challenge branch (challenge or challenge-YYYY-version) | |
| if [[ "$BRANCH" =~ ^challenge(-[0-9]{4}-.*)?$ ]]; then | |
| echo "is_challenge_branch=true" >> $GITHUB_OUTPUT | |
| echo "✅ Valid challenge branch detected: $BRANCH" | |
| # Extract branch suffix for versioning | |
| if [[ "$BRANCH" =~ ^challenge-([0-9]{4}-.+)$ ]]; then | |
| SUFFIX="${BASH_REMATCH[1]}" | |
| echo "branch_suffix=$SUFFIX" >> $GITHUB_OUTPUT | |
| echo "📋 Branch version: $SUFFIX" | |
| else | |
| echo "branch_suffix=main" >> $GITHUB_OUTPUT | |
| echo "📋 Main challenge branch (no suffix)" | |
| fi | |
| else | |
| echo "is_challenge_branch=false" >> $GITHUB_OUTPUT | |
| echo "ℹ️ Not a challenge branch: $BRANCH" | |
| echo "⏭️ Challenge processing will be skipped (valid patterns: 'challenge' or 'challenge-YYYY-version')" | |
| fi | |
| validate-host-config: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| is_valid: ${{ steps.validate.outputs.is_valid }} | |
| is_localhost: ${{ steps.validate.outputs.is_localhost }} | |
| host_url: ${{ steps.validate.outputs.host_url }} | |
| requires_self_hosted: ${{ steps.validate.outputs.requires_self_hosted }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Validate host_config.json | |
| id: validate | |
| run: | | |
| echo "is_valid=true" >> $GITHUB_OUTPUT | |
| echo "is_localhost=false" >> $GITHUB_OUTPUT | |
| echo "requires_self_hosted=false" >> $GITHUB_OUTPUT | |
| echo "" > validation_error.log | |
| if ! [ -f "github/host_config.json" ]; then | |
| echo "❌ host_config.json not found in github/ directory" | tee -a validation_error.log | |
| echo "is_valid=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| TOKEN=$(jq -r '.token' github/host_config.json) | |
| TEAM_PK=$(jq -r '.team_pk' github/host_config.json) | |
| HOST_URL=$(jq -r '.evalai_host_url' github/host_config.json) | |
| echo "host_url=$HOST_URL" >> $GITHUB_OUTPUT | |
| # Enhanced localhost detection (includes Docker networking hosts) | |
| if [[ "$HOST_URL" == *"127.0.0.1"* ]] || [[ "$HOST_URL" == *"localhost"* ]] || [[ "$HOST_URL" == *"0.0.0.0"* ]] || [[ "$HOST_URL" == *"host.docker.internal"* ]]; then | |
| echo "is_localhost=true" >> $GITHUB_OUTPUT | |
| echo "requires_self_hosted=true" >> $GITHUB_OUTPUT | |
| echo "🏠 Localhost/Docker networking server detected: $HOST_URL" | |
| echo "🤖 Self-hosted runner required for local development" | |
| echo "🐳 Docker networking will be used for container-to-host communication" | |
| echo "" | |
| echo "📋 Requirements: self-hosted runner + running local EvalAI server" | |
| echo "⚠️ Note: GitHub hosted runners cannot connect to localhost" | |
| echo "⚠️ This workflow will use your self-hosted runner instead" | |
| fi | |
| # Validate required fields | |
| if [[ -z "$TOKEN" || "$TOKEN" == "<evalai_user_auth_token>" ]]; then | |
| echo "❌ Invalid or missing token" | tee -a validation_error.log | |
| echo "is_valid=false" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ -z "$TEAM_PK" || "$TEAM_PK" == "<host_team_pk>" ]]; then | |
| echo "❌ Invalid or missing team_pk" | tee -a validation_error.log | |
| echo "is_valid=false" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ -z "$HOST_URL" || "$HOST_URL" == "<evalai_host_url>" ]]; then | |
| echo "❌ Invalid or missing evalai_host_url" | tee -a validation_error.log | |
| echo "is_valid=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Create issue if invalid | |
| if: steps.validate.outputs.is_valid == 'false' | |
| uses: peter-evans/create-issue-from-file@v4 | |
| with: | |
| title: "host_config.json validation failed" | |
| content-filepath: validation_error.log | |
| labels: | | |
| bug | |
| config | |
| self-hosted | |
| - name: Fail job if invalid | |
| if: steps.validate.outputs.is_valid == 'false' | |
| run: | | |
| echo "❌ host_config.json validation failed. See issue for details." | |
| exit 1 | |
| check-self-hosted-requirements: | |
| needs: validate-host-config | |
| if: needs.validate-host-config.outputs.requires_self_hosted == 'true' | |
| runs-on: self-hosted | |
| steps: | |
| - name: Self-hosted runner requirements check | |
| run: | | |
| echo "🏠 LOCAL DEVELOPMENT MODE DETECTED" | |
| echo "==================================" | |
| echo "" | |
| # Check 1: Verify Docker is available | |
| if ! docker info > /dev/null 2>&1; then | |
| echo "❌ Docker is not available or not running on this self-hosted runner" | |
| exit 1 | |
| else | |
| echo "✅ Docker is available" | |
| fi | |
| # Check 2: Test Docker Hub connectivity | |
| if ! curl -s --connect-timeout 5 https://hub.docker.com > /dev/null; then | |
| echo "⚠️ Warning: Cannot reach Docker Hub (network issue?)" | |
| else | |
| echo "✅ Docker Hub is accessible" | |
| fi | |
| echo "" | |
| echo "📋 Next job will run on this self-hosted runner and verify:" | |
| echo " • Docker engine is running" | |
| echo " • Can pull python:3.9-slim image" | |
| echo " • EvalAI server connectivity" | |
| process-evalai-challenge: | |
| needs: [detect-and-validate-challenge-branch, validate-host-config, check-self-hosted-requirements] | |
| if: | | |
| always() && | |
| needs.detect-and-validate-challenge-branch.outputs.is_challenge_branch == 'true' && | |
| needs.validate-host-config.outputs.is_valid == 'true' && | |
| (needs.check-self-hosted-requirements.result == 'success' || needs.check-self-hosted-requirements.result == 'skipped') | |
| runs-on: ${{ needs.validate-host-config.outputs.requires_self_hosted == 'true' && 'self-hosted' || 'ubuntu-latest' }} | |
| steps: | |
| - name: Checkout challenge branch | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} | |
| - name: Set up Python (GitHub-hosted only) | |
| if: needs.validate-host-config.outputs.requires_self_hosted != 'true' | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.9.21 | |
| - name: Prepare Docker environment (Self-hosted only) | |
| if: needs.validate-host-config.outputs.requires_self_hosted == 'true' | |
| run: | | |
| echo "🐳 PREPARING DOCKER ENVIRONMENT" | |
| echo "===============================" | |
| echo "Pulling Python 3.9 Docker image..." | |
| docker pull python:3.9-slim | |
| echo "✅ Docker image ready" | |
| - name: Install dependencies (GitHub-hosted) | |
| if: needs.validate-host-config.outputs.requires_self_hosted != 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f github/requirements.txt ]; then | |
| echo "📦 Installing dependencies from github/requirements.txt" | |
| pip install -r github/requirements.txt | |
| else | |
| echo "⚠️ No requirements.txt found in github/ directory" | |
| fi | |
| - name: Install dependencies (Self-hosted with Docker) | |
| if: needs.validate-host-config.outputs.requires_self_hosted == 'true' | |
| run: | | |
| echo "📦 Installing dependencies in Docker container" | |
| docker run --rm \ | |
| --add-host host.docker.internal:host-gateway \ | |
| -v "$(pwd):/workspace" \ | |
| -w /workspace \ | |
| python:3.9-slim bash -c " | |
| pip install --upgrade pip | |
| if [ -f github/requirements.txt ]; then | |
| echo '📦 Installing dependencies from github/requirements.txt' | |
| pip install -r github/requirements.txt | |
| else | |
| echo '⚠️ No requirements.txt found in github/ directory' | |
| fi | |
| " | |
| echo "✅ Dependencies installed in Docker" | |
| - name: Validate challenge configuration (GitHub-hosted) | |
| if: needs.validate-host-config.outputs.requires_self_hosted != 'true' | |
| run: | | |
| echo "🔍 VALIDATING CHALLENGE CONFIGURATION" | |
| echo "=====================================" | |
| python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} | |
| env: | |
| IS_VALIDATION: 'True' | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} | |
| - name: Validate challenge configuration (Self-hosted with Docker) | |
| if: needs.validate-host-config.outputs.requires_self_hosted == 'true' | |
| run: | | |
| echo "🔍 VALIDATING CHALLENGE CONFIGURATION (Docker)" | |
| echo "===============================================" | |
| docker run --rm \ | |
| --add-host host.docker.internal:host-gateway \ | |
| -v "$(pwd):/workspace" \ | |
| -w /workspace \ | |
| -e IS_VALIDATION='True' \ | |
| -e GITHUB_CONTEXT='${{ toJson(github) }}' \ | |
| -e GITHUB_REPOSITORY='${{ github.repository }}' \ | |
| -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ | |
| -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \ | |
| python:3.9-slim \ | |
| bash -c " | |
| pip install -r github/requirements.txt && | |
| python3 github/challenge_processing_script.py \$GIT_BRANCH | |
| " | |
| - name: Create or update challenge (GitHub-hosted) | |
| if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted != 'true' | |
| run: | | |
| echo "🚀 CREATING/UPDATING CHALLENGE" | |
| echo "==============================" | |
| python3 github/challenge_processing_script.py ${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }} | |
| env: | |
| IS_VALIDATION: 'False' | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }} | |
| - name: Create or update challenge (Self-hosted with Docker) | |
| if: github.event_name == 'push' && success() && needs.validate-host-config.outputs.requires_self_hosted == 'true' | |
| run: | | |
| echo "🚀 CREATING/UPDATING CHALLENGE (Docker)" | |
| echo "========================================" | |
| docker run --rm \ | |
| --add-host host.docker.internal:host-gateway \ | |
| -v "$(pwd):/workspace" \ | |
| -w /workspace \ | |
| -e IS_VALIDATION='False' \ | |
| -e GITHUB_CONTEXT='${{ toJson(github) }}' \ | |
| -e GITHUB_REPOSITORY='${{ github.repository }}' \ | |
| -e GITHUB_AUTH_TOKEN='${{ secrets.AUTH_TOKEN }}' \ | |
| -e GIT_BRANCH='${{ needs.detect-and-validate-challenge-branch.outputs.challenge_branch }}' \ | |
| python:3.9-slim \ | |
| bash -c " | |
| pip install -r github/requirements.txt && | |
| python3 github/challenge_processing_script.py \$GIT_BRANCH | |
| " |