cluster_redhat.yml

name: Cluster RedHat

on:
  push:
    branches:
      - develop
  workflow_dispatch:
    inputs:
      azure_region:
        description: 'Azure region to deploy resources'
        required: true
        default: 'centralus'
        type: choice
        options:
          - centralus
          - eastus
          - eastus2
          - westus
          - westus2
          - westus3
          - northcentralus
          - southcentralus
          - canadacentral
          - canadaeast
          - uksouth
          - ukwest
          - northeurope
          - westeurope
  pull_request:
    branches:
      - '*'

jobs:
  build-and-test-cluster-redhat:
    runs-on: self-hosted
    env:
      UNIQUE_ID: ${{ github.run_id }}-${{ github.run_number }}
      BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
      IP_ADDRESS: ""
      AZURE_IP: ""
      WINDOWS_IP: ""
      ENROLLMENT_TOKEN: ""
      ES_PASSWORD: ""
      KIBANA_PASSWORD: ""
      ELASTIC_AGENT_VERSION: "8.18.8"

    steps:
      - name: Generate random number
        shell: bash
        run: |
          RANDOM_NUM=$(shuf -i 1000000000-9999999999 -n 1)
          echo "UNIQUE_ID=${RANDOM_NUM}_cluster_redhat_${{ github.run_number }}" >> $GITHUB_ENV
          
      - name: Checkout repository
        uses: actions/checkout@v4.1.1
    
      - name: Setup environment variables
        run: |
          PUBLIC_IP=$(curl -s https://api.ipify.org)
          echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV

      - name: Get branch name
        shell: bash
        run: |
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
          else
            echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
          fi
                
      - name: Set the environment for docker-compose 
        run: |
          cd testing/v2/development
          # Get the UID and GID of the current user
          echo "HOST_UID=$(id -u)" > .env
          echo "HOST_GID=$(id -g)" >> .env
          cat .env

      - name: Build pipeline container
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} build pipeline --no-cache

      - name: Start pipeline container
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline
          
      - name: Install Python requirements
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            cd /home/lme-user/LME/testing/v2/installers/azure && \
            pip install -r requirements.txt
          "
          
      - name: Build Azure RHEL instance with Windows VM
        env:
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T \
            -e AZURE_CLIENT_ID \
            -e AZURE_CLIENT_SECRET \
            -e AZURE_TENANT_ID \
            -e AZURE_SUBSCRIPTION_ID \
            pipeline bash -c "
              cd /home/lme-user/LME/testing/v2/installers && \
              python3 ./azure/build_azure_linux_network.py \
                -g pipe-${{ env.UNIQUE_ID }} \
                -s ${{ env.IP_ADDRESS }}/32 \
                -vs Standard_E16d_v4 \
                -l ${{ inputs.azure_region || 'centralus' }} \
                -ast 23:00 \
                --use-rhel \
                -w \
                -y
            "

      - name: Install LME on Azure RHEL instance
        run: |
          cd testing/v2/development
          sleep 60
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            cd /home/lme-user/LME/testing/v2/installers && \
            IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
            ./install_v2/install_rhel.sh lme-user \$IP_ADDRESS  \"pipe-${{ env.UNIQUE_ID }}.password.txt\" ${{ env.BRANCH_NAME }}
          "

      - name: Wait for LME installation to complete
        run: |
          echo "Waiting for LME installation to fully complete..."
          sleep 300

      - name: Get Azure and Windows IP addresses
        run: |
          cd testing/v2/development
          AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt")
          echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV
          echo "Azure IP: $AZURE_IP"
          
          # Get Windows VM IP from Azure resource group - try multiple methods
          WINDOWS_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            # Try getting public IP directly
            az network public-ip show --resource-group pipe-${{ env.UNIQUE_ID }} --name ws1-public-ip --query ipAddress --output tsv 2>/dev/null || \
            # Try getting from VM network interfaces
            az vm show --resource-group pipe-${{ env.UNIQUE_ID }} --name ws1 --show-details --query publicIps --output tsv 2>/dev/null || \
            # Try listing all IPs in resource group and find ws1
            az vm list-ip-addresses --resource-group pipe-${{ env.UNIQUE_ID }} --query '[?virtualMachine.name==\`ws1\`].virtualMachine.network.publicIpAddresses[0].ipAddress' --output tsv 2>/dev/null || \
            echo ''
          ")
          
          if [ -z "$WINDOWS_IP" ]; then
            echo "Warning: Could not retrieve Windows VM public IP, using private IP instead"
            WINDOWS_IP="10.1.0.4"
          fi
          echo "WINDOWS_IP=$WINDOWS_IP" >> $GITHUB_ENV    
          echo "Windows IP: $WINDOWS_IP"
          
          echo "Azure IP: $AZURE_IP, Windows IP: $WINDOWS_IP"

      - name: Retrieve Elastic password 
        env:
          AZURE_IP: ${{ env.AZURE_IP }}
        run: |
          cd testing/v2/development
          ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$elastic'" | tail -n 1 | tr -d '\n')
          echo "::add-mask::$ES_PASSWORD"
          echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV
          echo "Elastic password retrieved successfully"
          KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$kibana_system'" | tail -n 1 | tr -d '\n')
          echo "::add-mask::$KIBANA_PASSWORD"
          echo "KIBANA_PASSWORD=$KIBANA_PASSWORD" >> $GITHUB_ENV
          echo "Kibana password retrieved successfully"

      - name: Install test requirements on Azure RHEL instance
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            cd /home/lme-user/LME/testing/v2/installers && \
            IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
            ssh lme-user@\$IP_ADDRESS  'whoami && hostname && \
            wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm && \
            sudo yum install -y ./google-chrome-stable_current_x86_64.rpm && \
            cd /home/lme-user/LME/testing/tests && \
            python3 -m venv venv && \
            source venv/bin/activate && \
            pip install -r requirements.txt '
          "

      - name: Retrieve Elastic policy ID and enrollment token
        env:
          KIBANA_URL: "https://localhost:5601" 
          ES_USERNAME: "elastic" 
          ES_PASSWORD: ${{ env.ES_PASSWORD }}
        run: |
          cd testing/v2/development
          
          # Retrieve policy ID
          POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            ssh lme-user@${{ env.AZURE_IP }} '
              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \
                -H \"kbn-xsrf: true\" \
                -H \"Content-Type: application/json\" |
              jq -r '.items[0].id'
            '
          ")
          echo "Retrieved Policy ID: $POLICY_ID"
          
          # Retrieve enrollment token using the policy ID
          ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            ssh lme-user@${{ env.AZURE_IP }} '
              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
                -H \"kbn-xsrf: true\" \
                -H \"Content-Type: application/json\" \
                -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" |
              jq -r .item.api_key
            '
          ")
          echo "Retrieved enrollment token: $ENROLLMENT_TOKEN"

          # Mask the enrollment token in logs and set it as an environment variable
          echo "::add-mask::$ENROLLMENT_TOKEN"
          echo "ENROLLMENT_TOKEN=$ENROLLMENT_TOKEN" >> $GITHUB_ENV
          echo "Policy ID and Enrollment Token retrieved successfully"

      - name: Install the Elastic Agent on Windows Azure VM
        env:
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T \
            -e AZURE_CLIENT_ID \
            -e AZURE_CLIENT_SECRET \
            -e AZURE_TENANT_ID \
            -e AZURE_SUBSCRIPTION_ID \
            pipeline bash -c "
              az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID && \
              cd /home/lme-user/LME/testing/v2/installers/lib/ && \
              chmod +x install_agent_windows_azure.sh && \
              ./install_agent_windows_azure.sh \
                --resource-group pipe-${{ env.UNIQUE_ID }} \
                --vm-name ws1 \
                --token ${{ env.ENROLLMENT_TOKEN }} \
                --version ${{ env.ELASTIC_AGENT_VERSION }} \
                --debug
            "

      - name: Check if the Windows Elastic agent is reporting 
        env:
          ES_PASSWORD: ${{ env.ES_PASSWORD }}
        run: |
          sleep 360
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
              ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
              'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh windows'
          "

      - name: Wait 5 minutes after Windows agent check
        run: |
          sleep 300

      - name: Run api tests on Azure RHEL instance
        env:
          ES_PASSWORD: ${{ env.ES_PASSWORD }}
          KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }}
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \
            echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \
            echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \
            echo elastic=\"$ES_PASSWORD\" >> .env && \
            source venv/bin/activate && \
            pytest -v api_tests/'
          "

      - name: Run selenium tests on Azure RHEL instance
        env:
          ES_PASSWORD: ${{ env.ES_PASSWORD }}
          KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }}
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \
            echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \
            echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \
            echo elastic=\"$ES_PASSWORD\" >> .env && \
            source venv/bin/activate && \
            pytest -v selenium_tests/'
          "
    
      - name: Cleanup Azure resources
        if: always()
        env:
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_SECRET: ${{ secrets.AZURE_SECRET }}
          AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
            az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_SECRET --tenant $AZURE_TENANT
            az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait
          "
          
      - name: Stop and remove containers
        if: always()
        run: |
          cd testing/v2/development
          docker compose -p ${{ env.UNIQUE_ID }} down
          docker system prune -af