Offline GitHub workflow #20
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Offline | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| azure_region: | |
| description: 'Azure region to deploy resources' | |
| required: true | |
| default: 'centralus' | |
| type: choice | |
| options: | |
| - centralus | |
| - eastus | |
| - eastus2 | |
| - westus | |
| - westus2 | |
| - westus3 | |
| - northcentralus | |
| - southcentralus | |
| - canadacentral | |
| - canadaeast | |
| - uksouth | |
| - ukwest | |
| - northeurope | |
| - westeurope | |
| pull_request: | |
| branches: | |
| - '*' | |
| jobs: | |
| offline: | |
| runs-on: self-hosted | |
| env: | |
| UNIQUE_ID: | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| IP_ADDRESS: "" | |
| U1_IP: "" | |
| U2_PRIVATE_IP: "10.1.0.6" | |
| U1_PASSWORD: "" | |
| ES_PASSWORD: "" | |
| KIBANA_PASSWORD: "" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4.1.1 | |
| - name: Set the environment for docker compose | |
| run: | | |
| cd testing/v2/development | |
| echo "HOST_UID=$(id -u)" > .env | |
| echo "HOST_GID=$(id -g)" >> .env | |
| PUBLIC_IP=$(curl -s https://api.ipify.org) | |
| echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV | |
| echo "UNIQUE_ID=$(openssl rand -hex 3 | head -c 6)-${{ github.run_number }}" >> $GITHUB_ENV | |
| - name: Get branch name | |
| shell: bash | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV | |
| else | |
| echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV | |
| fi | |
| - name: Start pipeline container | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline | |
| - name: Install Python requirements | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers/azure && \ | |
| pip install -r requirements.txt | |
| " | |
| - name: Build u1 Azure instance (bastion/build machine with internet) | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| -e AZURE_CLIENT_ID \ | |
| -e AZURE_CLIENT_SECRET \ | |
| -e AZURE_TENANT_ID \ | |
| -e AZURE_SUBSCRIPTION_ID \ | |
| pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| python3 ./azure/build_azure_linux_network.py \ | |
| -g pipe-${{ env.UNIQUE_ID }} \ | |
| -s ${{ env.IP_ADDRESS }}/32 \ | |
| -vs Standard_D8_v4 \ | |
| -l ${{ inputs.azure_region || 'centralus' }} \ | |
| -ast 23:00 \ | |
| -pub Canonical \ | |
| -io ubuntu-24_04-lts \ | |
| -is server \ | |
| -os 256 \ | |
| --no-prompt \ | |
| -y | |
| " | |
| - name: Get u1 IP and password | |
| run: | | |
| cd testing/v2/development | |
| U1_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt") | |
| echo "U1_IP=$U1_IP" >> $GITHUB_ENV | |
| U1_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.password.txt") | |
| echo "U1_PASSWORD=$U1_PASSWORD" >> $GITHUB_ENV | |
| echo "U1 IP: $U1_IP" | |
| - name: Create restrictive NSG for u2 (offline VM) | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| -e AZURE_CLIENT_ID \ | |
| -e AZURE_CLIENT_SECRET \ | |
| -e AZURE_TENANT_ID \ | |
| -e AZURE_SUBSCRIPTION_ID \ | |
| pipeline bash -c " | |
| az login --service-principal -u \$AZURE_CLIENT_ID -p \$AZURE_CLIENT_SECRET --tenant \$AZURE_TENANT_ID && \ | |
| # Create NSG for offline VM | |
| az network nsg create \ | |
| --resource-group pipe-${{ env.UNIQUE_ID }} \ | |
| --name NSG-offline && \ | |
| # Allow inbound traffic from VNet only (so U1 can SSH to U2) | |
| az network nsg rule create \ | |
| --resource-group pipe-${{ env.UNIQUE_ID }} \ | |
| --nsg-name NSG-offline \ | |
| --name allow-vnet-inbound \ | |
| --priority 1000 \ | |
| --direction Inbound \ | |
| --access Allow \ | |
| --protocol '*' \ | |
| --source-address-prefix 'VirtualNetwork' \ | |
| --destination-address-prefix 'VirtualNetwork' \ | |
| --source-port-range '*' \ | |
| --destination-port-ranges '*' && \ | |
| # Deny all other inbound traffic (blocks public access) | |
| az network nsg rule create \ | |
| --resource-group pipe-${{ env.UNIQUE_ID }} \ | |
| --nsg-name NSG-offline \ | |
| --name deny-public-inbound \ | |
| --priority 4000 \ | |
| --direction Inbound \ | |
| --access Deny \ | |
| --protocol '*' \ | |
| --source-address-prefix '*' \ | |
| --destination-address-prefix '*' \ | |
| --source-port-range '*' \ | |
| --destination-port-ranges '*' && \ | |
| # Allow outbound traffic to VNet only (local network communication) | |
| az network nsg rule create \ | |
| --resource-group pipe-${{ env.UNIQUE_ID }} \ | |
| --nsg-name NSG-offline \ | |
| --name allow-vnet-outbound \ | |
| --priority 1000 \ | |
| --direction Outbound \ | |
| --access Allow \ | |
| --protocol '*' \ | |
| --source-address-prefix 'VirtualNetwork' \ | |
| --destination-address-prefix 'VirtualNetwork' \ | |
| --source-port-range '*' \ | |
| --destination-port-ranges '*' && \ | |
| # Deny all internet outbound traffic | |
| az network nsg rule create \ | |
| --resource-group pipe-${{ env.UNIQUE_ID }} \ | |
| --nsg-name NSG-offline \ | |
| --name deny-internet-outbound \ | |
| --priority 4000 \ | |
| --direction Outbound \ | |
| --access Deny \ | |
| --protocol '*' \ | |
| --source-address-prefix '*' \ | |
| --destination-address-prefix 'Internet' \ | |
| --source-port-range '*' \ | |
| --destination-port-ranges '*' | |
| " | |
| - name: Create u2 VM (offline target machine - no public IP, same VNet) | |
| env: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| -e AZURE_CLIENT_ID \ | |
| -e AZURE_CLIENT_SECRET \ | |
| -e AZURE_TENANT_ID \ | |
| -e AZURE_SUBSCRIPTION_ID \ | |
| pipeline bash -c " | |
| az login --service-principal -u \$AZURE_CLIENT_ID -p \$AZURE_CLIENT_SECRET --tenant \$AZURE_TENANT_ID && \ | |
| az vm create \ | |
| --resource-group pipe-${{ env.UNIQUE_ID }} \ | |
| --name u2-offline \ | |
| --image Canonical:ubuntu-24_04-lts:server:latest \ | |
| --size Standard_D8_v4 \ | |
| --admin-username lme-user \ | |
| --admin-password '${{ env.U1_PASSWORD }}' \ | |
| --vnet-name VNet1 \ | |
| --subnet SNet1 \ | |
| --nsg NSG-offline \ | |
| --public-ip-address '' \ | |
| --private-ip-address ${{ env.U2_PRIVATE_IP }} \ | |
| --os-disk-size-gb 256 | |
| " | |
| - name: Wait for u1 SSH to be ready | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| echo 'Waiting for SSH to be available on ${{ env.U1_IP }}...' | |
| for i in {1..60}; do | |
| if timeout 5 bash -c 'cat < /dev/null > /dev/tcp/${{ env.U1_IP }}/22' 2>/dev/null; then | |
| echo 'SSH is ready!' | |
| exit 0 | |
| fi | |
| echo \"Attempt \$i/60: SSH not ready yet, waiting 5 seconds...\" | |
| sleep 5 | |
| done | |
| echo 'SSH did not become available within timeout' | |
| exit 1 | |
| " | |
| - name: Setup SSH keys for u1 | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| cd /home/lme-user/LME/testing/v2/installers && \ | |
| ./lib/copy_ssh_key.sh lme-user ${{ env.U1_IP }} pipe-${{ env.UNIQUE_ID }}.password.txt | |
| " | |
| - name: Setup SSH keys from u1 to u2 (via bastion) | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| # Install sshpass on u1 and generate SSH key | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'sudo apt-get update && sudo apt-get install -y sshpass' && \ | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh-keygen -t rsa -f ~/.ssh/id_rsa -N \"\" -q || true' && \ | |
| # Copy password to u1 for SSH key setup | |
| scp -o StrictHostKeyChecking=no /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.password.txt lme-user@${{ env.U1_IP }}:~/u2.password.txt && \ | |
| # Wait for u2 SSH to be ready (checking from u1) | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} ' | |
| echo \"Waiting for SSH on ${{ env.U2_PRIVATE_IP }}...\" | |
| for i in {1..60}; do | |
| if timeout 5 bash -c \"cat < /dev/null > /dev/tcp/${{ env.U2_PRIVATE_IP }}/22\" 2>/dev/null; then | |
| echo \"SSH is ready on u2!\" | |
| exit 0 | |
| fi | |
| echo \"Attempt \$i/60: u2 SSH not ready yet, waiting 5 seconds...\" | |
| sleep 5 | |
| done | |
| echo \"u2 SSH did not become available within timeout\" | |
| exit 1 | |
| ' && \ | |
| # Copy SSH key from u1 to u2 | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'sshpass -f ~/u2.password.txt ssh-copy-id -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }}' | |
| " | |
| - name: Verify u2 outbound connectivity is blocked | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| echo 'Verifying that outbound connectivity from u2 is blocked...' | |
| # SSH to u1, then to u2, and test HTTP connectivity (should fail) | |
| if ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \"timeout 10 curl -s --connect-timeout 5 https://www.google.com > /dev/null 2>&1\"'; then | |
| echo '✗ ERROR: Outbound connectivity is NOT blocked on u2!' | |
| exit 1 | |
| else | |
| echo '✓ Outbound connectivity confirmed blocked on u2 (HTTP test failed as expected)' | |
| fi | |
| " | |
| - name: Prepare offline resources on u1 | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'sudo apt-get update && sudo apt-get install -y git' && \ | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'cd ~ && git clone https://github.com/cisagov/LME.git && cd LME && git checkout ${{ env.BRANCH_NAME }}' && \ | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'cd ~/LME && ./scripts/prepare_offline.sh' | |
| " | |
| - name: Find offline tarball on u1 | |
| run: | | |
| cd testing/v2/development | |
| TARBALL_NAME=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ls -t ~/lme-offline-*.tar.gz 2>/dev/null | head -1 | xargs basename' | |
| ") | |
| echo "TARBALL_NAME=$TARBALL_NAME" >> $GITHUB_ENV | |
| echo "Found tarball: $TARBALL_NAME" | |
| - name: Copy offline tarball from u1 to u2 (via bastion) | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'scp -o StrictHostKeyChecking=no ~/lme-offline-*.tar.gz lme-user@${{ env.U2_PRIVATE_IP }}:~/' | |
| " | |
| - name: Extract offline tarball on u2 (via bastion) | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \"cd ~ && \ | |
| sudo mv lme-offline-*.tar.gz /var/ && \ | |
| tar -xzf /var/lme-offline-*.tar.gz -C ~/ && \ | |
| cd ~/LME\"' | |
| " | |
| - name: Run LME installer on u2 (via bastion) | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \"cd ~/LME && NON_INTERACTIVE=true AUTO_CREATE_ENV=true ./install.sh -o -d\"' | |
| " | |
| - name: Retrieve Elastic password (via bastion) | |
| run: | | |
| cd testing/v2/development | |
| echo "U2 Private IP: ${{ env.U2_PRIVATE_IP }}" | |
| ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \". /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \\\$elastic\"'" | tail -n 1 | tr -d '\n') | |
| #echo "::add-mask::$ES_PASSWORD" | |
| echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV | |
| KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \". /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \\\$kibana_system\"'" | tail -n 1 | tr -d '\n') | |
| #echo "::add-mask::$KIBANA_PASSWORD" | |
| echo "KIBANA_PASSWORD=$KIBANA_PASSWORD" >> $GITHUB_ENV | |
| echo "Kibana password retrieved successfully." | |
| #- name: Install test requirements on u2 (via bastion) | |
| # run: | | |
| # cd testing/v2/development | |
| # docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| # ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \"whoami && hostname && sudo apt-get update && \ | |
| # sudo apt-get install -y python3-venv wget && \ | |
| # wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ | |
| # sudo apt install -y ./google-chrome-stable_current_amd64.deb && \ | |
| # cd /home/lme-user/LME/testing/tests && \ | |
| # python3 -m venv venv && \ | |
| # source venv/bin/activate && \ | |
| # pip install -r requirements.txt\"' | |
| # " | |
| #- name: Run tests on u2 (via bastion) | |
| # env: | |
| # ES_PASSWORD: ${{ env.ES_PASSWORD }} | |
| # elastic: ${{ env.ES_PASSWORD }} | |
| # KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} | |
| # run: | | |
| # sleep 360 | |
| # cd testing/v2/development | |
| # docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " | |
| # ssh -o StrictHostKeyChecking=no lme-user@${{ env.U1_IP }} 'ssh -o StrictHostKeyChecking=no lme-user@${{ env.U2_PRIVATE_IP }} \"cd /home/lme-user/LME/testing/tests && \ | |
| # echo ELASTIC_PASSWORD=\\\"$ES_PASSWORD\\\" >> .env && \ | |
| # echo KIBANA_PASSWORD=\\\"$KIBANA_PASSWORD\\\" >> .env && \ | |
| # echo elastic=\\\"$ES_PASSWORD\\\" >> .env && \ | |
| # cat .env && \ | |
| # source venv/bin/activate && \ | |
| # pytest -v api_tests/linux_only/ selenium_tests/linux_only/ api_tests/connectivity/\"' | |
| # " | |
| #- name: Cleanup Azure resources | |
| # if: always() | |
| # env: | |
| # AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| # AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} | |
| # AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} | |
| # AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| # run: | | |
| # cd testing/v2/development | |
| # docker compose -p ${{ env.UNIQUE_ID }} exec -T \ | |
| # -e AZURE_CLIENT_ID \ | |
| # -e AZURE_CLIENT_SECRET \ | |
| # -e AZURE_TENANT_ID \ | |
| # -e AZURE_SUBSCRIPTION_ID \ | |
| # pipeline bash -c " | |
| # az login --service-principal -u \$AZURE_CLIENT_ID -p \$AZURE_CLIENT_SECRET --tenant \$AZURE_TENANT_ID && \ | |
| # az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait | |
| # " | |
| - name: Stop and remove containers | |
| if: always() | |
| run: | | |
| cd testing/v2/development | |
| docker compose -p ${{ env.UNIQUE_ID }} down | |
| docker system prune -af |