Merging v1.157

.github/workflows/extra.yml

@@ -0,0 +1,139 @@
+name: Test extra build
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
+      - 'ansible/roles/doca/**'
+      - 'ansible/roles/cuda/**'
+      - 'ansible/roles/lustre/**'
+      - '.github/workflows/extra.yml'
+  pull_request:
+    paths:
+      - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
+      - 'ansible/roles/doca/**'
+      - 'ansible/roles/cuda/**'
+      - 'ansible/roles/lustre/**'
+      - '.github/workflows/extra.yml'
+
+jobs:
+  doca:
+    name: extra-build
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
+      cancel-in-progress: true
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false # allow other matrix jobs to continue even if one fails
+      matrix: # build RL8, RL9
+        build:
+          - image_name: openhpc-extra-RL8
+            source_image_name_key: RL8 # key into environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
+            inventory_groups: doca,cuda,lustre
+            volume_size: 30 # needed for cuda
+          - image_name: openhpc-extra-RL9
+            source_image_name_key: RL9
+            inventory_groups: doca,cuda,lustre
+            volume_size: 30 # needed for cuda
+    env:
+      ANSIBLE_FORCE_COLOR: True
+      OS_CLOUD: openstack
+      CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings
+      ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Load current fat images into GITHUB_ENV
+        # see https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#example-of-a-multiline-string
+        run: |
+          {
+            echo 'FAT_IMAGES<<EOF'
+            cat environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
+            echo EOF
+          } >> "$GITHUB_ENV"
+
+      - name: Record settings
+        run: |
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+          echo FAT_IMAGES: ${FAT_IMAGES}
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir ~/.ssh
+          echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+        shell: bash
+
+      - name: Add bastion's ssh key to known_hosts
+        run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
+        shell: bash
+
+      - name: Install ansible etc
+        run: dev/setup-env.sh
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+        shell: bash
+
+      - name: Setup environment
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+
+      - name: Build fat image with packer
+        id: packer_build
+        run: |
+          set -x
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          cd packer/
+          packer init .
+
+          PACKER_LOG=1 packer build \
+          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
+          -var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \
+          -var "image_name=${{ matrix.build.image_name }}" \
+          -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
+          -var "volume_size=${{ matrix.build.volume_size }}" \
+          openstack.pkr.hcl
+
+      - name: Get created image names from manifest
+        id: manifest
+        run: |
+          . venv/bin/activate
+          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
+          while ! openstack image show -f value -c name $IMAGE_ID; do
+            sleep 5
+          done
+          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
+          echo $IMAGE_ID > image-id.txt
+          echo $IMAGE_NAME > image-name.txt
+
+      - name: Make image usable for further builds
+        run: |
+          . venv/bin/activate
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+
+      - name: Delete image for automatically-run workflows
+        run: |
+          . venv/bin/activate
+          openstack image delete "${{ steps.manifest.outputs.image-id }}"
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+
+      - name: Upload manifest artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-details-${{ matrix.build.image_name }}
+          path: |
+            ./image-id.txt
+            ./image-name.txt
+          overwrite: true

.github/workflows/fatimage.yml

@@ -15,36 +15,25 @@ jobs:
   openstack:
     name: openstack-imagebuild
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
       cancel-in-progress: true
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false # allow other matrix jobs to continue even if one fails
-      matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
-        os_version:
-          - RL8
-          - RL9
+      matrix: # build RL8, RL9
         build:
-          - openstack.openhpc
-          - openstack.openhpc-cuda
-        exclude:
-          - os_version: RL8
-            build: openstack.openhpc-cuda
+          - image_name: openhpc-RL8
+            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.qcow2
+            inventory_groups: control,compute,login,update
+          - image_name: openhpc-RL9
+            source_image_name: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2
+            inventory_groups: control,compute,login,update
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
-      SOURCE_IMAGES_MAP: |
-        {
-          "RL8": {
-            "openstack.openhpc": "rocky-latest-RL8",
-            "openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
-          },
-          "RL9": {
-            "openstack.openhpc": "rocky-latest-RL9",
-            "openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
-          }
-        }
+      ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2
@@ -90,13 +79,11 @@ jobs:
 
           PACKER_LOG=1 packer build \
           -on-error=${{ vars.PACKER_ON_ERROR }} \
-          -only=${{ matrix.build }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
+          -var "source_image_name=${{ matrix.build.source_image_name }}" \
+          -var "image_name=${{ matrix.build.image_name }}" \
+          -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
           openstack.pkr.hcl
-        env:
-          PKR_VAR_os_version: ${{ matrix.os_version }}
-          SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][matrix.build] }}
 
       - name: Get created image names from manifest
         id: manifest
@@ -107,14 +94,21 @@ jobs:
             sleep 5
           done
           IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
           echo $IMAGE_ID > image-id.txt
           echo $IMAGE_NAME > image-name.txt
 
+      - name: Make image usable for further builds
+        run: |
+          . venv/bin/activate
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+
       - name: Upload manifest artifact
         uses: actions/upload-artifact@v4
         with:
-          name: image-details-${{ matrix.build }}-${{ matrix.os_version }}
+          name: image-details-${{ matrix.build.image_name }}
           path: |
             ./image-id.txt
             ./image-name.txt
-          overwrite: true
+          overwrite: true

.github/workflows/nightly-cleanup.yml

@@ -1,17 +1,8 @@
 name: Cleanup CI clusters
 on:
   workflow_dispatch:
-      inputs:
-        ci_cloud:
-          description: 'Select the CI_CLOUD'
-          required: true
-          type: choice
-          options:
-            - LEAFCLOUD
-            - SMS
-            - ARCUS
   schedule:
-    - cron: '0 20 * * *'  # Run at 8PM - image sync runs at midnight
+    - cron: '0 21 * * *'  # Run at 9PM - image sync runs at midnight
 
 jobs:
   ci_cleanup:
@@ -52,20 +43,55 @@ jobs:
       - name: Find CI clusters
         run: |
           . venv/bin/activate
-          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq)
-          echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
+          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq || true)
+          echo "DEBUG: Raw CI clusters: $CI_CLUSTERS"
+
+          if [[ -z "$CI_CLUSTERS" ]]; then
+            echo "No matching CI clusters found."
+          else
+            # Flatten multiline value so can be passed as env var
+            CI_CLUSTERS_FORMATTED=$(echo "$CI_CLUSTERS" | tr '\n' ' ' | sed 's/ $//')
+            echo "DEBUG: Formatted CI clusters: $CI_CLUSTERS_FORMATTED"
+            echo "ci_clusters=$CI_CLUSTERS_FORMATTED" >> $GITHUB_ENV
+          fi
         shell: bash
 
       - name: Delete clusters if control node not tagged with keep
         run: |
           . venv/bin/activate
-          for cluster_prefix in ${CI_CLUSTERS}
+          if [[ -z ${ci_clusters} ]]; then
+            echo "No clusters to delete."
+            exit 0
+          fi
+
+          for cluster_prefix in ${ci_clusters}
           do
-            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
-            if [[ $TAGS =~ "keep" ]]; then
-              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+            echo "Processing cluster: $cluster_prefix"
+            # Get all servers with the matching name for control node
+            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
+            SERVER_COUNT=$(echo "$CONTROL_SERVERS" | jq length)
+
+            if [[ $SERVER_COUNT -gt 1 ]]; then
+              echo "Multiple servers found for control node '${cluster_prefix}-control'. Checking tags for each..."
+
+              for server in $(echo "$CONTROL_SERVERS" | jq -r '.[].ID'); do
+                # Get tags for each control node
+                TAGS=$(openstack server show "$server" --column tags --format value)
+
+                if [[ $TAGS =~ "keep" ]]; then
+                  echo "Skipping ${cluster_prefix} (server ${server}) - control instance is tagged as keep"
+                else
+                  ./dev/delete-cluster.py ${cluster_prefix} --force
+                fi
+              done
             else
-              yes | ./dev/delete-cluster.py ${cluster_prefix}
+              # If only one server, extract its tags and proceed
+              TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags')
+              if [[ $TAGS =~ "keep" ]]; then
+                echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+              else
+                ./dev/delete-cluster.py ${cluster_prefix} --force
+              fi
             fi
           done
         shell: bash