MGMT-23352: Fix disconnected deployment with dev-scripts

Fix disconnected deployment, enclave DNS/config, and Quay LVMS support:

Disconnected and enclave:
- Add BASE_DOMAIN=lab in dev-scripts config so CLUSTER_DOMAIN is enclave-test.lab
- Add mirror -> LZ IP to virsh network DNS after provisioning (mirror.<domain> resolution)
- Fix pullSecretPath to config/pull-secret.json (was .config/) for install-config.yaml
- Quay backend storage_path: /var/quaydata -> /datastorage/registry
- Increase VM extra disk size to 120G (configure_devscripts.sh)

Quay operator:
- Add quay_lvms.yaml: LVMS PVC, QuayRegistry, and deployment patch for registry storage
- Split tasks.yaml: ODF backend (existing QuayRegistry) vs LVMS (include quay_lvms.yaml)
- ODF branch: HPA managed true, task conditional on blockStorageBackend == 'odf'

Operator tasks (cluster API):
- acm_cis.yaml: Add KUBECONFIG to ClusterImageSet tasks (Get current, Delete, Create)
- clair_disconnected.yaml: Add KUBECONFIG to Clair tasks (Get Clair Pod, Execute Import in pod)

Author: danielerez

defaults/operators.yaml

@@ -112,9 +112,9 @@ operators:
 
   # AAP is a dependency for osac-operator
   - name: ansible-automation-platform-operator
-    defaultChannel: stable-2.5-cluster-scoped
+    defaultChannel: stable-2.6-cluster-scoped
     channels:
-    - name: stable-2.5-cluster-scoped
+    - name: stable-2.6-cluster-scoped
     namespace: ansible-aap
     source: cs-redhat-operator-index-v4-20
     disableTargetNamespace: true

operators/advanced-cluster-management/acm_cis.yaml

@@ -2,6 +2,8 @@
 # ACM CIS tasks: ClusterImageSet reconciliation
 
 - name: Get current ClusterImageSet
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
   kubernetes.core.k8s_info:
     api_version: hive.openshift.io/v1
     kind: ClusterImageSet
@@ -12,6 +14,8 @@
     r_desired_image_set: "{{ openshift_versions | map(attribute='version') | map('regex_replace', '^(.*)$', 'openshift-v\\1-disconnected') | list }}"
 
 - name: Delete existing ClusterImageSet
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
   kubernetes.core.k8s:
     state: absent
     definition:
@@ -27,6 +31,8 @@
     loop_var: imageset
 
 - name: Create ClusterImageSet
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
   kubernetes.core.k8s:
     state: present
     definition:

operators/quay-operator/clair_disconnected.yaml

@@ -1,4 +1,6 @@
 - name: Get Clair Pod object from quay-enterprise
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
   kubernetes.core.k8s_info:
     kind: Pod
     namespace: quay-enterprise
@@ -60,6 +62,8 @@
     dest: "/var/www/html/clair/updates.json.gz"
 
 - name: Get Clair Pod object from quay-enterprise
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
   kubernetes.core.k8s_info:
     kind: Pod
     namespace: quay-enterprise
@@ -73,6 +77,8 @@
   failed_when: clair_pod_list.resources | length == 0
 
 - name: Execute Import inside the Clair Pod
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
   ansible.builtin.shell: |
     {{ workingDir }}/bin/oc exec -n quay-enterprise {{ clair_pod.metadata.name }} -- \
     /bin/sh -c "

operators/quay-operator/quay_disconnected.yaml

@@ -33,8 +33,8 @@
       docker://registry-quay-quay-enterprise.apps.{{ clusterName }}.{{ baseDomain }} \
       --dest-tls-verify=false \
       --src-tls-verify=false \
-      --parallel-images 10 \
-      --parallel-layers 10 \
+      --parallel-images {{ 4 if quayBackend == 'LocalStorage' else 10 }} \
+      --parallel-layers {{ 1 if quayBackend == 'LocalStorage' else 10 }} \
       --retry-times 10 \
       --retry-delay 0 \
       --image-timeout 40m0s \

operators/quay-operator/quay_lvms.yaml

@@ -0,0 +1,99 @@
+---
+# LVMS-specific tasks for Quay: PVC, QuayRegistry, and deployment patch to mount registry storage.
+# Included from tasks.yaml only when blockStorageBackend == 'lvms'.
+
+- name: Create LVMS PVC for Quay Storage
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
+        name: quay-storage-pvc
+        namespace: quay-enterprise
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 300Gi
+        storageClassName: lvms-vg1
+
+- name: Ensure QuayRegistry is present (LVMS Backend)
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: quay.redhat.com/v1
+      kind: QuayRegistry
+      metadata:
+        name: registry
+        namespace: quay-enterprise
+      spec:
+        configBundleSecret: quay-config
+        components:
+        - kind: objectstorage
+          managed: false
+        - kind: quay
+          managed: true
+          overrides:
+            replicas: 1
+            resources:
+              limits:
+                cpu: "4"
+                memory: 12Gi
+              requests:
+                cpu: "{{ '2' if (disconnected | default(true)) else '1' }}"
+                memory: "{{ '6Gi' if (disconnected | default(true)) else '4Gi' }}"
+        - kind: horizontalpodautoscaler
+          managed: false
+  retries: 1
+  delay: 10
+  register: r_quay_lvms
+  until: r_quay_lvms is succeeded
+
+- name: Wait for Quay app deployment to exist (LVMS)
+  kubernetes.core.k8s_info:
+    api_version: apps/v1
+    kind: Deployment
+    namespace: quay-enterprise
+    name: registry-quay-app
+  register: quay_app_deployment
+  retries: 30
+  delay: 10
+  until: quay_app_deployment.resources | length > 0
+
+- name: Set quay container index for LVMS patch (first container is the app)
+  ansible.builtin.set_fact:
+    quay_container_index: 0
+  when: quay_app_deployment.resources | default([]) | length > 0
+
+- name: Check if Quay app deployment already has registry-storage volume (LVMS)
+  ansible.builtin.set_fact:
+    quay_registry_storage_already_patched: "{{ quay_app_deployment.resources[0].spec.template.spec.volumes | selectattr('name', 'equalto', 'registry-storage') | list | length > 0 }}"
+  when: quay_app_deployment.resources | default([]) | length > 0
+
+- name: Patch Quay app deployment to mount registry storage PVC (LVMS)
+  environment:
+    KUBECONFIG: "{{ workingDir }}/ocp-cluster/auth/kubeconfig"
+  ansible.builtin.shell: |
+    oc patch deployment registry-quay-app -n quay-enterprise --type=json -p='{{ quay_lvms_patch | to_json }}'
+  args:
+    executable: /bin/bash
+  vars:
+    quay_lvms_patch:
+      - op: add
+        path: /spec/template/spec/volumes/-
+        value:
+          name: registry-storage
+          persistentVolumeClaim:
+            claimName: quay-storage-pvc
+      - op: add
+        path: /spec/template/spec/containers/{{ quay_container_index }}/volumeMounts/-
+        value:
+          name: registry-storage
+          mountPath: /datastorage/registry
+  when:
+    - quay_app_deployment.resources | default([]) | length > 0
+    - quay_container_index is defined
+    - not (quay_registry_storage_already_patched | default(false))
+  register: quay_patch_result

operators/quay-operator/tasks.yaml

@@ -26,6 +26,9 @@
                    | combine(quayBackendRGWDefaults)
                    | combine(quayBackendRGWConfiguration))
                    if quayBackend == 'RadosGWStorage'
+                   else (quayBackendDefaults
+                   | combine(quayBackendLocalStorageConfiguration))
+                   if quayBackend == 'LocalStorage'
                    else quayBackendDefaults)
                    | to_yaml }}
           DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS:
@@ -57,7 +60,7 @@
           matcher:
             disable_updaters: true
 
-- name: Ensure QuayRegistry is present
+- name: Ensure QuayRegistry is present (ODF Backend)
   kubernetes.core.k8s:
     state: present
     definition:
@@ -70,7 +73,7 @@
         configBundleSecret: quay-config
         components:
         - kind: horizontalpodautoscaler
-          managed: "{{ true if blockStorageBackend == 'odf' else false }}"
+          managed: true
         - kind: quay
           managed: true
           overrides:
@@ -83,11 +86,15 @@
                 memory: "{{ '16Gi' if (disconnected | default(true)) else '2Gi' }}"
         - kind: objectstorage
           managed: false
+  when: blockStorageBackend == 'odf'
   retries: 60
   delay: 10
   register: r_quay
   until: r_quay is succeeded
 
+- name: Run LVMS-specific Quay tasks (PVC, QuayRegistry, deployment patch)
+  ansible.builtin.include_tasks: quay_lvms.yaml
+  when: blockStorageBackend == 'lvms'
 
 - name: Get router pod IP for Quay route access
   kubernetes.core.k8s_info:

playbooks/tasks/configure_hardware_ironic_setup.yaml

@@ -37,7 +37,7 @@
   ansible.builtin.wait_for:
     port: "{{ http_server_port }}"
     host: localhost
-    timeout: 10
+    timeout: 30
 
 - name: Verify ISO is accessible via temporary HTTP server
   ansible.builtin.uri:

scripts/configure_devscripts.sh

@@ -59,6 +59,16 @@ echo "Creating dev-scripts configuration at: $CONFIG_FILE"
 ENCLAVE_NUM_MASTERS="${ENCLAVE_NUM_MASTERS:-3}"
 ENCLAVE_NUM_LANDINGZONE="${ENCLAVE_NUM_LANDINGZONE:-1}"
 
+# VM extra disk size: 200G for disconnected (mirroring), 60G for connected
+DEPLOYMENT_MODE="${ENCLAVE_DEPLOYMENT_MODE:-disconnected}"
+if [ "$DEPLOYMENT_MODE" = "connected" ]; then
+    VM_EXTRADISKS_SIZE_VAL="60G"
+    MASTER_MEMORY_VAL=24576    # 24 GB for connected
+else
+    VM_EXTRADISKS_SIZE_VAL="200G"
+    MASTER_MEMORY_VAL=32768    # 32 GB for disconnected
+fi
+
 # Create configuration file
 cat > "$CONFIG_FILE" <<EOF
 #!/bin/bash
@@ -92,14 +102,14 @@ export NUM_EXTRA_WORKERS=0
 # =============================================================================
 
 # Master VMs need resources for OpenShift control plane nodes
-export MASTER_MEMORY=24576    # 24 GB RAM
+export MASTER_MEMORY=${MASTER_MEMORY_VAL}    # 24 GB (connected) or 32 GB (disconnected)
 export MASTER_DISK=120        # 120 GB disk
 export MASTER_VCPU=12         # 12 vCPUs
 
 # Extra disks for storage (used by LVMS for PersistentVolumes)
 export VM_EXTRADISKS=true
 export VM_EXTRADISKS_LIST="vdb"
-export VM_EXTRADISKS_SIZE="60G"
+export VM_EXTRADISKS_SIZE="${VM_EXTRADISKS_SIZE_VAL}"
 
 # =============================================================================
 # Landing Zone VM Specs
@@ -147,6 +157,9 @@ export CLUSTER_NAME="${ENCLAVE_CLUSTER_NAME}"
 # Cluster domain (for DNS)
 export CLUSTER_DOMAIN="${ENCLAVE_CLUSTER_NAME}.lab"
 
+# Base domain (so dev-scripts uses .lab not test.metalkube.org for CLUSTER_DOMAIN)
+export BASE_DOMAIN="lab"
+
 # Working directory (where VMs and configs are stored)
 export WORKING_DIR="/opt/dev-scripts"
 

scripts/generate_enclave_vars.sh

@@ -164,6 +164,8 @@ quayPassword: SuperPrivate123!
 # Option 1: External S3/RadosGW storage (RECOMMENDED for production)
 # Option 2: Local storage (NOT recommended for production)
 quayBackend: LocalStorage
+quayBackendLocalStorageConfiguration:
+  storage_path: /datastorage/registry
 
 # ============================================================================
 # Storage Backend
@@ -189,7 +191,7 @@ lvmsConfig: {}
 # Pull secret will be read from pullSecretPath
 pullSecret:
   auths: {}
-pullSecretPath: "{{ workingDir }}/.config/pull-secret.json"
+pullSecretPath: "{{ workingDir }}/config/pull-secret.json"
 
 # SSH public key path for cluster nodes
 sshPubPath: "{{ workingDir }}/.ssh/id_rsa.pub"
@@ -289,7 +291,7 @@ info "Generated configuration uses:"
 info "  - Worker IPs: ${WORKER_IP_START}-${WORKER_IP_END} (will be assigned during deployment)"
 info "  - Storage: LVMS with /dev/vda root disk"
 info "  - Registry: LocalStorage"
-info "  - Pull secret: Will use ~/.config/pull-secret.json on Landing Zone"
+info "  - Pull secret: Will use ~/config/pull-secret.json on Landing Zone"
 info "  - SSL certificates: Empty (self-signed will be generated)"
 echo ""
 info "Review config/global.yaml and config/certificates.yaml and adjust if needed before running Enclave Lab"

scripts/install_enclave.sh

@@ -291,8 +291,8 @@ if [ "$PULL_SECRET_FOUND" = true ]; then
     info "  Validated pull secret contains registry.redhat.io credentials"
 
     # Copy to Landing Zone
-    ssh $SSH_OPTS "$LZ_SSH" "mkdir -p ${LZ_ROOT_DIR}/.config"
-    scp $SSH_OPTS "$PULL_SECRET_SOURCE" "${LZ_SSH}:${LZ_ROOT_DIR}/.config/pull-secret.json"
+    ssh $SSH_OPTS "$LZ_SSH" "mkdir -p ${LZ_ROOT_DIR}/config"
+    scp $SSH_OPTS "$PULL_SECRET_SOURCE" "${LZ_SSH}:${LZ_ROOT_DIR}/config/pull-secret.json"
 
     success "Pull secret copied to Landing Zone"
 
@@ -307,7 +307,7 @@ import yaml
 import json
 
 # Read the pull secret
-with open('/home/cloud-user/.config/pull-secret.json', 'r') as f:
+with open('/home/cloud-user/config/pull-secret.json', 'r') as f:
     pull_secret = json.load(f)
 
 # Read config/global.yaml