wip: 🌱 Use irso after pivot

lib/common.sh

@@ -158,7 +158,7 @@ export CAPM3_BASE_URL="${CAPM3_BASE_URL:-metal3-io/cluster-api-provider-metal3}"
 export CAPM3REPO="${CAPM3REPO:-https://github.com/${CAPM3_BASE_URL}}"
 export CAPM3RELEASEBRANCH="${CAPM3RELEASEBRANCH:-main}"
 
-export USE_IRSO="${USE_IRSO:-false}"
+export USE_IRSO="true"
 export IRSOPATH="${IRSOPATH:-${M3PATH}/ironic-standalone-operator}"
 export IRSO_BASE_URL="${IRSO_BASE_URL:-metal3-io/ironic-standalone-operator}"
 export IRSOREPO="${IRSOREPO:-https://github.com/${IRSO_BASE_URL}}"

tests/roles/run_tests/tasks/move.yml

@@ -15,44 +15,72 @@
 
 - name: Fetch container logs (kind cluster)
   block:
-
   - name: Create directories for storing container logs (kind cluster)
     file:
       path: "/tmp/{{ CONTAINER_RUNTIME }}/{{ item }}"
       state: directory
     with_items:
     - "{{ ironic_containers }}"
     - "{{ general_containers }}"
-
   - name: Fetch container logs before pivoting (kind cluster)
     shell: "sudo {{ CONTAINER_RUNTIME }} logs {{ item }} > /tmp/{{ CONTAINER_RUNTIME }}/{{ item }}/stdout.log 2> /tmp/{{ CONTAINER_RUNTIME }}/{{ item }}/stderr.log"
     with_items:
     - "{{ ironic_containers }}"
     - "{{ general_containers }}"
 
-  - name: Remove ironic container from source cluster (kind cluster)
-    docker_container:
-      name: "{{ item }}"
-      state: absent
-    with_items: "{{ ironic_containers }}"
-
+- name: Remove ironic container from source cluster (kind cluster)
+  docker_container:
+    name: "{{ item }}"
+    state: absent
+  with_items: "{{ ironic_containers }}"
   when: BOOTSTRAP_CLUSTER == "kind"
-  become: yes
+  become: true
   become_user: root
 
 - name: Fetch container logs (minikube cluster)
   block:
-
   - name: Create directories for storing container logs (minikube cluster)
     file:
       path: "/tmp/{{ CONTAINER_RUNTIME }}/{{ item }}"
       state: directory
     with_items: "{{ general_containers }}"
-
   - name: Fetch container logs before pivoting (minikube cluster)
     shell: "sudo {{ CONTAINER_RUNTIME }} logs {{ item }} > /tmp/{{ CONTAINER_RUNTIME }}/{{ item }}/stdout.log 2> /tmp/{{ CONTAINER_RUNTIME }}/{{ item }}/stderr.log"
     with_items: "{{ general_containers }}"
+  become: true
+  become_user: root
+  when: BOOTSTRAP_CLUSTER == "minikube"
 
+- name: Remove Ironic CR from source cluster (minikube cluster)
+  kubernetes.core.k8s:
+    name: ironic
+    kind: Ironic
+    api_version: ironic.metal3.io/v1alpha1
+    state: absent
+  with_items: "{{ ironic_containers }}"
+
+- name: Remove IRSO from source cluster (minikube cluster)
+  shell: "make -C {{ IRSOPATH }} uninstall undeploy"
+  ignore_errors: true
+  when: BOOTSTRAP_CLUSTER == "minikube"
+
+- name: Label baremetalhost CRD to pivot.
+  shell: "kubectl label --overwrite crds baremetalhosts.metal3.io {{ item }}"
+  with_items:
+  - clusterctl.cluster.x-k8s.io=""
+  - clusterctl.cluster.x-k8s.io/move=""
+  - clusterctl.cluster.x-k8s.io/move-hierarchy=""
+
+- name: Fetch container logs (minikube cluster)
+  block:
+  - name: Create directories for storing container logs (minikube cluster)
+    file:
+      path: "/tmp/{{ CONTAINER_RUNTIME }}/{{ item }}"
+      state: directory
+    with_items: "{{ general_containers }}"
+  - name: Fetch container logs before pivoting (minikube cluster)
+    shell: "sudo {{ CONTAINER_RUNTIME }} logs {{ item }} > /tmp/{{ CONTAINER_RUNTIME }}/{{ item }}/stdout.log 2> /tmp/{{ CONTAINER_RUNTIME }}/{{ item }}/stderr.log"
+    with_items: "{{ general_containers }}"
   become: yes
   become_user: root
   when: BOOTSTRAP_CLUSTER == "minikube"
@@ -78,17 +106,124 @@
   - clusterctl.cluster.x-k8s.io=""
   - clusterctl.cluster.x-k8s.io/move=""
 
-- name: Obtain target cluster kubeconfig
+  # Install BMO
+- name: Install Baremetal Operator in target cluster
+  shell: "{{ BMOPATH }}/tools/deploy.sh -b {{ BMO_IRONIC_ARGS }}"
+  environment:
+    IRONIC_HOST: "{{ IRONIC_HOST }}"
+    IRONIC_HOST_IP: "{{ IRONIC_HOST_IP }}"
+    KUBECTL_ARGS: "{{ KUBECTL_ARGS }}"
+  args:
+    chdir: "{{ BMOPATH }}"
+
+  # Install IRSO and Ironic via IRSO
+- name: Set IPA_BASEURI in IRSO manager config
+  lineinfile:
+    path: "{{ IRSOPATH }}/config/manager/manager.env"
+    line: 'IPA_BASEURI=https://artifactory.nordix.org/artifactory/openstack-remote-cache/ironic-python-agent/dib'
+    create: yes
+
+- name: Install IRSO in target cluster
+  shell: "make install deploy IMG={{ IRSO_IMAGE }}"
+  args:
+    chdir: "{{ IRSOPATH }}"
+  environment:
+    KUBECONFIG: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+
+- name: Wait for IRSO controller manager to be available
+  kubernetes.core.k8s_info:
+    kind: Deployment
+    name: ironic-standalone-operator-controller-manager
+    namespace: ironic-standalone-operator-system
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    wait: true
+    wait_condition:
+      type: Available
+      status: "True"
+    wait_timeout: 60
+
+- name: Create ironic CA cert secret in target cluster
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: ironic-cacert
+        namespace: "{{ IRONIC_NAMESPACE }}"
+      type: kubernetes.io/tls
+      data:
+        tls.crt: "{{ lookup('file', IRONIC_CACERT_FILE) | b64encode }}"
+        tls.key: "{{ lookup('file', IRONIC_CAKEY_FILE) | b64encode }}"
+
+- name: Create ironic-credentials secret in target cluster
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: ironic-credentials
+        namespace: "{{ IRONIC_NAMESPACE }}"
+      type: Opaque
+      stringData:
+        username: "{{ lookup('env','IRONIC_USERNAME') | trim }}"
+        password: "{{ lookup('env','IRONIC_PASSWORD') | trim }}"
+
+- name: Create Ironic CR
+  set_fact:
+    ironic_spec:
+      apiVersion: ironic.metal3.io/v1alpha1
+      kind: Ironic
+      metadata:
+        name: ironic
+        namespace: "{{ IRONIC_NAMESPACE }}"
+      spec:
+        images:
+          deployRamdiskBranch: "{{ IPA_BRANCH }}"
+          deployRamdiskDownloader: "{{ IPA_DOWNLOADER_IMAGE }}"
+          ironic: "{{ IRONIC_IMAGE }}"
+          keepalived: "{{ IRONIC_KEEPALIVED_IMAGE }}"
+        version: "{{ IRSO_IRONIC_VERSION }}"
+        networking:
+          dhcp:
+            rangeBegin: "{{ CLUSTER_DHCP_RANGE_START }}"
+            rangeEnd: "{{ CLUSTER_DHCP_RANGE_END }}"
+            networkCIDR: "{{ BARE_METAL_PROVISIONER_NETWORK }}"
+          interface: "{{ BARE_METAL_PROVISIONER_INTERFACE }}"
+          ipAddress: "{{ CLUSTER_BARE_METAL_PROVISIONER_IP }}"
+          ipAddressManager: keepalived
+        deployRamdisk:
+          sshKey: "{{ SSH_PUB_KEY_CONTENT }}"
+
+- name: Add extraKernelParams for libvirt platform
+  set_fact:
+    ironic_spec: "{{ ironic_spec | combine({'spec': {'deployRamdisk': {'extraKernelParams': 'console=ttyS0'}}}, recursive=True) }}"
+  when: NODES_PLATFORM == 'libvirt'
+
+- name: Deploy Ironic CR in target cluster (retry if webhook not ready)
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    definition: "{{ ironic_spec }}"
+  register: ironic_create
+  retries: 10
+  delay: 3
+  until: ironic_create is succeeded
+
+- name: Wait for Ironic to be ready
   kubernetes.core.k8s_info:
-    kind: secrets
-    name: "{{ CLUSTER_NAME }}-kubeconfig"
-    namespace: "{{ NAMESPACE }}"
-  register: metal3_kubeconfig
-
-- name: Decode and save cluster kubeconfig
-  copy:
-    content: "{{ metal3_kubeconfig.resources[0].data.value | b64decode }}"
-    dest: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    kind: Ironic
+    name: ironic
+    namespace: "{{ IRONIC_NAMESPACE }}"
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    wait: true
+    wait_condition:
+      type: Ready
+      status: "True"
+    wait_timeout: "{{ (IRONIC_ROLLOUT_WAIT | default(10) | int * 60) }}"
 
 - name: Create namespace
   kubernetes.core.k8s:

tests/roles/run_tests/tasks/move_back.yml

@@ -24,14 +24,91 @@
     CONTAINER_RUNTIME: "{{ CONTAINER_RUNTIME }}"
   when: BOOTSTRAP_CLUSTER == "kind"
 
-- name: Install Ironic in Source cluster (Bootstrap Cluster is minikube)
-  shell: "{{ BMOPATH }}/tools/deploy.sh -i {{ BMO_IRONIC_ARGS }}"
-  environment:
-    IRONIC_HOST: "{{ IRONIC_HOST }}"
-    IRONIC_HOST_IP: "{{ IRONIC_HOST_IP }}"
+
+  # Install IRSO and Ironic via IRSO
+- name: Set IPA_BASEURI in IRSO manager config
+  lineinfile:
+    path: "{{ IRSOPATH }}/config/manager/manager.env"
+    line: 'IPA_BASEURI=https://artifactory.nordix.org/artifactory/openstack-remote-cache/ironic-python-agent/dib'
+    create: yes
   when: BOOTSTRAP_CLUSTER == "minikube"
+
+- name: Install and deploy IRSO
+  shell: "make install deploy IMG={{ IRSO_IMAGE }}"
   args:
-    chdir: "{{ BMOPATH }}"
+    chdir: "{{ IRSOPATH }}"
+  environment:
+    KUBECONFIG: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+  when: BOOTSTRAP_CLUSTER == "minikube"
+
+- name: Wait for IRSO controller manager to be available
+  kubernetes.core.k8s_info:
+    kind: Deployment
+    name: ironic-standalone-operator-controller-manager
+    namespace: ironic-standalone-operator-system
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    wait: true
+    wait_condition:
+      type: Available
+      status: "True"
+    wait_timeout: 60
+  when: BOOTSTRAP_CLUSTER == "minikube"
+
+- name: Set Ironic spec
+  when: BOOTSTRAP_CLUSTER == "minikube"
+  set_fact:
+    ironic_spec:
+      apiVersion: ironic.metal3.io/v1alpha1
+      kind: Ironic
+      metadata:
+        name: ironic
+        namespace: "{{ IRONIC_NAMESPACE }}"
+      spec:
+        images:
+          deployRamdiskBranch: "{{ IPA_BRANCH }}"
+          deployRamdiskDownloader: "{{ IPA_DOWNLOADER_IMAGE }}"
+          ironic: "{{ IRONIC_IMAGE }}"
+          keepalived: "{{ IRONIC_KEEPALIVED_IMAGE }}"
+          version: "{{ IRSO_IRONIC_VERSION }}"
+        networking:
+          dhcp:
+            rangeBegin: "{{ CLUSTER_DHCP_RANGE_START }}"
+            rangeEnd: "{{ CLUSTER_DHCP_RANGE_END }}"
+            networkCIDR: "{{ BARE_METAL_PROVISIONER_NETWORK }}"
+          interface: "{{ BARE_METAL_PROVISIONER_INTERFACE }}"
+          ipAddress: "{{ CLUSTER_BARE_METAL_PROVISIONER_IP }}"
+          ipAddressManager: keepalived
+        deployRamdisk:
+          sshKey: "{{ SSH_PUB_KEY_CONTENT }}"
+
+- name: Add extraKernelParams for libvirt platform
+  set_fact:
+    ironic_spec: "{{ ironic_spec | combine({'spec': {'deployRamdisk': {'extraKernelParams': 'console=ttyS0'}}}, recursive=True) }}"
+  when: NODES_PLATFORM == 'libvirt' and BOOTSTRAP_CLUSTER == "minikube"
+
+- name: Deploy Ironic using IRSO (retry if webhook not ready)
+  kubernetes.core.k8s:
+    state: present
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    definition: "{{ ironic_spec }}"
+  register: ironic_create
+  retries: 10
+  delay: 3
+  until: ironic_create is succeeded
+  when: BOOTSTRAP_CLUSTER == "minikube"
+
+- name: Wait for Ironic to be ready
+  when: BOOTSTRAP_CLUSTER == "minikube"
+  kubernetes.core.k8s_info:
+    kind: Ironic
+    name: ironic
+    namespace: "{{ IRONIC_NAMESPACE }}"
+    kubeconfig: "/tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml"
+    wait: true
+    wait_condition:
+      type: Ready
+      status: "True"
+    wait_timeout: "{{ (IRONIC_ROLLOUT_WAIT | default(10) | int * 60) }}"
 
 - name: Re-pivot everything back to source cluster
   shell: "clusterctl move --kubeconfig /tmp/kubeconfig-{{ CLUSTER_NAME }}.yaml --to-kubeconfig /home/$USER/.kube/config -n {{ NAMESPACE }} -v 10"

tests/roles/run_tests/vars/main.yml

@@ -16,7 +16,21 @@ WORKER_MACHINE_COUNT: "{{ lookup('env', 'WORKER_MACHINE_COUNT') | default(1, tru
 NUM_NODES: "{{ lookup('env', 'NUM_NODES') | default(2, true) }}"
 NODE_DRAIN_TIMEOUT: "{{ lookup('env', 'NODE_DRAIN_TIMEOUT') | default('0s', true) }}"
 BMOPATH: "{{ lookup('env', 'BMOPATH') }}"
+IRSOPATH: "{{ lookup('env', 'IRSOPATH') }}"
+IRONIC_ROLLOUT_WAIT: "{{ lookup('env', 'IRONIC_ROLLOUT_WAIT') | default(10, true) }}"
+IPA_DOWNLOADER_IMAGE: "{{ lookup('env', 'IPA_DOWNLOADER_IMAGE') | default('registry.nordix.org/metal3/ironic-python-agent-downloader:latest', true) }}"
+IRSO_IMAGE: "{{ lookup('env', 'IRSO_IMAGE') | default('registry.nordix.org/metal3/ironic-standalone-operator:latest', true) }}"
+IRONIC_IMAGE: "{{ lookup('env', 'IRONIC_IMAGE') | default('registry.nordix.org/metal3/ironic:latest', true) }}"
+IRONIC_KEEPALIVED_IMAGE: "{{ lookup('env', 'IRONIC_KEEPALIVED_IMAGE') | default('registry.nordix.org/metal3/ironic-keepalived:latest', true) }}"
+IRSO_IRONIC_VERSION: "{{ lookup('env', 'IRSO_IRONIC_VERSION') | default('latest', true) }}"
+CLUSTER_DHCP_RANGE_START: "{{ lookup('env', 'CLUSTER_DHCP_RANGE_START') | default('172.22.0.10', true) }}"
+CLUSTER_DHCP_RANGE_END: "{{ lookup('env', 'CLUSTER_DHCP_RANGE_END') | default('172.22.0.100', true) }}"
+BARE_METAL_PROVISIONER_NETWORK: "{{ lookup('env', 'BARE_METAL_PROVISIONER_NETWORK') | default('172.22.0.0/24', true) }}"
+BARE_METAL_PROVISIONER_INTERFACE: "{{ lookup('env', 'BARE_METAL_PROVISIONER_INTERFACE') | default('ironicendpoint', true) }}"
+CLUSTER_BARE_METAL_PROVISIONER_IP: "{{ lookup('env', 'CLUSTER_BARE_METAL_PROVISIONER_IP') | default('172.22.0.2', true) }}"
+IPA_BRANCH: "{{ lookup('env', 'IPA_BRANCH') | default('main', true) }}"
 IRONIC_DATA_DIR: "{{ lookup('env', 'IRONIC_DATA_DIR') }}"
+NODES_PLATFORM: "{{ lookup('env', 'NODES_PLATFORM') | default('libvirt', true) }}"
 KUBECONFIG_PATH: "/home/ubuntu/.kube/config"
 KUBERNETES_VERSION: "{{ lookup('env', 'KUBERNETES_VERSION') | default('v1.34.1', true) }}"
 UPGRADED_K8S_VERSION: "{{ lookup('env', 'UPGRADED_K8S_VERSION') | default('v1.34.1', true) }}"
@@ -72,6 +86,10 @@ CAPI_VERSION: "{{ lookup('env', 'CAPI_VERSION') | default('v1beta2', true) }}"
 CAPM3_VERSION: "{{ lookup('env', 'CAPM3_VERSION') | default('v1beta1', true) }}"
 IRONIC_IMAGE_DIR: "{{ lookup('env', 'IRONIC_IMAGE_DIR') | default('/opt/metal3-dev-env/ironic/html/images') }}"
 IRONIC_ENDPOINT_BRIDGE: "{{ lookup('env', 'BARE_METAL_PROVISIONER_INTERFACE') | default('provisioning', true) }}"
+IRONIC_CACERT_FILE: "{{ lookup('env', 'IRONIC_CACERT_FILE') | default(IRONIC_IMAGE_DIR + '/opt/metal3-dev-env/certs/ironic-ca.pem', true) }}"
+IRONIC_CAKEY_FILE: "{{ lookup('env', 'IRONIC_CAKEY_FILE') | default(IRONIC_IMAGE_DIR + '/opt/metal3-dev-env/certs/ironic-ca.key', true) }}"
+IRONIC_USERNAME: "{{ lookup('env', 'IRONIC_USERNAME') | default('admin', true) }}"
+IRONIC_PASSWORD: "{{ lookup('env', 'IRONIC_PASSWORD') | default('password', true) }}"
 UPGRADED_IMAGE_NAME: "{{ lookup('env', 'UPGRADED_IMAGE_NAME') }}"
 UPGRADED_RAW_IMAGE_NAME: "{{ lookup('env', 'UPGRADED_RAW_IMAGE_NAME') }}"