[ANSIENG-5796] Add KRaft migration rollback playbook with auto-phase detection

molecule/migration_rollback_phase2_converge.yml

@@ -0,0 +1,69 @@
+---
+# migration_rollback_phase2_converge.yml
+#
+# Molecule converge for the plaintext-rhel-rollback-phase2 scenario.
+#
+# Cluster state reached before rollback: PREMIGRATION (Phase 2)
+#   - KRaft controllers provisioned and running with zookeeper.connect set
+#   - Brokers untouched — still in pure ZooKeeper mode, no migration flag
+#
+# This is the state after kafka_controller.yml has run with kraft_migration: true
+# but BEFORE the broker restart that activates dual-write (migrate_to_dual_write).
+#
+# Rollback action: stop controllers only, no broker changes needed.
+
+# ── Step 1: Transform inventory groups ────────────────────────────────────────
+# Rename ZK-cluster groups so the KRaft group names can take their place.
+# After this transformation:
+#   - zookeeper    → original ZK node (was zookeeper_migration)
+#   - kafka_broker → original ZK-mode broker (was kafka_broker_migration)
+#   - kafka_controller → dedicated KRaft controller (was kafka_controller_migration)
+- name: Transform inventory groups for migration
+  hosts: kafka_broker
+  gather_facts: false
+  tasks:
+    - name: Rename ZK cluster groups out of the way
+      lineinfile:
+        path: "{{ inventory_dir }}/ansible_inventory.yml"
+        regexp: "{{ item }}:"
+        line: "{{ item }}_zoo:"
+      delegate_to: 127.0.0.1
+      run_once: true
+      loop:
+        - zookeeper
+        - kafka_broker
+
+    - name: Activate KRaft cluster groups
+      lineinfile:
+        path: "{{ inventory_dir }}/ansible_inventory.yml"
+        regexp: "{{ item }}_migration:"
+        line: "{{ item }}:"
+      delegate_to: 127.0.0.1
+      run_once: true
+      loop:
+        - kafka_controller
+        - kafka_broker
+        - zookeeper
+
+    - name: Refresh inventory
+      meta: refresh_inventory
+
+# ── Step 2: Pre-migration checks ─────────────────────────────────────────────
+- name: Run migration pre-flight checks
+  import_playbook: ../playbooks/migration_precheck.yml
+  vars:
+    kraft_migration: true
+
+# ── Step 3: Provision KRaft controllers (PREMIGRATION state) ─────────────────
+# kafka_controller.yml with kraft_migration: true configures controllers with
+# zookeeper.connect so they join the existing ZK quorum. Brokers are untouched.
+- name: Provision KRaft controllers
+  import_playbook: ../playbooks/kafka_controller.yml
+  vars:
+    kraft_migration: true
+
+# ── Step 4: Run Phase 2 rollback ──────────────────────────────────────────────
+- name: Run KRaft migration rollback
+  import_playbook: ../playbooks/kraft_migration_rollback.yml
+  vars:
+    kraft_migration: true

molecule/migration_rollback_phase3_converge.yml

@@ -0,0 +1,153 @@
+---
+# migration_rollback_phase3_converge.yml
+#
+# Molecule converge for the plaintext-rhel-rollback-phase3 scenario.
+#
+# Cluster state reached before rollback: HYBRID_DUAL_WRITE (Phase 3)
+#   - KRaft controllers running in dual-write mode (zookeeper.connect present)
+#   - Brokers have zookeeper.metadata.migration.enable=true and zookeeper.connect
+#   - Brokers do NOT have process.roles=broker
+#   - ZkMigrationState=1 via Jolokia
+#
+# This is the state after ZKtoKraftMigration.yml --tags migrate_to_dual_write.
+# The playbooks are imported directly (not via ZKtoKraftMigration.yml) to avoid
+# running the migrate_to_kraft and rollback plays that share the same file.
+#
+# Rollback action: stop controllers, clean ZK znodes, one rolling broker restart.
+
+# ── Step 1: Transform inventory groups ────────────────────────────────────────
+- name: Transform inventory groups for migration
+  hosts: kafka_broker
+  gather_facts: false
+  tasks:
+    - name: Rename ZK cluster groups out of the way
+      lineinfile:
+        path: "{{ inventory_dir }}/ansible_inventory.yml"
+        regexp: "{{ item }}:"
+        line: "{{ item }}_zoo:"
+      delegate_to: 127.0.0.1
+      run_once: true
+      loop:
+        - zookeeper
+        - kafka_broker
+
+    - name: Activate KRaft cluster groups
+      lineinfile:
+        path: "{{ inventory_dir }}/ansible_inventory.yml"
+        regexp: "{{ item }}_migration:"
+        line: "{{ item }}:"
+      delegate_to: 127.0.0.1
+      run_once: true
+      loop:
+        - kafka_controller
+        - kafka_broker
+        - zookeeper
+
+    - name: Refresh inventory
+      meta: refresh_inventory
+
+# ── Step 2: Pre-migration checks ─────────────────────────────────────────────
+- name: Run migration pre-flight checks
+  import_playbook: ../playbooks/migration_precheck.yml
+  vars:
+    kraft_migration: true
+
+# ── Step 3: Provision KRaft controllers ──────────────────────────────────────
+- name: Provision KRaft controllers
+  import_playbook: ../playbooks/kafka_controller.yml
+  vars:
+    kraft_migration: true
+
+# ── Step 4: Validate Jolokia endpoint is reachable ───────────────────────────
+# Mirrors the untagged inline play in ZKtoKraftMigration.yml (lines 8-42).
+# Ensures controllers are fully up and Jolokia is serving before broker migration.
+- name: Validate Jolokia endpoint access on controllers
+  hosts: kafka_controller
+  gather_facts: false
+  any_errors_fatal: true
+  tasks:
+    - name: Import variables role
+      import_role:
+        name: variables
+
+    - name: Test Jolokia ZkMigrationState endpoint access
+      uri:
+        url: "{{ 'https' if kafka_controller_jolokia_ssl_enabled | bool else 'http' }}://localhost:{{ kafka_controller_jolokia_port }}/jolokia/read/kafka.controller:type=KafkaController,name=ZkMigrationState"
+        validate_certs: false
+        return_content: true
+        status_code: 200
+        force_basic_auth: "{{ item.auth }}"
+        url_username: "{{ item.username if item.auth else omit }}"
+        url_password: "{{ item.password if item.auth else omit }}"
+      register: jolokia_auth_results
+      failed_when: false
+      retries: "{{ jolokia_endpoint_health_check_retries }}"
+      delay: "{{ jolokia_endpoint_health_check_delay }}"
+      loop:
+        - {name: "no_auth",   auth: false}
+        - {name: "basic_auth", auth: true, username: "{{ jolokia_user }}", password: "{{ jolokia_password }}"}
+
+    - name: Check if either auth method succeeded
+      set_fact:
+        jolokia_accessible: >-
+          {{ jolokia_auth_results.results
+             | map(attribute='status') | select('defined') | select('equalto', 200)
+             | list | length > 0 }}
+
+    - name: Fail if Jolokia endpoint is not reachable
+      fail:
+        msg: "ERROR: Unable to access Jolokia ZkMigrationState endpoint on controllers."
+      when: not jolokia_accessible | bool
+
+# ── Step 5: Migrate brokers to dual-write mode ───────────────────────────────
+# kafka_broker.yml with kraft_migration: true sets:
+#   zookeeper.metadata.migration.enable=true, zookeeper.connect, keeps no process.roles
+# Serial restart mirrors ZKtoKraftMigration.yml's deployment_strategy: serial.
+- name: Migrate brokers to dual-write mode
+  import_playbook: ../playbooks/kafka_broker.yml
+  vars:
+    kraft_migration: true
+    deployment_strategy: serial
+
+# ── Step 6: Wait for ZkMigrationState=1 (HYBRID_DUAL_WRITE confirmed) ────────
+- name: Wait for metadata migration to reach HYBRID_DUAL_WRITE
+  hosts: kafka_controller
+  gather_facts: false
+  any_errors_fatal: true
+  tasks:
+    - name: Import variables role
+      import_role:
+        name: variables
+
+    - name: Wait for ZkMigrationState=1 (no auth)
+      uri:
+        url: "{{ 'https' if kafka_controller_jolokia_ssl_enabled | bool else 'http' }}://localhost:{{ kafka_controller_jolokia_port }}/jolokia/read/kafka.controller:type=KafkaController,name=ZkMigrationState"
+        validate_certs: false
+        return_content: true
+        status_code: 200
+      retries: "{{ metadata_migration_retries }}"
+      delay: 90
+      until: (jolokia_output.content | from_json).value.Value == 1
+      register: jolokia_output
+      when: jolokia_auth_mode == "none"
+
+    - name: Wait for ZkMigrationState=1 (basic auth)
+      uri:
+        url: "{{ 'https' if kafka_controller_jolokia_ssl_enabled | bool else 'http' }}://localhost:{{ kafka_controller_jolokia_port }}/jolokia/read/kafka.controller:type=KafkaController,name=ZkMigrationState"
+        validate_certs: false
+        return_content: true
+        force_basic_auth: true
+        url_username: "{{ jolokia_user }}"
+        url_password: "{{ jolokia_password }}"
+        status_code: 200
+      retries: "{{ metadata_migration_retries }}"
+      delay: 90
+      until: (jolokia_output.content | from_json).value.Value == 1
+      register: jolokia_output
+      when: jolokia_auth_mode == "basic"
+
+# ── Step 7: Run Phase 3 rollback ──────────────────────────────────────────────
+- name: Run KRaft migration rollback
+  import_playbook: ../playbooks/kraft_migration_rollback.yml
+  vars:
+    kraft_migration: true