Merge branch 'main' into ntp_server_config

Author: hanyux-nv

crates/api-core/src/tests/dpu_reprovisioning.rs

@@ -1321,6 +1321,28 @@ pub enum ReprovisionState {
     BufferTime,
     VerifyFirmareVersions,
     WaitingForNetworkConfig,
+    PrepareHostBootRepair,
+    UnlockHostForBootRepair {
+        #[serde(default)]
+        unlock_host_state: UnlockHostState,
+    },
+    CheckHostBootConfig,
+    CheckHostBootConfigAfterHostReboot,
+    ConfigureHostBoot {
+        #[serde(default)]
+        retry_count: u32,
+    },
+    WaitingForHostBiosJob {
+        bios_config_info: BiosConfigInfo,
+    },
+    PollingHostBiosSetup {
+        #[serde(default)]
+        retry_count: u32,
+    },
+    SetHostBootOrder {
+        set_boot_order_info: SetBootOrderInfo,
+    },
+    LockHostAfterBootRepair,
     RebootHostBmc,
     RebootHost,
     NotUnderReprovision,
@@ -1797,7 +1819,9 @@ pub enum UefiSetupState {
 /// `bios_job_id` is `Some` while polling a vendor BIOS job (e.g. Dell). `None` only during
 /// `HandleBiosJobFailure` recovery from stuck PollingBiosSetup; non-Dell hosts reboot in
 /// `configure_host_bios` and never enter job-polling substates.
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+///
+/// Derived ordering is used by enclosing reprovision states to report the least advanced DPU.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 #[serde(rename_all = "lowercase")]
 pub struct BiosConfigInfo {
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1808,7 +1832,8 @@ pub struct BiosConfigInfo {
     pub retry_count: u32,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+/// Variant order follows BIOS job progression for derived reprovision-state comparisons.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 #[serde(tag = "state", rename_all = "lowercase")]
 pub enum BiosConfigState {
     WaitForBiosJobScheduled,
@@ -1821,7 +1846,8 @@ pub enum BiosConfigState {
     },
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+/// Derived ordering is used by enclosing reprovision states to report the least advanced DPU.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 #[serde(rename_all = "lowercase")]
 pub struct SetBootOrderInfo {
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1832,7 +1858,8 @@ pub struct SetBootOrderInfo {
     pub retry_count: u32,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
+/// Variant order follows boot-order job progression for derived reprovision-state comparisons.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 #[serde(tag = "state", rename_all = "lowercase")]
 pub enum SetBootOrderState {
     SetBootOrder,
@@ -2020,7 +2047,8 @@ pub enum HostPlatformConfigurationState {
     LockHost,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Default)]
+/// Variant order follows unlock progression for derived reprovision-state comparisons.
+#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord, Default)]
 #[serde(tag = "state", rename_all = "lowercase")]
 pub enum UnlockHostState {
     #[default]
@@ -2705,7 +2733,8 @@ impl<'r> FromRow<'r, PgRow> for MachineInterfaceSnapshot {
 
 // TODO: reconcile with site_explorer::PowerState. They are almost
 // identical but here we have Reset enum item.
-#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
+/// Variant order is a deterministic tie-breaker inside derived recovery-state comparisons.
+#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub enum PowerState {
     Off,
     On,
@@ -2972,6 +3001,10 @@ mod tests {
     // variant; the parsed value (PartialEq) is the whole assertion.
     #[test]
     fn test_json_deserialize_managed_host_states() {
+        let machine_id =
+            MachineId::from_str("fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng")
+                .unwrap();
+
         scenarios!(
             run = |s| serde_json::from_str::<ManagedHostState>(s).map_err(drop);
             "assigned booting with discovery image, default retry" {
@@ -2990,6 +3023,30 @@ mod tests {
                 }),
             }
 
+            "dpu reprovision host boot configure state" {
+                r#"{"state":"dpureprovision","dpu_states":{"states":{"fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng":{"configurehostboot":{"retry_count":2}}}}}"# => Yields(ManagedHostState::DPUReprovision {
+                    dpu_states: DpuReprovisionStates {
+                        states: HashMap::from([(
+                            machine_id,
+                            ReprovisionState::ConfigureHostBoot { retry_count: 2 },
+                        )]),
+                    },
+                }),
+            }
+
+            "dpu reprovision host boot unlock default state" {
+                r#"{"state":"dpureprovision","dpu_states":{"states":{"fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng":{"unlockhostforbootrepair":{}}}}}"# => Yields(ManagedHostState::DPUReprovision {
+                    dpu_states: DpuReprovisionStates {
+                        states: HashMap::from([(
+                            machine_id,
+                            ReprovisionState::UnlockHostForBootRepair {
+                                unlock_host_state: UnlockHostState::DisableLockdown,
+                            },
+                        )]),
+                    },
+                }),
+            }
+
             "host init polling bios setup, default retry" {
                 r#"{"state":"hostinit","machine_state":{"state":"pollingbiossetup"}}"# => Yields(ManagedHostState::HostInit {
                     machine_state: MachineState::PollingBiosSetup { retry_count: 0 },

crates/api-model/src/machine/mod.rs

@@ -165,11 +165,69 @@ write_files:
             /bin/rm /run/cloud-init.done
           fi
       }
+      function ensure_ethernet_link_type()
+      {
+          local log_file="/var/log/forge/link-type.log"
+          mkdir -p /var/log/forge
+
+          # Find the DPU config device before any management VRF setup depends on Ethernet.
+          local mst_device
+          mst_device="$(compgen -G '/dev/mst/mt*_pciconf0' | head -n 1)"
+          if [[ -z "${mst_device}" ]]
+          then
+            echo "No mst pciconf device found for LINK_TYPE normalization" | tee -a "${log_file}"
+            return
+          fi
+
+          # Non-VPI hardware may not expose LINK_TYPE, so treat query failures as non-fatal.
+          local link_config
+          if ! link_config="$(/usr/bin/mlxconfig -d "${mst_device}" q 2>&1)"
+          then
+            echo "LINK_TYPE query unsupported; continuing" | tee -a "${log_file}"
+            echo "${link_config}" | tee -a "${log_file}"
+            return
+          fi
+          echo "${link_config}" >> "${log_file}"
+
+          # Apply Ethernet mode only to exposed ports not already reporting Ethernet.
+          local link_type_keys=()
+          local link_type_key
+          local link_type_value
+          for link_type_key in LINK_TYPE_P1 LINK_TYPE_P2
+          do
+            link_type_value="$(printf '%s\n' "${link_config}" | awk -v key="${link_type_key}" '$1 == key { print $2; exit }')"
+            if [[ -n "${link_type_value}" && "${link_type_value}" != *"(2)"* ]]
+            then
+              link_type_keys+=("${link_type_key}")
+            fi
+          done
+
+          local link_type_changed=0
+          for link_type_key in "${link_type_keys[@]}"
+          do
+            echo "Normalizing ${link_type_key} to Ethernet" | tee -a "${log_file}"
+            if /usr/bin/mlxconfig -y -d "${mst_device}" set "${link_type_key}=2"
+            then
+              link_type_changed=1
+            else
+              echo "Failed to normalize ${link_type_key}; continuing" | tee -a "${log_file}"
+            fi
+          done
+
+          if [[ "${link_type_changed}" == "1" ]]
+          then
+            trap - EXIT
+            reboot
+            # Keep cloud-init incomplete; the controller restart path will re-enter PXE with the persisted LINK_TYPE setting.
+            exit 1
+          fi
+      }
       trap cleanup EXIT
       set -e
       date -s '@{{ seconds_since_epoch }}'
       hwclock --systohc
       /usr/bin/bfrec --capsule /lib/firmware/mellanox/boot/capsule/efi_sbkeysync.cap || true
+      ensure_ethernet_link_type
       /usr/bin/bfcfg
       ip vrf exec mgmt curl --retry 5 --retry-all-errors -v -o /opt/forge/forge_root.pem {{ pxe_url }}/api/v0/tls/root_ca
       echo "downloaded forge root cert"