Add delay before preingestion time checks

Author: hanyux-nv

crates/api-core/src/tests/host_bmc_firmware_test.rs

@@ -2339,6 +2339,7 @@ async fn test_preingestion_time_sync_ok(
         .into_inner();
 
     let addr = response.address.as_str();
+    let ip_addr = IpAddr::from_str(addr).unwrap();
     // Insert endpoint with current versions that are up to date
     insert_endpoint_version(&mut txn, addr, "6.00.30.00", "1.13.2", false).await?;
     txn.commit().await?;
@@ -2349,6 +2350,8 @@ async fn test_preingestion_time_sync_ok(
     // then check firmware and complete.
     mgr.run_single_iteration().await?;
 
+    // Second iteration applies site NTP servers and records when that
+    // succeeded, but does not check BMC time until the convergence wait elapses.
     mgr.run_single_iteration().await?;
 
     let actions = env.redfish_sim.actions_since(&timepoint);
@@ -2360,6 +2363,36 @@ async fn test_preingestion_time_sync_ok(
         "Expected SetNtpServers when site NTP is configured"
     );
 
+    let mut txn = pool.begin().await.unwrap();
+    let endpoints = db::explored_endpoints::find_all_by_ip(ip_addr, &mut txn).await?;
+    let endpoint = endpoints.first().expect("Endpoint should exist");
+    assert!(
+        matches!(
+            endpoint.preingestion_state,
+            PreingestionState::SetNtpServers {
+                set_at: Some(_),
+                attempts: 0
+            }
+        ),
+        "Expected SetNtpServers wait after applying NTP, got: {:?}",
+        endpoint.preingestion_state
+    );
+    txn.commit().await?;
+
+    // The next iteration should still wait for BMC NTP convergence.
+    mgr.run_single_iteration().await?;
+
+    let mut txn = pool.begin().await.unwrap();
+    db::explored_endpoints::set_preingestion_set_ntp_servers(
+        ip_addr,
+        Some(chrono::Utc::now() - chrono::TimeDelta::minutes(3)),
+        0,
+        &mut txn,
+    )
+    .await?;
+    txn.commit().await?;
+
+    // Once the convergence wait has elapsed, time sync and firmware checks complete.
     mgr.run_single_iteration().await?;
 
     let mut txn = pool.begin().await.unwrap();

crates/preingestion-manager/src/lib.rs

@@ -79,6 +79,8 @@ const INITIAL_BMC_RESET_MAX_ATTEMPTS: u32 = 3;
 
 /// How many times to attempt configuring site NTP servers before giving up.
 const SET_NTP_SERVERS_MAX_ATTEMPTS: u32 = 3;
+/// How long to wait for NTP servers to converge before checking time sync.
+const SET_NTP_SERVERS_CONVERGENCE_WAIT: chrono::TimeDelta = chrono::TimeDelta::minutes(1);
 
 pub struct PreingestionManager {
     static_info: Arc<PreingestionManagerStatic>,
@@ -1515,12 +1517,26 @@ impl PreingestionManagerStatic {
         set_at: Option<&DateTime<Utc>>,
         attempts: u32,
     ) -> PreingestionManagerResult<bool> {
-        if self.ntp_servers.is_empty()
-            || attempts >= SET_NTP_SERVERS_MAX_ATTEMPTS
-            || set_at.is_some()
-        {
+        if self.ntp_servers.is_empty() || attempts >= SET_NTP_SERVERS_MAX_ATTEMPTS {
+            tracing::info!(
+                "{} has no NTP servers configured or max attempts reached; running initial checks",
+                endpoint.address
+            );
+            return self.run_initial_checks(db, endpoint).await;
+        }
+
+        if let Some(set_at) = set_at {
+            let elapsed = Utc::now().signed_duration_since(*set_at);
+            if elapsed < SET_NTP_SERVERS_CONVERGENCE_WAIT {
+                tracing::info!(
+                    "{} waiting for BMC NTP servers to converge before checking time sync",
+                    endpoint.address
+                );
+                return Ok(false);
+            }
+
             tracing::info!(
-                "{} NTP setup is complete, unavailable, or exhausted; running initial checks",
+                "{} BMC NTP convergence wait complete; running initial checks",
                 endpoint.address
             );
             return self.run_initial_checks(db, endpoint).await;
@@ -1556,7 +1572,7 @@ impl PreingestionManagerStatic {
         }
 
         tracing::info!(
-            "{} set NTP servers; running initial checks on next iteration",
+            "{} set NTP servers; waiting for BMC time to converge",
             endpoint.address
         );
         db.with_txn(|txn| {