|
| 1 | +//! Durable, versioned lifecycle records for resumable multi-target deploys. |
| 2 | +
|
| 3 | +use std::fs; |
| 4 | +use std::path::PathBuf; |
| 5 | + |
| 6 | +use serde::{Deserialize, Serialize}; |
| 7 | + |
| 8 | +use crate::core::error::{Error, Result}; |
| 9 | +use crate::core::paths; |
| 10 | +use crate::core::phase_timing::PhaseTimingReport; |
| 11 | + |
| 12 | +const SCHEMA_VERSION: u32 = 1; |
| 13 | + |
| 14 | +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 15 | +pub struct DeployRunIdentity { |
| 16 | + pub source: String, |
| 17 | + pub artifact: String, |
| 18 | + pub components: Vec<String>, |
| 19 | + pub targets: Vec<String>, |
| 20 | + pub policy: String, |
| 21 | +} |
| 22 | + |
| 23 | +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 24 | +#[serde(rename_all = "snake_case")] |
| 25 | +pub enum DeployTargetStatus { |
| 26 | + Planned, |
| 27 | + Running, |
| 28 | + Succeeded, |
| 29 | + Failed, |
| 30 | +} |
| 31 | + |
| 32 | +#[derive(Debug, Clone, Serialize, Deserialize)] |
| 33 | +pub struct DeployTargetLifecycle { |
| 34 | + pub target: String, |
| 35 | + pub status: DeployTargetStatus, |
| 36 | + #[serde(default, skip_serializing_if = "Option::is_none")] |
| 37 | + pub error: Option<String>, |
| 38 | + #[serde(default, skip_serializing_if = "Option::is_none")] |
| 39 | + pub phase_timings: Option<PhaseTimingReport>, |
| 40 | +} |
| 41 | + |
| 42 | +#[derive(Debug, Clone, Serialize, Deserialize)] |
| 43 | +pub struct DeployLifecycleRun { |
| 44 | + pub schema_version: u32, |
| 45 | + pub id: String, |
| 46 | + pub identity: DeployRunIdentity, |
| 47 | + pub targets: Vec<DeployTargetLifecycle>, |
| 48 | +} |
| 49 | + |
| 50 | +impl DeployLifecycleRun { |
| 51 | + pub fn new(id: String, identity: DeployRunIdentity) -> Self { |
| 52 | + let targets = identity |
| 53 | + .targets |
| 54 | + .iter() |
| 55 | + .cloned() |
| 56 | + .map(|target| DeployTargetLifecycle { |
| 57 | + target, |
| 58 | + status: DeployTargetStatus::Planned, |
| 59 | + error: None, |
| 60 | + phase_timings: None, |
| 61 | + }) |
| 62 | + .collect(); |
| 63 | + Self { |
| 64 | + schema_version: SCHEMA_VERSION, |
| 65 | + id, |
| 66 | + identity, |
| 67 | + targets, |
| 68 | + } |
| 69 | + } |
| 70 | + |
| 71 | + pub fn resume(&mut self, identity: &DeployRunIdentity) -> Result<()> { |
| 72 | + if self.schema_version != SCHEMA_VERSION { |
| 73 | + return Err(Error::validation_invalid_argument( |
| 74 | + "resume", |
| 75 | + format!( |
| 76 | + "Deploy run '{}' uses unsupported schema version {}", |
| 77 | + self.id, self.schema_version |
| 78 | + ), |
| 79 | + None, |
| 80 | + None, |
| 81 | + )); |
| 82 | + } |
| 83 | + if &self.identity != identity { |
| 84 | + return Err(Error::validation_invalid_argument( |
| 85 | + "resume", |
| 86 | + format!("Deploy run '{}' identity does not exactly match the requested source, artifact, components, targets, and policy", self.id), |
| 87 | + None, |
| 88 | + None, |
| 89 | + )); |
| 90 | + } |
| 91 | + for target in &mut self.targets { |
| 92 | + if target.status == DeployTargetStatus::Running { |
| 93 | + // A process death leaves no reliable remote completion signal. Retry it. |
| 94 | + target.status = DeployTargetStatus::Planned; |
| 95 | + target.error = Some( |
| 96 | + "Recovered interrupted target; retrying from its durable checkpoint" |
| 97 | + .to_string(), |
| 98 | + ); |
| 99 | + } |
| 100 | + } |
| 101 | + Ok(()) |
| 102 | + } |
| 103 | + |
| 104 | + pub fn target_is_succeeded(&self, target: &str) -> bool { |
| 105 | + self.targets |
| 106 | + .iter() |
| 107 | + .any(|entry| entry.target == target && entry.status == DeployTargetStatus::Succeeded) |
| 108 | + } |
| 109 | + |
| 110 | + pub fn update_target( |
| 111 | + &mut self, |
| 112 | + target: &str, |
| 113 | + status: DeployTargetStatus, |
| 114 | + error: Option<String>, |
| 115 | + phase_timings: Option<PhaseTimingReport>, |
| 116 | + ) { |
| 117 | + if let Some(entry) = self.targets.iter_mut().find(|entry| entry.target == target) { |
| 118 | + entry.status = status; |
| 119 | + entry.error = error; |
| 120 | + entry.phase_timings = phase_timings; |
| 121 | + } |
| 122 | + } |
| 123 | +} |
| 124 | + |
| 125 | +pub(super) fn lifecycle_path(id: &str) -> Result<PathBuf> { |
| 126 | + Ok(paths::homeboy_data()? |
| 127 | + .join("deploy-runs") |
| 128 | + .join(format!("{id}.json"))) |
| 129 | +} |
| 130 | + |
| 131 | +pub(super) fn load(id: &str) -> Result<DeployLifecycleRun> { |
| 132 | + let path = lifecycle_path(id)?; |
| 133 | + let contents = fs::read_to_string(&path).map_err(|error| { |
| 134 | + Error::internal_io( |
| 135 | + error.to_string(), |
| 136 | + Some(format!("read deploy run {}", path.display())), |
| 137 | + ) |
| 138 | + })?; |
| 139 | + serde_json::from_str(&contents).map_err(|error| { |
| 140 | + Error::internal_io( |
| 141 | + error.to_string(), |
| 142 | + Some(format!("parse deploy run {}", path.display())), |
| 143 | + ) |
| 144 | + }) |
| 145 | +} |
| 146 | + |
| 147 | +pub(super) fn save(run: &DeployLifecycleRun) -> Result<()> { |
| 148 | + let path = lifecycle_path(&run.id)?; |
| 149 | + let parent = path.parent().expect("deploy lifecycle path has parent"); |
| 150 | + fs::create_dir_all(parent).map_err(|error| { |
| 151 | + Error::internal_io( |
| 152 | + error.to_string(), |
| 153 | + Some(format!("create {}", parent.display())), |
| 154 | + ) |
| 155 | + })?; |
| 156 | + let temporary = path.with_extension("json.tmp"); |
| 157 | + let contents = serde_json::to_vec_pretty(run).map_err(|error| { |
| 158 | + Error::internal_io(error.to_string(), Some("serialize deploy run".to_string())) |
| 159 | + })?; |
| 160 | + fs::write(&temporary, contents).map_err(|error| { |
| 161 | + Error::internal_io( |
| 162 | + error.to_string(), |
| 163 | + Some(format!("write {}", temporary.display())), |
| 164 | + ) |
| 165 | + })?; |
| 166 | + fs::rename(&temporary, &path).map_err(|error| { |
| 167 | + Error::internal_io( |
| 168 | + error.to_string(), |
| 169 | + Some(format!("commit {}", path.display())), |
| 170 | + ) |
| 171 | + }) |
| 172 | +} |
| 173 | + |
| 174 | +#[cfg(test)] |
| 175 | +mod tests { |
| 176 | + use super::*; |
| 177 | + use crate::test_support::with_isolated_home; |
| 178 | + |
| 179 | + fn identity() -> DeployRunIdentity { |
| 180 | + DeployRunIdentity { |
| 181 | + source: "refs/tags/v1.2.3@abc".to_string(), |
| 182 | + artifact: "sha256:123".to_string(), |
| 183 | + components: vec!["plugin".to_string()], |
| 184 | + targets: vec!["a".to_string(), "b".to_string()], |
| 185 | + policy: "tagged=true".to_string(), |
| 186 | + } |
| 187 | + } |
| 188 | + |
| 189 | + #[test] |
| 190 | + fn success_failure_and_skip_are_durable() { |
| 191 | + let mut run = DeployLifecycleRun::new("run".to_string(), identity()); |
| 192 | + run.update_target("a", DeployTargetStatus::Succeeded, None, None); |
| 193 | + run.update_target( |
| 194 | + "b", |
| 195 | + DeployTargetStatus::Failed, |
| 196 | + Some("boom".to_string()), |
| 197 | + None, |
| 198 | + ); |
| 199 | + assert!(run.target_is_succeeded("a")); |
| 200 | + assert!(!run.target_is_succeeded("b")); |
| 201 | + assert_eq!(run.targets[1].error.as_deref(), Some("boom")); |
| 202 | + } |
| 203 | + |
| 204 | + #[test] |
| 205 | + fn resume_recovers_interrupted_targets_without_redeploying_successes() { |
| 206 | + let mut run = DeployLifecycleRun::new("run".to_string(), identity()); |
| 207 | + run.update_target("a", DeployTargetStatus::Succeeded, None, None); |
| 208 | + run.update_target("b", DeployTargetStatus::Running, None, None); |
| 209 | + run.resume(&identity()).expect("matching resume"); |
| 210 | + assert!(run.target_is_succeeded("a")); |
| 211 | + assert_eq!(run.targets[1].status, DeployTargetStatus::Planned); |
| 212 | + } |
| 213 | + |
| 214 | + #[test] |
| 215 | + fn resume_refuses_any_identity_mismatch() { |
| 216 | + let run = DeployLifecycleRun::new("run".to_string(), identity()); |
| 217 | + let mut changed = identity(); |
| 218 | + changed.artifact = "sha256:changed".to_string(); |
| 219 | + let error = run.clone().resume(&changed).expect_err("must fail closed"); |
| 220 | + assert!(error.message.contains("does not exactly match")); |
| 221 | + } |
| 222 | + |
| 223 | + #[test] |
| 224 | + fn durable_record_round_trips_target_state_and_partial_timing_evidence() { |
| 225 | + with_isolated_home(|_| { |
| 226 | + let mut run = DeployLifecycleRun::new("run".to_string(), identity()); |
| 227 | + let mut timer = crate::core::phase_timing::PhaseTimer::new(); |
| 228 | + timer.record_failed("transfer", std::time::Duration::from_millis(1)); |
| 229 | + run.update_target( |
| 230 | + "b", |
| 231 | + DeployTargetStatus::Failed, |
| 232 | + Some("connection lost".to_string()), |
| 233 | + Some(timer.into_report()), |
| 234 | + ); |
| 235 | + save(&run).expect("persist run before retryable remote work"); |
| 236 | + |
| 237 | + let restored = load("run").expect("read durable run"); |
| 238 | + assert_eq!(restored.schema_version, SCHEMA_VERSION); |
| 239 | + assert_eq!(restored.targets[1].status, DeployTargetStatus::Failed); |
| 240 | + assert_eq!( |
| 241 | + restored.targets[1].error.as_deref(), |
| 242 | + Some("connection lost") |
| 243 | + ); |
| 244 | + assert_eq!( |
| 245 | + restored.targets[1] |
| 246 | + .phase_timings |
| 247 | + .as_ref() |
| 248 | + .and_then(|report| report.span("transfer")) |
| 249 | + .map(|span| span.status), |
| 250 | + Some(crate::core::phase_timing::PhaseStatus::Failed) |
| 251 | + ); |
| 252 | + }); |
| 253 | + } |
| 254 | +} |
0 commit comments