Fix issue with missing data (.blob) needed to for valm

Author: MichaelRyom

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "DSvClient"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 description = "VMware vCenter and ESXi patch downloading tool with checksum verification"
 authors = ["Michael Ryom"]

README.md

@@ -2,6 +2,42 @@
 
 Check out [https://michaelryom.dk/dsvclient-new-patch-downloading-tool-for-vcenter](https://michaelryom.dk/dsvclient-new-patch-downloading-tool-for-vcenter)
 
+## 📦 Version 0.7.0
+**Released:** April 11, 2026
+
+### Full VCSA Patch Downloads (Bug Fix)
+Earlier releases downloaded only the files referenced in `manifest-latest.xml`
+for VCSA sources. That manifest lists RPMs and the two `*-patch-scripts.zip`
+files, but it does **not** include the 24+ content-addressed container image
+layers (`.blob`) and container manifests (`.manifest`) that a VCSA patch ships
+with. Without those files, `software-packages stage --iso` on the appliance
+fails with `"The CD drive does not have a valid patch ISO or has an unsupported
+version"` because the appliance can't find the container layers referenced in
+the patch metadata.
+
+**What changed in 0.7.0:**
+- **New parser** — `XmlParser::parse_vcsa_rpm_manifest_json()` reads
+  `package-pool/rpm-manifest.json` and returns the complete file list that the
+  appliance's own patching code consumes.
+- **New downloader hook** — after `rpm-manifest.json` is downloaded, the VCSA
+  source loop now also parses it and queues every extra file it references
+  (blobs + container manifests + any additional RPMs) using the same URL layout
+  and dedup logic already used for the XML manifest entries.
+- **Shared queue back-end** — the per-file queueing logic in
+  `process_vcsa_manifest` was extracted into a shared helper
+  (`queue_vcsa_packages`) so the XML and JSON paths both go through the same
+  verification / dedup / concurrency machinery.
+- **Log output** — VCSA log lines now preserve the file extension for non-RPM
+  entries (`.blob`, `.manifest`) so they're readable in `DSvClient.log`.
+
+**No configuration changes required.** Existing `sources.toml` VCSA entries
+work as-is — the fix activates automatically whenever `rpm-manifest.json` is in
+the `files = [...]` list (which it already is in the default config).
+
+Result: a VCSA patch folder downloaded with 0.7.0 contains everything
+`software-packages stage --iso --acceptEulas` needs to validate and stage the
+patch on the appliance.
+
 ## 🔐 Version 0.6.0
 **Released:** August 27, 2025
 
@@ -165,4 +201,4 @@ Each platform release now includes:
 - **Improved config file handling** - Better integration with release packages
 
 ### Version Progression
-- v0.3.0 → v0.3.1 → v0.4.0 → v0.5.0 → v0.6.0 (current)
+- v0.3.0 → v0.3.1 → v0.4.0 → v0.5.0 → v0.6.0 → v0.7.0 (current)

src/downloader.rs

@@ -1,5 +1,5 @@
 use crate::config::AppConfig;
-use crate::parser::{Vendor, XmlParser};
+use crate::parser::{Vendor, VcsaPackage, XmlParser};
 use crate::process::{FileType, ProcessManager, Source};
 use crate::verify::VerificationManager;
 use anyhow::Result;
@@ -454,6 +454,24 @@ impl Downloader {
                                         Err(e) => warn!("Error reading manifest file: {}", e),
                                     }
                                 }
+
+                                // rpm-manifest.json lists additional files that are NOT referenced in
+                                // manifest-latest.xml - notably container image blobs (.blob) and
+                                // container manifests (.manifest). Without these, `software-packages
+                                // stage --iso` fails on the VCSA because the stage step can't find
+                                // the container layers referenced by the patch metadata.
+                                if file.ends_with("rpm-manifest.json") {
+                                    match tokio::fs::read_to_string(&target_path).await {
+                                        Ok(json_content) => {
+                                            if let Err(e) = this.process_vcsa_rpm_manifest_json(&json_content, &version, &processed_base_url).await {
+                                                warn!("Error processing rpm-manifest.json: {}", e);
+                                            } else {
+                                                info!("Successfully processed VCSA rpm-manifest.json");
+                                            }
+                                        }
+                                        Err(e) => warn!("Error reading rpm-manifest.json file: {}", e),
+                                    }
+                                }
                             }
                         } else {
                             warn!("VCSA source missing version or files: {}", url);
@@ -1482,21 +1500,45 @@ impl Downloader {
     }
 
     async fn process_vcsa_manifest(&self, content: &str, version: &str, base_url: &str) -> Result<()> {
-        let start_time = std::time::Instant::now();
         let packages = self.xml_parser.parse_vcsa_packages(content)?;
-        info!("Found {} packages in VCSA manifest for version {}", packages.len(), version);
+        info!("Found {} packages in VCSA manifest-latest.xml for version {}", packages.len(), version);
+        self.queue_vcsa_packages(packages, version, base_url, "manifest-latest.xml").await
+    }
+
+    /// Parse VCSA rpm-manifest.json and queue any files it lists that aren't
+    /// already covered by manifest-latest.xml (blobs, container manifests, and
+    /// any extra RPMs listed only in the JSON).
+    async fn process_vcsa_rpm_manifest_json(&self, content: &str, version: &str, base_url: &str) -> Result<()> {
+        let packages = self.xml_parser.parse_vcsa_rpm_manifest_json(content)?;
+        info!("Found {} packages in VCSA rpm-manifest.json for version {}", packages.len(), version);
+        self.queue_vcsa_packages(packages, version, base_url, "rpm-manifest.json").await
+    }
+
+    /// Shared back-end for processing a list of VcsaPackage entries: builds URLs,
+    /// skips already-processed / already-valid files, and spawns concurrent
+    /// download tasks.
+    async fn queue_vcsa_packages(
+        &self,
+        packages: Vec<VcsaPackage>,
+        version: &str,
+        base_url: &str,
+        source_label: &str,
+    ) -> Result<()> {
+        let start_time = std::time::Instant::now();
 
         let mut tasks = Vec::new();
         let max_concurrent = self.verifier.config.verification.max_concurrent_files();
         let file_semaphore = Arc::new(Semaphore::new(max_concurrent));
 
         for package in packages {
             let location = package.location.clone();
+            // pkg_info is only used for log output - use the filename with the
+            // package-pool/ prefix stripped, keeping the file extension so that
+            // non-RPM entries (like .blob / .manifest) are still readable in logs.
             let pkg_info = location
                 .strip_prefix("package-pool/")
-                .and_then(|s| s.strip_suffix(".rpm"))
                 .unwrap_or(&location);
-                
+
             let url = format!(
                 "{}/{}/{}",
                 base_url,
@@ -1598,8 +1640,8 @@ impl Downloader {
         }
 
         let duration = start_time.elapsed();
-        info!("VCSA manifest processing completed in {:?}: {}/{} packages successful, {} errors", 
-              duration, completed, total_tasks, errors);
+        info!("VCSA {} processing completed in {:?}: {}/{} packages successful, {} errors",
+              source_label, duration, completed, total_tasks, errors);
 
         Ok(())
     }

src/parser.rs

@@ -469,6 +469,70 @@ impl XmlParser {
         Ok(quick_xml::de::from_str(content)?)
     }
 
+    /// Parse the VCSA rpm-manifest.json file.
+    ///
+    /// This file is a JSON dict keyed by SHA256, with entries like:
+    ///
+    /// ```json
+    /// {
+    ///   "files": {
+    ///     "015dc347...c72565": {
+    ///       "relativepath": "015dc347...c72565.blob",
+    ///       "sha256val":    "015dc347...c72565",
+    ///       "type":         "blob"
+    ///     },
+    ///     "bba8b3bf...1aab5": {
+    ///       "relativepath": "Linux-PAM-1.5.3-6.ph4.x86_64.rpm",
+    ///       "sha256val":    "bba8b3bf...1aab5",
+    ///       "type":         "rpm"
+    ///     }
+    ///   }
+    /// }
+    /// ```
+    ///
+    /// rpm-manifest.json lists extra files that are NOT in manifest-latest.xml -
+    /// notably container image layers (`.blob`) and container manifests
+    /// (`.manifest`). These files are required for `software-packages stage --iso`
+    /// to succeed on the VCSA during appliance patching.
+    ///
+    /// The `relativepath` values in this file are bare filenames (no
+    /// `package-pool/` prefix), so we prepend `package-pool/` to match the URL
+    /// layout the downloader already uses for entries from the XML manifest.
+    pub fn parse_vcsa_rpm_manifest_json(&self, content: &str) -> Result<Vec<VcsaPackage>> {
+        let v: serde_json::Value = serde_json::from_str(content)?;
+        let files = v.get("files")
+            .and_then(|f| f.as_object())
+            .ok_or_else(|| anyhow::anyhow!("rpm-manifest.json is missing a top-level 'files' object"))?;
+
+        let mut packages = Vec::with_capacity(files.len());
+        for (_hash, entry) in files {
+            let relpath = entry.get("relativepath")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string();
+            if relpath.is_empty() {
+                continue;
+            }
+            let sha256 = entry.get("sha256val")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string();
+
+            packages.push(VcsaPackage {
+                name:        relpath.clone(),
+                // Prepend package-pool/ so existing URL-building in downloader.rs
+                // (format!("{base}/{version}/{location}")) resolves to the right URL.
+                location:    format!("package-pool/{}", relpath),
+                version:     String::new(),
+                arch:        String::new(),
+                checksum:    String::new(),
+                checksum256: sha256,
+            });
+        }
+
+        Ok(packages)
+    }
+
     pub fn parse_vcsa_packages(&self, content: &str) -> Result<Vec<VcsaPackage>> {
         let mut reader = quick_xml::Reader::from_str(content);
         let mut buf = Vec::new();