actually check sha256 for managing coman sqsh

Author: Panaetius

Cargo.lock

@@ -83,6 +83,7 @@ dirs = "6.0.0"
 iroh = "0.95.1"
 rand = "0.9.2"
 regex = "1.12.2"
+sha2 = "0.10.9"
 
 [build-dependencies]
 anyhow = "1.0.90"

coman/Cargo.toml

@@ -471,6 +471,8 @@ pub enum CscsSystemCommands {
     },
 }
 
+pub const COMAN_VERSION: &str = env!("CARGO_PKG_VERSION");
+
 const VERSION_MESSAGE: &str = concat!(
     env!("CARGO_PKG_VERSION"),
     "-",

coman/src/cli/app.rs

@@ -9,8 +9,8 @@ use firecrest_client::{
         get_compute_system_jobs, post_compute_system_job,
     },
     filesystem_api::{
-        delete_filesystem_ops_rm, get_filesystem_ops_download, get_filesystem_ops_ls, get_filesystem_ops_stat,
-        get_filesystem_ops_tail, post_filesystem_ops_mkdir, post_filesystem_ops_upload,
+        delete_filesystem_ops_rm, get_filesystem_ops_checksum, get_filesystem_ops_download, get_filesystem_ops_ls,
+        get_filesystem_ops_stat, get_filesystem_ops_tail, post_filesystem_ops_mkdir, post_filesystem_ops_upload,
         post_filesystem_transfer_download, post_filesystem_transfer_upload, put_filesystem_ops_chmod,
     },
     status_api::{get_status_systems, get_status_userinfo},
@@ -231,6 +231,11 @@ impl CscsApi {
             None => Ok(vec![]),
         }
     }
+    pub async fn checksum(&self, system_name: &str, path: PathBuf) -> Result<Option<String>> {
+        get_filesystem_ops_checksum(&self.client, system_name, path)
+            .await
+            .wrap_err("couldn't stat file")
+    }
     pub async fn stat_path(&self, system_name: &str, path: PathBuf) -> Result<Option<FileStat>> {
         let result = get_filesystem_ops_stat(&self.client, system_name, path)
             .await

coman/src/cscs/api_client/client.rs

@@ -18,6 +18,7 @@ use iroh::{Endpoint, EndpointId, SecretKey};
 use itertools::Itertools;
 use regex::Regex;
 use reqwest::Url;
+use sha2::{Digest, Sha256};
 use tokio::{
     fs::File,
     io::AsyncWriteExt,
@@ -26,6 +27,7 @@ use tokio::{
 
 use super::api_client::client::{EdfSpec, ScriptSpec};
 use crate::{
+    cli::app::COMAN_VERSION,
     config::{ComputePlatform, Config, get_data_dir},
     cscs::{
         api_client::{
@@ -396,6 +398,82 @@ async fn store_ssh_information(
     Ok(connection_name)
 }
 
+async fn maybe_download_latest_squash(current_system: &str, config: &Config) -> Result<PathBuf, eyre::Error> {
+    if let Some(path) = config.values.coman_squash_path.clone() {
+        return Ok(path);
+    }
+    //download from github for architecture
+    let system = config
+        .values
+        .cscs
+        .systems
+        .get(current_system)
+        .ok_or(eyre!("couldn't find architecture for system {}", current_system))?;
+    let architecture = system
+        .architecture
+        .first()
+        .ok_or(eyre!("no architecture set for {}", current_system))?;
+    let target_path = get_data_dir().join(format!("coman_{}.sqsh", architecture));
+    let arch = match architecture.as_str() {
+        "arm64" => "aarch64",
+        "amd64" => "x86_64",
+        _ => {
+            return Err(eyre!("unsupported architecture {}", architecture));
+        }
+    };
+    if target_path.exists() {
+        // if file exists, check checksum against newest version and delete if no match
+        let mut hasher = Sha256::new();
+        let mut file = std::fs::File::open(&target_path)?;
+        let _n = std::io::copy(&mut file, &mut hasher)?;
+        let hash = hasher.finalize();
+        let hash = format!("{hash:02x}");
+
+        let client = reqwest::Client::new();
+        let resp = client
+            .get(format!(
+                "https://api.github.com/repos/SwissDataScienceCenter/coman/releases/tags/v{}",
+                COMAN_VERSION
+            ))
+            .header("Accept", "application/vnd.github+json")
+            .header("User-Agent", format!("coman cli {COMAN_VERSION}"))
+            .send()
+            .await?;
+        if resp.status().is_success() {
+            let body = resp.text().await?;
+            let body: serde_json::Value = serde_json::from_str(&body)?;
+            if let Some(assets) = body["assets"].as_array() {
+                for asset in assets {
+                    if asset["name"].as_str() == Some(format!("coman_Linux-{arch}.sqsh").as_str())
+                        && let Some(remote_hash) = asset["digest"].as_str()
+                        && remote_hash.strip_prefix("sha256:").unwrap() != hash.as_str()
+                    {
+                        std::fs::remove_file(&target_path)?;
+                    }
+                }
+            }
+        }
+    }
+    if !target_path.exists() {
+        let url = format!(
+            "https://github.com/SwissDataScienceCenter/coman/releases/download/v{COMAN_VERSION}/coman_Linux-{arch}.sqsh"
+        );
+        let mut out = File::create(target_path.clone()).await?;
+        let resp = reqwest::get(url).await?;
+        match resp.error_for_status() {
+            Ok(resp) => {
+                let mut stream = resp.bytes_stream();
+                while let Some(chunk_result) = stream.next().await {
+                    let chunk = chunk_result?;
+                    out.write_all(&chunk).await?;
+                }
+                out.flush().await?;
+            }
+            Err(e) => return Err(eyre!("couldn't download coman squash file: {}", e)),
+        }
+    }
+    Ok(target_path)
+}
 async fn inject_coman_squash(
     api_client: &CscsApi,
     base_path: &Path,
@@ -406,50 +484,7 @@ async fn inject_coman_squash(
         return Ok(None);
     }
     let config = Config::new().unwrap();
-    let local_squash_path = match config.values.coman_squash_path.clone() {
-        Some(path) => path,
-        None => {
-            //download from github for architecture
-            let system = config
-                .values
-                .cscs
-                .systems
-                .get(current_system)
-                .ok_or(eyre!("couldn't find architecture for system {}", current_system))?;
-            let architecture = system
-                .architecture
-                .first()
-                .ok_or(eyre!("no architecture set for {}", current_system))?;
-            let target_path = get_data_dir().join(format!("coman_{}.sqsh", architecture));
-            if !target_path.exists() {
-                let url = match architecture.as_str() {
-                    "arm64" => {
-                        "https://github.com/SwissDataScienceCenter/coman/releases/latest/download/coman_Linux-aarch64.sqsh"
-                    }
-                    "amd64" => {
-                        "https://github.com/SwissDataScienceCenter/coman/releases/latest/download/coman_Linux-x86_64.sqsh"
-                    }
-                    _ => {
-                        return Err(eyre!("unsupported architecture {}", architecture));
-                    }
-                };
-                let mut out = File::create(target_path.clone()).await?;
-                let resp = reqwest::get(url).await?;
-                match resp.error_for_status() {
-                    Ok(resp) => {
-                        let mut stream = resp.bytes_stream();
-                        while let Some(chunk_result) = stream.next().await {
-                            let chunk = chunk_result?;
-                            out.write_all(&chunk).await?;
-                        }
-                        out.flush().await?;
-                    }
-                    Err(e) => return Err(eyre!("couldn't download coman squash file: {}", e)),
-                }
-            }
-            target_path
-        }
-    };
+    let local_squash_path = maybe_download_latest_squash(current_system, &config).await?;
     let target = base_path.join("coman.sqsh");
     let file_meta = std::fs::metadata(local_squash_path.clone())?;
 
@@ -459,13 +494,16 @@ async fn inject_coman_squash(
     #[cfg(target_family = "windows")]
     let size = file_meta.file_size() as usize;
 
-    let response = api_client.list_path(current_system, target.clone()).await;
-    if let Ok(existing) = response
-        && !existing.is_empty()
-    {
+    let mut hasher = Sha256::new();
+    let mut file = std::fs::File::open(&local_squash_path)?;
+    let _n = std::io::copy(&mut file, &mut hasher)?;
+    let hash = hasher.finalize();
+    let hash = format!("{hash:02x}");
+
+    let response = api_client.checksum(current_system, target.clone()).await;
+    if let Ok(Some(remote_hash)) = response {
         //squash file already present on remote, don't upload if it's the same
-        let entry = existing.first().unwrap();
-        if entry.size.unwrap_or_default() == size {
+        if hash == remote_hash {
             return Ok(Some(target));
         } else {
             // remove file before upload

coman/src/cscs/handlers.rs

@@ -5,10 +5,10 @@ use eyre::{Result, eyre};
 use crate::{
     client::FirecrestClient,
     types::{
-        DownloadFileResponse, DownloadFileResponseTransferDirectives, GetDirectoryLsResponse, GetFileStatResponse,
-        GetFileTailResponse, PostFileDownloadRequest, PostFileDownloadRequestTransferDirectives, PostFileUploadRequest,
-        PostMakeDirRequest, PostMkdirResponse, PutFileChmodRequest, PutFileChmodResponse, S3TransferRequest,
-        S3TransferResponse, UploadFileResponse,
+        DownloadFileResponse, DownloadFileResponseTransferDirectives, GetDirectoryLsResponse, GetFileChecksumResponse,
+        GetFileStatResponse, GetFileTailResponse, PostFileDownloadRequest, PostFileDownloadRequestTransferDirectives,
+        PostFileUploadRequest, PostMakeDirRequest, PostMkdirResponse, PutFileChmodRequest, PutFileChmodResponse,
+        S3TransferRequest, S3TransferResponse, UploadFileResponse,
     },
 };
 
@@ -238,3 +238,20 @@ pub async fn delete_filesystem_ops_rm(client: &FirecrestClient, system_name: &st
         .await?;
     Ok(())
 }
+
+pub async fn get_filesystem_ops_checksum(
+    client: &FirecrestClient,
+    system_name: &str,
+    path: PathBuf,
+) -> Result<Option<String>> {
+    let path = path.as_os_str().to_str().ok_or(eyre!("couldn't cast path to string"))?;
+    let response = client
+        .get(
+            format!("filesystem/{system_name}/ops/checksum").as_str(),
+            Some(vec![("path", path)]),
+        )
+        .await?;
+    let model: GetFileChecksumResponse = serde_json::from_str(response.as_str())?;
+
+    Ok(model.output.map(|o| o.checksum))
+}