draft - no merge

Cargo.toml

@@ -35,3 +35,5 @@ chrono = { version = "0.4", features = ["serde"] }
 url = { version = "2.5", features = ["serde"] }
 forc-util = "0.66.7"
 forc-pkg = "0.66.7"
+aws-sdk-s3 = "1.77"
+aws-config = "1.5.17"

src/api/search.rs

@@ -1,6 +1,6 @@
 use crate::{
     models::PackagePreview,
-    pinata::{ipfs_hash_to_abi_url, ipfs_hash_to_tgz_url},
+    file_uploader::pinata::{ipfs_hash_to_abi_url, ipfs_hash_to_tgz_url},
 };
 use serde::Serialize;
 use url::Url;

src/file_uploader/mod.rs

@@ -0,0 +1,142 @@
+pub mod pinata;
+pub mod s3;
+
+use crate::file_uploader::{pinata::PinataClient, s3::S3Client};
+use crate::handlers::upload::UploadError;
+use std::path::Path;
+use std::{fs::File, io::Read};
+
+pub struct FileUploader<'a, T: PinataClient, E: S3Client> {
+    pinata_client: &'a T,
+    s3_client: &'a E,
+}
+
+impl<'a, T: PinataClient, E: S3Client> FileUploader<'a, T, E> {
+    pub fn new(pinata_client: &'a T, s3_client: &'a E) -> Self {
+            Self {
+                pinata_client,
+                s3_client,
+            }
+        }
+
+    pub async fn upload_file(&self, path: &Path) -> Result<String, UploadError> {
+        let ipfs_hash = self
+            .pinata_client
+            .upload_file_to_ipfs(path)
+            .await?;
+
+        // Read file contents
+        let mut file = File::open(path).map_err(|_| UploadError::OpenFile)?;
+        let mut buffer = Vec::new();
+        file.read_to_end(&mut buffer)
+            .map_err(|_| UploadError::ReadFile)?;
+
+        // Upload to S3
+        self.s3_client
+            .upload_file_to_s3(path, ipfs_hash.clone())
+            .await?;
+
+        Ok(ipfs_hash)
+    }
+}
+
+#[cfg(test)]
+pub fn get_mock_file_uploader() -> FileUploader<'static, pinata::MockPinataClient, s3::MockS3Client> {
+    FileUploader::new(&pinata::MockPinataClient, &s3::MockS3Client)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::file_uploader::{pinata::MockPinataClient, s3::MockS3Client};
+    use std::io::Write;
+    use std::path::PathBuf;
+    use tempfile::NamedTempFile;
+
+    #[tokio::test]
+    async fn test_upload_file_success() {
+        let pinata_client = MockPinataClient;
+        let s3_client = MockS3Client;
+        let file_uploader = FileUploader::new(&pinata_client, &s3_client);
+
+        // Create a temporary file to simulate a real file upload
+        let mut temp_file = NamedTempFile::new().expect("Failed to create temp file");
+        temp_file
+            .write_all(b"Test file contents")
+            .expect("Failed to write to temp file");
+        let temp_path = temp_file.path().to_path_buf();
+
+        let result = file_uploader.upload_file(&temp_path).await;
+
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), "ABC123");
+    }
+
+    #[tokio::test]
+    async fn test_upload_file_ipfs_failure() {
+        struct FailingPinataClient;
+        impl PinataClient for FailingPinataClient {
+            async fn new() -> Result<Self, UploadError> {
+                Ok(FailingPinataClient)
+            }
+            async fn upload_file_to_ipfs(&self, _path: &Path) -> Result<String, UploadError> {
+                Err(UploadError::IpfsUploadFailed("IPFS error".to_string()))
+            }
+        }
+
+        let pinata_client = FailingPinataClient;
+        let s3_client = MockS3Client;
+        let file_uploader = FileUploader::new(&pinata_client, &s3_client);
+
+        let temp_path = PathBuf::from("fake_file.txt");
+
+        let result = file_uploader.upload_file(&temp_path).await;
+        assert!(result.is_err());
+        assert_eq!(result, Err(UploadError::IpfsUploadFailed("IPFS error".to_string())));
+    }
+
+    #[tokio::test]
+    async fn test_upload_file_s3_failure() {
+        struct FailingS3Client;
+        impl S3Client for FailingS3Client {
+            fn new() -> impl std::future::Future<Output = Result<Self, UploadError>> + Send {
+                async { Ok(FailingS3Client) }
+            }
+            async fn upload_file_to_s3(
+                &self,
+                _path: &Path,
+                _file_name: String,
+            ) -> Result<(), UploadError> {
+                Err(UploadError::S3UploadFailed("S3 error".to_string()))
+            }
+        }
+
+        let pinata_client = MockPinataClient;
+        let s3_client = FailingS3Client;
+        let file_uploader = FileUploader::new(&pinata_client, &s3_client);
+
+        // Create a temporary file
+        let mut temp_file = NamedTempFile::new().expect("Failed to create temp file");
+        temp_file
+            .write_all(b"Test file contents")
+            .expect("Failed to write to temp file");
+        let temp_path = temp_file.path().to_path_buf();
+
+        let result = file_uploader.upload_file(&temp_path).await;
+        assert!(result.is_err());
+        assert_eq!(result, Err(UploadError::S3UploadFailed("S3 error".to_string())));
+    }
+
+    #[tokio::test]
+    async fn test_upload_file_open_failure() {
+        let pinata_client = MockPinataClient;
+        let s3_client = MockS3Client;
+        let file_uploader = FileUploader::new(&pinata_client, &s3_client);
+
+        let non_existent_path = PathBuf::from("non_existent_file.txt");
+
+        let result = file_uploader.upload_file(&non_existent_path).await;
+        assert!(result.is_err());
+        assert!(matches!(result, Err(UploadError::OpenFile)));
+    }
+}

src/pinata/mod.rs → src/file_uploader/pinata.rs

@@ -24,7 +24,7 @@ impl PinataClient for PinataClientImpl {
         let (api_key, secret_api_key) =
             match (env::var("PINATA_API_KEY"), env::var("PINATA_API_SECRET")) {
                 (Ok(key), Ok(secret)) => (key, secret),
-                _ => return Err(UploadError::Ipfs),
+                _ => return Err(UploadError::IpfsUploadFailed("Missing API key".to_string())),
             };
         let api =
             PinataApi::new(api_key, secret_api_key).map_err(|_| UploadError::Authentication)?;
@@ -44,7 +44,7 @@ impl PinataClient for PinataClientImpl {
             .await
         {
             Ok(pinned_object) => Ok(pinned_object.ipfs_hash),
-            Err(_) => Err(UploadError::Ipfs),
+            Err(err) => Err(UploadError::IpfsUploadFailed(err.to_string())),
         }
     }
 }

src/file_uploader/s3.rs

@@ -0,0 +1,81 @@
+use crate::{handlers::upload::UploadError, util::load_env};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::{
+    config::{BehaviorVersion, Region, Credentials},
+    primitives::ByteStream,
+};
+use std::{env, fs::File, io::Read, path::Path};
+
+pub trait S3Client: Sized {
+    fn new() -> impl std::future::Future<Output = Result<Self, UploadError>> + Send;
+    fn upload_file_to_s3(
+        &self,
+        path: &Path,
+        file_name: String,
+    ) -> impl std::future::Future<Output = Result<(), UploadError>> + Send;
+}
+
+pub struct S3ClientImpl {
+    s3_client: Client,
+    bucket_name: String,
+}
+
+impl S3Client for S3ClientImpl {
+    async fn new() -> Result<Self, UploadError> {
+        load_env();
+
+        // TODO: verify this locally
+
+        let bucket_name = env::var("S3_BUCKET_NAME")
+            .map_err(|_| UploadError::S3UploadFailed("Missing S3_BUCKET_NAME".to_string()))?;
+        let bucket_region = env::var("S3_BUCKET_REGION")
+            .map_err(|_| UploadError::S3UploadFailed("Missing S3_BUCKET_REGION".to_string()))?;
+
+        let shared_config = aws_config::defaults(BehaviorVersion::v2024_03_28())
+            .region(Region::new(bucket_region))
+//            .profile_name("forcpub")
+            .load()
+            .await;
+        let s3_client = Client::new(&shared_config);
+
+        Ok(S3ClientImpl {
+            s3_client,
+            bucket_name,
+        })
+    }
+
+    /// Uploads a file at the given path to an S3 bucket.
+    async fn upload_file_to_s3(&self, path: &Path, file_name: String) -> Result<(), UploadError> {
+        // Read file contents
+        let mut file = File::open(path).map_err(|_| UploadError::OpenFile)?;
+        let mut buffer = Vec::new();
+        file.read_to_end(&mut buffer)
+            .map_err(|_| UploadError::ReadFile)?;
+
+        // Upload to S3
+        self.s3_client
+            .put_object()
+            .bucket(&self.bucket_name)
+            .key(&file_name)
+            .body(ByteStream::from(buffer))
+            .send()
+            .await
+            .map_err(|e| UploadError::S3UploadFailed(format!("{:?}", e)))?;
+        Ok(())
+    }
+}
+
+/// A mock implementation of the S3Client trait for testing.
+#[cfg(test)]
+pub struct MockS3Client;
+
+#[cfg(test)]
+impl S3Client for MockS3Client {
+    async fn new() -> Result<Self, UploadError> {
+        Ok(MockS3Client)
+    }
+
+    async fn upload_file_to_s3(&self, _path: &Path, _file_name: String) -> Result<(), UploadError> {
+        Ok(())
+    }
+}

src/handlers/upload.rs

@@ -1,5 +1,6 @@
+use crate::file_uploader::FileUploader;
+use crate::file_uploader::{pinata::PinataClient, s3::S3Client};
 use crate::models::NewUpload;
-use crate::pinata::PinataClient;
 use flate2::{
     Compression,
     {read::GzDecoder, write::GzEncoder},
@@ -41,6 +42,9 @@ pub enum UploadError {
     #[error("Failed to open file.")]
     OpenFile,
 
+    #[error("Failed to read file.")]
+    ReadFile,
+
     #[error("Failed to copy files.")]
     CopyFiles,
 
@@ -53,8 +57,11 @@ pub enum UploadError {
     #[error("Failed to authenticate.")]
     Authentication,
 
-    #[error("Failed to upload to IPFS.")]
-    Ipfs,
+    #[error("Failed to upload to IPFS. Err: {0}")]
+    IpfsUploadFailed(String),
+
+    #[error("Failed to upload to S3. Err: {0}")]
+    S3UploadFailed(String),
 
     #[error("Failed to generate bytecode ID. Err: {0}")]
     BytecodeId(String),
@@ -75,13 +82,13 @@ pub enum UploadError {
 /// 3. Storing the source code tarball and ABI file in IPFS
 ///
 /// Returns a [NewUpload] with the necessary information to store in the database.
-pub async fn handle_project_upload(
-    upload_dir: &Path,
+pub async fn handle_project_upload<'a>(
+    upload_dir: &'a Path,
     upload_id: &Uuid,
     orig_tarball_path: &PathBuf,
     forc_path: &Path,
     forc_version: String,
-    pinata_client: &impl PinataClient,
+    file_uploader: &FileUploader<'a, impl PinataClient, impl S3Client>,
 ) -> Result<NewUpload, UploadError> {
     let unpacked_dir = upload_dir.join(UNPACKED_DIR);
     let release_dir = unpacked_dir.join(RELEASE_DIR);
@@ -148,9 +155,7 @@ pub async fn handle_project_upload(
     enc.finish().map_err(|_| UploadError::CopyFiles)?;
 
     // Store the tarball in IPFS.
-    let tarball_ipfs_hash = pinata_client
-        .upload_file_to_ipfs(&final_tarball_path)
-        .await?;
+    let tarball_ipfs_hash = file_uploader.upload_file(&final_tarball_path).await?;
 
     fn find_file_in_dir_by_suffix(dir: &Path, suffix: &str) -> Option<PathBuf> {
         let dir = fs::read_dir(dir).ok()?;
@@ -174,7 +179,7 @@ pub async fn handle_project_upload(
 
     // Store the ABI in IPFS.
     let abi_ipfs_hash = match find_file_in_dir_by_suffix(&release_dir, "-abi.json") {
-        Some(abi_path) => Some(pinata_client.upload_file_to_ipfs(&abi_path).await?),
+        Some(abi_path) => Some(file_uploader.upload_file(&abi_path).await?),
         None => None,
     };
 
@@ -244,7 +249,7 @@ pub fn install_forc_at_path(forc_version: &str, forc_path: &Path) -> Result<(),
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::pinata::MockPinataClient;
+    use crate::file_uploader::get_mock_file_uploader;
     use serial_test::serial;
 
     #[tokio::test]
@@ -260,15 +265,15 @@ mod tests {
         let forc_path = fs::canonicalize(forc_path.clone()).expect("forc path ok");
         install_forc_at_path(forc_version, &forc_path).expect("forc installed");
 
-        let mock_client = MockPinataClient::new().await.expect("mock pinata client");
+        let mock_file_uploader = get_mock_file_uploader();
 
         let result = handle_project_upload(
             &upload_dir,
             &upload_id,
             &orig_tarball_path,
             &forc_path,
             forc_version.to_string(),
-            &mock_client,
+            &mock_file_uploader,
         )
         .await
         .expect("result ok");

src/lib.rs

@@ -4,6 +4,6 @@ pub mod github;
 pub mod handlers;
 pub mod middleware;
 pub mod models;
-pub mod pinata;
+pub mod file_uploader;
 pub mod schema;
 pub mod util;

src/main.rs

@@ -14,13 +14,14 @@ use forc_pub::api::{
     ApiResult, EmptyResponse,
 };
 use forc_pub::db::Database;
+use forc_pub::file_uploader::s3::{S3Client, S3ClientImpl};
 use forc_pub::github::handle_login;
 use forc_pub::handlers::publish::handle_publish;
 use forc_pub::handlers::upload::{handle_project_upload, install_forc_at_path, UploadError};
 use forc_pub::middleware::cors::Cors;
 use forc_pub::middleware::session_auth::{SessionAuth, SESSION_COOKIE_NAME};
 use forc_pub::middleware::token_auth::TokenAuth;
-use forc_pub::pinata::{PinataClient, PinataClientImpl};
+use forc_pub::file_uploader::{FileUploader, pinata::{PinataClient, PinataClientImpl}};
 use forc_pub::util::validate_or_format_semver;
 use rocket::{
     data::Capped,
@@ -166,13 +167,15 @@ async fn upload_project(
         .map_err(|_| ApiError::Upload(UploadError::SaveFile))?;
 
     // Handle the project upload and store the metadata in the database.
+    let s3_client = S3ClientImpl::new().await?;
+    let file_uploader = FileUploader::new(pinata_client.inner(), &s3_client);
     let upload_entry = handle_project_upload(
         &upload_dir,
         &upload_id,
         &orig_tarball_path,
         &forc_path,
         forc_version.to_string(),
-        pinata_client.inner(),
+        &file_uploader,
     )
     .await?;