Merge pull request #996 from benjamin-747/main

fix: remove tls server support & comment failed tests

Author: genedna

Cargo.toml

@@ -58,10 +58,9 @@ sha1 = "0.10.6"
 futures = "0.3.30"
 futures-util = "0.3.30"
 go-defer = "0.1.0"
-russh = "0.51.1"
+russh = "0.52"
 axum = "0.8.1"
 axum-extra = "0.10.0"
-axum-server = "0.7.1"
 tower-http = "0.6.1"
 tower = "0.5.2"
 hex = "0.4.3"
@@ -83,7 +82,10 @@ tempfile = "3.19.0"
 home = "0.5.9"
 ring = "0.17.8"
 cedar-policy = "4.3.1"
-secp256k1 = "0.30.0"
+secp256k1 = "0.30"
+pgp = "0.15"
+async-std = "1.13"
+openssl = "0.10"
 oauth2 = "5.0.0"
 base64 = "0.22.1"
 encoding_rs = "0.8.31"
@@ -100,6 +102,8 @@ byteorder = "1.5.0"
 bincode = "2.0"
 once_cell = "1.21"
 testcontainers = "0.23"
+scopeguard = "1.2.0"
+serial_test = "3.2.0"
 
 [profile.release]
 debug = true

ceres/src/lfs/handler.rs

@@ -177,6 +177,7 @@ pub async fn lfs_delete_lock(
 pub async fn lfs_process_batch(
     context: &Context,
     request: BatchRequest,
+    listen_addr: &str,
 ) -> Result<BatchResponse, GitLFSError> {
     let objects = request.objects;
 
@@ -211,12 +212,14 @@ pub async fn lfs_process_batch(
         };
         let enable_split = meta.splited && config.local.enable_split;
         let file_exist = file_storage.exist_object(&meta.oid, enable_split).await;
-        let hostname = &config.url;
         let download_url = file_storage
-            .download_url(&meta.oid, hostname)
+            .download_url(&meta.oid, listen_addr)
+            .await
+            .unwrap();
+        let upload_url = file_storage
+            .upload_url(&meta.oid, listen_addr)
             .await
             .unwrap();
-        let upload_url = file_storage.upload_url(&meta.oid, hostname).await.unwrap();
 
         response_objects.push(ResponseObject::new(
             &meta,
@@ -242,6 +245,7 @@ pub async fn lfs_process_batch(
 pub async fn lfs_fetch_chunk_ids(
     context: &Context,
     oid: &str,
+    listen_addr: &str,
 ) -> Result<Vec<ChunkDownloadObject>, GitLFSError> {
     let config = context.config.lfs.clone();
 
@@ -269,7 +273,6 @@ pub async fn lfs_fetch_chunk_ids(
         ));
     }
     let mut response_objects = Vec::<ChunkDownloadObject>::new();
-    let hostname = context.config.lfs.url.clone();
 
     for relation in relations {
         // Reuse RequestArgs to create a link
@@ -280,7 +283,7 @@ pub async fn lfs_fetch_chunk_ids(
         };
         let download_url = context
             .lfs_file_stg()
-            .download_url(&req_obj.oid, &hostname)
+            .download_url(&req_obj.oid, listen_addr)
             .await
             .unwrap();
         response_objects.push(ChunkDownloadObject {

common/src/config.rs

@@ -38,6 +38,19 @@ impl Config {
         Config::from_config(config)
     }
 
+    pub fn mock() -> Self {
+        Self {
+            base_dir: PathBuf::new(),
+            log: LogConfig::default(),
+            database: DbConfig::default(),
+            monorepo: MonoConfig::default(),
+            pack: PackConfig::default(),
+            authentication: AuthConfig::default(),
+            lfs: LFSConfig::default(),
+            oauth: None,
+        }
+    }
+
     pub fn load_str(content: &str) -> Result<Self, c::ConfigError> {
         let builder = c::Config::builder()
             .add_source(c::File::from_str(content, FileFormat::Toml))
@@ -399,19 +412,19 @@ impl PackConfig {
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct LFSConfig {
-    pub url: String,
     pub storage_type: StorageTypeEnum,
     pub local: LFSLocalConfig,
     pub aws: LFSAwsConfig,
+    pub ssh: LFSSshConfig,
 }
 
 impl Default for LFSConfig {
     fn default() -> Self {
         Self {
-            url: "http://localhost:8000".to_string(),
             storage_type: StorageTypeEnum::LocalFs,
             local: LFSLocalConfig::default(),
             aws: LFSAwsConfig::default(),
+            ssh: LFSSshConfig::default(),
         }
     }
 }
@@ -441,6 +454,19 @@ pub struct LFSAwsConfig {
     pub s3_access_key_id: String,
     pub s3_secret_access_key: String,
 }
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct LFSSshConfig {
+    pub http_url: String,
+}
+
+impl Default for LFSSshConfig {
+    fn default() -> Self {
+        Self {
+            http_url: "http://localhost:8000".to_string(),
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone, Default)]
 pub struct OauthConfig {
     pub github_client_id: String,

common/src/model.rs

@@ -3,9 +3,12 @@ use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
 #[derive(Args, Clone, Debug)]
-pub struct CommonOptions {
+pub struct CommonHttpOptions {
     #[arg(long, default_value_t = String::from("127.0.0.1"))]
     pub host: String,
+
+    #[arg(short = 'p', long, default_value_t = 8000)]
+    pub port: u16,
 }
 
 #[derive(Args, Clone, Debug, Default)]

config/config.toml

@@ -80,12 +80,13 @@ clean_cache_after_decode = true
 channel_message_size = 1_000_000
 
 [lfs]
-# LFS Server url
-url = "http://localhost:8000"
-
 # lfs file storage type, support values can be "local_fs" or "aws_s3"
 storage_type = "local_fs"
 
+[lfs.ssh]
+# The lfs ssh transport protocol still relies on http to send files
+http_url = "http://localhost:8000"
+
 [lfs.local]
 # set the local path of the lfs storage
 lfs_file_path = "${base_dir}/lfs"

gateway/Cargo.toml

@@ -18,7 +18,6 @@ gemini = { workspace = true }
 vault = { workspace = true }
 
 axum = { workspace = true }
-axum-server = { workspace = true, features = ["tls-rustls"] }
 tower = { workspace = true }
 tracing = { workspace = true }
 serde = { workspace = true }

gateway/src/api/mod.rs

@@ -7,6 +7,5 @@ mod model;
 #[derive(Clone)]
 pub struct MegaApiServiceState {
     pub inner: MonoApiServiceState,
-    pub port: u16,
     pub p2p: P2pOptions,
 }

gateway/src/https_server.rs

@@ -1,10 +1,5 @@
-use std::net::SocketAddr;
-use std::path::PathBuf;
-use std::str::FromStr;
-
 use axum::routing::get;
 use axum::{http, Router};
-use axum_server::tls_rustls::RustlsConfig;
 use clap::Args;
 
 use quinn::rustls;
@@ -13,7 +8,7 @@ use tower_http::cors::{Any, CorsLayer};
 use tower_http::decompression::RequestDecompressionLayer;
 use tower_http::trace::TraceLayer;
 
-use common::model::{CommonOptions, P2pOptions};
+use common::model::{CommonHttpOptions, P2pOptions};
 use jupiter::context::Context;
 use mono::api::lfs::lfs_router;
 use mono::api::MonoApiServiceState;
@@ -24,65 +19,15 @@ use crate::api::{github_router, MegaApiServiceState};
 #[derive(Args, Clone, Debug)]
 pub struct HttpOptions {
     #[clap(flatten)]
-    pub common: CommonOptions,
+    pub common: CommonHttpOptions,
 
     #[clap(flatten)]
     pub p2p: P2pOptions,
-
-    #[arg(long, default_value_t = 8000)]
-    pub http_port: u16,
-}
-
-#[derive(Args, Clone, Debug)]
-pub struct HttpsOptions {
-    #[clap(flatten)]
-    pub common: CommonOptions,
-
-    #[clap(flatten)]
-    pub p2p: P2pOptions,
-
-    #[arg(long, default_value_t = 443)]
-    pub https_port: u16,
-
-    #[arg(long, value_name = "FILE")]
-    pub https_key_path: PathBuf,
-
-    #[arg(long, value_name = "FILE")]
-    pub https_cert_path: PathBuf,
-}
-
-pub async fn https_server(context: Context, options: HttpsOptions) {
-    let HttpsOptions {
-        common: CommonOptions { host, .. },
-        https_key_path,
-        https_cert_path,
-        https_port,
-        p2p,
-    } = options.clone();
-
-    rustls::crypto::ring::default_provider()
-        .install_default()
-        .expect("Failed to install rustls crypto provider");
-
-    check_run_with_p2p(context.clone(), options.p2p.clone());
-
-    let app = app(context, host.clone(), https_port, p2p.clone()).await;
-
-    let server_url = format!("{}:{}", host, https_port);
-    let addr = SocketAddr::from_str(&server_url).unwrap();
-    let config = RustlsConfig::from_pem_file(https_cert_path.to_owned(), https_key_path.to_owned())
-        .await
-        .unwrap();
-    axum_server::bind_rustls(addr, config)
-        .serve(app.into_make_service())
-        .await
-        .unwrap();
 }
 
 pub async fn http_server(context: Context, options: HttpOptions) {
     let HttpOptions {
-        common: CommonOptions { host, .. },
-        http_port,
+        common: CommonHttpOptions { host, port, .. },
         p2p,
     } = options.clone();
 
@@ -92,38 +37,33 @@ pub async fn http_server(context: Context, options: HttpOptions) {
 
     check_run_with_p2p(context.clone(), options.p2p.clone());
 
-    let app = app(context, host.clone(), http_port, p2p.clone()).await;
+    let app = app(context, host.clone(), port, p2p.clone()).await;
 
-    let server_url = format!("{}:{}", host, http_port);
+    let server_url = format!("{}:{}", host, port);
 
-    let addr = SocketAddr::from_str(&server_url).unwrap();
-    let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
+    let listener = tokio::net::TcpListener::bind(server_url).await.unwrap();
     axum::serve(listener, app.into_make_service())
         .await
         .unwrap();
 }
 
 pub async fn app(context: Context, host: String, port: u16, p2p: P2pOptions) -> Router {
     let state = AppState {
-        host,
+        host: host.clone(),
         port,
         context: context.clone(),
     };
 
-    let mega_api_state = MegaApiServiceState {
-        inner: MonoApiServiceState {
-            context: context.clone(),
-            oauth_client: None,
-            store: None,
-        },
-        p2p,
-        port,
-    };
-
     let mono_api_state = MonoApiServiceState {
         context: context.clone(),
         oauth_client: None,
         store: None,
+        listen_addr: format!("http://{}:{}", host, port),
+    };
+
+    let mega_api_state = MegaApiServiceState {
+        inner: mono_api_state.clone(),
+        p2p,
     };
 
     pub fn mega_routers() -> Router<MegaApiServiceState> {

jupiter/src/context.rs

@@ -49,7 +49,7 @@ impl Context {
     pub fn mock() -> Self {
         Context {
             services: Service::mock(),
-            config: Arc::new(Config::default()),
+            config: Arc::new(Config::mock()),
         }
     }
 }

libra/Cargo.toml

@@ -58,6 +58,6 @@ pager = "0.16.0"
 
 [dev-dependencies]
 tempfile = { workspace = true }
-serial_test = "3.2.0"
+serial_test = { workspace = true }
 tokio = { workspace = true, features = ["macros", "process"] }
 tracing-test = "0.2.4"

mega/Cargo.toml

@@ -40,6 +40,7 @@ mimalloc = "0.1.43"
 
 [dev-dependencies]
 tempfile = { workspace = true }
-serial_test = "3.1.1"
+serial_test = { workspace = true }
 lazy_static = { workspace = true }
 assert_cmd = "2.0.16"
+scopeguard = { workspace = true }

mega/config.toml

@@ -79,12 +79,13 @@ clean_cache_after_decode = true
 channel_message_size = 1_000_000
 
 [lfs]
-# LFS Server url
-url = "http://localhost:8000"
-
 # lfs file storage type, support values can be "local_fs" or "aws_s3"
 storage_type = "local_fs"
 
+[lfs.ssh]
+# The lfs ssh transport protocol still relies on http to send files
+http_url = "http://localhost:8000"
+
 [lfs.local]
 # set the local path of the lfs storage
 lfs_file_path = "${base_dir}/lfs"