feat: add metadata to entry object and store it in database && update filepath (#1609)

* add ext metadata

* add ext metadata

* fix

* fix fmt

* fix clippy

* fix clippy

Author: pleasedontbreak123

Cargo.toml

@@ -33,7 +33,7 @@ bellatrix = { path = "orion-server/bellatrix" }
 context = { path = "context" }
 neptune = { path = "neptune" }
 
-git-internal = "0.1.0"
+ git-internal = "0.2.1"
 
 #====
 anyhow = "1.0.100"
@@ -51,11 +51,12 @@ tokio-stream = "0.1.17"
 tokio-util = "0.7.16"
 async-trait = "0.1.89"
 async-stream = "0.3.6"
+async-recursion = "1.1.1"
 futures = "0.3.31"
 futures-util = "0.3.31"
 axum = { version = "0.8.6", features = ["macros", "json"] }
 axum-extra = "0.10.3"
-russh = "0.52.1"
+russh = "0.54.6"
 tower-http = "0.6.6"
 tower = "0.5.2"
 tower-sessions = { version = "0.12.3", features = ["memory-store"] }

ceres/Cargo.toml

@@ -15,6 +15,7 @@ common = { workspace = true }
 jupiter = { workspace = true }
 callisto = { workspace = true }
 git-internal = { workspace = true }
+
 neptune = { workspace = true }
 bellatrix = { workspace = true }
 
@@ -32,6 +33,7 @@ chrono = { workspace = true }
 futures = { workspace = true }
 bytes = { workspace = true }
 async-trait = { workspace = true }
+async-recursion = { workspace = true }
 rand = { workspace = true }
 ring = { workspace = true }
 hex = { workspace = true }

ceres/src/api_service/import_api_service.rs

@@ -8,6 +8,7 @@ use tokio::sync::Mutex;
 use async_trait::async_trait;
 use git_internal::errors::GitError;
 use git_internal::hash::SHA1;
+use git_internal::internal::metadata::{EntryMeta, MetaAttached};
 use git_internal::internal::object::commit::Commit;
 use git_internal::internal::object::tree::{Tree, TreeItem, TreeItemMode};
 use git_internal::internal::pack::entry::Entry;
@@ -443,12 +444,21 @@ impl ApiHandler for ImportApiService {
                 Commit::from_tree_id(new_root_id, vec![parent_id], &payload.commit_message);
             let new_commit_id = new_commit.id.to_string();
 
-            let mut entries: Vec<Entry> = Vec::new();
+            let mut entries: Vec<MetaAttached<Entry, EntryMeta>> = Vec::new();
             for t in updated_trees.iter().cloned() {
-                entries.push(Entry::from(t));
+                entries.push(MetaAttached {
+                    inner: Entry::from(t),
+                    meta: EntryMeta::new(),
+                });
             }
-            entries.push(Entry::from(new_blob.clone()));
-            entries.push(Entry::from(new_commit.clone()));
+            entries.push(MetaAttached {
+                inner: Entry::from(new_blob.clone()),
+                meta: EntryMeta::new(),
+            });
+            entries.push(MetaAttached {
+                inner: Entry::from(new_commit.clone()),
+                meta: EntryMeta::new(),
+            });
             git_storage
                 .save_entry(self.repo.repo_id, entries)
                 .await
@@ -788,6 +798,9 @@ impl ImportApiService {
             tag_name: name,
             tagger: tagger_info,
             message: message.unwrap_or_default(),
+            pack_id: String::new(),
+            pack_offset: 0,
+
             created_at: chrono::Utc::now().naive_utc(),
         }
     }

ceres/src/api_service/mono_api_service.rs

@@ -52,6 +52,7 @@ use bytes::Bytes;
 use common::utils::MEGA_BRANCH_NAME;
 use git_internal::errors::GitError;
 use git_internal::hash::SHA1;
+use git_internal::internal::metadata::EntryMeta;
 use git_internal::internal::object::blob::Blob;
 use git_internal::internal::object::commit::Commit;
 use git_internal::internal::object::tree::{Tree, TreeItem, TreeItemMode};
@@ -283,7 +284,7 @@ impl ApiHandler for MonoApiService {
             let save_trees: Vec<mega_tree::ActiveModel> = save_trees
                 .into_iter()
                 .map(|save_t| {
-                    let mut tree_model: mega_tree::Model = save_t.into_mega_model();
+                    let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new());
                     tree_model.commit_id.clone_from(&new_commit_id);
                     tree_model.into()
                 })
@@ -390,7 +391,7 @@ impl ApiHandler for MonoApiService {
             let save_trees: Vec<mega_tree::ActiveModel> = save_trees
                 .into_iter()
                 .map(|save_t| {
-                    let mut tree_model: mega_tree::Model = save_t.into_mega_model();
+                    let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new());
                     tree_model.commit_id.clone_from(&new_commit_id);
                     tree_model.into()
                 })
@@ -1045,6 +1046,8 @@ impl MonoApiService {
             tag_name: name,
             tagger: tagger_info,
             message: message.unwrap_or_default(),
+            pack_id: String::new(),
+            pack_offset: 0,
             created_at: chrono::Utc::now().naive_utc(),
         }
     }
@@ -1229,7 +1232,7 @@ impl MonoApiService {
             .clone()
             .into_iter()
             .map(|save_t| {
-                let mut tree_model: mega_tree::Model = save_t.into_mega_model();
+                let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new());
                 tree_model.commit_id.clone_from(&new_commit_id);
                 tree_model.into()
             })

ceres/src/pack/import_repo.rs

@@ -1,3 +1,6 @@
+use async_recursion::async_recursion;
+use async_trait::async_trait;
+use futures::StreamExt;
 use std::{
     collections::{HashMap, HashSet},
     path::PathBuf,
@@ -7,9 +10,6 @@ use std::{
         atomic::{AtomicUsize, Ordering},
     },
 };
-
-use async_trait::async_trait;
-use futures::StreamExt;
 use tokio::sync::{
     Mutex,
     mpsc::{self},
@@ -18,6 +18,7 @@ use tokio_stream::wrappers::ReceiverStream;
 
 use callisto::{mega_tree, raw_blob, sea_orm_active_enums::RefTypeEnum};
 use common::errors::MegaError;
+use git_internal::internal::metadata::{EntryMeta, MetaAttached};
 use git_internal::{
     errors::GitError,
     internal::{
@@ -65,14 +66,23 @@ impl RepoHandler for ImportRepo {
 
     async fn post_receive_pack(&self) -> Result<(), MegaError> {
         let _guard = self.shared.lock().await;
+        self.traverses_tree_and_update_filepath().await?;
         self.attach_to_monorepo_parent().await
     }
 
-    async fn save_entry(&self, entry_list: Vec<Entry>) -> Result<(), MegaError> {
+    async fn save_entry(
+        &self,
+        entry_list: Vec<MetaAttached<Entry, EntryMeta>>,
+    ) -> Result<(), MegaError> {
         let storage = self.storage.git_db_storage();
         storage.save_entry(self.repo.repo_id, entry_list).await
     }
 
+    async fn update_pack_id(&self, temp_pack_id: &str, pack_id: &str) -> Result<(), MegaError> {
+        let storage = self.storage.git_db_storage();
+        storage.update_pack_id(temp_pack_id, pack_id).await
+    }
+
     async fn check_entry(&self, _: &Entry) -> Result<(), GitError> {
         Ok(())
     }
@@ -96,7 +106,10 @@ impl RepoHandler for ImportRepo {
                 match model {
                     Ok(m) => {
                         let c: Commit = Commit::from_git_model(m);
-                        let entry = c.into();
+                        let entry = MetaAttached {
+                            inner: c.into(),
+                            meta: EntryMeta::new(),
+                        };
                         entry_tx.send(entry).await.unwrap();
                     }
                     Err(err) => eprintln!("Error: {err:?}"),
@@ -109,7 +122,10 @@ impl RepoHandler for ImportRepo {
                 match model {
                     Ok(m) => {
                         let t: Tree = Tree::from_git_model(m);
-                        let entry = t.into();
+                        let entry = MetaAttached {
+                            inner: t.into(),
+                            meta: EntryMeta::new(),
+                        };
                         entry_tx.send(entry).await.unwrap();
                     }
                     Err(err) => eprintln!("Error: {err:?}"),
@@ -139,7 +155,26 @@ impl RepoHandler for ImportRepo {
                             // TODO handle storage type
                             let data = m.data.unwrap_or_default();
                             let b: Blob = Blob::from_content_bytes(data);
-                            let entry: Entry = b.into();
+                            // let blob_with_data = storage.get_blobs_by_hashes(repo_id,vec![b.id.to_string()]).await?.iter().next().unwrap();
+                            let blob_with_data = storage
+                                .get_blobs_by_hashes(repo_id, vec![b.id.to_string()])
+                                .await
+                                .expect("get_blobs_by_hashes failed")
+                                .into_iter()
+                                .next()
+                                .expect("blob metadata not found");
+
+                            let meta_data = EntryMeta {
+                                pack_id: Some(blob_with_data.pack_id.clone()),
+                                pack_offset: Some(blob_with_data.pack_offset as usize),
+                                file_path: Some(blob_with_data.file_path.clone()),
+                                is_delta: Some(blob_with_data.is_delta_in_pack),
+                            };
+
+                            let entry = MetaAttached {
+                                inner: b.into(),
+                                meta: meta_data,
+                            };
                             sender_clone.send(entry).await.unwrap();
                         }
                         Err(err) => eprintln!("Error: {err:?}"),
@@ -154,7 +189,10 @@ impl RepoHandler for ImportRepo {
             let tags = storage.get_tags_by_repo_id(repo_id).await.unwrap();
             for m in tags.into_iter() {
                 let c: Tag = Tag::from_git_model(m);
-                let entry: Entry = c.into();
+                let entry = MetaAttached {
+                    inner: c.into(),
+                    meta: EntryMeta::new(),
+                };
                 entry_tx.send(entry).await.unwrap();
             }
             drop(entry_tx);
@@ -256,7 +294,13 @@ impl RepoHandler for ImportRepo {
                 Some(&entry_tx),
             )
             .await;
-            entry_tx.send(c.into()).await.unwrap();
+            entry_tx
+                .send(MetaAttached {
+                    inner: c.into(),
+                    meta: EntryMeta::new(),
+                })
+                .await
+                .unwrap();
         }
         drop(entry_tx);
 
@@ -285,6 +329,34 @@ impl RepoHandler for ImportRepo {
             .await
     }
 
+    async fn get_blob_metadata_by_hashes(
+        &self,
+        hashes: Vec<String>,
+    ) -> Result<HashMap<String, EntryMeta>, MegaError> {
+        let models = self
+            .storage
+            .git_db_storage()
+            .get_blobs_by_hashes(self.repo.repo_id, hashes)
+            .await?;
+
+        let map = models
+            .into_iter()
+            .map(|blob| {
+                (
+                    blob.blob_id.clone(),
+                    EntryMeta {
+                        pack_id: Some(blob.pack_id.clone()),
+                        pack_offset: Some(blob.pack_offset as usize),
+                        file_path: Some(blob.file_path.clone()),
+                        is_delta: Some(blob.is_delta_in_pack),
+                    },
+                )
+            })
+            .collect::<HashMap<String, EntryMeta>>();
+
+        Ok(map)
+    }
+
     async fn update_refs(&self, refs: &RefCommand) -> Result<(), GitError> {
         let storage = self.storage.git_db_storage();
         match refs.command_type {
@@ -324,9 +396,65 @@ impl RepoHandler for ImportRepo {
             .await
             .unwrap()
     }
+
+    async fn traverses_tree_and_update_filepath(&self) -> Result<(), MegaError> {
+        //let (current_head, refs) = self.head_hash().await;
+        let (current_head, _refs) = self.refs_with_head_hash().await;
+        let commit = Commit::from_git_model(
+            self.storage
+                .git_db_storage()
+                .get_commit_by_hash(self.repo.repo_id, &current_head)
+                .await?
+                .unwrap(),
+        );
+
+        let root_tree = Tree::from_git_model(
+            self.storage
+                .git_db_storage()
+                .get_tree_by_hash(self.repo.repo_id, &commit.tree_id.to_string())
+                .await?
+                .unwrap()
+                .clone(),
+        );
+        self.traverses_and_update_filepath(root_tree, PathBuf::new())
+            .await?;
+        Ok(())
+    }
 }
 
 impl ImportRepo {
+    #[async_recursion]
+    async fn traverses_and_update_filepath(
+        &self,
+        tree: Tree,
+        path: PathBuf,
+    ) -> Result<(), MegaError> {
+        for item in tree.tree_items {
+            if item.is_tree() {
+                let tree = Tree::from_git_model(
+                    self.storage
+                        .git_db_storage()
+                        .get_tree_by_hash(self.repo.repo_id, &item.id.to_string())
+                        .await?
+                        .unwrap()
+                        .clone(),
+                );
+
+                // 递归调用
+                self.traverses_and_update_filepath(tree, path.join(item.name))
+                    .await?;
+            } else {
+                let id = item.id.to_string();
+                self.storage
+                    .git_db_storage()
+                    .update_git_blob_filepath(&id, path.join(item.name).to_str().unwrap())
+                    .await?;
+            }
+        }
+
+        Ok(())
+    }
+
     // attach import repo to monorepo parent tree
     pub(crate) async fn attach_to_monorepo_parent(&self) -> Result<(), MegaError> {
         // 1. find branch command
@@ -379,7 +507,7 @@ impl ImportRepo {
         let save_trees: Vec<mega_tree::ActiveModel> = save_trees
             .into_iter()
             .map(|tree| {
-                let mut model: mega_tree::Model = tree.into_mega_model();
+                let mut model: mega_tree::Model = tree.into_mega_model(EntryMeta::new());
                 model.commit_id = new_commit.id.to_string();
                 model.into()
             })