Skip to content

feat: add metadata to entry object and store it in database && update filepath #1609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Nov 5, 2025
Merged

feat: add metadata to entry object and store it in database && update filepath #1609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e00d9e2
add ext metadata
pleasedontbreak123 Oct 31, 2025
9731b5c
add ext metadata
pleasedontbreak123 Nov 2, 2025
4b68a5a
fix
pleasedontbreak123 Nov 2, 2025
7ea684f
Merge branch 'web3infra-foundation:main' into main
pleasedontbreak123 Nov 4, 2025
7771ba7
fix fmt
pleasedontbreak123 Nov 4, 2025
6f553e9
fix clippy
pleasedontbreak123 Nov 4, 2025
323280a
fix clippy
pleasedontbreak123 Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ bellatrix = { path = "orion-server/bellatrix" }
context = { path = "context" }
neptune = { path = "neptune" }

git-internal = "0.1.0"
git-internal = "0.2.1"
Copy link

Copilot AI Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Line 36 has an extra leading space before git-internal. The alignment should match the surrounding lines (no leading space).

Copilot uses AI. Check for mistakes.

#====
anyhow = "1.0.100"
Expand All @@ -51,11 +51,12 @@ tokio-stream = "0.1.17"
tokio-util = "0.7.16"
async-trait = "0.1.89"
async-stream = "0.3.6"
async-recursion = "1.1.1"
futures = "0.3.31"
futures-util = "0.3.31"
axum = { version = "0.8.6", features = ["macros", "json"] }
axum-extra = "0.10.3"
russh = "0.52.1"
russh = "0.54.6"
tower-http = "0.6.6"
tower = "0.5.2"
tower-sessions = { version = "0.12.3", features = ["memory-store"] }
Expand Down
2 changes: 2 additions & 0 deletions ceres/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ common = { workspace = true }
jupiter = { workspace = true }
callisto = { workspace = true }
git-internal = { workspace = true }

neptune = { workspace = true }
bellatrix = { workspace = true }

Expand All @@ -32,6 +33,7 @@ chrono = { workspace = true }
futures = { workspace = true }
bytes = { workspace = true }
async-trait = { workspace = true }
async-recursion = { workspace = true }
rand = { workspace = true }
ring = { workspace = true }
hex = { workspace = true }
Expand Down
21 changes: 17 additions & 4 deletions ceres/src/api_service/import_api_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use tokio::sync::Mutex;
use async_trait::async_trait;
use git_internal::errors::GitError;
use git_internal::hash::SHA1;
use git_internal::internal::metadata::{EntryMeta, MetaAttached};
use git_internal::internal::object::commit::Commit;
use git_internal::internal::object::tree::{Tree, TreeItem, TreeItemMode};
use git_internal::internal::pack::entry::Entry;
Expand Down Expand Up @@ -443,12 +444,21 @@ impl ApiHandler for ImportApiService {
Commit::from_tree_id(new_root_id, vec![parent_id], &payload.commit_message);
let new_commit_id = new_commit.id.to_string();

let mut entries: Vec<Entry> = Vec::new();
let mut entries: Vec<MetaAttached<Entry, EntryMeta>> = Vec::new();
for t in updated_trees.iter().cloned() {
entries.push(Entry::from(t));
entries.push(MetaAttached {
inner: Entry::from(t),
meta: EntryMeta::new(),
});
}
entries.push(Entry::from(new_blob.clone()));
entries.push(Entry::from(new_commit.clone()));
entries.push(MetaAttached {
inner: Entry::from(new_blob.clone()),
meta: EntryMeta::new(),
});
entries.push(MetaAttached {
inner: Entry::from(new_commit.clone()),
meta: EntryMeta::new(),
});
git_storage
.save_entry(self.repo.repo_id, entries)
.await
Expand Down Expand Up @@ -788,6 +798,9 @@ impl ImportApiService {
tag_name: name,
tagger: tagger_info,
message: message.unwrap_or_default(),
pack_id: String::new(),
pack_offset: 0,

Copy link

Copilot AI Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Extra blank line at line 803 should be removed to maintain consistent code formatting.

Suggested change

Copilot uses AI. Check for mistakes.
created_at: chrono::Utc::now().naive_utc(),
}
}
Expand Down
9 changes: 6 additions & 3 deletions ceres/src/api_service/mono_api_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ use bytes::Bytes;
use common::utils::MEGA_BRANCH_NAME;
use git_internal::errors::GitError;
use git_internal::hash::SHA1;
use git_internal::internal::metadata::EntryMeta;
use git_internal::internal::object::blob::Blob;
use git_internal::internal::object::commit::Commit;
use git_internal::internal::object::tree::{Tree, TreeItem, TreeItemMode};
Expand Down Expand Up @@ -283,7 +284,7 @@ impl ApiHandler for MonoApiService {
let save_trees: Vec<mega_tree::ActiveModel> = save_trees
.into_iter()
.map(|save_t| {
let mut tree_model: mega_tree::Model = save_t.into_mega_model();
let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new());
tree_model.commit_id.clone_from(&new_commit_id);
tree_model.into()
})
Expand Down Expand Up @@ -390,7 +391,7 @@ impl ApiHandler for MonoApiService {
let save_trees: Vec<mega_tree::ActiveModel> = save_trees
.into_iter()
.map(|save_t| {
let mut tree_model: mega_tree::Model = save_t.into_mega_model();
let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new());
tree_model.commit_id.clone_from(&new_commit_id);
tree_model.into()
})
Expand Down Expand Up @@ -1045,6 +1046,8 @@ impl MonoApiService {
tag_name: name,
tagger: tagger_info,
message: message.unwrap_or_default(),
pack_id: String::new(),
pack_offset: 0,
created_at: chrono::Utc::now().naive_utc(),
}
}
Expand Down Expand Up @@ -1229,7 +1232,7 @@ impl MonoApiService {
.clone()
.into_iter()
.map(|save_t| {
let mut tree_model: mega_tree::Model = save_t.into_mega_model();
let mut tree_model: mega_tree::Model = save_t.into_mega_model(EntryMeta::new());
tree_model.commit_id.clone_from(&new_commit_id);
tree_model.into()
})
Expand Down
148 changes: 138 additions & 10 deletions ceres/src/pack/import_repo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
use async_recursion::async_recursion;
use async_trait::async_trait;
use futures::StreamExt;
use std::{
collections::{HashMap, HashSet},
path::PathBuf,
Expand All @@ -7,9 +10,6 @@ use std::{
atomic::{AtomicUsize, Ordering},
},
};

use async_trait::async_trait;
use futures::StreamExt;
use tokio::sync::{
Mutex,
mpsc::{self},
Expand All @@ -18,6 +18,7 @@ use tokio_stream::wrappers::ReceiverStream;

use callisto::{mega_tree, raw_blob, sea_orm_active_enums::RefTypeEnum};
use common::errors::MegaError;
use git_internal::internal::metadata::{EntryMeta, MetaAttached};
use git_internal::{
errors::GitError,
internal::{
Expand Down Expand Up @@ -65,14 +66,23 @@ impl RepoHandler for ImportRepo {

async fn post_receive_pack(&self) -> Result<(), MegaError> {
let _guard = self.shared.lock().await;
self.traverses_tree_and_update_filepath().await?;
self.attach_to_monorepo_parent().await
}

async fn save_entry(&self, entry_list: Vec<Entry>) -> Result<(), MegaError> {
async fn save_entry(
&self,
entry_list: Vec<MetaAttached<Entry, EntryMeta>>,
) -> Result<(), MegaError> {
let storage = self.storage.git_db_storage();
storage.save_entry(self.repo.repo_id, entry_list).await
}

async fn update_pack_id(&self, temp_pack_id: &str, pack_id: &str) -> Result<(), MegaError> {
let storage = self.storage.git_db_storage();
storage.update_pack_id(temp_pack_id, pack_id).await
}

async fn check_entry(&self, _: &Entry) -> Result<(), GitError> {
Ok(())
}
Expand All @@ -96,7 +106,10 @@ impl RepoHandler for ImportRepo {
match model {
Ok(m) => {
let c: Commit = Commit::from_git_model(m);
let entry = c.into();
let entry = MetaAttached {
inner: c.into(),
meta: EntryMeta::new(),
};
entry_tx.send(entry).await.unwrap();
}
Err(err) => eprintln!("Error: {err:?}"),
Expand All @@ -109,7 +122,10 @@ impl RepoHandler for ImportRepo {
match model {
Ok(m) => {
let t: Tree = Tree::from_git_model(m);
let entry = t.into();
let entry = MetaAttached {
inner: t.into(),
meta: EntryMeta::new(),
};
entry_tx.send(entry).await.unwrap();
}
Err(err) => eprintln!("Error: {err:?}"),
Expand Down Expand Up @@ -139,7 +155,26 @@ impl RepoHandler for ImportRepo {
// TODO handle storage type
let data = m.data.unwrap_or_default();
let b: Blob = Blob::from_content_bytes(data);
let entry: Entry = b.into();
// let blob_with_data = storage.get_blobs_by_hashes(repo_id,vec![b.id.to_string()]).await?.iter().next().unwrap();
Copy link

Copilot AI Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove commented-out code on line 158. The working implementation is already in place below, so this line serves no purpose.

Suggested change
// let blob_with_data = storage.get_blobs_by_hashes(repo_id,vec![b.id.to_string()]).await?.iter().next().unwrap();

Copilot uses AI. Check for mistakes.
let blob_with_data = storage
.get_blobs_by_hashes(repo_id, vec![b.id.to_string()])
.await
.expect("get_blobs_by_hashes failed")
.into_iter()
.next()
.expect("blob metadata not found");

let meta_data = EntryMeta {
pack_id: Some(blob_with_data.pack_id.clone()),
pack_offset: Some(blob_with_data.pack_offset as usize),
file_path: Some(blob_with_data.file_path.clone()),
is_delta: Some(blob_with_data.is_delta_in_pack),
};

let entry = MetaAttached {
inner: b.into(),
meta: meta_data,
};
sender_clone.send(entry).await.unwrap();
}
Err(err) => eprintln!("Error: {err:?}"),
Expand All @@ -154,7 +189,10 @@ impl RepoHandler for ImportRepo {
let tags = storage.get_tags_by_repo_id(repo_id).await.unwrap();
for m in tags.into_iter() {
let c: Tag = Tag::from_git_model(m);
let entry: Entry = c.into();
let entry = MetaAttached {
inner: c.into(),
meta: EntryMeta::new(),
};
entry_tx.send(entry).await.unwrap();
}
drop(entry_tx);
Expand Down Expand Up @@ -256,7 +294,13 @@ impl RepoHandler for ImportRepo {
Some(&entry_tx),
)
.await;
entry_tx.send(c.into()).await.unwrap();
entry_tx
.send(MetaAttached {
inner: c.into(),
meta: EntryMeta::new(),
})
.await
.unwrap();
}
drop(entry_tx);

Expand Down Expand Up @@ -285,6 +329,34 @@ impl RepoHandler for ImportRepo {
.await
}

async fn get_blob_metadata_by_hashes(
&self,
hashes: Vec<String>,
) -> Result<HashMap<String, EntryMeta>, MegaError> {
let models = self
.storage
.git_db_storage()
.get_blobs_by_hashes(self.repo.repo_id, hashes)
.await?;

let map = models
.into_iter()
.map(|blob| {
(
blob.blob_id.clone(),
EntryMeta {
pack_id: Some(blob.pack_id.clone()),
pack_offset: Some(blob.pack_offset as usize),
file_path: Some(blob.file_path.clone()),
is_delta: Some(blob.is_delta_in_pack),
},
)
})
.collect::<HashMap<String, EntryMeta>>();

Ok(map)
}

async fn update_refs(&self, refs: &RefCommand) -> Result<(), GitError> {
let storage = self.storage.git_db_storage();
match refs.command_type {
Expand Down Expand Up @@ -324,9 +396,65 @@ impl RepoHandler for ImportRepo {
.await
.unwrap()
}

async fn traverses_tree_and_update_filepath(&self) -> Result<(), MegaError> {
//let (current_head, refs) = self.head_hash().await;
Copy link

Copilot AI Nov 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove commented-out code on line 401. If this code might be needed later, document why with a TODO comment or remove it entirely as it's captured in version control.

Suggested change
//let (current_head, refs) = self.head_hash().await;

Copilot uses AI. Check for mistakes.
let (current_head, _refs) = self.refs_with_head_hash().await;
let commit = Commit::from_git_model(
self.storage
.git_db_storage()
.get_commit_by_hash(self.repo.repo_id, &current_head)
.await?
.unwrap(),
);

let root_tree = Tree::from_git_model(
self.storage
.git_db_storage()
.get_tree_by_hash(self.repo.repo_id, &commit.tree_id.to_string())
.await?
.unwrap()
.clone(),
);
self.traverses_and_update_filepath(root_tree, PathBuf::new())
.await?;
Ok(())
}
}

impl ImportRepo {
#[async_recursion]
async fn traverses_and_update_filepath(
&self,
tree: Tree,
path: PathBuf,
) -> Result<(), MegaError> {
for item in tree.tree_items {
if item.is_tree() {
let tree = Tree::from_git_model(
self.storage
.git_db_storage()
.get_tree_by_hash(self.repo.repo_id, &item.id.to_string())
.await?
.unwrap()
.clone(),
);

// 递归调用
self.traverses_and_update_filepath(tree, path.join(item.name))
.await?;
} else {
let id = item.id.to_string();
self.storage
.git_db_storage()
.update_git_blob_filepath(&id, path.join(item.name).to_str().unwrap())
.await?;
}
}

Ok(())
}

// attach import repo to monorepo parent tree
pub(crate) async fn attach_to_monorepo_parent(&self) -> Result<(), MegaError> {
// 1. find branch command
Expand Down Expand Up @@ -379,7 +507,7 @@ impl ImportRepo {
let save_trees: Vec<mega_tree::ActiveModel> = save_trees
.into_iter()
.map(|tree| {
let mut model: mega_tree::Model = tree.into_mega_model();
let mut model: mega_tree::Model = tree.into_mega_model(EntryMeta::new());
model.commit_id = new_commit.id.to_string();
model.into()
})
Expand Down
Loading