feat(git): SHA256 repository support

This wires the unstable libgit2 SHA256 support into Cargo.

SHA256 repositories usage are gated behind `-Zgit=sha256`.

Before looking at a repo,
Cargo now try to guess whether this git dep is SHA1 or SHA256
from these places (in this order):

* locked rev in Cargo.lock
* local db (with or without `-sha256` suffix)
* Create a detached remote and probe its object format

What works and doesn't:

* Git CLI and libgit2 interop works
* SHA1 and SHA256 git db coexist (via `-sha256` dir suffix)
* `-Zgit=sha256` gates during early fetch paths,
  so even have local db cached you cannot use without Z flag
* gitoxide hasn't yet supported

Some known issues and regressions:

* Probing adds a silent extra round-trip on every first fetch,
  even for SHA1 repos.
  An alternative is to assume SHA1 and retry on mismatch,
  though it has cost of a wasted fetch attempt for SHA256 repos.

Author: weihanglo

src/cargo/sources/git/source.rs

@@ -8,6 +8,7 @@ use crate::sources::IndexSummary;
 use crate::sources::RecursivePathSource;
 use crate::sources::git::utils::GitDatabase;
 use crate::sources::git::utils::GitRemote;
+use crate::sources::git::utils::probe_remote_object_format;
 use crate::sources::git::utils::rev_to_oid;
 use crate::sources::source::MaybePackage;
 use crate::sources::source::QueryKind;
@@ -47,14 +48,17 @@ use url::Url;
 /// │  │  └── e33d1ac/
 /// │  ├── log-c58e1db3de7c154d-shallow/
 /// │  │  └── 11eda98/
+/// │  └── foo-9f1e8cfc50c5ba7d-sha256/
+/// │     └── cfc50c5/
 /// └── db/
 ///    ├── gimli-a0d193bd15a5ed96/
-///    └── log-c58e1db3de7c154d-shallow/
+///    ├── log-c58e1db3de7c154d-shallow/
+///    └── foo-9f1e8cfc50c5ba7d-sha256/
 /// ```
 ///
 /// For more on Git cache directory, see ["Cargo Home"] in The Cargo Book.
 ///
-/// For more on the directory format `<pkg>-<hash>[-shallow]`, see [`ident`]
+/// For more on the directory format `<pkg>-<hash>[-shallow][-sha256]`, see [`ident`]
 /// and [`ident_shallow`].
 ///
 /// ## Locked to a revision
@@ -171,23 +175,81 @@ impl<'gctx> GitSource<'gctx> {
     }
 
     fn mark_used(&self) -> CargoResult<()> {
+        let format = match &*self.locked_rev.borrow() {
+            Revision::Locked(oid) => oid.object_format(),
+            _ => unreachable!("locked_rev must be resolved before mark_used"),
+        };
+        let ident = self.ident_for_format(format);
         self.gctx
             .deferred_global_last_use()?
             .mark_git_checkout_used(global_cache_tracker::GitCheckout {
-                encoded_git_name: self.ident,
+                encoded_git_name: ident,
                 short_name: self.short_id.borrow().expect("update before download"),
                 size: None,
             });
         Ok(())
     }
 
+    fn ident_for_format(&self, format: git2::ObjectFormat) -> InternedString {
+        match format {
+            git2::ObjectFormat::Sha1 => self.ident,
+            git2::ObjectFormat::Sha256 => format!("{}-sha256", self.ident).into(),
+        }
+    }
+
+    /// Determines the Git object format for this remote.
+    ///
+    /// This may probe the remote repository if needed.
+    fn object_format_hint(&self) -> CargoResult<Option<git2::ObjectFormat>> {
+        if let Revision::Locked(oid) = &*self.locked_rev.borrow() {
+            return Ok(Some(oid.object_format()));
+        }
+
+        let git_db_path = self.gctx.git_db_path();
+        self.gctx
+            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &git_db_path);
+        let git_db_path = git_db_path.as_path_unlocked();
+
+        for format in [git2::ObjectFormat::Sha1, git2::ObjectFormat::Sha256] {
+            let ident = self.ident_for_format(format);
+            let path = git_db_path.join(ident);
+            if path.exists()
+                && let Ok(db) = self.remote.db_at(&path)
+            {
+                return Ok(Some(db.object_format()));
+            }
+        }
+
+        Ok(None)
+    }
+
     /// Fetch and return a [`GitDatabase`] with the resolved revision
     /// for this source,
     ///
     /// This won't fetch anything if the required revision is
     /// already available locally.
     pub(crate) fn fetch_db(&self, is_submodule: bool) -> CargoResult<(GitDatabase, git2::Oid)> {
-        let db_path = self.gctx.git_db_path().join(&self.ident);
+        let mut is_update_status_shown = false;
+
+        let format = if let Some(format) = self.object_format_hint()? {
+            format
+        } else {
+            if !is_update_status_shown {
+                is_update_status_shown = true;
+                self.show_update_status(is_submodule)?;
+            }
+            if let Some(offline_flag) = self.gctx.offline_flag() {
+                anyhow::bail!(
+                    "can't checkout from '{}': you are in the offline mode ({offline_flag})",
+                    self.remote.url()
+                );
+            }
+
+            trace!("probing git source `{:?}`", self.remote);
+            probe_remote_object_format(self.source_id.borrow().url(), self.gctx)?
+        };
+
+        let db_path = self.gctx.git_db_path().join(self.ident_for_format(format));
         let db_path = db_path.into_path_unlocked();
 
         let db = self.remote.db_at(&db_path).ok();
@@ -226,14 +288,22 @@ impl<'gctx> GitSource<'gctx> {
                     );
                 }
 
-                self.show_update_status(is_submodule)?;
+                if !is_update_status_shown {
+                    self.show_update_status(is_submodule)?;
+                }
 
                 trace!("updating git source `{:?}`", self.remote);
 
                 let locked_rev = locked_rev.clone().into();
                 let manifest_reference = self.source_id.borrow().git_reference().unwrap();
-                self.remote
-                    .checkout(&db_path, db, manifest_reference, &locked_rev, self.gctx)?
+                self.remote.checkout(
+                    &db_path,
+                    db,
+                    manifest_reference,
+                    &locked_rev,
+                    format,
+                    self.gctx,
+                )?
             }
         };
         Ok((db, actual_rev))
@@ -287,10 +357,11 @@ impl<'gctx> GitSource<'gctx> {
         // Check out `actual_rev` from the database to a scoped location on the
         // filesystem. This will use hard links and such to ideally make the
         // checkout operation here pretty fast.
+        let ident = self.ident_for_format(actual_rev.object_format());
         let checkout_path = self
             .gctx
             .git_checkouts_path()
-            .join(&self.ident)
+            .join(ident)
             .join(short_id.as_str());
         let checkout_path = checkout_path.into_path_unlocked();
         db.copy_to(actual_rev, &checkout_path, self.gctx, self.quiet)?;
@@ -365,6 +436,7 @@ fn ident(id: &SourceId) -> String {
 
 /// Like [`ident()`], but appends `-shallow` to it, turning
 /// `proto://host/path/repo` into `repo-<hash-of-url>-shallow`.
+/// SHA256 repositories add the `-sha256` suffix on top of this identifier.
 ///
 /// It's important to separate shallow from non-shallow clones for reasons of
 /// backwards compatibility --- older cargo's aren't necessarily handling

src/cargo/sources/git/utils.rs

@@ -14,7 +14,9 @@ use crate::util::{GlobalContext, IntoUrl, MetricsCounter, Progress, network};
 use anyhow::{Context as _, anyhow};
 use cargo_util::{ProcessBuilder, paths};
 use cargo_util_terminal::Verbosity;
-use git2::{ErrorClass, ObjectType, Oid};
+use git2::ErrorClass;
+use git2::ObjectType;
+use git2::Oid;
 use http::{Request, StatusCode};
 use tracing::{debug, info};
 use url::Url;
@@ -112,6 +114,7 @@ impl GitRemote {
         db: Option<GitDatabase>,
         manifest_reference: &GitReference,
         reference: &GitReference,
+        object_format: git2::ObjectFormat,
         gctx: &GlobalContext,
     ) -> CargoResult<(GitDatabase, git2::Oid)> {
         if let Some(mut db) = db {
@@ -137,7 +140,7 @@ impl GitRemote {
             paths::remove_dir_all(into)?;
         }
         paths::create_dir_all(into)?;
-        let mut repo = init(into, true)?;
+        let mut repo = init(into, true, object_format)?;
         fetch(
             &mut repo,
             self.url(),
@@ -212,6 +215,11 @@ impl GitDatabase {
         self.repo.revparse_single(&oid.to_string()).is_ok()
     }
 
+    /// Gets the object format for this database.
+    pub fn object_format(&self) -> git2::ObjectFormat {
+        self.repo.object_format()
+    }
+
     /// [`resolve_ref`]s this reference with this database.
     pub fn resolve(&self, r: &GitReference) -> CargoResult<git2::Oid> {
         resolve_ref(r, &self.repo)
@@ -471,7 +479,10 @@ impl<'a> GitCheckout<'a> {
                 Err(..) => {
                     let path = parent.workdir().unwrap().join(child.path());
                     let _ = paths::remove_dir_all(&path);
-                    init(&path, false)?
+                    // Inherit the parent repo's object format.
+                    // Git does not support mixed-format submodules as of Git 2.54.
+                    // https://github.com/git/git/blob/94f057755b7941b321fd11fec1b2e3ca5313a4e0/object-file.c#L1747
+                    init(&path, false, head.object_format())?
                 }
             };
             // Fetch submodule database and checkout to target revision
@@ -890,6 +901,9 @@ fn reset(repo: &git2::Repository, obj: &git2::Object<'_>, gctx: &GlobalContext)
 ///
 /// The callback is provided a fetch options, which can be used by the actual
 /// git fetch.
+///
+/// NOTE: The auth/certificate_check setup is duplicated in
+/// [`probe_remote_object_format`]. Keep them in sync.
 pub fn with_fetch_options(
     git_config: &git2::Config,
     url: &str,
@@ -981,6 +995,67 @@ pub fn with_fetch_options(
     })
 }
 
+/// Probes a remote for its object format (SHA-1 vs SHA-256) without fetching.
+///
+/// NOTE: The auth/certificate_check setup is duplicated from
+/// [`with_fetch_options`] above. Keep them in sync.
+pub(crate) fn probe_remote_object_format(
+    remote_url: &Url,
+    gctx: &GlobalContext,
+) -> CargoResult<git2::ObjectFormat> {
+    let git_config = git2::Config::open_default()?;
+    let ssh_config = gctx.net_config()?.ssh.as_ref();
+    let config_known_hosts = ssh_config.and_then(|ssh| ssh.known_hosts.as_ref());
+    let diagnostic_home_config = gctx.diagnostic_home_config();
+    let remote_url = remote_url.as_str();
+    network::retry::with_retry(gctx, || {
+        // Hack: libgit2 disallows overriding the error from check_cb since v1.8.0,
+        // so we store the error additionally and unwrap it later
+        let mut check_cb_result = Ok(());
+        let auth_result = with_authentication(gctx, remote_url, &git_config, |f| {
+            let port = Url::parse(remote_url).ok().and_then(|url| url.port());
+            let mut rcb = git2::RemoteCallbacks::new();
+            rcb.credentials(f);
+            rcb.certificate_check(|cert, host| {
+                match super::known_hosts::certificate_check(
+                    gctx,
+                    cert,
+                    host,
+                    port,
+                    config_known_hosts,
+                    &diagnostic_home_config,
+                ) {
+                    Ok(status) => Ok(status),
+                    Err(e) => {
+                        check_cb_result = Err(e);
+                        // This is not really used because it'll be overridden by libgit2
+                        // See https://github.com/libgit2/libgit2/commit/9a9f220119d9647a352867b24b0556195cb26548
+                        Err(git2::Error::from_str(
+                            "invalid or unknown remote ssh hostkey",
+                        ))
+                    }
+                }
+            });
+
+            let proxy_options = git2::ProxyOptions::new();
+            // FIXME: Remove this comment when https://github.com/libgit2/libgit2/pull/7195 merges
+            // This doesn't respect insteadOf from global gitconfig
+            // If libgit2 can't get this fixed timely, we need to switch to
+            // probing a temporary on-disk repo.
+            let mut remote = git2::Remote::create_detached(remote_url)?;
+            let mut conn =
+                remote.connect_auth(git2::Direction::Fetch, Some(rcb), Some(proxy_options))?;
+            // Query while connected — libgit2's local transport frees the
+            // repo handle on disconnect, causing a SIGSEGV if queried after.
+            Ok(conn.remote().object_format()?)
+        });
+        if auth_result.is_err() {
+            check_cb_result?;
+        }
+        auth_result
+    })
+}
+
 /// Attempts to fetch the given git `reference` for a Git repository.
 ///
 /// This is the main entry for git clone/fetch. It does the followings:
@@ -1084,11 +1159,17 @@ pub fn fetch(
     }
 
     debug!("doing a fetch for {remote_url}");
+    let format = repo.object_format();
     let result = if let Some(true) = gctx.net_config()?.git_fetch_with_cli {
+        ensure_sha256_allowed(format, gctx)?;
         fetch_with_cli(repo, remote_url, &refspecs, tags, shallow, gctx)
     } else if gctx.cli_unstable().gitoxide.map_or(false, |git| git.fetch) {
+        if matches!(format, git2::ObjectFormat::Sha256) {
+            anyhow::bail!("gitoxide does not yet support SHA256 repositories");
+        }
         fetch_with_gitoxide(repo, remote_url, refspecs, tags, shallow, gctx)
     } else {
+        ensure_sha256_allowed(format, gctx)?;
         fetch_with_libgit2(repo, remote_url, refspecs, tags, shallow, gctx)
     };
 
@@ -1100,6 +1181,15 @@ pub fn fetch(
     result
 }
 
+fn ensure_sha256_allowed(format: git2::ObjectFormat, gctx: &GlobalContext) -> CargoResult<()> {
+    if matches!(format, git2::ObjectFormat::Sha256)
+        && !gctx.cli_unstable().git.map_or(false, |git| git.sha256)
+    {
+        anyhow::bail!("SHA256 git repositories require `-Zgit=sha256` to be enabled");
+    }
+    Ok(())
+}
+
 /// `gitoxide` uses shallow locks to assure consistency when fetching to and to avoid races, and to write
 /// files atomically.
 /// Cargo has its own lock files and doesn't need that mechanism for race protection, so a stray lock means
@@ -1438,6 +1528,7 @@ fn clean_repo_temp_files(repo: &git2::Repository) {
 /// Reinitializes a given Git repository. This is useful when a Git repository
 /// seems corrupted and we want to start over.
 fn reinitialize(repo: &mut git2::Repository) -> CargoResult<()> {
+    let format = repo.object_format();
     // Here we want to drop the current repository object pointed to by `repo`,
     // so we initialize temporary repository in a sub-folder, blow away the
     // existing git folder, and then recreate the git repo. Finally we blow away
@@ -1446,7 +1537,7 @@ fn reinitialize(repo: &mut git2::Repository) -> CargoResult<()> {
     debug!("reinitializing git repo at {:?}", path);
     let tmp = path.join("tmp");
     let bare = !repo.path().ends_with(".git");
-    *repo = init(&tmp, false)?;
+    *repo = init(&tmp, false, format)?;
     for entry in path.read_dir()? {
         let entry = entry?;
         if entry.file_name().to_str() == Some("tmp") {
@@ -1455,19 +1546,20 @@ fn reinitialize(repo: &mut git2::Repository) -> CargoResult<()> {
         let path = entry.path();
         drop(paths::remove_file(&path).or_else(|_| paths::remove_dir_all(&path)));
     }
-    *repo = init(&path, bare)?;
+    *repo = init(&path, bare, format)?;
     paths::remove_dir_all(&tmp)?;
     Ok(())
 }
 
 /// Initializes a Git repository at `path`.
-fn init(path: &Path, bare: bool) -> CargoResult<git2::Repository> {
+fn init(path: &Path, bare: bool, format: git2::ObjectFormat) -> CargoResult<git2::Repository> {
     let mut opts = git2::RepositoryInitOptions::new();
     // Skip anything related to templates, they just call all sorts of issues as
     // we really don't want to use them yet they insist on being used. See #6240
     // for an example issue that comes up.
     opts.external_template(false);
     opts.bare(bare);
+    opts.object_format(format);
     Ok(git2::Repository::init_opts(&path, &opts)?)
 }