|
| 1 | +// This is free and unencumbered software released into the public domain. |
| 2 | + |
| 3 | +use alloc::{boxed::Box, string::String, vec, vec::Vec}; |
| 4 | +use bitcache_core::{ |
| 5 | + Blob, BlobMetadata, Bytes, Id, ListOptions, ListOrder, Repository, RepositoryError, Stream, |
| 6 | + futures_util::{StreamExt, stream}, |
| 7 | +}; |
| 8 | +use reqwest::{Client, RequestBuilder, Url, header}; |
| 9 | +use serde::Deserialize; |
| 10 | + |
| 11 | +const GITHUB_API: &str = "https://api.github.com"; |
| 12 | +const GITHUB_API_VERSION: &str = "2022-11-28"; |
| 13 | +const GITHUB_JSON: &str = "application/vnd.github+json"; |
| 14 | +const GITHUB_RAW: &str = "application/vnd.github.raw+json"; |
| 15 | +const USER_AGENT: &str = concat!("bitcache-git/", env!("CARGO_PKG_VERSION")); |
| 16 | +const BITCACHE_DIR: &str = ".bitcache"; |
| 17 | + |
| 18 | +/// A read-only Bitcache repository stored in a Git repository. |
| 19 | +/// |
| 20 | +/// Currently only public GitHub repositories are supported. Bitcache blobs are |
| 21 | +/// read from the repository's `.bitcache/` directory and must use the flat |
| 22 | +/// hexadecimal filename layout of `bitcache-fs`. |
| 23 | +#[derive(Clone, Debug)] |
| 24 | +pub enum GitRepository { |
| 25 | + /// A public repository hosted on GitHub. |
| 26 | + GitHub { |
| 27 | + owner: String, |
| 28 | + repo: String, |
| 29 | + branch: String, |
| 30 | + }, |
| 31 | +} |
| 32 | + |
| 33 | +#[derive(Clone, Debug)] |
| 34 | +struct GitHubBlob { |
| 35 | + id: Id, |
| 36 | + sha: String, |
| 37 | + size: Option<u64>, |
| 38 | +} |
| 39 | + |
| 40 | +#[derive(Debug, Deserialize)] |
| 41 | +struct GitHubTree { |
| 42 | + tree: Vec<GitHubTreeEntry>, |
| 43 | + truncated: bool, |
| 44 | +} |
| 45 | + |
| 46 | +#[derive(Debug, Deserialize)] |
| 47 | +struct GitHubTreeEntry { |
| 48 | + path: String, |
| 49 | + #[serde(rename = "type")] |
| 50 | + kind: String, |
| 51 | + sha: String, |
| 52 | + size: Option<u64>, |
| 53 | +} |
| 54 | + |
| 55 | +impl GitRepository { |
| 56 | + /// Creates a read-only repository backed by a public GitHub repository. |
| 57 | + pub fn github( |
| 58 | + owner: impl Into<String>, |
| 59 | + repo: impl Into<String>, |
| 60 | + branch: impl Into<String>, |
| 61 | + ) -> Self { |
| 62 | + Self::GitHub { |
| 63 | + owner: owner.into(), |
| 64 | + repo: repo.into(), |
| 65 | + branch: branch.into(), |
| 66 | + } |
| 67 | + } |
| 68 | + |
| 69 | + fn github_parts(&self) -> (&str, &str, &str) { |
| 70 | + match self { |
| 71 | + Self::GitHub { |
| 72 | + owner, |
| 73 | + repo, |
| 74 | + branch, |
| 75 | + } => (owner, repo, branch), |
| 76 | + } |
| 77 | + } |
| 78 | + |
| 79 | + fn github_url(&self, resource: &str, reference: &str) -> Url { |
| 80 | + let (owner, repo, _) = self.github_parts(); |
| 81 | + let mut url = Url::parse(GITHUB_API).expect("the GitHub API URL is valid"); |
| 82 | + url.path_segments_mut() |
| 83 | + .expect("the GitHub API URL can contain path segments") |
| 84 | + .extend(["repos", owner, repo, "git", resource, reference]); |
| 85 | + url |
| 86 | + } |
| 87 | + |
| 88 | + fn github_request(&self, url: Url, accept: &'static str) -> RequestBuilder { |
| 89 | + github_client() |
| 90 | + .get(url) |
| 91 | + .header(header::ACCEPT, accept) |
| 92 | + .header(header::USER_AGENT, USER_AGENT) |
| 93 | + .header("X-GitHub-Api-Version", GITHUB_API_VERSION) |
| 94 | + } |
| 95 | + |
| 96 | + async fn github_tree( |
| 97 | + &self, |
| 98 | + reference: &str, |
| 99 | + recursive: bool, |
| 100 | + ) -> Result<GitHubTree, RepositoryError> { |
| 101 | + let mut url = self.github_url("trees", reference); |
| 102 | + if recursive { |
| 103 | + url.query_pairs_mut().append_pair("recursive", "1"); |
| 104 | + } |
| 105 | + |
| 106 | + let tree = self |
| 107 | + .github_request(url, GITHUB_JSON) |
| 108 | + .send() |
| 109 | + .await |
| 110 | + .map_err(other_error)? |
| 111 | + .error_for_status() |
| 112 | + .map_err(other_error)? |
| 113 | + .json::<GitHubTree>() |
| 114 | + .await |
| 115 | + .map_err(other_error)?; |
| 116 | + if tree.truncated { |
| 117 | + return Err(other_message( |
| 118 | + "GitHub returned a truncated tree; the Bitcache repository cannot be enumerated completely", |
| 119 | + )); |
| 120 | + } |
| 121 | + Ok(tree) |
| 122 | + } |
| 123 | + |
| 124 | + async fn github_bitcache_tree(&self) -> Result<Option<GitHubTree>, RepositoryError> { |
| 125 | + let (_, _, branch) = self.github_parts(); |
| 126 | + let root = self.github_tree(branch, false).await?; |
| 127 | + let Some(bitcache) = root |
| 128 | + .tree |
| 129 | + .into_iter() |
| 130 | + .find(|entry| entry.kind == "tree" && entry.path == BITCACHE_DIR) |
| 131 | + else { |
| 132 | + return Ok(None); |
| 133 | + }; |
| 134 | + self.github_tree(&bitcache.sha, true).await.map(Some) |
| 135 | + } |
| 136 | + |
| 137 | + async fn collect_blobs( |
| 138 | + &self, |
| 139 | + options: &ListOptions, |
| 140 | + ) -> Result<Vec<GitHubBlob>, RepositoryError> { |
| 141 | + let Some(tree) = self.github_bitcache_tree().await? else { |
| 142 | + return Ok(Vec::new()); |
| 143 | + }; |
| 144 | + let mut blobs = tree |
| 145 | + .tree |
| 146 | + .into_iter() |
| 147 | + .filter_map(|entry| { |
| 148 | + if entry.kind != "blob" { |
| 149 | + return None; |
| 150 | + } |
| 151 | + if entry.path.contains('/') { |
| 152 | + return None; |
| 153 | + } |
| 154 | + let id = Id::from_hex(&entry.path).ok()?; |
| 155 | + options.matches(&id).then_some(GitHubBlob { |
| 156 | + id, |
| 157 | + sha: entry.sha, |
| 158 | + size: entry.size, |
| 159 | + }) |
| 160 | + }) |
| 161 | + .collect::<Vec<_>>(); |
| 162 | + |
| 163 | + blobs.sort_unstable_by(|left, right| left.id.cmp(&right.id)); |
| 164 | + if options.order == Some(ListOrder::Descending) { |
| 165 | + blobs.reverse(); |
| 166 | + } |
| 167 | + if let Some(limit) = options.limit { |
| 168 | + blobs.truncate(limit); |
| 169 | + } |
| 170 | + Ok(blobs) |
| 171 | + } |
| 172 | + |
| 173 | + async fn find_blob(&self, id: &Id) -> Result<Option<GitHubBlob>, RepositoryError> { |
| 174 | + Ok(self |
| 175 | + .collect_blobs(&ListOptions::new().with_prefix(id.to_hex().as_str())) |
| 176 | + .await? |
| 177 | + .into_iter() |
| 178 | + .find(|blob| &blob.id == id)) |
| 179 | + } |
| 180 | + |
| 181 | + async fn fetch_blob(&self, blob: GitHubBlob) -> Result<Blob, RepositoryError> { |
| 182 | + let bytes = self |
| 183 | + .github_request(self.github_url("blobs", &blob.sha), GITHUB_RAW) |
| 184 | + .send() |
| 185 | + .await |
| 186 | + .map_err(other_error)? |
| 187 | + .error_for_status() |
| 188 | + .map_err(other_error)? |
| 189 | + .bytes() |
| 190 | + .await |
| 191 | + .map_err(other_error)?; |
| 192 | + let size = blob.size.unwrap_or(bytes.len() as u64); |
| 193 | + Ok(Blob::new_unchecked(blob.id, bytes).with_metadata(BlobMetadata::new(size))) |
| 194 | + } |
| 195 | +} |
| 196 | + |
| 197 | +impl Repository for GitRepository { |
| 198 | + type Error = RepositoryError; |
| 199 | + |
| 200 | + async fn contains(&self, id: &Id) -> Result<bool, Self::Error> { |
| 201 | + Ok(self.find_blob(id).await?.is_some()) |
| 202 | + } |
| 203 | + |
| 204 | + async fn get(&self, id: &Id) -> Result<Option<Blob>, Self::Error> { |
| 205 | + match self.find_blob(id).await? { |
| 206 | + Some(blob) => self.fetch_blob(blob).await.map(Some), |
| 207 | + None => Ok(None), |
| 208 | + } |
| 209 | + } |
| 210 | + |
| 211 | + async fn get_len(&self, id: &Id) -> Result<Option<u64>, Self::Error> { |
| 212 | + match self.find_blob(id).await? { |
| 213 | + Some(blob) => match blob.size { |
| 214 | + Some(size) => Ok(Some(size)), |
| 215 | + None => Ok(Some(self.fetch_blob(blob).await?.len())), |
| 216 | + }, |
| 217 | + None => Ok(None), |
| 218 | + } |
| 219 | + } |
| 220 | + |
| 221 | + async fn put(&mut self, _data: Bytes) -> Result<Id, Self::Error> { |
| 222 | + Err(RepositoryError::UnsupportedOperation) |
| 223 | + } |
| 224 | + |
| 225 | + async fn remove(&mut self, _id: &Id) -> Result<bool, Self::Error> { |
| 226 | + Err(RepositoryError::UnsupportedOperation) |
| 227 | + } |
| 228 | + |
| 229 | + async fn clear(&mut self) -> Result<(), Self::Error> { |
| 230 | + Err(RepositoryError::UnsupportedOperation) |
| 231 | + } |
| 232 | + |
| 233 | + fn list(&self, options: ListOptions) -> impl Stream<Item = Result<Id, Self::Error>> { |
| 234 | + let repository = self.clone(); |
| 235 | + stream::once(async move { |
| 236 | + match repository.collect_blobs(&options).await { |
| 237 | + Ok(blobs) => stream::iter( |
| 238 | + blobs |
| 239 | + .into_iter() |
| 240 | + .map(|blob| Ok(blob.id)) |
| 241 | + .collect::<Vec<_>>(), |
| 242 | + ), |
| 243 | + Err(error) => stream::iter(vec![Err(error)]), |
| 244 | + } |
| 245 | + }) |
| 246 | + .flatten() |
| 247 | + .boxed() |
| 248 | + } |
| 249 | +} |
| 250 | + |
| 251 | +fn github_client() -> &'static Client { |
| 252 | + static CLIENT: std::sync::LazyLock<Client> = std::sync::LazyLock::new(Client::new); |
| 253 | + &CLIENT |
| 254 | +} |
| 255 | + |
| 256 | +fn other_error(error: impl core::error::Error + Send + Sync + 'static) -> RepositoryError { |
| 257 | + RepositoryError::Other(Box::new(error)) |
| 258 | +} |
| 259 | + |
| 260 | +fn other_message(message: &'static str) -> RepositoryError { |
| 261 | + other_error(std::io::Error::other(message)) |
| 262 | +} |
0 commit comments