Skip to content

Commit 5798926

Browse files
authored
Support for read-only Storage (#798)
Why: * We may implement `Storage` for certain read-only s3-like, such as HTTP * We'll need this for [Tigris support](https://github.com/earth-mover/icechunk/blob/main/design-docs/009-tigris-support.md)
1 parent 95e16da commit 5798926

File tree

5 files changed

+32
-1
lines changed

5 files changed

+32
-1
lines changed

icechunk/src/repository.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ pub enum RepositoryErrorKind {
8888
ConcurrencyError(#[from] JoinError),
8989
#[error("main branch cannot be deleted")]
9090
CannotDeleteMain,
91+
#[error("the storage used by this Icechunk repository is read-only: {0}")]
92+
ReadonlyStorage(String),
9193
}
9294

9395
pub type RepositoryError = ICError<RepositoryErrorKind>;
@@ -685,6 +687,12 @@ impl Repository {
685687

686688
#[instrument(skip(self))]
687689
pub async fn writable_session(&self, branch: &str) -> RepositoryResult<Session> {
690+
if !self.storage.can_write() {
691+
return Err(RepositoryErrorKind::ReadonlyStorage(
692+
"Cannot create writable_session".to_string(),
693+
)
694+
.into());
695+
}
688696
let ref_data =
689697
fetch_branch_tip(self.storage.as_ref(), &self.storage_settings, branch)
690698
.await?;

icechunk/src/storage/logging.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ impl Storage for LoggingStorage {
5353
fn default_settings(&self) -> Settings {
5454
self.backend.default_settings()
5555
}
56+
57+
fn can_write(&self) -> bool {
58+
self.backend.can_write()
59+
}
60+
5661
async fn fetch_config(
5762
&self,
5863
settings: &Settings,

icechunk/src/storage/mod.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,9 @@ pub trait Storage: fmt::Debug + fmt::Display + private::Sealed + Sync + Send {
302302
fn default_settings(&self) -> Settings {
303303
Default::default()
304304
}
305+
306+
fn can_write(&self) -> bool;
307+
305308
async fn fetch_config(&self, settings: &Settings)
306309
-> StorageResult<FetchConfigResult>;
307310
async fn update_config(
@@ -624,6 +627,7 @@ pub fn new_s3_storage(
624627
bucket,
625628
prefix,
626629
credentials.unwrap_or(S3Credentials::FromEnv),
630+
true,
627631
)?;
628632
Ok(Arc::new(st))
629633
}
@@ -645,6 +649,7 @@ pub fn new_tigris_storage(
645649
bucket,
646650
prefix,
647651
credentials.unwrap_or(S3Credentials::FromEnv),
652+
true,
648653
)?;
649654
Ok(Arc::new(st))
650655
}

icechunk/src/storage/object_store.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,10 @@ impl private::Sealed for ObjectStorage {}
296296
#[async_trait]
297297
#[typetag::serde]
298298
impl Storage for ObjectStorage {
299+
fn can_write(&self) -> bool {
300+
true
301+
}
302+
299303
#[instrument(skip(self))]
300304
fn default_settings(&self) -> Settings {
301305
self.backend.default_settings()

icechunk/src/storage/s3.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ pub struct S3Storage {
5151
credentials: S3Credentials,
5252
bucket: String,
5353
prefix: String,
54+
can_write: bool,
5455

5556
#[serde(skip)]
5657
/// We need to use OnceCell to allow async initialization, because serde
@@ -134,6 +135,7 @@ impl S3Storage {
134135
bucket: String,
135136
prefix: Option<String>,
136137
credentials: S3Credentials,
138+
can_write: bool,
137139
) -> Result<S3Storage, StorageError> {
138140
let client = OnceCell::new();
139141
Ok(S3Storage {
@@ -142,6 +144,7 @@ impl S3Storage {
142144
bucket,
143145
prefix: prefix.unwrap_or_default(),
144146
credentials,
147+
can_write,
145148
})
146149
}
147150

@@ -290,6 +293,10 @@ impl private::Sealed for S3Storage {}
290293
#[async_trait]
291294
#[typetag::serde]
292295
impl Storage for S3Storage {
296+
fn can_write(&self) -> bool {
297+
self.can_write
298+
}
299+
293300
#[instrument(skip(self, _settings))]
294301
async fn fetch_config(
295302
&self,
@@ -791,14 +798,15 @@ mod tests {
791798
"bucket".to_string(),
792799
Some("prefix".to_string()),
793800
credentials,
801+
true,
794802
)
795803
.unwrap();
796804

797805
let serialized = serde_json::to_string(&storage).unwrap();
798806

799807
assert_eq!(
800808
serialized,
801-
r#"{"config":{"region":"us-west-2","endpoint_url":"http://localhost:9000","anonymous":false,"allow_http":true},"credentials":{"s3_credential_type":"static","access_key_id":"access_key_id","secret_access_key":"secret_access_key","session_token":"session_token","expires_after":null},"bucket":"bucket","prefix":"prefix"}"#
809+
r#"{"config":{"region":"us-west-2","endpoint_url":"http://localhost:9000","anonymous":false,"allow_http":true},"credentials":{"s3_credential_type":"static","access_key_id":"access_key_id","secret_access_key":"secret_access_key","session_token":"session_token","expires_after":null},"bucket":"bucket","prefix":"prefix","can_write":true}"#
802810
);
803811

804812
let deserialized: S3Storage = serde_json::from_str(&serialized).unwrap();
@@ -817,6 +825,7 @@ mod tests {
817825
"bucket".to_string(),
818826
Some("prefix".to_string()),
819827
S3Credentials::FromEnv,
828+
true,
820829
)
821830
.unwrap();
822831

0 commit comments

Comments
 (0)