Add ZipStore storage backend.

This commit adds a ZipStore storage backend as described in the
specification zarr-developers/zarr-specs#311 .
Note that the implementation loads the entire zip archive into memory so
care must be taken to ensure the zip archive is not too big to fit into
the machine's memory. To use a ZipStore impelementation that does not
load the archive into memory see `examples/zipstore.ml`.

Author: zoj613

dune-project

@@ -30,6 +30,7 @@
      (and (>= 4.14.0)))
    (yojson (>= 1.6.0))
    (stdint (>= 0.7.2))
+   (zipc (>= 0.2.0))
    (checkseum (>= 0.4.0))
    (odoc :with-doc)
    (ounit2 :with-test)

examples/dune

@@ -4,12 +4,6 @@
  (ocamlopt_flags (:standard -O3))
  (libraries zarr-eio camlzip))
 
-(executable
- (name inmemory_zipstore)
- (modules inmemory_zipstore)
- (ocamlopt_flags (:standard -O3))
- (libraries zarr-lwt zipc))
-
 (executable
  (name picos_fs_store)
  (modules picos_fs_store)

examples/inmemory_zipstore.ml

@@ -3,6 +3,12 @@ module MemoryStore = struct
   let create = Zarr.Memory.create
 end
 
+module ZipStore = struct
+  module Z = Zarr.Zip.Make(Deferred)
+  include Zarr.Storage.Make(Z)
+  let with_open = Z.with_open
+end
+
 module FilesystemStore = struct
   module FS = struct
     module Deferred = Deferred

zarr-eio/src/storage.ml

@@ -7,6 +7,37 @@ module MemoryStore : sig
   (** [create ()] returns a new In-memory Zarr store type. *)
 end
 
+module ZipStore : sig
+  (** An Eio-aware Zip file storage backend for a Zarr v3 hierarchy. *)
+
+  include Zarr.Storage.STORE with type 'a Deferred.t = 'a
+  val with_open :
+    ?level:[ `None | `Fast | `Default | `Best ] ->
+    ?perm:Unix.file_perm ->
+    [< `Read_only | `Read_write ] ->
+    string ->
+    (t -> 'a) ->
+    'a
+  (** [with_open mode p f] opens the zip archive at path [p] and applies
+      function [f] to its open handle and writes any changes back to the zip
+      archive if [mode] is [`Read_write], otherwise discards them at exit.
+      If [p] does not exist, a handle to an empty zip archive is opened.
+      Note that this function loads the entire zip archive into memory, so care
+      must be taken to ensure that the compressed file contents can fit into
+      memory. For now it does not handle ZIP64. ZIP64 is needed if your ZIP
+      archive or decompressed file sizes exceed 2{^32}-1 bytes or if you need
+      more than 65535 archive members.
+
+      {ul 
+      {- [level] is the DEFLATE algorithm compression level used when writing
+        data to the store and defaults to [`Default]. Choose [`None] for no
+        compression, [`Fast] for best speed, [`Best] for high compression rate
+        and [`Default] for a mix of good speed and compression rate.}
+      {- [perm] is the file permission to use when opening an existing zip file
+        and defaults to [0o700].}
+      } *)
+end
+
 module FilesystemStore : sig
   (** A local filesystem storage backend for a Zarr V3 hierarchy. *)
 

zarr-eio/src/storage.mli

@@ -143,6 +143,11 @@ let _ =
         (Zarr.Storage.Not_a_filesystem_store fn)
         (fun () -> FilesystemStore.open_store ~env fn);
 
+      (* test with non-existant archive *)
+      let zpath = tmp_dir ^ ".zip" in
+      ZipStore.with_open `Read_write zpath (fun z -> test_storage (module ZipStore) z);
+      (* test just opening the now exisitant archive created by the previous test. *)
+      ZipStore.with_open `Read_only zpath (fun _ -> ZipStore.Deferred.return_unit);
       test_storage (module MemoryStore) @@ MemoryStore.create ();
       test_storage (module FilesystemStore) s)
   ])

zarr-eio/test/test_eio.ml

@@ -3,6 +3,12 @@ module MemoryStore = struct
   let create = Zarr.Memory.create
 end
 
+module ZipStore = struct
+  module Z = Zarr.Zip.Make(Deferred)
+  include Zarr.Storage.Make(Z)
+  let with_open = Z.with_open
+end
+
 module FilesystemStore = struct
   module FS = struct
     module Deferred = Deferred

zarr-lwt/src/storage.ml

@@ -7,6 +7,37 @@ module MemoryStore : sig
   (** [create ()] returns a new In-memory Zarr store type. *)
 end
 
+module ZipStore : sig
+  (** An Lwt-aware Zip file storage backend for a Zarr v3 hierarchy. *)
+
+  include Zarr.Storage.STORE with type 'a Deferred.t = 'a Lwt.t
+  val with_open :
+    ?level:[ `None | `Fast | `Default | `Best ] ->
+    ?perm:Unix.file_perm ->
+    [< `Read_only | `Read_write ] ->
+    string ->
+    (t -> 'a Lwt.t) ->
+    'a Lwt.t
+  (** [with_open mode p f] opens the zip archive at path [p] and applies
+      function [f] to its open handle and writes any changes back to the zip
+      archive if [mode] is [`Read_write], otherwise discards them at exit.
+      If [p] does not exist, a handle to an empty zip archive is opened.
+      Note that this function loads the entire zip archive into memory, so care
+      must be taken to ensure that the compressed file contents can fit into
+      memory. For now it does not handle ZIP64. ZIP64 is needed if your ZIP
+      archive or decompressed file sizes exceed 2{^32}-1 bytes or if you need
+      more than 65535 archive members.
+
+      {ul 
+      {- [level] is the DEFLATE algorithm compression level used when writing
+        data to the store and defaults to [`Default]. Choose [`None] for no
+        compression, [`Fast] for best speed, [`Best] for high compression rate
+        and [`Default] for a mix of good speed and compression rate.}
+      {- [perm] is the file permission to use when opening an existing zip file
+        and defaults to [0o700].}
+      } *)
+end
+
 module FilesystemStore : sig
   (** A local filesystem storage backend for a Zarr V3 hierarchy. *)
 

zarr-lwt/src/storage.mli

@@ -141,8 +141,11 @@ let _ =
         (Zarr.Storage.Not_a_filesystem_store fn)
         (fun () -> FilesystemStore.open_store fn);
 
-      Lwt_main.run @@
-      Lwt.join 
-        [test_storage (module MemoryStore) @@ MemoryStore.create ()
+      let zpath = tmp_dir ^ ".zip" in
+      Lwt_main.run @@ Lwt.join 
+        [ZipStore.with_open `Read_write zpath (fun z -> test_storage (module ZipStore) z)
+         (* test just opening the now exisitant archive created by the previous test. *)
+        ;ZipStore.with_open `Read_only zpath (fun _ -> ZipStore.Deferred.return_unit)
+        ;test_storage (module MemoryStore) @@ MemoryStore.create ()
         ;test_storage (module FilesystemStore) s])
 ])

zarr-lwt/test/test_lwt.ml

@@ -3,6 +3,12 @@ module MemoryStore = struct
   let create = Zarr.Memory.create
 end
 
+module ZipStore = struct
+  module Z = Zarr.Zip.Make(Deferred)
+  include Zarr.Storage.Make(Z)
+  let with_open = Z.with_open
+end
+
 module FilesystemStore = struct
   module F = struct
     module Deferred = Deferred