Merge pull request #14 from JuliaDataCubes/fix_noncontig

read into non-contig memory

Author: meggart

.github/workflows/CI.yml

@@ -14,6 +14,7 @@ jobs:
       matrix:
         version:
           - '1.6'
+          - '1'
         os:
           - ubuntu-latest
           - macOS-latest

src/axisarrays/axisindices.jl

@@ -1,4 +1,4 @@
-using .AxisIndices: AbstractAxis,AxisIndices
+using .AxisIndices: AbstractAxis,AxisIndicesArray
 
 valfromaxis(ax::AbstractAxis) = keys(ax)
 

src/datasets/netcdf.jl

@@ -30,8 +30,19 @@ for m in [:haschunks, :eachchunk]
 end
 ))
 end
+function check_contig(x)
+   isa(x,Array) || (isa(x,SubArray) && Base.iscontiguous(x))
+end
 writeblock!(v::NetCDFVariable, aout, r::AbstractUnitRange...) = NetCDF.open(a->writeblock!(a,aout,r...), v.filename, v.varname, mode=NC_WRITE)
-readblock!(v::NetCDFVariable, aout, r::AbstractUnitRange...) = NetCDF.open(a->readblock!(a,aout,r...), v.filename, v.varname)
+function readblock!(v::NetCDFVariable, aout, r::AbstractUnitRange...) 
+  if check_contig(aout)
+    NetCDF.open(a->readblock!(a,aout,r...), v.filename, v.varname)
+  else
+    aouttemp = Array(aout)
+    NetCDF.open(a->readblock!(a,aouttemp,r...), v.filename, v.varname)
+    aout .= aouttemp
+  end
+end
 iscompressed(v::NetCDFVariable) = NetCDF.open(v->v.compress > 0, v.filename, v.varname)
 
 Base.size(v::NetCDFVariable) = v.size

test/Artifacts.toml

@@ -0,0 +1,2 @@
+[ncar]
+git-tree-sha1 = "b3ad9125b25731444e8118c0a808aa2d3fc0eb1e"

test/Project.toml

@@ -7,5 +7,6 @@ DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 NamedDims = "356022a1-0364-5f58-8944-0da4b18d706f"
 NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"

test/datasets.jl

@@ -5,11 +5,24 @@ end
 
 using NetCDF, Zarr
 
-@testset "Reading NetCDF" begin
+using Pkg.Artifacts
 import Downloads
-p = Downloads.download("https://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc")
+# This is the path to the Artifacts.toml we will manipulate
+artifact_toml =  joinpath(@__DIR__,"Artifacts.toml")
+ncar_hash = artifact_hash("ncar", artifact_toml)
+if ncar_hash == nothing || !artifact_exists(ncar_hash)
+  oldhash = ncar_hash
+  ncar_hash = create_artifact() do artifact_dir     
+    Downloads.download("https://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc",joinpath(artifact_dir,"ncar.nc"))
+  end
+  if oldhash !== nothing
+    unbind_artifact!(artifact_toml, "ncar")
+  end
+  bind_artifact!(artifact_toml, "ncar", ncar_hash)
+end
+p2 = joinpath(artifact_path(ncar_hash),"ncar.nc")
 
-p2 = mv(p,string(tempname(),".nc"))
+@testset "Reading NetCDF" begin
 
 ds_nc = YAXArrayBase.to_dataset(p2)
 vn = get_varnames(ds_nc)