Merge pull request #11 from loleg/master

Remote loading of data packages

Author: loleg

README.md

@@ -23,13 +23,25 @@ Initially this package is primarily targeted and depends on [TableSchema.jl](htt
 
 :construction: This package is pre-release and under heavy development. Please visit the [issues page](https://github.com/frictionlessdata/datapackage-jl/issues) to contribute and make suggestions. For questions that need to a real time response, reach out via [Gitter](https://gitter.im/frictionlessdata/chat). Thanks! :construction:
 
+Please visit [our wiki](https://github.com/frictionlessdata/datapackage-jl/wiki) for a list of related projects that we are tracking, and suggest use cases there or as enhancement [issues](https://github.com/frictionlessdata/datapackage-jl/issues).
+
 # Usage
 
-Please visit [our wiki](https://github.com/frictionlessdata/datapackage-jl/wiki) for a list of related projects that we are tracking, and suggest use cases there or as enhancement [issues](https://github.com/frictionlessdata/datapackage-jl/issues).
+Install *tableschema-jl* (temporarily obtained from GitHub until released):
+
+`$ julia -e 'Pkg.clone("https://github.com/frictionlessdata/tableschema-jl")'`
+
+You may also need to explicitly install *HTTP*:
+
+`$ julia -e 'Pkg.add("HTTP")'`
+
+Update to the latest version with:
+
+`$ julia -e 'Pkg.update("TableSchema")'`
 
-Install *tableschema-jl* first:
+Clone this repository and enter it:
 
-`julia -e 'Pkg.clone("https://github.com/loleg/TableSchema.jl")'`
+`$ git clone https://github.com/frictionlessdata/datapackage-jl && cd datapackage-jl`
 
 See *examples* folder and unit tests in [runtests.jl](test/runtests.jl) for current usage, e.g.:
 

REQUIRE

@@ -1 +1,2 @@
 julia 0.6
+HTTP

examples/datahub.jl

@@ -0,0 +1,17 @@
+include("../src/DataPackage.jl")
+using DataPackage
+import DataPackage: Package, read
+
+data_url = "https://datahub.io/core/pharmaceutical-drug-spending/datapackage.json"
+
+# to load Data Package into storage
+package = Package(data_url)
+
+# to load only tabular data
+resources = package.resources
+for resource in resources
+    if resource.profile == "tabular-data-resource"
+        data = read(resource)
+        println(data)
+    end
+end

examples/infer.jl

@@ -7,7 +7,7 @@ import DataPackage: add_resource
 
 # Initialise a resource from a CSV file
 # Source: https://opendata.swiss/en/dataset/kennzahlen-der-schweizer-pflegeheime
-res = Resource("data/health/hopitaux_ch_2016_Sample_fr.csv", false, "hopitaux")
+res = Resource("data/health/hopitaux_ch_2016_Sample_fr.csv", strict=false, name="hopitaux")
 
 # Add this resource to a blank data package
 p = Package()

examples/remote.jl

@@ -0,0 +1,24 @@
+using TableSchema
+import TableSchema: read, validate
+
+include("../src/DataPackage.jl")
+using DataPackage
+import DataPackage: Resource, add_resource, get_table, read
+
+# Initialise a resource from a remote file
+REMOTE_URL = "https://raw.githubusercontent.com/frictionlessdata/tableschema-jl/master/data/data_simple.csv"
+res = Resource(REMOTE_URL, strict=false, name="sample")
+
+# Add this resource to a blank data package
+p = Package()
+add_resource(p, res)
+
+# Read the data from the table
+data = read(res)
+println( "The number of cells is ", length(data[:,1]) )
+println( "Column 1 is called ", res.schema.fields[1].name )
+println( join([ row for row in data[:,1] ], ", ", " and ") )
+
+# Validate the table
+table = get_table(res)
+if validate(table); println("The table is valid according to the Schema"); end

src/DataPackage.jl

@@ -17,6 +17,9 @@ using JSON
 
 using TableSchema
 import TableSchema: Schema
+import TableSchema: read, infer, validate, is_empty
+
+import HTTP: request
 
 include("exceptions.jl")
 include("resource.jl")

src/package.jl

@@ -13,20 +13,33 @@ mutable struct Package
         resources = []
         if haskey(d, "resources")
             for r in d["resources"]
-                t = Resource(r, strict)
-                push!(resources, t)
+                if !isempty(r)
+                    t = Resource(r, strict=strict)
+                    push!(resources, t)
+                end
             end
         end
         new(d, strict, [], resources)
     end
 
     Package(filename::String, strict::Bool=false) =
-        Package(JSON.parsefile(filename), strict)
+        Package(fetch_json(filename), strict)
 
     Package(strict::Bool=false) =
         Package(Dict(), strict)
 end
 
+function fetch_json(filename::String)
+    if match(r"^https?://", filename) !== nothing
+        req = request("GET", filename)
+        j = JSON.parse(req.body)
+    else
+        j = JSON.parsefile(filename)
+    end
+    isempty(j) && println("JSON could not be loaded")
+    j
+end
+
 add_resource(p::Package, r::Resource) =
     push!(p.resources, r)
 

src/resource.jl

@@ -13,22 +13,22 @@ mutable struct Resource
     schema::Schema
     errors::Array{PackageError}
 
-    function Resource(d::Dict, strict::Bool=false)
+    function Resource(d::Dict ; strict::Bool=false)
         schema = haskey(d, "schema") ?
-            Schema(d["schema"], strict) : nothing
+            Schema(d["schema"], strict) : Schema()
         name = haskey(d, "name") ?
             d["name"] : nothing
         path = haskey(d, "path") ?
             d["path"] : nothing
         profile = haskey(d, "profile") ?
             d["profile"] : nothing
         dialect = haskey(d, "dialect") ?
-            d["dialect"] : nothing
+            d["dialect"] : Dict()
 
         new(d, strict, name, path, profile, dialect, schema, [])
     end
 
-    function Resource(path::String, strict::Bool=false, name::String=nothing)
+    function Resource(path::String ; strict::Bool=false, name::String=nothing)
         name = isempty(name) ? path.split('/')[-1] : name
         new(
             Dict(), strict, name, path, "tabular-data-resource", Dict(), Schema(), []
@@ -48,14 +48,14 @@ function fields_to_descriptor(s::Schema)
 end
 
 function get_table(r::Resource)
-    if TableSchema.is_empty(r.schema)
+    if is_empty(r.schema)
         s = Schema()
         t = Table(r.path)
-        tr = TableSchema.read(t, cast=false)
-        TableSchema.infer(s, tr, t.headers)
+        tr = read(t, cast=false)
+        infer(s, tr, t.headers)
         s.errors = []
         s.descriptor = fields_to_descriptor(s)
-        TableSchema.validate(s, r.strict)
+        validate(s, r.strict)
         r.schema = t.schema = s
         t
     else
@@ -66,7 +66,7 @@ end
 function read(r::Resource)
     if r.profile == "tabular-data-resource"
         t = get_table(r)
-        TableSchema.read(t, cast=false)
+        read(t, cast=false)
     else
         throw(ErrorException("Not supported"))
     end

test/read.jl

@@ -18,4 +18,12 @@
         @test data[2,1] == "London"
     end
 
+    @testset "Read remote data package" begin
+        p = Package("https://raw.githubusercontent.com/frictionlessdata/datapackage-jl/master/data/cities/datapackage.json")
+        r = get_resource(p, "cities")
+        r.path = "https://raw.githubusercontent.com/frictionlessdata/datapackage-jl/master/data/cities/cities.csv"
+        data = read(r)
+        @test data[2,1] == "London"
+    end
+
 end