Develop (#4)

* classes, tests

* fix ci

* fix ci

* fix ci

* fix ci

Author: wchatx

.github/workflows/push.yml

@@ -38,4 +38,4 @@ jobs:
         python -m venv venv
         source venv/bin/activate
         python setup.py install
-        python -c "from esridumpgdf import layer_to_gdf"
+        python -c "from esridumpgdf import Layer, Service"

README.md

@@ -1,10 +1,16 @@
 # esridumpgdf
 
-Export ArcGIS Map and Feature services to GeoDataFrame
-
-
+Simple functions using [pyesridump](https://github.com/openaddresses/pyesridump) 
+and [geopandas](https://github.com/geopandas/geopandas) to create GeoDataFrames.  
 
 ## Install
 ```
 pip install git+https://github.com/wchatx/esridumpgdf.git
-```
+```
+
+## Usage
+For exporting a single Map or Feature service to GeoDataFrame:
+```python
+
+```
+

esridumpgdf/__init__.py

@@ -1,26 +1,2 @@
-from pandas import to_datetime
-from geopandas import GeoDataFrame
-from esridump.dumper import EsriDumper
-
-
-def layer_to_gdf(url: str, **kwargs) -> GeoDataFrame:
-    """
-    Export an ArcGIS Server Map or Feature service to GeoDataFrame
-
-    ::
-        from esridumpgdf import layer_to_gdf
-        layer = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/911CallsHotspot/MapServer/1'
-        gdf = layer_to_gdf(layer)
-
-    :param url: url of the esri layer
-    :return: GeoDataFrame
-    """
-    crs = kwargs.get('crs') or 4326
-    layer = EsriDumper(url, outSR=crs, **kwargs)
-    gdf = GeoDataFrame.from_features(features=layer, crs=crs)
-    for field in layer.get_metadata()['fields']:
-        if field['type'] == 'esriFieldTypeOID':
-            gdf.set_index(field['name'], inplace=True)
-        if field['type'] == 'esriFieldTypeDate':
-            gdf[field['name']] = to_datetime(gdf[field['name']], unit='ms')
-    return gdf
+from .layer import Layer
+from .service import Service

esridumpgdf/base.py

@@ -0,0 +1,11 @@
+from requests import Session
+
+
+class Base(object):
+    def __init__(self, url, **kwargs):
+        self.url = url
+        self.session = kwargs.get('session') or Session()
+
+    @property
+    def meta(self) -> dict:
+        return self.session.get(self.url, params=dict(f='pjson')).json()

esridumpgdf/layer.py

@@ -0,0 +1,27 @@
+from pandas import to_datetime
+from geopandas import GeoDataFrame
+from esridump.dumper import EsriDumper
+
+from .base import Base
+
+
+class Layer(Base):
+    def __init__(self, url: str, **kwargs):
+        super(Layer, self).__init__(url, **kwargs)
+
+    def to_gdf(self, **kwargs) -> GeoDataFrame:
+        crs = kwargs.get('crs') or 4326
+        layer = EsriDumper(self.url, outSR=crs, **kwargs)
+
+        if self.meta['type'] == 'Group Layer':
+            raise Exception('Provided URL is a Group Layer. This is currently unsupported')
+
+        gdf = GeoDataFrame.from_features(features=layer, crs=crs)
+        for field in self.meta['fields']:
+            if field['type'] == 'esriFieldTypeOID':
+                gdf.set_index(field['name'], inplace=True)
+            if field['type'] == 'esriFieldTypeDate':
+                gdf[field['name']] = to_datetime(gdf[field['name']], unit='ms')
+            if field['type'] == 'esriFieldTypeInteger':
+                gdf[field['name']] = gdf[field['name']].astype('Int64')
+        return gdf

esridumpgdf/service.py

@@ -0,0 +1,36 @@
+from collections import OrderedDict
+from typing import List, Dict
+
+from geopandas import GeoDataFrame
+
+from .base import Base
+from .layer import Layer
+
+
+class Service(Base):
+    def __init__(self, url, **kwargs):
+        super(Service, self).__init__(url, **kwargs)
+
+    @property
+    def layers(self) -> List[dict]:
+        return self.meta['layers']
+
+    @property
+    def tables(self) -> List[dict]:
+        return self.meta['tables']
+
+    def to_gdfs(self, include_tables=True, **kwargs) -> Dict[str, GeoDataFrame]:
+        """
+        Export a complete ArcGIS Server Map or Feature service to list of GeoDataFrames
+
+        """
+        layers = [layer for layer in self.meta['layers'] if layer['type'] != 'Group Layer']
+        tables = self.meta['tables']
+        gdfs = OrderedDict(
+            {layer['name']: Layer(f'{self.url}/{layer["id"]}', **kwargs).to_gdf()
+             for layer in sorted(layers, key=lambda _: _['name'])}
+        )
+        if tables and include_tables:
+            gdfs.update({table['name']: Layer(f'{self.url}/{table["id"]}').to_gdf() for table in tables})
+            gdfs = sorted(gdfs.items())
+        return gdfs

tests.py

@@ -1,13 +1,21 @@
-from esridumpgdf import layer_to_gdf
+from collections import OrderedDict
+
+from esridumpgdf import Layer, Service
 
 
 def test_layer_to_gdf():
     layer = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/911CallsHotspot/MapServer/1'
-    gdf = layer_to_gdf(layer)
+    gdf = Layer(layer).to_gdf()
 
     assert gdf.shape[0]
     assert gdf.index.name == 'FID'
+    assert gdf.geometry.name == 'geometry'
+
+
+def test_service_to_gdfs():
+    service = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/Wildfire/MapServer'
+    gdfs = Service(service).to_gdfs()
 
+    assert gdfs
+    assert isinstance(gdfs, OrderedDict)
 
-if __name__ == '__main__':
-    test_layer_to_gdf()