Develop (#8)

Author: wchatx

.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '**'
+  pull_request: {}
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Get Poetry
+        uses: abatilo/actions-poetry@v2.1.4
+        with:
+          poetry-version: 1.1.12
+      - name: Install
+        run: poetry install
+      - name: Run tests
+        run: poetry run pytest tests.py

.github/workflows/test.yml

@@ -137,4 +137,5 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-.idea
+.idea
+.DS_Store

.gitignore

@@ -0,0 +1,34 @@
+# Remember to update the version in the pyproject.toml.
+repos:
+-   repo: https://github.com/timothycrosley/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+-   repo: https://github.com/psf/black
+    rev: 21.11b1
+    hooks:
+    - id: black
+      args:
+        - --line-length=88
+        - --include='\.pyi?$'
+
+        - --exclude="""\.git |
+          \.__pycache__|
+          \.hg|
+          \.mypy_cache|
+          \.tox|
+          \.venv|
+          _build|
+          buck-out|
+          build|
+          dist"""
+# flake8
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+    - id: flake8
+      args:
+        - "--max-line-length=88"
+        - "--max-complexity=18"
+        - "--select=B,C,E,F,W,T4,B9"
+        - "--ignore=E203,E266,E501,W503,F403,F401,E402"

.pre-commit-config.yaml

@@ -1,2 +1,4 @@
+# flake8: noqa
 from .layer import Layer
 from .service import Service
+from .site import Site

Makefile

@@ -0,0 +1,36 @@
+import json
+from copy import deepcopy
+
+from pandas import Series
+from requests import Session
+
+
+class Base(object):
+    current_version: float = None
+    _supported_services = ["MapServer", "FeatureServer"]
+    _supported_types = ["Feature Layer", "Table"]
+
+    def __init__(self, url, session: Session = None):
+        super(Base, self).__init__()
+        self.url = url
+        self._session = session or Session()
+        self._session.params.update(dict(f="json"))
+
+        self.meta = self._session.get(self.url).json()
+        self.meta["url"] = self.url
+        self.current_version = self.meta["currentVersion"]
+
+    def __repr__(self) -> str:
+        return Series(self.meta).to_string()
+
+    def _repr_html_(self) -> str:
+        session = deepcopy(self._session)
+        session.params.pop("f")
+        text = session.get(self.url).text
+        disabled = "Services Directory has been disabled"
+        if disabled in text:
+            return f"""
+            <b>{disabled}</b>\n
+            <pre>{json.dumps(self.meta, indent=2)}</pre>
+            """
+        return text

esridumpgdf/__init__.py

@@ -1,44 +1,38 @@
-from typing import Union, Dict
-from collections import OrderedDict
+from typing import Dict, Union
 
-from pandas import to_datetime
-from geopandas import GeoDataFrame
 from esridump.dumper import EsriDumper
+from geopandas import GeoDataFrame
+from pandas import to_datetime
 
-from .base import Base
+from ._base import Base
 
 
 class Layer(Base):
     def __init__(self, url: str, **kwargs):
-        super(Layer, self).__init__(url, **kwargs)
+        super(Layer, self).__init__(url)
+        self.crs = kwargs.get("crs") or 4326
+        self.layer = EsriDumper(self.url, outSR=str(self.crs), **kwargs)
 
-    @property
-    def type(self) -> str:
-        return self.meta['type']
-
-    def to_gdf(self, **kwargs) -> Union[GeoDataFrame, Dict[str, GeoDataFrame]]:
+    def to_gdf(
+        self, columns: list = None
+    ) -> Union[GeoDataFrame, Dict[str, GeoDataFrame]]:
         """
-        Export layer to GeoDataFrame
+        Export an ArcGIS Server layer to GeoDataFrame
 
-        :param kwargs: extra keyword arguments provided to pyesridump's EsriDumper class
+        :param columns: list of column names, optional
+            Optionally specify the column names to include in the output frame.
+            This does not overwrite the property names of the input, but can
+            ensure a consistent output format.
         :return:
         """
-        crs = kwargs.get('crs') or 4326
-        layer = EsriDumper(self.url, outSR=crs, **kwargs)
-
-        if self.type == 'Group Layer':
-            gdfs = OrderedDict(
-                {layer['name']: Layer(f'{"/".join(self.url.split("/")[:-1])}/{layer["id"]}', **kwargs).to_gdf()
-                 for layer in self.meta['subLayers']}
-            )
-            return gdfs
-
-        gdf = GeoDataFrame.from_features(features=layer, crs=crs)
-        for field in self.meta['fields']:
-            if field['type'] == 'esriFieldTypeOID':
-                gdf.set_index(field['name'], inplace=True)
-            if field['type'] == 'esriFieldTypeDate':
-                gdf[field['name']] = to_datetime(gdf[field['name']], unit='ms')
-            if field['type'] == 'esriFieldTypeInteger':
-                gdf[field['name']] = gdf[field['name']].astype('Int64')
+        gdf = GeoDataFrame.from_features(
+            features=self.layer, crs=self.crs, columns=columns
+        )
+        for field in self.meta["fields"]:
+            if field["type"] == "esriFieldTypeOID":
+                gdf.set_index(field["name"], inplace=True)
+            if field["type"] == "esriFieldTypeDate":
+                gdf[field["name"]] = to_datetime(gdf[field["name"]], unit="ms")
+            if field["type"] == "esriFieldTypeInteger":
+                gdf[field["name"]] = gdf[field["name"]].astype("Int64")
         return gdf

esridumpgdf/_base.py

@@ -1,37 +1,96 @@
-from collections import OrderedDict
-from typing import List, Dict
+from typing import Dict, Iterable, List
 
 from geopandas import GeoDataFrame
+from pandas import DataFrame, concat
 
-from .base import Base
+from ._base import Base
 from .layer import Layer
 
 
 class Service(Base):
-    def __init__(self, url, **kwargs):
-        super(Service, self).__init__(url, **kwargs)
+    def __init__(self, url):
+        self.url = url
+        super(Service, self).__init__(self.url)
 
-    @property
-    def layers(self) -> List[dict]:
-        return self.meta['layers']
-
-    @property
-    def tables(self) -> List[dict]:
-        return self.meta['tables']
-
-    def to_gdfs(self, include_tables: bool = True, **kwargs) -> Dict[str, GeoDataFrame]:
+    def layers(self, include_tables: bool = True) -> DataFrame:
         """
-        Export a complete ArcGIS Server Map or Feature service to GeoDataFrames
+        Get Service layers.
 
-        :param include_tables: whether to include attribute-only tables
-        :param kwargs: extra keyword arguments provided to pyesridump's EsriDumper class
+        :param include_tables: include Service tables
         :return:
         """
-        layers = [layer for layer in self.layers if layer['type'] != 'Group Layer']
-        gdfs = OrderedDict(
-            {layer['name']: Layer(f'{self.url}/{layer["id"]}', **kwargs).to_gdf()
-             for layer in sorted(layers, key=lambda _: _['name'])}
+        layers = DataFrame(
+            data=[
+                {**layer, "url": f'{self.url}/{layer["id"]}'}
+                for layer in self.meta["layers"]
+            ]
         )
-        if self.tables and include_tables:
-            gdfs.update({table['name']: Layer(f'{self.url}/{table["id"]}').to_gdf() for table in self.tables})
+        layers.set_index("id", inplace=True)
+
+        if include_tables and self.meta["tables"]:
+            tables = DataFrame(
+                data=[
+                    {**table, "url": f'{self.url}/{table["id"]}'}
+                    for table in self.meta["tables"]
+                ]
+            )
+            tables.set_index("id", inplace=True)
+            layers = concat([layers, tables])
+
+        return layers
+
+    def to_gpkg(
+        self,
+        filename: str,
+        index: bool = True,
+        schema: dict = None,
+        include_tables: bool = True,
+        **kwargs,
+    ) -> str:
+        """
+        Export an ArcGIS Server Map or Feature service to geopackage
+
+        :param filename: File path or file handle to write to.
+        :param index: If True, write index into one or more columns (for MultiIndex).
+            Default None writes the index into one or more columns only if
+            the index is named, is a MultiIndex, or has a non-integer data
+            type. If False, no index is written.
+        :param schema: If specified, the schema dictionary is passed to Fiona to
+            better control how the file is written.
+        :param include_tables: include Service tables
+        :param kwargs: extra keyword arguments provided to the EsriDumper class
+        :return: provided filename
+        """
+        layers = self.layers(include_tables).to_dict(orient="records")
+        for layer in layers:
+            if layer["type"] in self._supported_types:
+                Layer(layer["url"], **kwargs).to_gdf().to_file(
+                    filename,
+                    driver="GPKG",
+                    index=index,
+                    schema=schema,
+                    layer=layer["name"],
+                )
+        return filename
+
+    def to_gdfs(
+        self, include_tables: bool = True, **kwargs
+    ) -> List[Dict[str, GeoDataFrame]]:
+        """
+        Export an ArcGIS Server Map or Feature service to GeoDataFrames
+
+        :param include_tables: include Service tables
+        :param kwargs: extra keyword arguments provided to the EsriDumper class
+        :return: list of dicts with layer names and layer GeoDataFrames
+        """
+        layers = self.layers(include_tables).to_dict(orient="records")
+        gdfs = []
+        for layer in layers:
+            if layer["type"] in self._supported_types:
+                gdfs.append(
+                    {
+                        "name": layer["name"],
+                        "gdf": Layer(layer["url"], **kwargs).to_gdf(),
+                    }
+                )
         return gdfs