Store attributes brute-force

Author: ridoo

importer_datapackage/handlers/datapackage/handler.py

@@ -5,20 +5,18 @@
 
 import importlib.resources as pkg_resources
 
-from io import BytesIO
 from pathlib import Path
 from osgeo import ogr
-from PIL import Image
 
 from django.conf import settings
+from django.db.utils import DataError
 from django.db.models import Q
 
 from geonode.base.models import ResourceBase
 from geonode.layers.models import Dataset
 from geonode.resource.manager import resource_manager
 from geonode.resource.models import ExecutionRequest
 from geonode.resource.enumerator import ExecutionRequestAction as exa
-from geonode.geoserver.helpers import set_attributes
 from geonode.utils import set_resource_default_links
 
 from importer.handlers.common.vector import BaseVectorFileHandler
@@ -100,12 +98,13 @@ def prepare_import(self, files, execution_id, **kwargs):
             process_rows(resource)
 
         folder = Path(_file).parent
-        mapper = TabularDataHelper(package)
+        mapper = TabularDataHelper(package, self.fixup_name)
         vrt_file = mapper.write_vrt_file(f"{package.name}.vrt", folder)
 
         # update base file to be imported by ogr2ogr
         prepared_files = {
             "base_file": str(vrt_file),
+            # "sld_file": str(self.load_local_resource("fake.sld")),
             "package_file": _file,
         }
         files.update(prepared_files)
@@ -118,21 +117,21 @@ def prepare_import(self, files, execution_id, **kwargs):
             execution_id=str(execution_id), input_params=_input
         )
 
-    def can_handle_sld_file():
-        return False
+    # def can_handle_sld_file():
+    #     return True
 
     def get_ogr2ogr_driver(self):
         return ogr.GetDriverByName("VRT")
 
-    def handle_sld_file(self, saved_dataset: Dataset, _exec: ExecutionRequest):
-        sld_file = self.load_local_resource("fake.sld")
-        resource_manager.exec(
-            "set_style",
-            None,
-            instance=saved_dataset,
-            sld_file=sld_file,
-            sld_uploaded=True,
-        )
+    # def handle_sld_file(self, saved_dataset: Dataset, _exec: ExecutionRequest):
+    #     sld_file = self.load_local_resource("fake.sld")
+    #     resource_manager.exec(
+    #         "set_style",
+    #         None,
+    #         instance=saved_dataset,
+    #         sld_file=sld_file,
+    #         sld_uploaded=True,
+    #     )
 
     def create_geonode_resource(
         self,
@@ -190,10 +189,33 @@ def create_geonode_resource(
         package_file = _files.get("package_file")
         package = Package(package_file)
 
-        mapper = TabularDataHelper(package)
-        attribute_map = mapper.parse_attribute_map(layer_name, self.fixup_name)
-        set_attributes(saved_dataset, attribute_map, _overwrite)
-
+        mapper = TabularDataHelper(package, self.fixup_name)
+        attributes = saved_dataset.attribute_set.all()
+        attribute_map = mapper.parse_attribute_map(layer_name)
+        for la in attributes:
+            for attribute in attribute_map:
+                field, ftype, description, label, display_order = attribute
+                if field == la.attribute:
+                    try:
+                        la.description = description
+                        la.attribute_label = label
+                        la.save()
+                    except DataError as e:
+                        logger.error(f"Cannot save attribute {field} for layer {saved_dataset.name}: {e}")
+
+                    continue
+
+        saved_dataset.set_bbox_polygon(
+            # for 'non-spatial' data we apply world bbox
+            bbox=[
+                -179.0, # west
+                -89.0, # south,
+                179.0, # east,
+                89.0, #n north,
+            ],
+            srid="EPSG:4326",
+        )
+        saved_dataset.save()
         saved_dataset.refresh_from_db()
         return saved_dataset
 

importer_datapackage/handlers/datapackage/mapper.py

@@ -1,22 +1,26 @@
+import logging
 from pathlib import Path
-from functools import reduce
+from collections.abc import Callable
 
 import xml.etree.cElementTree as ET
 
+logger = logging.getLogger(__name__)
+
 class TabularDataHelper():
 
-    def __init__(self, package):
+    def __init__(self, package, fixup_name: Callable):
         self.package = package
         self.resources = package.resources or []
+        self.fixup_name = fixup_name
 
 
-    def parse_attribute_map(self, resource_name: str, fixup_name) -> list:
+    def parse_attribute_map(self, resource_name: str) -> list:
         """ Of type: [ [field, ftype, description, label, display_oder], ... ]"""
 
         resource = self.package.get_resource(resource_name)
         schema = resource.schema
         attribute_map = [
-            [fixup_name(field.name), _parse_field_type(field)[0], field.description, field.title or field.name, None]
+            [self.fixup_name(field.name), _parse_field_type(field)[0], field.description, field.title or field.name, None]
             for field in schema.fields
         ]
         return attribute_map
@@ -31,23 +35,38 @@ def write_vrt_file(self, filename: str, folder: Path):
             layer = ET.SubElement(root, "OGRVRTLayer", name=resource.name)
             source = Path(folder, resource.path) if folder else resource.path
             ET.SubElement(layer, "SrcDataSource").text = str(source)
+            ET.SubElement(layer, "ExtentXMin").text = "-89.0"
+            ET.SubElement(layer, "ExtentYMin").text = "-179"
+            ET.SubElement(layer, "ExtentXMax").text = "89"
+            ET.SubElement(layer, "ExtentYMax").text = "179"
+
             schema = resource.schema
             for field in schema.fields:
                 (type, subtype) = _parse_field_type(field)
-                normalized_fieldname = normalize(field.name).lower()
-                ET.SubElement(layer, "Field", src=field.name, name=normalized_fieldname, type=type, subtype=subtype,)
+                normalized_fieldname = self.fixup_name(field.name)
+
+                ET.SubElement(
+                    layer, "Field", 
+                    src=field.name or "", 
+                    name=normalized_fieldname or "",
+                    # alternativeName=field.title or "", # GDAL >=3.7
+                    # comment=field.description or "", # GDAL >=3.7
+                    type=type,
+                    subtype=subtype,
+                )
 
         # write VRT file
         vrt_filename = Path(folder, filename) if folder else filename
         tree = ET.ElementTree(root)
         ET.indent(tree, space="  ")
-        tree.write(vrt_filename)
+        try:
+            with open(vrt_filename, "wb") as f:
+                tree.write(f, encoding="UTF-8")
+        except TypeError:
+            logger.error(f"Could not create VRT file '{vrt_filename}'", exc_info=True)
 
         return vrt_filename
 
-def normalize(fieldname):
-    return fieldname.replace(" ", "_") if fieldname else fieldname
-    
 
 def _parse_field_type(field) -> tuple: