Let extensions deal with changes

Author: avdata99

ckanext/datapusher_plus/interfaces.py

@@ -54,6 +54,24 @@ def after_upload(
         """
         pass
 
-    def datastore_column_changed(self, changes, old_headers, new_headers):
-        """ While we are updating the datastore, we detect column changes """
+    def datastore_before_update(self, existing_info, new_headers):
+        """ We are about to update the datastore
+
+            :param existing_info: The existing information in the datastore.
+            Empty if the resource is new.
+            Something like:
+                {
+                'Header 1': {'label': 'Header 1'},
+                'Header 2': {'label': 'Header 2'},
+                ...
+                }
+
+            :param new_headers: The new headers that are to be added to the datastore.
+            Something like:
+                [
+                {'id': 'Header 1', 'type': 'text', 'info': {'label': 'Header 1'}},
+                {'id': 'Header 2', 'type': 'numeric', 'info': {'label': 'Header 2'}},
+                ...
+                ]
+        """
         pass

ckanext/datapusher_plus/jobs.py

@@ -153,7 +153,7 @@ def send_resource_to_datastore(
     """
     Stores records in CKAN datastore
     """
-    log.info(f"\n\nSending data to datastore {resource_id}\n{headers}\n")
+    log.info(f"Sending data to datastore {resource_id}")
 
     if resource_id:
         # used to create the "main" resource
@@ -1016,30 +1016,10 @@ def _push_to_datastore(task_id, input, dry_run=False, temp_dir=None):
     # Maintain data dictionaries from matching column names
     # if data dictionary already exists for this resource as
     # we want to preserve the user's data dictionary curations
-    # Also, detect column changes
-    changed = []
-    if existing_info:
-        for h in headers_dicts:
-            if h["id"] in existing_info:
-                h["info"] = existing_info[h["id"]]
-                # create columns with types user requested
-                type_override = existing_info[h["id"]].get("type_override")
-                if type_override in list(type_mapping.values()):
-                    h["type"] = type_override
-            else:
-                # new column
-                changed.append({'new': h["id"]})
-
-        new_headers_ids = [h["id"] for h in headers_dicts]
-        for ei in existing_info:
-            if ei not in new_headers_ids:
-                # deleted column
-                changed.append({'deleted': ei})
-
-    # send all plugins the column_changes
-    if changed:
-        for plugin in plugins.PluginImplementations(interfaces.IDataPusher):
-            plugin.datastore_column_changed(changed, old_headers=existing_info, new_headers=headers_dicts)
+
+    # Notify plugins the datastore will be updated
+    for plugin in plugins.PluginImplementations(interfaces.IDataPusher):
+        plugin.datastore_before_update(existing_info=existing_info, new_headers=headers_dicts)
 
     logger.info(
         "Determined headers and types: {headers}...".format(headers=headers_dicts)