Enhance timestamp handling for single item workflows; address #22

Author: jefferya

rootfs/var/www/leaf-isle-bagger/drupal/api.py

@@ -65,3 +65,14 @@ def get_node_by_format(session, server, item_id):
     )
     response.raise_for_status()
     return response
+
+def media_associated_with_node_endpoint(id):
+    return f"node/{id}/media?_format=json"
+
+#
+def get_associated_media_by_format(session, server, id):
+    response = session.get(
+        urljoin(server, media_associated_with_node_endpoint(id))
+    )
+    response.raise_for_status()
+    return response

rootfs/var/www/leaf-isle-bagger/drupal/utilities.py

@@ -60,6 +60,29 @@ def id_list_from_arg(session, args):
     return node_list
 
 
+# Query Media because media updates are not reflected in associated Node change timestamps 
+# exclude Drupal Media not attached to a Drupal Node
+# Apply only to the single node case
+def single_node_merge_with_media(session, server, node_list, node_id):
+
+    try:
+        node_id = node_id if isinstance(node_id, int) else int(node_id)
+    except ValueError:
+        logging.error(f"Invalid node id {node_id}")
+    else:
+        # Get assocaiated Media
+        associated_media_json = drupalApi.get_associated_media_by_format(session, server, node_id)
+        associated_media = json.loads(associated_media_json.content)
+        for media in associated_media:
+            media_changed = media["changed"][0]["value"] if ("changed" in media) else None
+            node = node_list.get(node_id, None)
+            if media_changed is not None and node is not None and node["changed"] < media_changed:
+                # media changed but the parent node did not change
+                node_list[node_id]["changed"] = media_changed
+                logging.info(f"  Updating node list changed date : {node_list}")                           
+
+
+
 # query media as media changes are not reflected as node revisions
 # exclude Drupal Media not attached to a Drupal Node
 def id_list_merge_with_media(session, args, node_list):

rootfs/var/www/leaf-isle-bagger/leaf-bagger.py

@@ -80,18 +80,23 @@ def process(args, session):
     # a list of resources to preserve
     node_list = {}
 
+    # get a list of Drupal Node IDs either from specified ID or via a list
     if args.force_single_node:
         node_list = drupalUtilities.id_list_from_arg(session, args)
-        logging.info(node_list)
+        logging.info(f"AIP: Drupal node before media inclusion - {node_list}")
+        # inspect associated Drupal Media for changes
+        # a Media change does not transitively update the associated Node change timestamp
+        # if Media changed but not the associated Node then add associated Node ID to the list
+        drupalUtilities.single_node_merge_with_media(session, args.server, node_list, args.force_single_node)
+        logging.info(f"AIP: Drupal node with media changes - {node_list}")
     else:
         # get a list of Drupal Node IDs changed since a given optional date
         # or a single node then force update
         node_list = drupalUtilities.id_list_from_nodes(session, args)
         logging.info(f"AIP: Drupal nodes before media inclusion - {node_list}")
-
-        # inspect Drupal Media for changes
-        # a Media change is does not transitively change the associated Node change timestamp)
-        # if Media changed then add associated Node ID to the list
+        # inspect associated Drupal Media for changes
+        # a Media change does not transitively update the associated Node change timestamp)
+        # if Media changed but not the associated Node then add associated Node ID to the list
         drupalUtilities.id_list_merge_with_media(session, args, node_list)
         logging.info(f"AIP: Drupal nodes with media changes - {node_list}")
 

rootfs/var/www/leaf-isle-bagger/swift/utilities.py

@@ -93,8 +93,8 @@ def validate(node_list, swift_container):
                         ):
                             logging.error(
                                 (
-                                    f"id:[{aip_id}] - mismatched modification timestamp [{src_value['changed']}]"
-                                    f" : {dst['headers']['x-object-meta-last-mod-timestamp']}"
+                                    f"id:[{aip_id}] - mismatched modification timestamp local[{src_value['changed']}]"
+                                    f" : swift[{dst['headers']['x-object-meta-last-mod-timestamp']}]"
                                 )
                             )
                             break

rootfs/var/www/leaf-isle-bagger/tests/test_drupal.py

@@ -152,3 +152,23 @@ def test_drupal_node_change_without_media(mocker):
     drupalUtilities.id_list_merge_with_media(_session, args, node_list)
     assert node_list[1]
     assert node_list[1]["changed"] == "2025-01-02"
+
+
+# Test the update of a single node with associated media 
+# test that the date list captures the media date not the node date
+def test_drupal_node_change_without_media(mocker):
+    node_id = 9999
+    node_list = {9999: {'changed': '2022-05-18T13:35:49+00:00', 'content_type': 'application/zip'}}
+    mocker.patch(
+        "argparse.ArgumentParser.parse_args",
+        return_value=argparse.Namespace(force_single_node=node_id, server="http://example.com"),
+    )
+    args = argparse.ArgumentParser.parse_args()
+    _adapter.register_uri(
+        "GET",
+        f"{args.server}/{drupalApi.media_associated_with_node_endpoint(node_id)}",
+        text='[ { "changed": [{"value": "2022-05-18T13:35:52+00:00"}], "field_media_of": [{"target_id": 9999}] } ]',
+    )
+    drupalUtilities.single_node_merge_with_media(_session, args.server, node_list, args.force_single_node)
+    assert node_list[node_id]
+    assert node_list[node_id]["changed"] == "2022-05-18T13:35:52+00:00"