Adapting merge code to derivative datasets layouts

Author: bclenet

bids_prov/merge.py

@@ -5,6 +5,7 @@
     from JSON files into one RDF graph.
 """
 
+from pathlib import Path
 import json
 from argparse import ArgumentParser
 from io import StringIO
@@ -18,6 +19,18 @@
 from bids import BIDSLayout
 from bids.layout.models import BIDSFile, BIDSJSONFile
 
+def get_associated_sidecar(layout: BIDSLayout, data_file: BIDSFile) -> dict:
+    """ This function is a workaround to BIDSFile.get_associations function not working with
+        derivative datasets.
+
+        Return the associated sidecar of a BIDSFile in a given BIDSLayout.
+    """
+    filename = Path(data_file.path)
+    extensions = ''.join(filename.suffixes)
+    sidecar_filename = str(filename).replace(extensions, '.json')
+    
+    return layout.get_file(sidecar_filename)
+
 def filter_provenance_group(files: list, group: str) -> list:
     """ Filter a given BIDSFile list, returning the sub-list containig the BIDS
     `prov` entity equal to group
@@ -74,7 +87,6 @@ def get_dataset_entity_record(description_file: BIDSJSONFile) -> dict:
         AND
         - at least one of the objects of the GeneratedBy has a Id field.
     """
-
     metadata = description_file.get_dict()
 
     # Provenance Entity record for the dataset
@@ -92,8 +104,10 @@ def get_dataset_entity_record(description_file: BIDSJSONFile) -> dict:
 
     return entity
 
-def get_entity_record(data_file: BIDSFile) -> dict:
-    """ Return an Entity provenance record from metadata of a BIDS file. """
+def get_entity_record(layout: BIDSLayout, data_file: BIDSFile) -> dict:
+    """ Return an Entity provenance record from metadata of a BIDS file
+        in a given BIDSLayout.
+    """
 
     # Provenance Entity record for the data file
     entity = {
@@ -103,7 +117,12 @@ def get_entity_record(data_file: BIDSFile) -> dict:
     }
 
     # Get provenance-related metadata
-    metadata = data_file.get_metadata()
+    ## metadata = data_file.get_metadata()
+    sidecar = get_associated_sidecar(layout, data_file)
+    if sidecar is None:
+        return None
+
+    metadata = sidecar.get_dict()
     if 'GeneratedBy' in metadata:
         entity['GeneratedBy'] = metadata['GeneratedBy']
     if 'Digest' in metadata:
@@ -113,11 +132,14 @@ def get_entity_record(data_file: BIDSFile) -> dict:
 
     return entity
 
-def get_sidecar_entity_record(data_file: BIDSFile) -> dict:
-    """ Return an Entity provenance record for the sidecar of a BIDS file, given its metadata. """
+def get_sidecar_entity_record(layout: BIDSLayout, data_file: BIDSFile) -> dict:
+    """ Return an Entity provenance record for the sidecar of a BIDS file, in a given BIDSLayout.
+    """
 
     # Get sidecar associated with the data_file
-    sidecar = data_file.get_associations()[0]
+    sidecar = get_associated_sidecar(layout, data_file)
+    if sidecar is None:
+        return None
 
     # Provenance Entity record for the sidecar JSON file
     entity = {
@@ -127,26 +149,25 @@ def get_sidecar_entity_record(data_file: BIDSFile) -> dict:
     }
 
     # Get provenance-related metadata
-    metadata = data_file.get_metadata()
+    metadata = sidecar.get_dict()
     if 'SidecarGeneratedBy' in metadata:
         entity['GeneratedBy'] = metadata['SidecarGeneratedBy']
         return entity
 
     return None
 
-def get_entities_in_group(input_graph: dict, group: str = None) -> list:
+def get_linked_entities(input_graph: dict) -> list:
     """ Return the Ids of Entity provenance records from the provenance graph
-        and belonging to the same provenance group.
+        that were either used or generated by an Activity.
 
         Arguments:
             - input_graph, dict: JSON-LD graph containing provenance records
-            - group, str: label of the provenance group
         Return:
             - list: list of Entity provenance records from the graph that
-            belong to the group
+            were either used or generated by an Activity
     """
 
-    # Open file & create graph from it
+    # Create RDF graph from input JSON-LD
     graph = Dataset()
     graph.parse(StringIO(json.dumps(jsonld.expand(input_graph))), format='json-ld')
 
@@ -180,7 +201,7 @@ def get_entities_in_group(input_graph: dict, group: str = None) -> list:
     used_entities = [s.n3(graph.namespace_manager).replace('<', '').replace('>', '')
         for s, _, _ in graph.query(query)]
 
-    # Return all prov:Entity in the group
+    # Return all linked prov:Entity
     return list(set(used_entities + generated_entities))
 
 def merge_records(layout: BIDSLayout, group: str = None) -> dict:
@@ -212,10 +233,11 @@ def merge_records(layout: BIDSLayout, group: str = None) -> dict:
 
     # Get provenance metadata from other JSON files in the dataset
     for data_file in get_described_files(layout):
-        entity = get_entity_record(data_file)
-        base_provenance['Records']['Entities'].append(entity)
-    for sidecar in get_described_sidecars(layout):
-        entity = get_sidecar_entity_record(sidecar)
+        entity = get_entity_record(layout, data_file)
+        if entity is not None:
+            base_provenance['Records']['Entities'].append(entity)
+    for data_file in get_described_sidecars(layout):
+        entity = get_sidecar_entity_record(layout, data_file)
         if entity is not None:
             base_provenance['Records']['Entities'].append(entity)
     for dataset in get_described_datasets(layout):
@@ -224,7 +246,7 @@ def merge_records(layout: BIDSLayout, group: str = None) -> dict:
             base_provenance['Records']['Entities'].append(entity)
 
     # Filter on provenance group
-    entities_in_group = get_entities_in_group(base_provenance, group)
+    entities_in_group = get_linked_entities(base_provenance)
     entities = []
     for entity in base_provenance['Records']['Entities']:
         if entity['Id'] in entities_in_group:
@@ -239,6 +261,8 @@ def entry_point():
     parser = ArgumentParser()
     parser.add_argument('--dataset', '-d', type=str, default='.',
         help='The path to the input BIDS dataset.')
+    parser.add_argument('--derivative', action='store_true',
+        help='Set this option to specify the dataset is a BIDS derivative dataset.')
     parser.add_argument('--output_file', '-o', type=str, required=True,
         help='Output JSON-LD file containing the provenance graph for the input dataset.')
     parser.add_argument('--group', '-g', type=str,
@@ -248,7 +272,10 @@ def entry_point():
     # Write output JSON-LD file
     with open(arguments.output_file, 'w', encoding = 'utf-8') as file:
         file.write(
-            json.dumps(merge_records(BIDSLayout(arguments.dataset), arguments.group), indent = 2)
+            json.dumps(merge_records(
+                BIDSLayout(arguments.dataset, is_derivative=arguments.derivative),
+                arguments.group
+                ), indent = 2)
         )
 
 if __name__ == '__main__':