Provenance using sidecar json field GeneratedBy

Author: bclenet

examples/dcm2niix/README.md

@@ -13,12 +13,14 @@ datalad install --recursive https://github.com/psychoinformatics-de/hirni-demo.g
 
 ## Experiment #1
 
-The aim of the experiment is to describe the provenance records inside several files. Here we use sidecars and modality agnostic files inside the `prov/` directory, as follows:
+The aim of the experiment is to describe the provenance records inside several files.
+Here we use sidecars and modality agnostic files inside the `prov/` directory, as follows:
 ```
 .
 ├── prov
-│   ├── environments.prov.jsonld
-│   └── software.prov.jsonld
+│   ├── prov-dcm2niix_.prov.json
+│   ├── prov-dcm2niix_env.prov.json
+│   └── prov-dcm2niix_soft.prov.json
 └── sub_02
     ├── ses_20130717141500
     │   └── anat
@@ -28,6 +30,16 @@ The aim of the experiment is to describe the provenance records inside several f
             └── sub-02_ses-20140425155335_task-oneback_run-1_bold.prov.jsonld
 ```
 
+We introduce the following BIDS suffixes that are currently not existing:
+* `soft`:
+* `env`:
+
+We introduce the following BIDS entity that is currently not existing:
+* `prov`
+    * Full name: Provenance traces
+    * Format: `prov-<label>`
+    * Definition: A grouping of provenance traces. Defining multiple provenance traces groups is appropriate when several processings have been performed on data.
+
 * `sub-02_ses-20130717141500_T1w.prov.jsonld` and `sub-02_ses-20140425155335_task-oneback_run-1_bold.prov.jsonld` are sidecars defining provenance for the corresponding `.nii` files.
 * `environments.prov.jsonld` mutualises the declaration of software environments objects for lower level prov files
 * `software.prov.jsonld` mutualises the declaration of software pieces objects for lower level prov files

examples/dcm2niix/code/exp1_merge_prov.py

@@ -1,28 +1,59 @@
 #!/usr/bin/python
 # coding: utf-8
 
-""" Merge available prov JSON-LD files into one RDF graph """
+""" Merge available prov JSON files into one RDF graph """
 
 import json
 from pyld import jsonld
 
 # List of available prov files
-prov_files = [
-	'prov/software.prov.jsonld',
-	'prov/environments.prov.jsonld',
-	'sub_02/ses_20130717141500/anat/sub-02_ses-20130717141500_T1w.prov.jsonld',
-	'sub_02/ses_20140425155335/func/sub-02_ses-20140425155335_task-oneback_run-1_bold.prov.jsonld'
+prov_soft_files = [
+	'prov/prov-dcm2niix_ses-01_soft.prov.json',
+	'prov/prov-dcm2niix_ses-02_soft.prov.json'
+]
+prov_env_files = [
+	'prov/prov-dcm2niix_ses-01_env.prov.json',
+	'prov/prov-dcm2niix_ses-02_env.prov.json'
+]
+sidecar_files = [
+	'sub_02/ses_20130717141500/anat/sub-02_ses-20130717141500_T1w.json',
+	'sub_02/ses_20140425155335/func/sub-02_ses-20140425155335_task-oneback_run-1_bold.json'
 ]
 
-# Generate RDF graph of each file using pyld
-rdf_graph = ''
-for prov_file in prov_files:
+# Base jsonld
+with open('prov/base.prov.jsonld', encoding = 'utf-8') as file:
+	base_provenance = json.load(file)
+
+# Parse Software
+for prov_file in prov_soft_files:
 	with open(prov_file, encoding = 'utf-8') as file:
-		rdf_graph += f'# {prov_file}\n'
-		rdf_graph += jsonld.normalize(
-			jsonld.flatten(json.load(file)), {'algorithm': 'URDNA2015', 'format': 'application/n-quads'})
-		rdf_graph += '\n'
-
-# Write RDF graph
-with open('prov/experiment_1/merged_provenance.ttl', 'w' , encoding = 'utf-8') as file:
-	file.write(rdf_graph)
+		data = json.load(file)
+		for key, value in data.items():
+			value['Id'] = key
+			base_provenance['Records']['Software'].append(value)
+
+# Parse Environments
+for prov_file in prov_env_files:
+	with open(prov_file, encoding = 'utf-8') as file:
+		data = json.load(file)
+		for key, value in data.items():
+			value['Id'] = key
+		 	# /!\ Workaround: environments are added in the Entities list because
+		 	# the Environments term is not defined in the BIDS Prov context yet
+			base_provenance['Records']['Entities'].append(value)
+
+# Parse Sidecar files
+for sidecar_file in sidecar_files:
+	with open(sidecar_file, encoding = 'utf-8') as file:
+		data = json.load(file)
+		if 'GeneratedBy' in data:
+			activity = data['GeneratedBy']
+			base_provenance['Records']['Activities'].append(activity)
+			base_provenance['Records']['Entities'].append({
+				"Id": f"bids::{sidecar_file}",
+				"GeneratedBy": activity["Id"]
+				})
+
+# Write jsonld
+with open('prov/experiment_1/merged_provenance.prov.jsonld', 'w', encoding = 'utf-8') as file:
+	file.write(json.dumps(base_provenance, indent = 2))

examples/dcm2niix/prov/base.prov.jsonld

@@ -0,0 +1,9 @@
+{
+  "@context": "https://purl.org/nidash/bidsprov/context.json",
+  "BIDSProvVersion": "0.0.1",
+  "Records": {
+    "Software": [],
+    "Activities": [],
+    "Entities": []
+  }
+}

examples/dcm2niix/prov/environments.prov.jsonld

@@ -0,0 +1,79 @@
+{
+  "@context": "https://purl.org/nidash/bidsprov/context.json",
+  "BIDSProvVersion": "0.0.1",
+  "Records": {
+    "Software": [
+      {
+        "Label": "dcm2niix",
+        "Version": "v1.0.20220720",
+        "Id": "bids::prov/dcm2niix-v1.0.20220720"
+      },
+      {
+        "Label": "fmriprep",
+        "Version": "v8.0.5",
+        "Id": "bids::prov/fmriprep-v8.0.5"
+      },
+      {
+        "Label": "dcm2niix",
+        "Version": "v1.0.20231025",
+        "Id": "bids::prov/dcm2niix-v1.0.20231025"
+      },
+      {
+        "Label": "fmriprep",
+        "Version": "v8.0.5",
+        "Id": "bids::prov/fmriprep-v8.0.5"
+      }
+    ],
+    "Activities": [
+      {
+        "Id": "urn:conversion-00f3a18f",
+        "Type": "Activity",
+        "Label": "Conversion",
+        "Command": "dcm2niix -o . -f sub-%i/ses-%t/anat/sub-%i_ses-%t_T1w sourcedata/hirni-demo/acq1/dicoms/example-dicom-structural-master/dicoms",
+        "AssociatedWith": "bids::prov/dcm2niix-v1.0.20220720",
+        "Used": [
+          "bids::prov/fedora-35",
+          {
+            "Id": "bids::sourcedata/hirni-demo/acq1/dicoms/example-dicom-structural-master/dicoms",
+            "Type": "Entity",
+            "Label": "Structural MRI DICOMs"
+          }
+        ]
+      },
+      {
+        "Id": "urn:conversion-5a66f5be",
+        "Type": "Activity",
+        "Label": "Conversion",
+        "Command": "dcm2niix -o . -f sub-%i/ses-%t/func/sub-%i_ses-%t_task-oneback_run-1_bold sourcedata/hirni-demo/acq2/dicoms/example-dicom-functional-master/dicoms",
+        "AssociatedWith": "bids::prov/dcm2niix-v1.0.20231025",
+        "Used": [
+          "bids::prov/fedora-36",
+          {
+            "Id": "bids::sourcedata/hirni-demo/acq2/dicoms/example-dicom-functional-master/dicoms",
+            "Type": "Entity",
+            "Label": "Functional MRI DICOMs"
+          }
+        ]
+      }
+    ],
+    "Entities": [
+      {
+        "Label": "Fedora release 35 (Thirty Five)",
+        "Id": "bids::prov/fedora-35"
+      },
+      {
+        "Label": "Fedora release 36 (Thirty Six)",
+        "OperatingSystem": "GNU/Linux 6.2.15-100.fc36.x86_64",
+        "Id": "bids::prov/fedora-36"
+      },
+      {
+        "Id": "bids::sub_02/ses_20130717141500/anat/sub-02_ses-20130717141500_T1w.json",
+        "GeneratedBy": "urn:conversion-00f3a18f"
+      },
+      {
+        "Id": "bids::sub_02/ses_20140425155335/func/sub-02_ses-20140425155335_task-oneback_run-1_bold.json",
+        "GeneratedBy": "urn:conversion-5a66f5be"
+      }
+    ]
+  }
+}