Bluecloudfix

mdingestion/command/harvest.py

@@ -54,4 +54,7 @@ def _harvest(self, identifier, fromdate=None, clean=False, limit=None, dry_run=F
                            unit=' records',
                            total=_harvester.total(),
                            disable=silent):
-            _harvester.write_record(record, pretty_print=True)
+            try:
+                _harvester.write_record(record, pretty_print=True)
+            except Exception:
+                logging.exception(f"Harvesting of {identifier} failed.")

mdingestion/community/bluecloud.py

@@ -31,7 +31,7 @@ def _find(self, name):
     def update(self, doc):
         doc.discipline = ['Marine Science']
         doc.description = self._find('Abstract')
-        doc.source = self._find('OnlineResourceUrl')
+        doc.source = self.source()
         doc.publication_year = self._find('Last_Update')
         doc.contributor = self._find('Organisations')
         doc.contact = ['blue-cloud-support@maris.nl']
@@ -43,7 +43,17 @@ def update(self, doc):
         doc.keywords = self.keywords()
         doc.publisher = self.publishers()
 
-    # TODO: fix list problem in json parser = json.py in Carsten weiss wo
+    def source(self):
+        source = []
+        oru_urls = self._find("OnlineResourceUrl")
+        for oru in oru_urls:
+            if "http" in oru:
+                url = oru
+            else:
+                url = f"https://data.blue-cloud.org{oru}"
+            source.append(url)
+        return source
+
     def instruments(self):
         instruments = []
         instruments.extend(self._find('Instruments'))