feat(fao-open-knowledge): improve metadata parsing and update test cases for consistency

Author: lpi-tn

tests/document_collector_hub/plugins_test/test_fao_open_knowledge.py

@@ -32,7 +32,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
                 "dc.contributor.author": [
@@ -41,7 +41,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
                 "dc.description.abstract": [
@@ -50,7 +50,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
                 "dc.identifier.doi": [
@@ -59,7 +59,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
                 "dc.date.available": [
@@ -68,7 +68,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
                 "dc.date.lastModified": [
@@ -77,7 +77,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
                 "fao.taxonomy.type": [
@@ -86,7 +86,7 @@ def setUp(self):
                         "language": "en",
                         "authority": "FAO",
                         "confidence": 1,
-                        "place": "Rome",
+                        "place": 0,
                     }
                 ],
             },
@@ -140,17 +140,22 @@ def test_run_success(self, mock_get_metadata, mock_get_bundle, mock_get_pdf):
         # Simulate a successful run with valid PDF and metadata
         mock_get_metadata.return_value = self.item
         mock_get_bundle.return_value = [self.bundle]
-        mock_get_pdf.return_value = "PDF content extracted."
+        mock_get_pdf.return_value = "PDF content extracted. Lorem ispum"
         result = self.collector.run([self.doc])
         self.assertEqual(len(result), 1)
         doc_result = result[0]
         self.assertIsNone(doc_result.error_info)
         self.assertIsInstance(doc_result.document, WeLearnDocument)
-        self.assertEqual(doc_result.document.full_content, "PDF content extracted.")
+        self.assertEqual(
+            doc_result.document.full_content, "PDF content extracted. Lorem ispum"
+        )
         self.assertEqual(doc_result.document.title, "FAO Document Title")
         self.assertEqual(doc_result.document.description, "A description.")
         self.assertEqual(doc_result.document.details["doi"], "10.1234/fao.5678")
-        self.assertEqual(doc_result.document.details["license_url"], "cc-by-4.0")
+        self.assertEqual(
+            doc_result.document.details["license_url"],
+            "https://creativecommons.org/licenses/by/4.0/",
+        )
         self.assertEqual(doc_result.document.details["type"], "Report")
         self.assertTrue(
             doc_result.document.details["contrent_from_pdf"]
@@ -238,19 +243,6 @@ def test_run_embargo(self, mock_get_metadata, mock_get_bundle, mock_get_pdf):
         self.assertIn("unauthorized state", result[0].error_info)
         self.assertTrue(result[0].is_error)
 
-    # @patch.object(FAOOpenKnowledgeCollector, "_get_pdf_content")
-    # @patch.object(FAOOpenKnowledgeCollector, "get_bundle_json")
-    # @patch.object(FAOOpenKnowledgeCollector, "get_metadata_json")
-    # def test_run_pydantic_validation_error(
-    #     self, mock_get_metadata, mock_get_bundle, mock_get_pdf
-    # ):
-    #     # Simulate pydantic validation error
-    #     mock_get_metadata.side_effect = pydantic.ValidationError([], "error")
-    #     result = self.collector.run([self.doc])
-    #     self.assertEqual(len(result), 1)
-    #     self.assertIn("validation error", result[0].error_info)
-    #     self.assertTrue(result[0].is_error)
-
     @patch.object(FAOOpenKnowledgeCollector, "_get_pdf_content")
     @patch.object(FAOOpenKnowledgeCollector, "get_bundle_json")
     @patch.object(FAOOpenKnowledgeCollector, "get_metadata_json")

welearn_datastack/data/source_models/fao_open_knowledge.py

@@ -91,3 +91,8 @@ class MetadataEntry(BaseModel):
     authority: str | None
     confidence: int | None
     place: int
+
+
+#
+# class MetadataEntries(BaseModel):
+#     metadata: list[MetadataEntry]

welearn_datastack/plugins/rest_requesters/fao_open_knowledge.py

@@ -1,6 +1,7 @@
 import io
 import logging
 import os
+from collections import defaultdict
 from datetime import datetime
 from typing import Any
 
@@ -201,39 +202,46 @@ def _extract_external_sdgs(sdgs_str: list[MetadataEntry]) -> list[int]:
         return ret
 
     def _extract_details(self, fao_document: Item) -> dict:
-        parsed_metadata: dict[str, MetadataEntry] = {}
+        parsed_metadata: defaultdict[str, list[MetadataEntry]] = defaultdict(list)
         for metadata in fao_document.metadata:
             try:
-                parsed_metadata[metadata] = MetadataEntry.model_validate(
-                    fao_document.metadata.get(metadata)
+                mds = fao_document.metadata.get(metadata)
+
+                parsed_metadata[metadata].extend(
+                    [MetadataEntry.model_validate(md) for md in mds]
                 )
-            except pydantic.ValidationError:
-                logger.warning(f"Cannot parse metadata entry: {metadata}")
+            except pydantic.ValidationError as e:
+                logger.warning(f"Cannot parse metadata entry: {metadata}: {e}")
                 continue
-        empty_entry = MetadataEntry(
-            value="", language="", authority=None, confidence=-1, place=0
-        )
-        date_format = "yyyy-MM-ddTHH:mm:ssZ"
-        publication_date = parsed_metadata.get("dc.date.available", empty_entry).value
-        update_date = parsed_metadata.get("dc.date.lastModified", empty_entry).value
+        empty_entry = [
+            MetadataEntry(value="", language="", authority=None, confidence=-1, place=0)
+        ]
+        date_format = "%Y-%m-%dT%H:%M:%SZ"
+        [publication_date] = parsed_metadata.get("dc.date.available", empty_entry)
+        [update_date] = parsed_metadata.get("dc.date.lastModified", empty_entry)
+        [isbn] = parsed_metadata.get("dc.identifier.isbn", empty_entry)
+        [doi] = parsed_metadata.get("dc.identifier.doi", empty_entry)
+        [type_] = parsed_metadata.get("fao.taxonomy.type", empty_entry)
         ret: dict[str, Any] = {
             "publication_date": (
                 None
-                if not publication_date
-                else datetime.strptime(publication_date, date_format).timestamp()
+                if not publication_date.value
+                else datetime.strptime(publication_date.value, date_format).timestamp()
             ),
             "update_date": (
                 None
-                if not update_date
-                else datetime.strptime(update_date, date_format).timestamp()
+                if not update_date.value
+                else datetime.strptime(update_date.value, date_format).timestamp()
             ),
-            "isbn": parsed_metadata.get("dc.identifier.isbn", empty_entry).value,
+            "isbn": isbn.value,
             "license_url": self._extract_licence(fao_document),
             "authors": self._extract_authors(fao_document),
-            "external_sdg": self._extract_external_sdgs(fao_document.metadata),
+            "external_sdg": self._extract_external_sdgs(
+                parsed_metadata.get("fao.sdgs", [])
+            ),
             "contrent_from_pdf": True,
-            "doi": parsed_metadata.get("dc.identifier.doi", empty_entry).value,
-            "type": parsed_metadata.get("fao.taxonomy.type", empty_entry).value,
+            "doi": doi.value,
+            "type": type_.value,
         }
         return ret