feat(fao_open_knowledge): implement serialization for dataclass instances and enhance document processing

Author: lpi-tn

welearn_datastack/modules/computed_metadata.py

@@ -3,6 +3,7 @@
 import os
 import re
 from collections import deque
+from dataclasses import asdict, is_dataclass
 from functools import cache
 
 from lingua import LanguageDetectorBuilder
@@ -268,3 +269,34 @@ def compute_readability(
         document.full_content, document.lang
     )
     return document
+
+
+def is_dataclass_instance(obj):
+    return is_dataclass(obj) and not isinstance(obj, type)
+
+
+def _inner_serialize_dataclass(value):
+    match value:
+        case list():
+            return [_inner_serialize_dataclass(item) for item in value]
+        case dict():
+            return {k: _inner_serialize_dataclass(v) for k, v in value.items()}
+    if is_dataclass_instance(value):
+        return asdict(value)
+    return value
+
+
+def serialize_dataclass_instance(document: WeLearnDocument) -> WeLearnDocument:
+    for detail_key, detail_value in document.details.items():
+        match detail_value:
+            case list():
+                document.details[detail_key] = [
+                    _inner_serialize_dataclass(item) for item in detail_value
+                ]
+            case dict():
+                for k, v in detail_value.items():
+                    detail_value[k] = _inner_serialize_dataclass(v)
+                document.details[detail_key] = detail_value
+            case _:
+                document.details[detail_key] = _inner_serialize_dataclass(detail_value)
+    return document

welearn_datastack/nodes_workflow/DocumentHubCollector/document_collector.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import uuid
+from dataclasses import asdict
 from typing import Dict, List, Tuple
 from uuid import UUID
 
@@ -16,6 +17,7 @@
     compute_duration,
     compute_readability,
     identify_document_language,
+    serialize_dataclass_instance,
 )
 from welearn_datastack.modules.validation import validate_non_null_fields_document
 from welearn_datastack.plugins.interface import IPlugin
@@ -91,6 +93,7 @@ def main() -> None:
         identify_document_language(doc)
         compute_duration(doc)
         compute_readability(doc)
+        serialize_dataclass_instance(doc)
         flag_modified(doc, "details")
 
     db_session.add_all(states)

welearn_datastack/plugins/rest_requesters/fao_open_knowledge.py

@@ -133,7 +133,8 @@ def _extract_authors(fao_document: Item) -> list[AuthorDetails]:
         contributors_names: list[str] = []
         for contributor_entry in messy_authors:
             for name in contributor_entry.value.split(";"):
-                contributors_names.append(name.strip())
+                if name.strip():
+                    contributors_names.append(name.strip())
 
         for contributor in contributors_names:
             ret.append(AuthorDetails(name=contributor, misc=""))