feat(fao-open-knowledge): implement FAO Open Knowledge data models and URL collector

Author: lpi-tn

welearn_datastack/collectors/fao_open_knowledge_collector.py

@@ -0,0 +1,53 @@
+import logging
+from typing import List
+
+import requests  # type: ignore
+from requests.adapters import HTTPAdapter  # type: ignore
+from welearn_database.data.models import Corpus, WeLearnDocument
+
+from welearn_datastack import constants
+from welearn_datastack.data.source_models.fao_open_knowledge import FaoOKModel
+from welearn_datastack.data.url_collector import URLCollector
+from welearn_datastack.utils_.http_client_utils import get_new_https_session
+
+logger = logging.getLogger(__name__)
+
+
+class FAOOpenKnowledgeURLCollector(URLCollector):
+    related_corpus = "fao-open-knowledge"
+
+    def __init__(self, corpus: Corpus):
+        self.corpus = corpus
+
+        self.api_base_url = "https://openknowledge.fao.org/server/api/"
+        self.application_base_url = "https://openknowledge.fao.org/"
+        self.headers = constants.HEADERS
+
+    def _extract_fao_ok_urls(
+        self, fao_ok_api_response: FaoOKModel
+    ) -> List[WeLearnDocument]:
+        urls: List[WeLearnDocument] = []
+        for item in fao_ok_api_response.embedded.items:
+            document = WeLearnDocument(
+                url=self.application_base_url + f"items/{item.uuid}",
+                external_id=item.uuid,
+                corpus=self.corpus,
+            )
+            urls.append(document)
+        return urls
+
+    def collect(self) -> List[WeLearnDocument]:
+        session = get_new_https_session()
+
+        discover_url = f"{self.api_base_url}discover/browses/dateavailable/items"
+        params = {
+            "sort": "default,DESC",
+            "page": 0,
+            "size": 50,
+        }
+        fao_ok_resp = session.get(url=discover_url, headers=self.headers, params=params)
+        fao_ok_resp.raise_for_status()
+        fao_ok_response = FaoOKModel.model_validate(fao_ok_resp.json())
+
+        urls = self._extract_fao_ok_urls(fao_ok_response)
+        return urls

welearn_datastack/data/source_models/fao_open_knowledge.py

@@ -0,0 +1,101 @@
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class Bundles(BaseModel):
+    href: str
+
+
+class MappedCollections(BaseModel):
+    href: str
+
+
+class OwningCollection(BaseModel):
+    href: str
+
+
+class Relationships(BaseModel):
+    href: str
+
+
+class Version(BaseModel):
+    href: str
+
+
+class TemplateItemOf(BaseModel):
+    href: str
+
+
+class Thumbnail(BaseModel):
+    href: str
+
+
+class Relateditemlistconfigs(BaseModel):
+    href: str
+
+
+class Self(BaseModel):
+    href: str
+
+
+class _Links(BaseModel):
+    bundles: Bundles
+    mappedCollections: MappedCollections
+    owningCollection: OwningCollection
+    relationships: Relationships
+    version: Version
+    templateItemOf: TemplateItemOf
+    thumbnail: Thumbnail
+    relateditemlistconfigs: Relateditemlistconfigs
+    self: Self
+
+
+class Item(BaseModel):
+    id: str
+    uuid: str
+    name: str
+    handle: str
+    metadata: dict[str, Any]
+    inArchive: bool
+    discoverable: bool
+    withdrawn: bool
+    lastModified: str
+    entityType: Any
+    type: str
+    _links: _Links
+
+
+class Embedded(BaseModel):
+    items: list[Item]
+
+
+class Next(BaseModel):
+    href: str
+
+
+class Last(BaseModel):
+    href: str
+
+
+class Self1(BaseModel):
+    href: str
+
+
+class Links1(BaseModel):
+    next: Next
+    last: Last
+    self: Self1
+
+
+class Page(BaseModel):
+    number: int
+    size: int
+    totalPages: int
+    totalElements: int
+
+
+class FaoOKModel(BaseModel):
+    embedded: Embedded = Field(..., alias="_embedded")
+    links: Links1 = Field(..., alias="_links")
+    page: Page

welearn_datastack/nodes_workflow/URLCollectors/node_fao_open_knowledge_collect.py

@@ -0,0 +1,53 @@
+import logging
+import os
+
+from welearn_database.data.models import Corpus
+
+from welearn_datastack.collectors.fao_open_knowledge_collector import (
+    FAOOpenKnowledgeURLCollector,
+)
+from welearn_datastack.nodes_workflow.URLCollectors.nodes_helpers.collect import (
+    insert_urls,
+)
+from welearn_datastack.utils_.database_utils import create_db_session
+
+log_level: int = logging.getLevelName(os.getenv("LOG_LEVEL", "INFO"))
+log_format: str = os.getenv(
+    "LOG_FORMAT", "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s"
+)
+
+if not isinstance(log_level, int):
+    raise ValueError(f"Log level is not recognized : '{log_level}'")
+
+logging.basicConfig(
+    level=logging.getLevelName(log_level),
+    format=log_format,
+)
+logger = logging.getLogger(__name__)
+
+
+if __name__ == "__main__":
+    logger.info("FAOOpenKnowledgeURLCollector collector starting...")
+    session = create_db_session()
+    corpus: Corpus | None = (
+        session.query(Corpus)
+        .filter_by(source_name=FAOOpenKnowledgeURLCollector.related_corpus)
+        .one_or_none()
+    )
+
+    if corpus is None:
+        raise ValueError(
+            f"Corpus {FAOOpenKnowledgeURLCollector.related_corpus} not found"
+        )
+
+    fao_ok_collector = FAOOpenKnowledgeURLCollector(corpus=corpus)
+
+    urls = fao_ok_collector.collect()
+
+    logger.info("URLs retrieved : '%s'", len(urls))
+    insert_urls(
+        session=session,
+        urls=urls,
+    )
+
+    logger.info("FAOOpenKnowledgeURLCollector collector ended")