feat(directory-client): fill in FDP-related fields during publishing

tools/directory/dev/.env_example

@@ -1,6 +1,6 @@
 TARGET="https://your-server"
-USERNAME=****
-PASSWORD=****
+DIR_USERNAME=****
+DIR_PASSWORD=****
 DIRECTORY="name of the BBMRI-ERIC schema"
 NN_SCHEMA_PREFIX="National nodes are stored as two letter (country) codes might have a schema prefix"
 PID_SERVICE="dummy"

tools/directory/dev/dev.py

@@ -8,22 +8,23 @@
 import os
 
 from dotenv import load_dotenv
-from tools.directory.src.molgenis_emx2.directory_client.directory import Directory
-from tools.directory.src.molgenis_emx2.directory_client.directory_client import (
+from directory.src.molgenis_emx2.directory_client.directory import Directory
+from directory.src.molgenis_emx2.directory_client.directory_client import (
     DirectorySession,
 )
-from tools.directory.src.molgenis_emx2.directory_client.pid_service import (
+from directory.src.molgenis_emx2.directory_client.pid_service import (
     DummyPidService,
     NoOpPidService,
     PidService,
 )
 
 # Get credentials from .env
 load_dotenv('./tools/directory/dev/.env')
+load_dotenv()
 
 target = os.getenv("TARGET")
-username = os.getenv("USERNAME")
-password = os.getenv("PASSWORD")
+username = os.getenv("DIR_USERNAME")
+password = os.getenv("DIR_PASSWORD")
 directory_schema = os.getenv("DIRECTORY")
 pid_service_type = os.getenv(
     "PID_SERVICE"

tools/directory/src/molgenis_emx2/directory_client/directory.py

@@ -1,7 +1,14 @@
 from typing import List, Optional
 
+from molgenis_emx2_pyclient.exceptions import NoSuchTableException
+
 from .directory_client import AttributesRequest, DirectorySession
-from .errors import DirectoryError, ErrorReport, requests_error_handler
+from .errors import (
+    DirectoryError,
+    DirectoryWarning,
+    ErrorReport,
+    requests_error_handler,
+)
 from .model import ExternalServerNode, FileIngestNode, Node
 from .pid_manager import PidManagerFactory
 from .pid_service import BasePidService
@@ -110,15 +117,73 @@ def _init_state(self, nodes: List[Node], report: ErrorReport) -> PublishingState
             matching_attrs=["exact_mapping", "ntbt_mapping"],
         )
 
+        self.printer.print("📦 Retrieving FDP catalog and publisher")
+        try:
+            catalogs = self.session.get(
+                table='Catalogs', schema=self.session.directory_schema
+            )
+            catalog_id = self._get_catalog_id(catalogs)
+        except NoSuchTableException:
+            self.printer.print_warning(
+                DirectoryWarning(' Table Catalogs not found'), indent=1
+            )
+            catalog_id = ''
+
+        try:
+            publishers = self.session.get(
+                table="Publishers", schema=self.session.directory_schema
+            )
+            publisher_id = self._get_publisher_id(publishers)
+        except NoSuchTableException:
+            self.printer.print_warning(
+                DirectoryWarning(' Table Publishers not found'), indent=1
+            )
+            publisher_id = ''
+
         return PublishingState(
             existing_data=published_data,
             quality_info=quality_info,
             eu_node_data=eu_node_data,
             diseases=diseases,
+            catalog_id=catalog_id,
+            publisher_id=publisher_id,
             nodes=nodes,
             report=report,
         )
 
+    def _get_catalog_id(self, catalogs: list) -> str:
+        if len(catalogs) == 0:
+            self.printer.print_warning(
+                DirectoryWarning(' No FDP catalog found'), indent=1
+            )
+            catalog_id = ''
+        else:
+            catalog_id = catalogs[0]['id']
+            if len(catalogs) > 1:
+                self.printer.print_warning(
+                    DirectoryWarning(
+                        f' More than one FDP catalog found, selecting {catalog_id}'
+                    ),
+                    indent=1,
+                )
+        return catalog_id
+
+    def _get_publisher_id(self, publishers: list) -> str:
+        if len(publishers) > 0 and 'BBMRI-ERIC' in [
+            publisher['name'] for publisher in publishers
+        ]:
+            publisher_id = "BBMRI-ERIC"
+        else:
+            self.printer.print_warning(
+                DirectoryWarning(
+                    " Publisher 'BBMRI-ERIC' not found, "
+                    "not setting Collections.publisher field"
+                ),
+                indent=1,
+            )
+            publisher_id = ""
+        return publisher_id
+
     async def _prepare_nodes(self, nodes, state):
         for node in nodes:
             self.printer.print_node_title(node)

tools/directory/src/molgenis_emx2/directory_client/publication_preparer.py

@@ -55,6 +55,8 @@ def _transform_node(self, node_data: NodeData, state: PublishingState):
                 existing_biobanks=state.existing_data.biobanks,
                 eu_node_data=state.eu_node_data,
                 diseases=state.diseases,
+                catalog_id=state.catalog_id,
+                publisher_id=state.publisher_id,
             ).transform()
             if warnings:
                 state.report.add_node_warnings(node_data.node, warnings)

tools/directory/src/molgenis_emx2/directory_client/publisher.py

@@ -25,6 +25,8 @@ class PublishingState:
     nodes: List[Node]
     report: ErrorReport
     diseases: OntologyTable
+    catalog_id: str
+    publisher_id: str
     data_to_publish: MixedData = field(init=False)
 
     def __post_init__(self):

tools/directory/src/molgenis_emx2/directory_client/transformer.py

@@ -21,13 +21,18 @@ def __init__(
         existing_biobanks: Table,
         eu_node_data: NodeData,
         diseases: OntologyTable,
+        catalog_id: str,
+        publisher_id: str,
     ):
         self.node_data = node_data
         self.quality = quality
         self.printer = printer
         self.existing_biobanks = existing_biobanks.rows_by_id
         self.eu_node_data = eu_node_data
         self.category_mapper = CategoryMapper(diseases)
+        self.catalog_id = catalog_id
+        self.publisher_id = publisher_id
+        self.diseases = diseases
 
         self.warnings = []
 
@@ -43,6 +48,8 @@ def transform(self):
         7. Add biobank labels (biobank name) to collections
         8. Adds the combined qualities to collections
         9. Sets the collections' categories
+        10. Adds all non-withdrawn collections to the main FDP catalog
+        11. Fill in FDP-related fields for collections
 
         """
         self._set_national_node_code()
@@ -54,6 +61,9 @@ def transform(self):
         self._set_biobank_labels()
         self._set_combined_qualities()
         self._set_collection_categories()
+        if self.catalog_id:
+            self._set_catalog_membership()
+        self._set_collection_fdp_fields()
         return self.warnings
 
     def _set_commercial_use_bool(self):
@@ -188,3 +198,29 @@ def _set_withdrawn(self):
             ):
                 for row in table.rows:
                     row["withdrawn"] = True
+
+    def _set_catalog_membership(self):
+        """
+        Adds all non-withdrawn collections to the FDP catalog
+        """
+        self.printer.print("Setting FDP catalog membership")
+        for row in self.node_data.collections.rows:
+            if not row['withdrawn']:
+                row['catalog'] = self.catalog_id
+
+    def _set_collection_fdp_fields(self):
+        """
+        Fill in FDP-related fields for collections
+        """
+        self.printer.print("Setting FDP-related fields for collections")
+        for row in self.node_data.collections.rows:
+            if self.publisher_id:
+                row['publisher'] = self.publisher_id
+            row['language'] = 'English'
+            row[
+                'license'
+            ] = 'https://ejp-rd-vp.github.io/resource-metadata-schema/license/v1.0.txt'
+            keywords = []
+            for diagnosis in row['diagnosis_available']:
+                keywords.append(self.diseases.rows_by_id[diagnosis]['label'])
+            row['keywords'] = keywords

tools/directory/tests/conftest.py

@@ -6,7 +6,7 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.model import (
+from src.molgenis_emx2.directory_client.model import (
     Node,
     NodeData,
     Source,

tools/directory/tests/test_categories.py

@@ -3,8 +3,8 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.categories import Category, CategoryMapper
-from molgenis_emx2.directory_client.model import OntologyTable
+from src.molgenis_emx2.directory_client.categories import Category, CategoryMapper
+from src.molgenis_emx2.directory_client.model import OntologyTable
 
 
 @pytest.fixture

tools/directory/tests/test_directory.py

@@ -3,17 +3,19 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.directory import Directory
-from molgenis_emx2.directory_client.errors import DirectoryError, ErrorReport
-from molgenis_emx2.directory_client.model import ExternalServerNode, Node
-from molgenis_emx2.directory_client.publisher import PublishingState
+from src.molgenis_emx2.directory_client.directory import Directory
+from src.molgenis_emx2.directory_client.errors import DirectoryError, ErrorReport
+from src.molgenis_emx2.directory_client.model import ExternalServerNode, Node
+from src.molgenis_emx2.directory_client.publisher import PublishingState
 
 pytest_plugins = ("pytest_asyncio",)
 
 
 @pytest.fixture
 def report_init():
-    with patch("molgenis_emx2.directory_client.directory.ErrorReport") as report_mock:
+    with patch(
+        "src.molgenis_emx2.directory_client.directory.ErrorReport"
+    ) as report_mock:
         yield report_mock
 
 
@@ -95,6 +97,8 @@ def _setup_state(nodes: List[Node], eric: Directory, report_init):
         nodes=nodes,
         report=report,
         diseases=MagicMock(),
+        catalog_id=MagicMock(),
+        publisher_id=MagicMock()
     )
     eric._init_state = MagicMock()
     eric._init_state.return_value = state

tools/directory/tests/test_errors.py

@@ -1,13 +1,13 @@
 import pytest
 import requests.exceptions
 
-from molgenis_emx2.directory_client.errors import (
+from src.molgenis_emx2.directory_client.errors import (
     DirectoryError,
     DirectoryWarning,
     ErrorReport,
     requests_error_handler,
 )
-from molgenis_emx2.directory_client.model import Node
+from src.molgenis_emx2.directory_client.model import Node
 
 
 def test_warning():

tools/directory/tests/test_model.py

@@ -1,7 +1,7 @@
 # noinspection PyProtectedMember
 from unittest.mock import MagicMock
 
-from molgenis_emx2.directory_client.model import (
+from src.molgenis_emx2.directory_client.model import (
     ExternalServerNode,
     Node,
     NodeData,

tools/directory/tests/test_model_fitting.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.model_fitting import ModelFitter
+from src.molgenis_emx2.directory_client.model_fitting import ModelFitter
 
 
 @pytest.fixture

tools/directory/tests/test_pid_manager.py

@@ -3,18 +3,18 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.model import Table, TableType
-from molgenis_emx2.directory_client.pid_manager import (
+from src.molgenis_emx2.directory_client.model import Table, TableType
+from src.molgenis_emx2.directory_client.pid_manager import (
     NoOpPidManager,
     PidManager,
     PidManagerFactory,
 )
-from molgenis_emx2.directory_client.pid_service import (
+from src.molgenis_emx2.directory_client.pid_service import (
     DummyPidService,
     NoOpPidService,
     Status,
 )
-from molgenis_emx2.directory_client.printer import Printer
+from src.molgenis_emx2.directory_client.printer import Printer
 
 
 @pytest.fixture

tools/directory/tests/test_pid_service.py

@@ -3,8 +3,8 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.errors import DirectoryError
-from molgenis_emx2.directory_client.pid_service import (
+from src.molgenis_emx2.directory_client.errors import DirectoryError
+from src.molgenis_emx2.directory_client.pid_service import (
     DummyPidService,
     NoOpPidService,
     PidService,

tools/directory/tests/test_printer.py

@@ -1,12 +1,12 @@
 import textwrap
 
-from molgenis_emx2.directory_client.errors import (
+from src.molgenis_emx2.directory_client.errors import (
     DirectoryError,
     DirectoryWarning,
     ErrorReport,
 )
-from molgenis_emx2.directory_client.model import Node
-from molgenis_emx2.directory_client.printer import Printer
+from src.molgenis_emx2.directory_client.model import Node
+from src.molgenis_emx2.directory_client.printer import Printer
 
 
 def test_indentation(capsys):

tools/directory/tests/test_publication_preparer.py

@@ -2,16 +2,16 @@
 
 import pytest
 
-from molgenis_emx2.directory_client.errors import DirectoryWarning, ErrorReport
-from molgenis_emx2.directory_client.model import Node
-from molgenis_emx2.directory_client.publication_preparer import PublicationPreparer
+from src.molgenis_emx2.directory_client.errors import DirectoryWarning, ErrorReport
+from src.molgenis_emx2.directory_client.model import Node
+from src.molgenis_emx2.directory_client.publication_preparer import PublicationPreparer
 
 
 @pytest.fixture
 def validator_init():
     with (
         patch(
-            "molgenis_emx2.directory_client.publication_preparer.Validator"
+            "src.molgenis_emx2.directory_client.publication_preparer.Validator"
         ) as validator_mock
     ):
         yield validator_mock
@@ -20,15 +20,15 @@ def validator_init():
 @pytest.fixture
 def model_fitter_init():
     with patch(
-        "molgenis_emx2.directory_client.publication_preparer.ModelFitter"
+        "src.molgenis_emx2.directory_client.publication_preparer.ModelFitter"
     ) as model_fitter_mock:
         yield model_fitter_mock
 
 
 @pytest.fixture
 def transformer_init():
     with patch(
-        "molgenis_emx2.directory_client.publication_preparer.Transformer"
+        "src.molgenis_emx2.directory_client.publication_preparer.Transformer"
     ) as transformer_mock:
         yield transformer_mock