test&bugfix(all): start test of interface with ETS, fix a bug in DwCA export

grololo06 · grololo06 · commit 77da554e2c78 · 2026-02-13T10:41:41.000+01:00
diff --git a/QA/py/pytest.ini b/QA/py/pytest.ini
@@ -1,3 +1,3 @@
 [pytest]
 asyncio_mode = auto
-asyncio_default_fixture_loop_scope = function
+asyncio_default_fixture_loop_scope = module
diff --git a/QA/py/tests/api_wrappers.py b/QA/py/tests/api_wrappers.py
@@ -27,7 +27,7 @@ def api_file_import(
     return rsp
 
 
-def api_wait_for_stable_job(fastapi, job_id, max_wait=200):
+def api_wait_for_stable_job(fastapi, job_id, max_wait=300):
     url = JOB_QUERY_URL.format(job_id=job_id)
     waited = 0
     while True:
diff --git a/QA/py/tests/emodnet_ref.py b/QA/py/tests/emodnet_ref.py
@@ -252,7 +252,21 @@
 m106_mn04_n6_sml	m106_mn04_n6_sml	m106_mn04_n6_sml_78418	MachineObservation	Oncaeidae	urn:lsid:marinespecies.org:taxname:128586	Animalia	present
 m106_mn04_n6_sml	m106_mn04_n6_sml	m106_mn04_n6_sml_56693	MachineObservation	Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	present
 """
-_occurence_with_absent = ""
+_occurence_with_absent = r"""m106_mn01_n1_sml	m106_mn01_n1_sml	m106_mn01_n1_sml_56693	MachineObservation		Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	absent
+m106_mn01_n2_sml	m106_mn01_n2_sml	m106_mn01_n2_sml_45072	MachineObservation		Cyclopoida	urn:lsid:marinespecies.org:taxname:1101	Animalia	absent
+m106_mn01_n2_sml	m106_mn01_n2_sml	m106_mn01_n2_sml_78418	MachineObservation		Oncaeidae	urn:lsid:marinespecies.org:taxname:128586	Animalia	absent
+m106_mn01_n2_sml	m106_mn01_n2_sml	m106_mn01_n2_sml_56693	MachineObservation		Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	absent
+m106_mn01_n3_sml	m106_mn01_n3_sml	m106_mn01_n3_sml_45072	MachineObservation		Cyclopoida	urn:lsid:marinespecies.org:taxname:1101	Animalia	absent
+m106_mn01_n3_sml	m106_mn01_n3_sml	m106_mn01_n3_sml_78418	MachineObservation		Oncaeidae	urn:lsid:marinespecies.org:taxname:128586	Animalia	absent
+{p1}_m106_mn04_n4_sml	{p1}_m106_mn04_n4_sml	{p1}_m106_mn04_n4_sml_45072	MachineObservation		Cyclopoida	urn:lsid:marinespecies.org:taxname:1101	Animalia	absent
+{p1}_m106_mn04_n4_sml	{p1}_m106_mn04_n4_sml	{p1}_m106_mn04_n4_sml_56693	MachineObservation		Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	absent
+m106_mn04_n5_sml	m106_mn04_n5_sml	m106_mn04_n5_sml_45072	MachineObservation		Cyclopoida	urn:lsid:marinespecies.org:taxname:1101	Animalia	absent
+m106_mn04_n5_sml	m106_mn04_n5_sml	m106_mn04_n5_sml_56693	MachineObservation		Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	absent
+m106_mn04_n6_sml	m106_mn04_n6_sml	m106_mn04_n6_sml_56693	MachineObservation		Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	absent
+{p2}_m106_mn04_n4_sml	{p2}_m106_mn04_n4_sml	{p2}_m106_mn04_n4_sml_45072	MachineObservation		Cyclopoida	urn:lsid:marinespecies.org:taxname:1101	Animalia	absent
+{p2}_m106_mn04_n4_sml	{p2}_m106_mn04_n4_sml	{p2}_m106_mn04_n4_sml_78418	MachineObservation		Oncaeidae	urn:lsid:marinespecies.org:taxname:128586	Animalia	absent
+{p2}_m106_mn04_n4_sml	{p2}_m106_mn04_n4_sml	{p2}_m106_mn04_n4_sml_56693	MachineObservation		Actinopterygii	urn:lsid:marinespecies.org:taxname:10194	Animalia	absent
+"""
 _emofs = r"""
 id	occurrenceID	measurementValue	measurementType	measurementUnit	measurementValueID	measurementTypeID	measurementUnitID
 m106_mn01_n1_sml	m106_mn01_n1_sml_1	2	Count (in assayed sample) of biological entity specified elsewhere			http://vocab.nerc.ac.uk/collection/P01/current/OCOUNT01/	
diff --git a/QA/py/tests/test_export_emodnet.py b/QA/py/tests/test_export_emodnet.py
@@ -3,10 +3,12 @@
 
 # noinspection PyPackageRequirements
 from io import BytesIO
+from typing import Any, Dict, List, Union
 from zipfile import ZipFile
 
 import pytest
 from starlette import status
+from starlette.testclient import TestClient
 
 from tests.api_wrappers import (
     api_wait_for_stable_job,
@@ -94,12 +96,7 @@ def exportable_collection(fastapi, admin_or_creator):
             "with_computations": ["ABO", "CNC", "BIV"],
         }
     )
-    rsp = fastapi.post(
-        COLLECTION_EXPORT_EMODNET_URL, headers=admin_or_creator, json=req
-    )
-    assert rsp.status_code == status.HTTP_200_OK
-    job_id = rsp.json()["job_id"]
-    api_wait_for_stable_job(fastapi, job_id)
+    job_id = export_collection(fastapi, req, admin_or_creator)
     rsp = api_check_job_failed(fastapi, job_id, "5 error(s) during run")
     json = rsp.json()
     assert json["errors"] == [
@@ -139,58 +136,84 @@ def exportable_collection(fastapi, admin_or_creator):
     add_concentration_data(fastapi, prj_id, "mn04")
 
     # Update the collection to fill in missing data
-    url = COLLECTION_QUERY_URL.format(collection_id=coll_id)
-    rsp = fastapi.get(url, headers=admin_or_creator)
-    assert rsp.status_code == status.HTTP_200_OK
-    the_coll = rsp.json()
-    url = COLLECTION_UPDATE_URL.format(collection_id=coll_id)
-    the_coll[
-        "abstract"
-    ] = """
+    def update_cb(the_coll):
+        the_coll[
+            "abstract"
+        ] = """
 This series is part of the long term planktonic monitoring of
     # Villefranche-sur-mer, which is one of the oldest and richest in the world.
     # The data collection and processing has been funded by several projects
     # over its lifetime. It is currently supported directly by the Institut de la Mer
     # de Villefranche (IMEV), as part of its long term monitoring effort.
     """
-    the_coll["license"] = (
-        "CC BY 4.0"  # Would do nothing as the license comes from the underlying project
+        the_coll["license"] = (
+            "CC BY 4.0"  # Would do nothing as the license comes from the underlying project
+        )
+        user_doing_all = {
+            "id": REAL_USER_ID,
+            # TODO: below is redundant with ID and ignored, but fails validation (http 422) if not set
+            "email": "creator",
+            "name": "User Creating Projects",
+            "organisation": "OrgTest",
+        }
+        the_coll["creator_users"] = [user_doing_all]
+        the_coll["creator_organisations"] = [
+            "Institut de la Mer de Villefranche (IMEV)"
+        ]
+        the_coll["contact_user"] = {
+            "id": ORDINARY_USER3_USER_ID,
+            # TODO: below is redundant with ID and ignored, but fails validation (http 422) if not set
+            "email": "?",
+            "name": ".",
+            "organisation": "OrgTest",
+        }
+        the_coll["provider_user"] = user_doing_all
+
+    the_coll = update_collection(fastapi, coll_id, admin_or_creator, update_cb)
+
+    # Read-back for org Ids, we cannot fix creators order without them
+    def update_cb2(read):
+        station_id = read["creator_organisations"][0]["id"]
+        real_user_id = read["creator_users"][0]["id"]
+        read["display_order"] = {  # This is EML order
+            "creators": [f"{real_user_id}_u", f"{station_id}_o"],
+        }
+
+    update_collection(fastapi, coll_id, admin_or_creator, update_cb2)
+
+    # Store in ref
+    all_colls[str(admin_or_creator)] = the_coll
+    yield the_coll
+
+
+def export_collection(
+    fastapi: TestClient,
+    req: Dict[str, Union[None, bool, List[Any], Dict[str, str], List[str]]],
+    admin_or_creator,
+) -> Any:
+    rsp = fastapi.post(
+        COLLECTION_EXPORT_EMODNET_URL, headers=admin_or_creator, json=req
     )
-    user_doing_all = {
-        "id": REAL_USER_ID,
-        # TODO: below is redundant with ID and ignored, but fails validation (http 422) if not set
-        "email": "creator",
-        "name": "User Creating Projects",
-        "organisation": "OrgTest",
-    }
-    the_coll["creator_users"] = [user_doing_all]
-    the_coll["creator_organisations"] = ["Institut de la Mer de Villefranche (IMEV)"]
-    the_coll["contact_user"] = {
-        "id": ORDINARY_USER3_USER_ID,
-        # TODO: below is redundant with ID and ignored, but fails validation (http 422) if not set
-        "email": "?",
-        "name": ".",
-        "organisation": "OrgTest",
-    }
-    the_coll["provider_user"] = user_doing_all
-    rsp = fastapi.put(url, headers=admin_or_creator, json=the_coll)
     assert rsp.status_code == status.HTTP_200_OK
-    # Read-back for org Ids, we cannot fix creators order without them
+    job_id = rsp.json()["job_id"]
+    assert rsp.json()["errors"] == []
+    api_wait_for_stable_job(fastapi, job_id)
+    return job_id
+
+
+def update_collection(fastapi, coll_id, who, callback):
+    # Query
     url = COLLECTION_QUERY_URL.format(collection_id=coll_id)
-    rsp = fastapi.get(url, headers=admin_or_creator)
+    rsp = fastapi.get(url, headers=who)
     assert rsp.status_code == status.HTTP_200_OK
-    read = rsp.json()
-    station_id = read["creator_organisations"][0]["id"]
-    real_user_id = read["creator_users"][0]["id"]
+    the_coll = rsp.json()
+    # Update
+    callback(the_coll)
+    # Put
     url = COLLECTION_UPDATE_URL.format(collection_id=coll_id)
-    read["display_order"] = {  # This is EML order
-        "creators": [f"{real_user_id}_u", f"{station_id}_o"],
-    }
-    rsp = fastapi.put(url, headers=admin_or_creator, json=read)
+    rsp = fastapi.put(url, headers=who, json=the_coll)
     assert rsp.status_code == status.HTTP_200_OK
-    # Store in ref
-    all_colls[str(admin_or_creator)] = the_coll
-    yield the_coll
+    return the_coll
 
 
 def make_project_exportable(prj_id, fastapi, who):
@@ -211,11 +234,7 @@ def test_emodnet_export(fastapi, exportable_collection, admin_or_creator, fixed_
     prj_id, prj_id2 = sorted(exportable_collection["project_ids"])
     req = _req_tmpl.copy()
     req.update({"collection_id": coll_id, "with_computations": ["ABO", "CNC", "BIV"]})
-    rsp = fastapi.post(
-        COLLECTION_EXPORT_EMODNET_URL, headers=admin_or_creator, json=req
-    )
-    assert rsp.status_code == status.HTTP_200_OK
-    job_id = rsp.json()["job_id"]
+    job_id = export_collection(fastapi, req, admin_or_creator)
     api_wait_for_stable_job(fastapi, job_id)
     job_status = api_check_job_ok(fastapi, job_id)
 
@@ -261,17 +280,12 @@ def test_emodnet_export(fastapi, exportable_collection, admin_or_creator, fixed_
         "Stats: predicted:2 validated:19 produced to zip:21 not produced (M):0 not produced (P):0",
     ]
     assert warns == ref_warns
-    assert rsp.json()["errors"] == []
-    # job_id = rsp.json()["job_id"]
 
     # Download the result zip
     url = JOB_DOWNLOAD_URL.format(job_id=job_id)
     # Ensure it's not public
     rsp = fastapi.get(url)
     assert rsp.status_code == status.HTTP_403_FORBIDDEN
-    # But the creator can get it
-    # rsp = fastapi.get(url, headers=REAL_USER_AUTH)
-    # assert rsp.status_code == status.HTTP_200_OK
 
     # Admin/owner can get it
     rsp = fastapi.get(url, headers=admin_or_creator)
@@ -292,10 +306,7 @@ def test_emodnet_export_with_absent(
             "with_computations": ["ABO", "CNC", "BIV"],
         }
     )
-    rsp = fastapi.post(COLLECTION_EXPORT_EMODNET_URL, headers=ADMIN_AUTH, json=req)
-    assert rsp.status_code == status.HTTP_200_OK
-    job_id = rsp.json()["job_id"]
-    api_wait_for_stable_job(fastapi, job_id)
+    job_id = export_collection(fastapi, req, ADMIN_AUTH)
     api_check_job_ok(fastapi, job_id)
     dl_url = JOB_DOWNLOAD_URL.format(job_id=job_id)
     rsp = fastapi.get(dl_url, headers=ADMIN_AUTH)
@@ -309,10 +320,7 @@ def test_emodnet_export_no_comp(
     prj_id, prj_id2 = sorted(exportable_collection["project_ids"])
     req = _req_tmpl.copy()
     req.update({"collection_id": coll_id})
-    rsp = fastapi.post(COLLECTION_EXPORT_EMODNET_URL, headers=ADMIN_AUTH, json=req)
-    assert rsp.status_code == status.HTTP_200_OK
-    job_id = rsp.json()["job_id"]
-    api_wait_for_stable_job(fastapi, job_id)
+    job_id = export_collection(fastapi, req, ADMIN_AUTH)
     api_check_job_ok(fastapi, job_id)
     dl_url = JOB_DOWNLOAD_URL.format(job_id=job_id)
     rsp = fastapi.get(dl_url, headers=ADMIN_AUTH)
@@ -338,10 +346,7 @@ def test_emodnet_export_recast1(
             },
         }
     )
-    rsp = fastapi.post(COLLECTION_EXPORT_EMODNET_URL, headers=ADMIN_AUTH, json=req)
-    assert rsp.status_code == status.HTTP_200_OK
-    job_id = rsp.json()["job_id"]
-    api_wait_for_stable_job(fastapi, job_id)
+    job_id = export_collection(fastapi, req, ADMIN_AUTH)
     job_status = api_check_job_ok(fastapi, job_id)
     assert "wrns" in job_status["result"], job_status
     warns = job_status["result"]["wrns"]
@@ -370,10 +375,7 @@ def test_emodnet_export_recast2(
             },
         }
     )
-    rsp = fastapi.post(COLLECTION_EXPORT_EMODNET_URL, headers=ADMIN_AUTH, json=req)
-    assert rsp.status_code == status.HTTP_200_OK
-    job_id = rsp.json()["job_id"]
-    api_wait_for_stable_job(fastapi, job_id)
+    job_id = export_collection(fastapi, req, ADMIN_AUTH)
     job_status = api_check_job_ok(fastapi, job_id)
     warns = job_status["result"]["wrns"]
     assert "Not produced due to non-match" not in str(warns)
diff --git a/QA/py/tests/test_taxoserver.py b/QA/py/tests/test_taxoserver.py
@@ -0,0 +1,62 @@
+from starlette import status
+
+SEARCH_WORMS_URL = "/searchworms/{}"
+
+ACARTIA_RSP = [
+    {
+        "aphia_id": 104108,
+        "name": "Acartia",
+        "rank": "Genus",
+        "status": "accepted",
+        "lineage": {
+            "104108": {
+                "AphiaID": 104074,
+                "rank": "Family",
+                "scientificname": "Acartiidae",
+            },
+            "104074": {"AphiaID": 1100, "rank": "Order", "scientificname": "Calanoida"},
+            "1100": {
+                "AphiaID": 155877,
+                "rank": "Superorder",
+                "scientificname": "Gymnoplea",
+            },
+            "155877": {
+                "AphiaID": 155876,
+                "rank": "Infraclass",
+                "scientificname": "Neocopepoda",
+            },
+            "155876": {"AphiaID": 1080, "rank": "Class", "scientificname": "Copepoda"},
+            "1080": {
+                "AphiaID": 845959,
+                "rank": "Superclass",
+                "scientificname": "Multicrustacea",
+            },
+            "845959": {
+                "AphiaID": 1066,
+                "rank": "Subphylum",
+                "scientificname": "Crustacea",
+            },
+            "1066": {"AphiaID": 1065, "rank": "Phylum", "scientificname": "Arthropoda"},
+            "1065": {"AphiaID": 2, "rank": "Kingdom", "scientificname": "Animalia"},
+            "2": {"AphiaID": 1, "rank": "Superdomain", "scientificname": "Biota"},
+        },
+        "id": 80116,
+    }
+]
+
+
+def test_search_worms_name(fastapi, mocker):
+    # Mock the 'call' method of EcoTaxoServerClient
+    mock_call = mocker.patch("providers.EcoTaxoServer.EcoTaxoServerClient.call")
+    mock_response = mocker.Mock()
+    mock_response.json.return_value = ACARTIA_RSP
+    mock_call.return_value = mock_response
+
+    url = SEARCH_WORMS_URL.format("Acartia")
+    # Unauthenticated call
+    rsp = fastapi.get(url)
+    assert rsp.status_code == status.HTTP_200_OK
+    assert rsp.json() == ACARTIA_RSP
+
+    # Verify the mock was called correctly
+    mock_call.assert_called_with("/wormstaxon/Acartia", {}, "get")
diff --git a/QA/py/tools/dbBuildSQL.py b/QA/py/tools/dbBuildSQL.py
@@ -248,6 +248,8 @@ def create(self):
 MAILSERVICE_SALT = "mailservice_salt"
 SENDER_ACCOUNT = senderemail@testsendermailtest.com,senderpwd,senderdns
 TAXOSERVER_URL = http://ecotaxoserver.dev.com
+TAXOSERVER_INSTANCE_ID = 123455
+TAXOSERVER_SHARED_SECRET = qsdf3fsdtre5665TY
 INSTANCE_ID = EcoTaxa.01
 USER_EMAIL_VERIFICATION = off
 ACCOUNT_VALIDATION = off
diff --git a/py/API_operations/exports/DarwinCore.py b/py/API_operations/exports/DarwinCore.py
diff --git a/py/BO/Sample.py b/py/BO/Sample.py
