ecotaxa/ecotaxa_front#843: Reconcile individual_biovolume with individual_volume, use formulas from projects during calculations.

Author: grololo06

QA/py/pg_files/upgrade_prod.sql

@@ -1152,7 +1152,7 @@ CREATE TABLE projects_variables (
     project_id INTEGER NOT NULL,
     subsample_coef VARCHAR,
     total_water_volume VARCHAR,
-    individual_biovolume VARCHAR,
+    individual_volume VARCHAR,
     PRIMARY KEY (project_id),
     FOREIGN KEY(project_id) REFERENCES projects (projid)
 );

QA/py/tests/test_export_sci.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Dict
 
 from starlette import status
 
@@ -11,19 +12,33 @@
 from tests.test_import import DATA_DIR, do_import
 from tests.test_jobs import get_job_and_wait_until_ok
 from tests.test_objectset_query import _prj_query
+from tests.test_update_prj import PROJECT_UPDATE_URL
 
 formulae = {"subsample_coef": "1/ssm.sub_part",
             "total_water_volume": "sam.tot_vol",  # Volumes are in m3 already for this data
             "individual_volume": "4.0/3.0*math.pi*(math.sqrt(obj.area/math.pi)*ssm.pixel)**3"}
 
 
-def test_export_sci(config, database, fastapi, caplog):
+def set_formulae_in_project(fastapi, prj_id: int, prj_formulae: Dict):
+    from tests.test_project_vars import BODC_VARS_KEY
+    read_url = PROJECT_QUERY_URL.format(project_id=prj_id, manage=True)
+    rsp = fastapi.get(read_url, headers=ADMIN_AUTH)
+    assert rsp.status_code == status.HTTP_200_OK
+    prj_json = rsp.json()
+    prj_json[BODC_VARS_KEY] = prj_formulae
+    upd_url = PROJECT_UPDATE_URL.format(project_id=prj_id)
+    rsp = fastapi.put(upd_url, headers=ADMIN_AUTH, json=prj_json)
+    assert rsp.status_code == status.HTTP_200_OK
+
+
+def test_export_abundances(config, database, fastapi, caplog):
     caplog.set_level(logging.FATAL)
 
     # Admin imports the project, which is an export expected result
     from tests.test_import import test_import
     path = str(DATA_DIR / "ref_exports" / "bak_all_images")
     prj_id = test_import(config, database, caplog, "TSV sci export", path=path)
+    set_formulae_in_project(fastapi, prj_id, formulae)  # Note: This is _not_ needed for abundances
 
     # Validate all, otherwise empty report
     obj_ids = _prj_query(fastapi, CREATOR_AUTH, prj_id)
@@ -106,6 +121,8 @@ def test_export_conc_biovol(config, database, fastapi, caplog):
     prj_id = test_import(config, database, caplog, "SCISUM project")
     # Add a sample spanning 2 days
     test_import_a_bit_more_skipping(config, database, caplog, "SCISUM project")
+    # Store computation variables
+    set_formulae_in_project(fastapi, prj_id, formulae)
     # Add some data for calculations
     add_concentration_data(fastapi, prj_id)
     # Add a sample with weird data in free columns
@@ -129,8 +146,7 @@ def test_export_conc_biovol(config, database, fastapi, caplog):
     req = _req_tmpl.copy()
     req.update({"project_id": prj_id,
                 "exp_type": "CNC",
-                "sum_subtotal": "S",
-                "formulae": formulae})
+                "sum_subtotal": "S"})
     req_and_filters = {"filters": filters,
                        "request": req}
     rsp = fastapi.post(OBJECT_SET_EXPORT_URL, headers=ADMIN_AUTH, json=req_and_filters)
@@ -144,8 +160,7 @@ def test_export_conc_biovol(config, database, fastapi, caplog):
     req = _req_tmpl.copy()
     req.update({"project_id": prj_id,
                 "exp_type": "BIV",
-                "sum_subtotal": "S",
-                "formulae": formulae})
+                "sum_subtotal": "S"})
     req_and_filters = {"filters": filters,
                        "request": req}
     rsp = fastapi.post(OBJECT_SET_EXPORT_URL, headers=ADMIN_AUTH, json=req_and_filters)
@@ -163,7 +178,7 @@ def test_export_conc_biovol(config, database, fastapi, caplog):
     # log = get_log_file(fastapi, job_id)
 
 
-def test_export_sci_filtered_by_taxo(config, database, fastapi, caplog):
+def test_export_abundances_filtered_by_taxo(config, database, fastapi, caplog):
     """ Simulate calls to export with an active filter """
     caplog.set_level(logging.FATAL)
 
@@ -172,6 +187,7 @@ def test_export_sci_filtered_by_taxo(config, database, fastapi, caplog):
     from tests.test_import import test_import
     path = str(DATA_DIR / "ref_exports" / "bak_all_images")
     prj_id = test_import(config, database, caplog, "TSV sci export filtered", path=path)
+    set_formulae_in_project(fastapi, prj_id, formulae) # Not needed
 
     # Validate all, otherwise empty report
     obj_ids = _prj_query(fastapi, CREATOR_AUTH, prj_id)
@@ -198,7 +214,7 @@ def test_export_sci_filtered_by_taxo(config, database, fastapi, caplog):
     download_and_check(fastapi, job_id, "abundances_by_sample_filtered_on_cat", only_hdr=True)
 
 
-def test_export_sci_filtered_by_sample(config, database, fastapi, caplog):
+def test_export_abundances_filtered_by_sample(config, database, fastapi, caplog):
     """ Simulate calls to export with an active filter """
     caplog.set_level(logging.FATAL)
 
@@ -207,6 +223,7 @@ def test_export_sci_filtered_by_sample(config, database, fastapi, caplog):
     from tests.test_import import test_import
     path = str(DATA_DIR / "ref_exports" / "bak_all_images")
     prj_id = test_import(config, database, caplog, "TSV sci export filtered", path=path)
+    set_formulae_in_project(fastapi, prj_id, formulae)
 
     # Validate all, otherwise empty report
     obj_ids = _prj_query(fastapi, CREATOR_AUTH, prj_id)

QA/py/tests/test_project_vars.py

@@ -71,7 +71,7 @@ def test_project_vars(config, database, fastapi, caplog):
     assert rsp.status_code == status.HTTP_200_OK, rsp.reason + str(rsp.text)
     # Good format update with good key but no val
     vars["subsample_coef"] = "  "
-    vars["individual_biovolume"] = " "
+    vars["individual_volume"] = " "
     vars["total_water_volume"] = None
     rsp = fastapi.put(url, headers=ADMIN_AUTH, json=upd_json)
     assert rsp.status_code == status.HTTP_200_OK, rsp.reason + str(rsp.text)

QA/py/tests/test_update_prj.py

@@ -258,7 +258,7 @@ def test_update_prj_bodc_vars(config, database, fastapi, caplog):
 
     vars = {"subsample_coef": "1",
             "total_water_volume": "2",
-            "individual_biovolume": "3"}
+            "individual_volume": "3"}
     settings_json[BODC_VARS_KEY] = vars
     # Now it should be OK
     rsp = fastapi.put(upd_url, headers=ADMIN_AUTH, json=settings_json)

py/API_operations/exports/ForProject.py

@@ -659,7 +659,12 @@ def create_sci_summary(self, src_project: Project) -> int:
         # The specialized SQL builder operates from the object set
         aug_qry = ObjectSetQueryPlus(object_set)
         aug_qry.remap_categories(req.pre_mapping)
-        aug_qry.set_formulae(req.formulae)
+        # Formulae default from the project but are overriden by the query
+        formulae: Dict[str, str] = {}
+        if src_project.variables is not None:
+            formulae.update(src_project.variables.to_dict())
+        formulae.update(req.formulae)
+        aug_qry.set_formulae(formulae)
         # Set common aliases, not all of them is always used
         aug_qry.set_aliases({"sam.orig_id": "sampleid",
                              "acq.orig_id": "acquisid",

py/BO/Project.py

@@ -89,7 +89,7 @@ def __init__(self, project: Project):
         self.viewers: List[User] = []
         self.annotators: List[User] = []
         self.managers: List[User] = []
-        # Formulas AKA variables
+        # Formulas AKA variables, used to compute BODC quantities
         self.bodc_variables: Dict[str, str] = {}
 
     def get_preset(self) -> ClassifIDListT:

py/BO/ProjectVars.py

@@ -15,7 +15,7 @@
 TYPES_PER_VAR = {
     "subsample_coef": (Vocabulary.subsampling_coefficient, Units.dimensionless),
     "total_water_volume": (Vocabulary.volume_sampled, Units.cubic_metres),
-    "individual_biovolume": (Vocabulary.biovolume, Units.cubic_millimetres_per_cubic_metre)
+    "individual_volume": (Vocabulary.volume_mm3, Units.cubic_millimetres)
 }
 # ...but below should prevent any de-sync
 assert KNOWN_PROJECT_VARS == set(TYPES_PER_VAR.keys())

py/BO/Vocabulary.py

@@ -29,6 +29,8 @@ class Vocabulary(object):
                             "http://vocab.nerc.ac.uk/collection/P01/current/CVOLUKNB/")
     concentrations: Final = Term("Abundance of biological entity specified elsewhere per unit volume of the water body",
                                  "http://vocab.nerc.ac.uk/collection/P01/current/SDBIOL01/")
+    volume_mm3: Final = Term("Volume in mm³",
+                             "http://vocab.nerc.ac.uk/collection/P01/current/SDBIOL01/")
 
 
 class Units(Vocabulary):
@@ -41,3 +43,5 @@ class Units(Vocabulary):
                                                     "http://vocab.nerc.ac.uk/collection/P06/current/CMCM/")
     number_per_cubic_metre: Final = Term("Number per cubic metre",
                                          "http://vocab.nerc.ac.uk/collection/P06/current/UPMM/")
+    cubic_millimetres: Final = Term("Cubic millimetres",
+                                         "http://vocab.nerc.ac.uk/collection/P06/current/?/")

py/DB/ProjectVariables.py

@@ -11,7 +11,7 @@
 from .helpers.ORM import relationship
 from .helpers.Postgres import VARCHAR, INTEGER
 
-KNOWN_PROJECT_VARS = {"subsample_coef", "total_water_volume", "individual_biovolume"}
+KNOWN_PROJECT_VARS = {"subsample_coef", "total_water_volume", "individual_volume"}
 
 
 class ProjectVariables(Model):
@@ -25,15 +25,15 @@ class ProjectVariables(Model):
     subsample_coef = Column(VARCHAR)
     # Python expression, the calculation result complies to http://vocab.nerc.ac.uk/collection/P01/current/VOLWBSMP/
     total_water_volume = Column(VARCHAR)
-    # Python expression, the calculation result complies to http://vocab.nerc.ac.uk/collection/P01/current/CVOLUKNB/
-    individual_biovolume = Column(VARCHAR)
+    # Python expression, the implied unit is mm3
+    individual_volume = Column(VARCHAR)
 
     # The relationship(s) are created in Relations.py but the typing here helps IDE
     project: relationship
 
     def __str__(self):
         return "{0}:{1}{2}{3}".format(self.project_id, self.subsample_coef, self.total_water_volume,
-                                      self.individual_biovolume)
+                                      self.individual_volume)
 
     def load_from_dict(self, vars_dict: Dict[str, Optional[str]]) -> 'ProjectVariables':
         """ Load self from a dict with proper keys """

py/cmds/db_upg/versions/521c25353fa0_new_report_tables.py

@@ -19,7 +19,7 @@ def upgrade():
     sa.Column('project_id', sa.INTEGER(), nullable=False),
     sa.Column('subsample_coef', sa.VARCHAR(), nullable=True),
     sa.Column('total_water_volume', sa.VARCHAR(), nullable=True),
-    sa.Column('individual_biovolume', sa.VARCHAR(), nullable=True),
+    sa.Column('individual_volume', sa.VARCHAR(), nullable=True),
     sa.ForeignKeyConstraint(['project_id'], ['projects.projid'], ),
     sa.PrimaryKeyConstraint('project_id')
     )