Merge branch 'main' into fix/180-suppress-warning

Author: jimmysway

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: check-merge-conflict
@@ -10,7 +10,7 @@ repos:
       - id: detect-private-key
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.1
+    rev: v0.11.10
     hooks:
       - id: ruff
       - id: ruff-format

process_report/institute_list.yaml

@@ -99,3 +99,6 @@
 -   display_name: The Synergist
     domains:
     - thesynergist.org
+-   display_name: CipherSonic Labs
+    domains:
+    - ciphersoniclabs.io

process_report/invoices/NERC_total_invoice.py

@@ -49,10 +49,6 @@ def output_s3_archive_key(self):
         return f"Invoices/{self.invoice_month}/Archive/NERC-{self.invoice_month}-Total-Invoice {util.get_iso8601_time()}.csv"
 
     def _prepare_export(self):
-        def _lower_col(data):
-            if data:
-                return str.lower(data)
-
         included_institutions = list()
         institute_list = util.load_institute_list()
         for institute_info in institute_list.root:
@@ -64,5 +60,5 @@ def _lower_col(data):
         ]
         self.export_data = self.export_data[
             self.export_data[invoice.INSTITUTION_FIELD].isin(included_institutions)
-            | (self.export_data[invoice.GROUP_MANAGED_FIELD].apply(_lower_col) == "yes")
+            | self.export_data[invoice.GROUP_MANAGED_FIELD]
         ]

process_report/invoices/invoice.py

@@ -55,6 +55,7 @@
 PI_BALANCE_FIELD = "PI Balance"
 PROJECT_NAME_FIELD = "Project"
 GROUP_MANAGED_FIELD = "MGHPCC Managed"
+CLUSTER_NAME_FIELD = "Cluster Name"
 ###
 
 

process_report/processors/add_institution_processor.py

@@ -34,9 +34,9 @@ def _add_institution(self):
             if pandas.isna(pi_name):
                 logger.info(f"Project {row[invoice.PROJECT_FIELD]} has no PI")
             else:
-                self.data.at[
-                    i, invoice.INSTITUTION_FIELD
-                ] = util.get_institution_from_pi(institute_map, pi_name)
+                self.data.at[i, invoice.INSTITUTION_FIELD] = (
+                    util.get_institution_from_pi(institute_map, pi_name)
+                )
 
     def _process(self):
         self._add_institution()

process_report/processors/coldfront_fetch_processor.py

@@ -7,13 +7,15 @@
 import requests
 
 from process_report.invoices import invoice
-from process_report.processors import processor
+from process_report.processors import processor, validate_billable_pi_processor
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
 
 CF_ATTR_ALLOCATED_PROJECT_NAME = "Allocated Project Name"
+CF_ATTR_ALLOCATED_PROJECT_ID = "Allocated Project ID"
+CF_ATTR_INSTITUTION_SPECIFIC_CODE = "Institution-Specific Code"
 
 
 @dataclass
@@ -49,58 +51,69 @@ def coldfront_client(self):
         session.headers.update(headers)
         return session
 
-    def _get_project_list(self):
-        return self.data[invoice.PROJECT_FIELD].unique()
+    def _get_project_id_list(self):
+        """Returns list of project IDs from billable clusters"""
+        nonbillable_cluster_mask = ~self.data[invoice.CLUSTER_NAME_FIELD].isin(
+            validate_billable_pi_processor.NONBILLABLE_CLUSTERS
+        )
+        return self.data[nonbillable_cluster_mask][invoice.PROJECT_ID_FIELD].unique()
 
-    def _fetch_coldfront_allocation_api(self, allocation_list):
+    def _fetch_coldfront_allocation_api(self):
         coldfront_api_url = os.environ.get(
             "COLDFRONT_URL", "https://coldfront.mss.mghpcc.org/api/allocations"
         )
-        api_query_str = "&".join(
-            [
-                f"attr_{CF_ATTR_ALLOCATED_PROJECT_NAME}={project}"
-                for project in allocation_list
-            ]
-        )
-        r = self.coldfront_client.get(f"{coldfront_api_url}?{api_query_str}")
+        r = self.coldfront_client.get(f"{coldfront_api_url}?all=true")
 
         return r.json()
 
     def _get_allocation_data(self, coldfront_api_data):
+        """Returns a mapping of project IDs to a dict of project name, PI name, and institution code."""
         allocation_data = {}
-        for project, project_dict in coldfront_api_data.items():
-            allocation_data[project] = {
-                invoice.PI_FIELD: project_dict["project"]["pi"],
-                invoice.INSTITUTION_ID_FIELD: project_dict["attributes"][
-                    "Institution-Specific Code"
-                ],
-            }
+        for project_dict in coldfront_api_data:
+            try:
+                # Allow allocation to not have institute code
+                project_id = project_dict["attributes"][CF_ATTR_ALLOCATED_PROJECT_ID]
+                project_name = project_dict["attributes"][
+                    CF_ATTR_ALLOCATED_PROJECT_NAME
+                ]
+                pi_name = project_dict["project"]["pi"]
+                institute_code = project_dict["attributes"].get(
+                    CF_ATTR_INSTITUTION_SPECIFIC_CODE, "N/A"
+                )
+                allocation_data[project_id] = {
+                    invoice.PROJECT_FIELD: project_name,
+                    invoice.PI_FIELD: pi_name,
+                    invoice.INSTITUTION_ID_FIELD: institute_code,
+                }
+            except KeyError:
+                continue
+
         return allocation_data
 
     def _validate_allocation_data(self, allocation_data):
         missing_projects = (
-            set(self._get_project_list())
+            set(self._get_project_id_list())
             - set(allocation_data.keys())
             - set(self.nonbillable_projects)
         )
         missing_projects = list(missing_projects)
         missing_projects.sort()  # Ensures order for testing purposes
         if missing_projects:
-            logger.warning(
+            raise ValueError(
                 f"Projects {missing_projects} not found in Coldfront and are billable! Please check the project names"
             )
 
     def _apply_allocation_data(self, allocation_data):
-        for project, data in allocation_data.items():
-            mask = self.data[invoice.PROJECT_FIELD] == project
+        for project_id, data in allocation_data.items():
+            mask = self.data[invoice.PROJECT_ID_FIELD] == project_id
+            self.data.loc[mask, invoice.PROJECT_FIELD] = data[invoice.PROJECT_FIELD]
             self.data.loc[mask, invoice.PI_FIELD] = data[invoice.PI_FIELD]
             self.data.loc[mask, invoice.INSTITUTION_ID_FIELD] = data[
                 invoice.INSTITUTION_ID_FIELD
             ]
 
     def _process(self):
-        project_allocations_list = self._get_project_list()
-        api_data = self._fetch_coldfront_allocation_api(project_allocations_list)
+        api_data = self._fetch_coldfront_allocation_api()
         allocation_data = self._get_allocation_data(api_data)
         self._validate_allocation_data(allocation_data)
         self._apply_allocation_data(allocation_data)

process_report/processors/prepayment_processor.py

@@ -66,12 +66,12 @@ def _get_prepay_group_dict(self):
         for _, group_info in self.prepay_contacts.iterrows():
             group_name = group_info[invoice.PREPAY_GROUP_NAME_FIELD]
             prepay_group_dict[group_name] = dict()
-            prepay_group_dict[group_name][
-                invoice.PREPAY_GROUP_CONTACT_FIELD
-            ] = group_info[invoice.PREPAY_GROUP_CONTACT_FIELD]
-            prepay_group_dict[group_name][invoice.PREPAY_MANAGED_FIELD] = group_info[
-                invoice.PREPAY_MANAGED_FIELD
-            ]
+            prepay_group_dict[group_name][invoice.PREPAY_GROUP_CONTACT_FIELD] = (
+                group_info[invoice.PREPAY_GROUP_CONTACT_FIELD]
+            )
+            prepay_group_dict[group_name][invoice.PREPAY_MANAGED_FIELD] = (
+                group_info[invoice.PREPAY_MANAGED_FIELD].lower() == "yes"
+            )
             prepay_group_dict[group_name][invoice.GROUP_BALANCE_FIELD] = 0
             prepay_group_dict[group_name][invoice.PREPAY_PROJECT_FIELD] = []
 

process_report/processors/validate_billable_pi_processor.py

@@ -10,8 +10,18 @@
 logging.basicConfig(level=logging.INFO)
 
 
+NONBILLABLE_CLUSTERS = ["ocp-test"]
+
+
 @dataclass
 class ValidateBillablePIsProcessor(processor.Processor):
+    """
+    This processor validates the billable PIs and projects in the data,
+    and determines if a project is billable or not.
+
+    Every project belonging to ocp-test is nonbillable.
+    """
+
     nonbillable_pis: list[str]
     nonbillable_projects: list[str]
 
@@ -37,9 +47,13 @@ def _str_to_lowercase(data):
         nonbillable_projects_lowercase = [
             project.lower() for project in nonbillable_projects
         ]
-        return ~data[invoice.PI_FIELD].isin(nonbillable_pis) & ~data[
-            invoice.PROJECT_FIELD
-        ].apply(_str_to_lowercase).isin(nonbillable_projects_lowercase)
+        return (
+            ~data[invoice.PI_FIELD].isin(nonbillable_pis)
+            & ~data[invoice.PROJECT_FIELD]
+            .apply(_str_to_lowercase)
+            .isin(nonbillable_projects_lowercase)
+            & ~data[invoice.CLUSTER_NAME_FIELD].isin(NONBILLABLE_CLUSTERS)
+        )
 
     def _process(self):
         self.data[invoice.IS_BILLABLE_FIELD] = self._get_billables(

process_report/tests/unit/invoices/test_nerc_total_invoice.py

@@ -0,0 +1,69 @@
+from unittest import TestCase, mock
+import pandas
+
+from process_report.tests import util as test_utils
+from process_report.institute_list_models import InstituteList
+
+
+class TestNERCTotalInvoice(TestCase):
+    def _get_test_invoice(
+        self,
+        institutions,
+        is_billable=None,
+        missing_pi=None,
+        group_managed=None,
+    ):
+        if not is_billable:
+            is_billable = [True for _ in range(len(institutions))]
+
+        if not missing_pi:
+            missing_pi = [False for _ in range(len(institutions))]
+
+        if not group_managed:
+            group_managed = [True for _ in range(len(institutions))]
+
+        return pandas.DataFrame(
+            {
+                "Institution": institutions,
+                "Is Billable": is_billable,
+                "Missing PI": missing_pi,
+                "MGHPCC Managed": group_managed,
+            }
+        )
+
+    @mock.patch("process_report.util.load_institute_list")
+    def test_prepare_export(self, mock_load_institute_list):
+        """Basic test for coverage of the _prepare_export method."""
+        mock_load_institute_list.return_value = InstituteList.model_validate(
+            [
+                {
+                    "display_name": "Institution A",
+                    "domains": ["a.edu"],
+                    "include_in_nerc_total_invoice": True,
+                },
+                {
+                    "display_name": "Institution B",
+                    "domains": ["b.edu"],
+                    "include_in_nerc_total_invoice": True,
+                },
+                {
+                    "display_name": "Institution C",
+                    "domains": ["c.edu"],
+                    "include_in_nerc_total_invoice": False,
+                },
+            ]
+        )
+
+        test_invoice = self._get_test_invoice(
+            institutions=["Institution A", "Institution B", "Institution C"],
+            group_managed=[False] * 3,  # Institution C should not be included
+        )
+        answer_invoice = test_invoice.copy()[0:2]
+
+        test_inv = test_utils.new_nerc_total_invoice(
+            data=test_invoice,
+            invoice_month="2025-01",
+        )
+        test_inv._prepare_export()
+        output_invoice = test_inv.export_data
+        self.assertTrue(output_invoice.equals(answer_invoice))