Allow supplemental API data to be passed to ColdfrontFetchProcessor

This allows passing data for billable projects not currently in Coldfront
Namely bare metal projects

Added unit test and updated e2e test

Author: QuanMPhm

process_report/loader.py

@@ -208,5 +208,13 @@ def get_nonbillable_timed_projects(self) -> list[tuple[str, str]]:
             ].itertuples(index=False, name=None)
         )
 
+    def get_supplement_api_data(self) -> list[dict]:
+        supplemental_data = []
+        if invoice_settings.supplement_api_data_filepath:
+            with open(invoice_settings.supplement_api_data_filepath) as f:
+                supplemental_data = yaml.safe_load(f)
+
+        return supplemental_data
+
 
 loader = Loader()

process_report/processors/coldfront_fetch_processor.py

@@ -33,6 +33,9 @@ class ColdfrontFetchProcessor(processor.Processor):
         default_factory=loader.get_nonbillable_projects
     )
     coldfront_data_filepath: str = invoice_settings.coldfront_api_filepath
+    supplement_api_data: list[dict] = field(
+        default_factory=lambda: loader.get_supplement_api_data()
+    )
 
     @functools.cached_property
     def coldfront_client(self):
@@ -96,7 +99,8 @@ def _get_coldfront_api_data(self):
     def _get_allocation_data(self, coldfront_api_data):
         """Returns a mapping of (project ID, cluster name) tupels to a dict of project name, PI name, and institution code."""
         allocation_data = {}
-        for project_dict in coldfront_api_data:
+        combined_api_data = coldfront_api_data + self.supplement_api_data
+        for project_dict in combined_api_data:
             try:
                 # Allow allocation to not have institute code
                 project_id = project_dict["attributes"][CF_ATTR_ALLOCATED_PROJECT_ID]

process_report/settings.py

@@ -8,6 +8,7 @@
 class Settings(BaseSettings):
     # Coldfront info
     coldfront_api_filepath: str | None = None
+    supplement_api_data_filepath: str | None = None
     keycloak_client_id: str | None = None
     keycloak_client_secret: str | None = None
 

process_report/tests/e2e/test_data/test_PI.csv

@@ -1,4 +1,5 @@
 PI,First Invoice Month,Initial Credits,1st Month Used,2nd Month Used
+pi4@harvard.edu,2025-06,1000,400.00,0
 pi5@harvard.edu,2025-06,0.00,0.00,0.00
 pi6@mit.edu,2025-06,0.00,0.00,0.00
 pi4@example.edu,2025-06,0.00,0.00,0.00

process_report/tests/e2e/test_data/test_invoices/test_NERC OpenShift 2025-04.csv

@@ -3,4 +3,5 @@ Invoice Month,Project - Allocation,Project - Allocation ID,Manager (PI),Cluster
 2024-01,P1ID,P1ID,,shift,,,,,280,OpenStack GPUA100SXM4,0.013,100
 2024-01,P2ID,P2ID,,shift,,,,,280,OpenShift CPU,0.013,200
 2024-01,P3ID,P3ID,,shift,,,,,280,OpenShift CPU,0.013,300
+2024-01,P4-supplement,P4-supplement,,shift,,,,,300,OpenShift CPU,0.013,400
 2024-01,P9ID,P9ID,,shift,,,,,280,OpenShift CPU,0.013,3000

process_report/tests/e2e/test_data/test_supplement_api_data.yaml

@@ -0,0 +1,7 @@
+- project:
+    pi: pi4@harvard.edu
+  attributes:
+    Allocated Project ID: P4-supplement
+    Allocated Project Name: P4-supplement-name
+  resource:
+    name: shift

process_report/tests/e2e/test_e2e_pipeline.py

@@ -113,6 +113,9 @@ def _prepare_pipeline_execution(
     env = os.environ.copy()
     env["INVOICE_MONTH"] = INVOICE_MONTH
     env["COLDFRONT_API_FILEPATH"] = str(test_files["test_coldfront_api_data.json"])
+    env["SUPPLEMENT_API_DATA_FILEPATH"] = str(
+        test_files["test_supplement_api_data.yaml"]
+    )
     env["FETCH_FROM_S3"] = "false"
     env["UPLOAD_TO_S3"] = "false"
     env["invoice_path_template"] = str(test_files["test_invoice_dir"])

process_report/tests/unit/processors/test_coldfront_fetch_processor.py

@@ -259,3 +259,48 @@ def test_missing_project_cluster_tuples(self, mock_get_allocation_data):
         assert str(cm.value) == (
             f"Projects {expected_missing} not found in Coldfront and are billable! Please check the project names"
         )
+
+    @mock.patch(
+        "process_report.processors.coldfront_fetch_processor.ColdfrontFetchProcessor._fetch_coldfront_allocation_api",
+    )
+    def test_supplement_api_data_used_when_coldfront_missing(
+        self, mock_get_allocation_data
+    ):
+        """Supplement API rows are applied to invoice in _get_allocation_data()."""
+        mock_get_allocation_data.return_value = self._get_mock_allocation_data(
+            ["P1"],
+            ["PI1"],
+            ["IC1"],
+            ["stack"],
+        )
+
+        # Supplemental data should follow same structure as Coldfront data,
+        # only missing "Is Course?" and "Institution-Specific Code" fields
+        supplemental_df = self._get_mock_allocation_data(
+            ["P2"],
+            ["PI2"],
+            ["Delete Institude Code"],
+            ["stack"],
+        )
+        del supplemental_df[0]["attributes"]["Institution-Specific Code"]
+
+        test_invoice = self._get_test_invoice(
+            ["P1", "P2"], cluster_name=["stack", "stack"]
+        )
+
+        expected_invoice = self._get_test_invoice(
+            ["P1", "P2"],
+            ["P1-name", "P2-name"],
+            ["PI1", "PI2"],
+            ["IC1", "N/A"],
+            ["stack", "stack"],
+            [False, False],
+        )
+
+        test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
+            data=test_invoice, supplement_api_data=supplemental_df
+        )
+        test_coldfront_fetch_proc.process()
+        output_invoice = test_coldfront_fetch_proc.data
+
+        assert output_invoice.equals(expected_invoice)

process_report/tests/util.py

@@ -63,15 +63,23 @@ def new_coldfront_fetch_processor(
     data=None,
     nonbillable_projects=None,
     coldfront_data_filepath=None,
+    supplement_api_data=None,
 ):
     if data is None:
         data = pandas.DataFrame()
     if nonbillable_projects is None:
         nonbillable_projects = pandas.DataFrame(
             columns=["Project Name", "Cluster", "Is Timed", "Is Billable Override"]
         )
+    if supplement_api_data is None:
+        supplement_api_data = []
     return coldfront_fetch_processor.ColdfrontFetchProcessor(
-        invoice_month, data, name, nonbillable_projects, coldfront_data_filepath
+        invoice_month,
+        data,
+        name,
+        nonbillable_projects,
+        coldfront_data_filepath,
+        supplement_api_data,
     )