Merge pull request #275 from jimmysway/feature-274/updating-invoicing-schema

Add billable override to nonbillable projects

Author: QuanMPhm

process_report/invoices/invoice.py

@@ -28,6 +28,7 @@
 NONBILLABLE_PROJECT_NAME = "Project Name"
 NONBILLABLE_CLUSTER_NAME = "Cluster"
 NONBILLABLE_IS_TIMED = "Timed"
+NONBILLABLE_IS_BILLABLE_OVERRIDE = "Is Billable Override"
 
 ### Invoice field names
 INVOICE_DATE_FIELD = "Invoice Month"

process_report/loader.py

@@ -120,10 +120,12 @@ def get_nonbillable_pis(self) -> list[str]:
     def get_nonbillable_projects(self) -> pandas.DataFrame:
         """
         Returns dataframe of nonbillable projects for current invoice month
-        The dataframe has 3 columns: Project Name, Cluster, Is Timed
+        The dataframe has 4 columns: Project Name, Cluster, Is Timed, Is Billable Override
         1. Project Name: Name of the nonbillable project
         2. Cluster: Name of the cluster for which the project is nonbillable, or None meaning all clusters
         3. Is Timed: Boolean indicating if the nonbillable status is time-bound
+        4. Is Billable Override: Optional boolean override from projects.yaml
+           indicating whether matching projects should be treated as billable
         """
 
         def _is_in_time_range(timed_object) -> bool:
@@ -140,33 +142,41 @@ def _is_in_time_range(timed_object) -> bool:
         for project in projects_dict:
             project_name = project["name"]
             cluster_list = project.get("clusters")
+            is_billable = project.get("is_billable", False)
 
             if project.get("start"):
                 if not _is_in_time_range(project):
                     continue
 
                 if cluster_list:
                     for cluster in cluster_list:
-                        project_list.append((project_name, cluster["name"], True))
+                        project_list.append(
+                            (project_name, cluster["name"], True, is_billable)
+                        )
                 else:
-                    project_list.append((project_name, None, True))
+                    project_list.append((project_name, None, True, is_billable))
             elif cluster_list:
                 for cluster in cluster_list:
                     cluster_start_time = cluster.get("start")
                     if cluster_start_time:
                         if _is_in_time_range(cluster):
-                            project_list.append((project_name, cluster["name"], True))
+                            project_list.append(
+                                (project_name, cluster["name"], True, is_billable)
+                            )
                     elif not cluster_start_time:
-                        project_list.append((project_name, cluster["name"], False))
+                        project_list.append(
+                            (project_name, cluster["name"], False, is_billable)
+                        )
             else:
-                project_list.append((project_name, None, False))
+                project_list.append((project_name, None, False, is_billable))
 
         return pandas.DataFrame(
             project_list,
             columns=[
                 invoice.NONBILLABLE_PROJECT_NAME,
                 invoice.NONBILLABLE_CLUSTER_NAME,
                 invoice.NONBILLABLE_IS_TIMED,
+                invoice.NONBILLABLE_IS_BILLABLE_OVERRIDE,
             ],
         )
 

process_report/processors/validate_billable_pi_processor.py

@@ -78,7 +78,10 @@ def _apply_lowercase(data: pandas.DataFrame, col) -> pandas.DataFrame:
     nonbillable_cluster_mask = ~merged_data[invoice.CLUSTER_NAME_FIELD].isin(
         NONBILLABLE_CLUSTERS
     )
-    return cluster_agnostic_mask & cluster_specific_mask & nonbillable_cluster_mask
+    billable_override_mask = merged_data[invoice.NONBILLABLE_IS_BILLABLE_OVERRIDE]
+    return (
+        cluster_agnostic_mask & cluster_specific_mask & nonbillable_cluster_mask
+    ) | billable_override_mask
 
 
 @dataclass

process_report/settings.py

@@ -24,7 +24,7 @@ class Settings(BaseSettings):
     prepay_debits_remote_filepath: str = "Prepay/prepay_debits.csv"
 
     # Local input files
-    nonbillable_pis_filepath: str = "pi.txt"
+    nonbillable_pis_filepath: str = "pi.yaml"
     nonbillable_projects_filepath: str = "projects.yaml"
     prepay_projects_filepath: str = "prepaid_projects.csv"
     prepay_credits_filepath: str = "prepaid_credits.csv"

process_report/tests/unit/processors/test_coldfront_fetch_processor.py

@@ -113,6 +113,7 @@ def test_coldfront_project_not_found(self, mock_get_allocation_data):
                 "Project Name": ["P3"],
                 "Cluster": [None],
                 "Is Timed": [False],
+                "Is Billable Override": [False],
             }
         )
         test_invoice = self._get_test_invoice(
@@ -189,6 +190,7 @@ def test_is_course_default_false(self, mock_get_allocation_data):
                 "Project Name": ["P3"],
                 "Cluster": [None],
                 "Is Timed": [False],
+                "Is Billable Override": [False],
             }
         )
         test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(

process_report/tests/unit/processors/test_validate_billable_pi_processor.py

@@ -43,6 +43,7 @@ def test_remove_nonbillables(self):
                     "ocp-prod",
                 ],  # P1 is cluster-agnostic, P8-bm should be nonbillable, P9 should be billable because its on bm cluster in test invoice
                 "Is Timed": [False, False, False],
+                "Is Billable Override": [False, False, False],
             }
         )
         institutions = ["Test University"] * len(pis)
@@ -67,6 +68,34 @@ def test_remove_nonbillables(self):
         output = validate_billable_pi_proc.data
         assert output[output["Is Billable"]].equals(data.iloc[[3, 4, 5, 9]])
 
+    def test_billable_override_marks_project_billable(self):
+        test_data = pandas.DataFrame(
+            {
+                "Manager (PI)": ["PI1"],
+                "Project - Allocation": ["ProjectA"],
+                "Cluster Name": ["stack"],
+                "Institution": ["Test University"],
+                "Is Course": [False],
+            }
+        )
+        nonbillable_projects = pandas.DataFrame(
+            {
+                "Project Name": ["ProjectA"],
+                "Cluster": ["stack"],
+                "Is Timed": [False],
+                "Is Billable Override": [True],
+            }
+        )
+
+        validate_billable_pi_proc = test_utils.new_validate_billable_pi_processor(
+            data=test_data,
+            nonbillable_projects=nonbillable_projects,
+        )
+        validate_billable_pi_proc.process()
+        output = validate_billable_pi_proc.data
+
+        assert output["Is Billable"].tolist() == [True]
+
     def test_empty_pi_name(self):
         test_data = pandas.DataFrame(
             {

process_report/tests/unit/test_util.py

@@ -84,6 +84,7 @@ def setUp(self):
             },
             {
                 "name": "ProjectD",
+                "is_billable": True,
                 "clusters": [
                     {"name": "Cluster1", "start": "2023-05", "end": "2023-09"},
                     {"name": "Cluster2", "start": "2023-05", "end": "2023-11"},
@@ -114,6 +115,30 @@ def test_timed_projects(self):
         ]
         assert excluded_projects == expected_projects
 
+    def test_get_nonbillable_projects_loads_yaml_into_expected_dataframe(self):
+        # This verifies the loader translates the YAML fixture into the
+        # internal dataframe shape, including the Is Billable Override column.
+        nonbillable_projects = loader.get_nonbillable_projects()
+
+        expected_projects = pandas.DataFrame(
+            [
+                ("ProjectA", "Cluster1", False, False),
+                ("ProjectA", "Cluster2", False, False),
+                ("ProjectB", "Cluster1", True, False),
+                ("ProjectD", "Cluster1", True, True),
+                ("ProjectD", "Cluster2", True, True),
+                ("ProjectE", None, False, False),
+            ],
+            columns=[
+                "Project Name",
+                "Cluster",
+                "Timed",
+                "Is Billable Override",
+            ],
+        )
+
+        assert nonbillable_projects.equals(expected_projects)
+
 
 class TestValidateRequiredEnvVars(TestCase):
     @mock.patch.dict(

process_report/tests/util.py

@@ -68,7 +68,7 @@ def new_coldfront_fetch_processor(
         data = pandas.DataFrame()
     if nonbillable_projects is None:
         nonbillable_projects = pandas.DataFrame(
-            columns=["Project Name", "Cluster", "Is Timed"]
+            columns=["Project Name", "Cluster", "Is Timed", "Is Billable Override"]
         )
     return coldfront_fetch_processor.ColdfrontFetchProcessor(
         invoice_month, data, name, nonbillable_projects, coldfront_data_filepath
@@ -110,7 +110,7 @@ def new_validate_billable_pi_processor(
         nonbillable_pis = []
     if nonbillable_projects is None:
         nonbillable_projects = pandas.DataFrame(
-            columns=["Project Name", "Cluster", "Is Timed"]
+            columns=["Project Name", "Cluster", "Is Timed", "Is Billable Override"]
         )
 
     return validate_billable_pi_processor.ValidateBillablePIsProcessor(