Resolved future warnings regarding dtype compatibility

Rebased ontop of latest upstream branch

Updated dtypes in pipeline and in unit tests to strings instead of objects

Author: jimmysway

process_report/invoices/pi_specific_invoice.py

@@ -85,24 +85,28 @@ def _get_pi_dataframe(self, data, pi):
                 axis=1,
             )
 
-        # Add a row containing sums for certain columns
-        column_sums = []
-        sum_columns_list = []
+        # Create new row with proper dtypes instead of concatenating with None
+        # Calculate totals first
+        column_sums = {}
         for column_name in self.TOTAL_COLUMN_LIST:
             if column_name in pi_projects.columns:
-                column_sums.append(pi_projects[column_name].sum())
-                sum_columns_list.append(column_name)
+                column_sums[column_name] = pi_projects[column_name].sum()
 
-        # Add a row with None values (this will convert int64 columns to float64 and bool to object)
-        pi_projects.loc[len(pi_projects)] = None
+        # Use pandas.concat with future-compatible approach
+        # Create an empty row first, then populate it
+        pi_projects = pi_projects.copy()  # Ensure we have a copy
+        new_index = len(pi_projects)
+
+        # Add empty row by reindexing
+        pi_projects = pi_projects.reindex(range(len(pi_projects) + 1))
 
         # Set Invoice Month and totals - add Invoice Month column if it doesn't exist
         if invoice.INVOICE_DATE_FIELD not in pi_projects.columns:
-            pi_projects[invoice.INVOICE_DATE_FIELD] = None
+            pi_projects[invoice.INVOICE_DATE_FIELD] = ""
 
-        pi_projects.loc[pi_projects.index[-1], invoice.INVOICE_DATE_FIELD] = "Total"
-        for col, val in zip(sum_columns_list, column_sums):
-            pi_projects.loc[pi_projects.index[-1], col] = val
+        pi_projects.loc[new_index, invoice.INVOICE_DATE_FIELD] = "Total"
+        for col, val in column_sums.items():
+            pi_projects.loc[new_index, col] = val
 
         # Add dollar sign to certain columns
         for column_name in self.DOLLAR_COLUMN_LIST:
@@ -111,7 +115,14 @@ def _get_pi_dataframe(self, data, pi):
                     lambda data: data if pandas.isna(data) else f"${data}"
                 )
 
-        pi_projects.fillna("", inplace=True)
+        # Fill NaN values selectively - only fill non-numeric columns with empty strings
+        # Keep numeric columns as they are to preserve their dtypes
+        for col in pi_projects.columns:
+            if not pandas.api.types.is_numeric_dtype(pi_projects[col]):
+                pi_projects[col] = pi_projects[col].fillna("")
+
+        # Convert any remaining pandas NA values to empty strings for template compatibility
+        pi_projects = pi_projects.fillna("")
 
         return pi_projects
 

process_report/process_report.py

@@ -440,6 +440,8 @@ def merge_csv(files):
             dtype={
                 COST_FIELD: pandas.ArrowDtype(pyarrow.decimal128(12, 2)),
                 RATE_FIELD: str,
+                PI_FIELD: "string",  # Use pandas string dtype for proper string handling
+                INSTITUTION_ID_FIELD: "string",
             },
         )
         dataframes.append(dataframe)

process_report/processors/coldfront_fetch_processor.py

@@ -116,6 +116,14 @@ def _validate_allocation_data(self, allocation_data):
             )
 
     def _apply_allocation_data(self, allocation_data):
+        # Convert columns to string dtype to handle string values properly
+        if invoice.PI_FIELD in self.data.columns:
+            self.data[invoice.PI_FIELD] = self.data[invoice.PI_FIELD].astype("string")
+        if invoice.INSTITUTION_ID_FIELD in self.data.columns:
+            self.data[invoice.INSTITUTION_ID_FIELD] = self.data[
+                invoice.INSTITUTION_ID_FIELD
+            ].astype("string")
+
         for project_id, data in allocation_data.items():
             mask = self.data[invoice.PROJECT_ID_FIELD] == project_id
             self.data.loc[mask, invoice.PROJECT_FIELD] = data[invoice.PROJECT_FIELD]

process_report/processors/discount_processor.py

@@ -51,9 +51,26 @@ def apply_discount_on_project(remaining_discount_amount, project_i, project):
             remaining_project_balance = project[pi_balance_field]
             applied_discount = min(remaining_project_balance, remaining_discount_amount)
             invoice.at[project_i, discount_field] = applied_discount
-            invoice.at[project_i, pi_balance_field] -= applied_discount
+
+            # Convert applied_discount to the same dtype as the balance columns
+            pi_balance_dtype = invoice[pi_balance_field].dtype
+            balance_dtype = invoice[balance_field].dtype
+
+            applied_discount_pi = applied_discount
+            if hasattr(applied_discount, "astype"):
+                applied_discount_pi = applied_discount.astype(pi_balance_dtype)
+            elif not isinstance(applied_discount, pi_balance_dtype.type):
+                applied_discount_pi = pi_balance_dtype.type(applied_discount)
+
+            applied_discount_balance = applied_discount
+            if hasattr(applied_discount, "astype"):
+                applied_discount_balance = applied_discount.astype(balance_dtype)
+            elif not isinstance(applied_discount, balance_dtype.type):
+                applied_discount_balance = balance_dtype.type(applied_discount)
+
+            invoice.at[project_i, pi_balance_field] -= applied_discount_pi
             if self.IS_DISCOUNT_BY_NERC:
-                invoice.at[project_i, balance_field] -= applied_discount
+                invoice.at[project_i, balance_field] -= applied_discount_balance
             remaining_discount_amount -= applied_discount
             return remaining_discount_amount
 

process_report/tests/unit/processors/test_coldfront_fetch_processor.py

@@ -69,6 +69,12 @@ def test_coldfront_fetch(self, mock_get_allocation_data):
             ["PI1", "PI1", "PI1", "", "PI12"],
             ["IC1", "IC1", "", "", "IC2"],
         )
+        # Convert columns to string dtype to match processor output
+        answer_invoice["Manager (PI)"] = answer_invoice["Manager (PI)"].astype("string")
+        answer_invoice["Institution - Specific Code"] = answer_invoice[
+            "Institution - Specific Code"
+        ].astype("string")
+
         test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
             data=test_invoice
         )
@@ -123,6 +129,12 @@ def test_nonbillable_clusters(self, mock_get_allocation_data):
             ["IC1", "IC2", "", ""],
             ["ocp-prod", "stack", "ocp-test", "ocp-test"],
         )
+        # Convert columns to string dtype to match processor output
+        answer_invoice["Manager (PI)"] = answer_invoice["Manager (PI)"].astype("string")
+        answer_invoice["Institution - Specific Code"] = answer_invoice[
+            "Institution - Specific Code"
+        ].astype("string")
+
         test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
             data=test_invoice
         )