Add royalty mechanism

process_report/invoices/invoice.py

@@ -76,6 +76,7 @@ class InvoiceColumn:
 CREDIT_CODE_FIELD = "Credit Code"
 SUBSIDY_FIELD = "Subsidy"
 BALANCE_FIELD = "Balance"
+ROYALTY_FIELD = "Royalty"
 ###
 
 ### Internally used field names
@@ -86,6 +87,7 @@ class InvoiceColumn:
 GROUP_MANAGED_FIELD = "MGHPCC Managed"
 CLUSTER_NAME_FIELD = "Cluster Name"
 IS_COURSE_FIELD = "Is Course"
+IS_EXTERNALLY_FUNDED_FIELD = "Is Externally Funded"
 ###
 
 ### Initialized Column objects
@@ -142,6 +144,10 @@ class InvoiceColumn:
 IS_COURSE_COLUMN = InvoiceColumn(
     name=IS_COURSE_FIELD, dtype=BOOL_FIELD_TYPE, default_value=False
 )
+IS_EXTERNALLY_FUNDED_COLUMN = InvoiceColumn(
+    name=IS_EXTERNALLY_FUNDED_FIELD, dtype=BOOL_FIELD_TYPE, default_value=False
+)  # TODO: We are fine with this default?
+ROYALTY_COLUMN = InvoiceColumn(name=ROYALTY_FIELD, dtype=BALANCE_FIELD_TYPE)
 ###
 
 

process_report/invoices/royalty_invoice.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+
+import process_report.invoices.invoice as invoice
+
+
+@dataclass
+class RoyaltyInvoice(invoice.Invoice):
+    name: str = "Royalties"
+    export_columns_list = [  # TODO: Confirm list of information we want to include in royalty report
+        invoice.INVOICE_DATE_FIELD,
+        invoice.PROJECT_FIELD,
+        invoice.PI_FIELD,
+        invoice.CLUSTER_NAME_FIELD,
+        invoice.INSTITUTION_FIELD,
+        invoice.SU_TYPE_FIELD,
+        invoice.BALANCE_FIELD,
+        invoice.ROYALTY_FIELD,
+    ]
+
+    def _prepare_export(self):
+        self.export_data = self.data[~self.data[invoice.ROYALTY_FIELD].isna()]

process_report/loader.py

@@ -215,5 +215,10 @@ def get_nonbillable_timed_projects(self) -> list[tuple[str, str]]:
             ].itertuples(index=False, name=None)
         )
 
+    @functools.lru_cache
+    def get_royalty_exempt_institutions_list(self) -> tuple[str]:
+        with open(invoice_settings.royalty_exempt_institutions_filepath) as f:
+            return tuple(f.read().splitlines())
+
 
 loader = Loader()

process_report/process_report.py

@@ -18,6 +18,7 @@
     MOCA_prepaid_invoice,
     prepay_credits_snapshot,
     ocp_test_invoice,
+    royalty_invoice,
 )
 from process_report.processors import (
     coldfront_fetch_processor,
@@ -31,6 +32,7 @@
     bu_subsidy_processor,
     prepayment_processor,
     validate_cluster_name_processor,
+    royalty_processor,
 )
 
 PROCESSING_ORDER = [
@@ -45,6 +47,7 @@
     new_pi_credit_processor.NewPICreditProcessor,
     bu_subsidy_processor.BUSubsidyProcessor,
     prepayment_processor.PrepaymentProcessor,
+    royalty_processor.RoyaltyProcessor,
 ]
 
 
@@ -97,6 +100,7 @@ def main():
             MOCA_prepaid_invoice.MOCAPrepaidInvoice,
             prepay_credits_snapshot.PrepayCreditsSnapshot,
             ocp_test_invoice.OcpTestInvoice,
+            royalty_invoice.RoyaltyInvoice,
         ],
         invoice_settings.upload_to_s3,
     )

process_report/processors/coldfront_fetch_processor.py

@@ -25,6 +25,7 @@
 CF_ATTR_ALLOCATED_PROJECT_ID = "Allocated Project ID"
 CF_ATTR_INSTITUTION_SPECIFIC_CODE = "Institution-Specific Code"
 CF_ATTR_IS_COURSE = "Is Course?"
+CF_ATTR_IS_EXTERNALLY_FUNDED = "Is Externally Funded"
 
 
 @dataclass
@@ -34,7 +35,10 @@ class ColdfrontFetchProcessor(processor.Processor):
     )
     coldfront_data_filepath: str = invoice_settings.coldfront_api_filepath
 
-    initializes_columns = (invoice.IS_COURSE_COLUMN,)
+    initializes_columns = (
+        invoice.IS_COURSE_COLUMN,
+        invoice.IS_EXTERNALLY_FUNDED_COLUMN,
+    )
     operates_on_columns = (
         *initializes_columns,
         invoice.PROJECT_COLUMN,
@@ -125,12 +129,19 @@ def _get_allocation_data(self, coldfront_api_data):
                     project_dict["attributes"].get(CF_ATTR_IS_COURSE, "No").lower()
                     == "yes"
                 )
+                is_externally_funded = (
+                    project_dict["project"]["attributes"]
+                    .get(CF_ATTR_IS_EXTERNALLY_FUNDED, "No")
+                    .lower()
+                    == "yes"
+                )
                 allocation_data[(project_id, cluster_name)] = {
                     invoice.PROJECT_FIELD: project_name,
                     invoice.PI_FIELD: pi_name,
                     invoice.INSTITUTION_ID_FIELD: institute_code,
                     invoice.CLUSTER_NAME_FIELD: cluster_name,
                     invoice.IS_COURSE_FIELD: is_course,
+                    invoice.IS_EXTERNALLY_FUNDED_FIELD: is_externally_funded,
                 }
             except KeyError:
                 continue
@@ -164,6 +175,9 @@ def _apply_allocation_data(self, allocation_data):
                 invoice.INSTITUTION_ID_FIELD
             ]
             self.data.loc[mask, invoice.IS_COURSE_FIELD] = data[invoice.IS_COURSE_FIELD]
+            self.data.loc[mask, invoice.IS_EXTERNALLY_FUNDED_FIELD] = data[
+                invoice.IS_EXTERNALLY_FUNDED_FIELD
+            ]
 
     def _process(self):
         api_data = self._get_coldfront_api_data()

process_report/processors/prepayment_processor.py

@@ -17,7 +17,7 @@
 
 @dataclass
 class PrepaymentProcessor(discount_processor.DiscountProcessor):
-    IS_DISCOUNT_BY_NERC = True
+    IS_DISCOUNT_BY_NERC = False
     PREPAY_DEBITS_S3_FILEPATH = "Prepay/prepay_debits.csv"
 
     initializes_columns = (

process_report/processors/royalty_processor.py

@@ -0,0 +1,44 @@
+from decimal import Decimal
+import logging
+from dataclasses import dataclass, field
+
+from process_report.loader import loader
+from process_report.settings import invoice_settings
+from process_report.invoices import invoice
+from process_report.processors import processor
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+@dataclass
+class RoyaltyProcessor(processor.Processor):
+    """
+    Given a percentage royalty rate and list of exemept institutions, creates a new `Royalty` column equal to `Balance` * royalty_rate
+    """
+
+    royalty_rate: Decimal = invoice_settings.royalty_rate
+    royalty_exempt_institution_list: tuple[str] = field(
+        default_factory=loader.get_royalty_exempt_institutions_list
+    )
+
+    initializes_columns = (invoice.ROYALTY_COLUMN,)
+    operates_on_columns = (
+        *initializes_columns,
+        invoice.INSTITUTION_COLUMN,
+        invoice.IS_EXTERNALLY_FUNDED_COLUMN,
+        invoice.BALANCE_COLUMN,
+    )
+
+    def _process(self):
+        non_moc_member_mask = ~self.data[invoice.INSTITUTION_FIELD].isin(
+            self.royalty_exempt_institution_list
+        )
+        externally_funded_mask = self.data[invoice.INSTITUTION_FIELD].isin(
+            self.royalty_exempt_institution_list
+        ) & (self.data[invoice.IS_EXTERNALLY_FUNDED_FIELD] == True)  # noqa: E712
+
+        self.data[invoice.ROYALTY_FIELD] = (
+            self.data[invoice.BALANCE_FIELD] * self.royalty_rate
+        ).where(non_moc_member_mask | externally_funded_mask)

process_report/settings.py

@@ -30,6 +30,10 @@ class Settings(BaseSettings):
     prepay_credits_filepath: str = "prepaid_credits.csv"
     prepay_contacts_filepath: str = "prepaid_contacts.csv"
 
+    # Royalty configuration
+    royalty_rate: Decimal = Decimal("0.00")
+    royalty_exempt_institutions_filepath: str = "royalty_exempt_institutions.txt"
+
     # nerc_rates info
     new_pi_credit_amount: Decimal | None = None
     limit_new_pi_credit_to_partners: bool | None = None

process_report/tests/base.py

@@ -43,6 +43,8 @@
     "MGHPCC Managed": BOOL_FIELD_TYPE,
     "Cluster Name": STRING_FIELD_TYPE,
     "Is Course": BOOL_FIELD_TYPE,
+    "Is Externally Funded": BOOL_FIELD_TYPE,
+    "Royalty": BALANCE_FIELD_TYPE,
 }
 
 

process_report/tests/e2e/test_data/test_coldfront_api_data.json

@@ -2,7 +2,10 @@
     {
         "id": 1,
         "project": {
-            "pi": "pi1@bu.edu"
+            "pi": "pi1@bu.edu",
+            "attributes": {
+                "Is Externally Funded": "Yes"
+            }
         },
         "resource": {
             "name": "shift"
@@ -15,7 +18,8 @@
     {
         "id": 1,
         "project": {
-            "pi": "pi1@bu.edu"
+            "pi": "pi1@bu.edu",
+            "attributes": {}
         },
         "resource": {
             "name": "shift"
@@ -28,7 +32,8 @@
     {
         "id": 1,
         "project": {
-            "pi": "pi2@harvard.edu"
+            "pi": "pi2@harvard.edu",
+            "attributes": {}
         },
         "resource": {
             "name": "shift"

process_report/tests/e2e/test_e2e_pipeline.py

@@ -21,6 +21,7 @@
     "MOCA-A_Prepaid_Groups-2025-06-Invoice.csv",
     "NERC_Prepaid_Group-Credits-2025-06.csv",
     "OCP_TEST 2025-06.csv",
+    "Royalties 2025-06.csv",
 ]
 
 EXPECTED_DIRECTORIES = ["pi_invoices"]
@@ -127,6 +128,8 @@ def _prepare_pipeline_execution(
     env["nonbillable_pis_filepath"] = str(test_files["test_pi.yaml"])
     env["nonbillable_projects_filepath"] = str(test_files["test_projects.yaml"])
 
+    env["ROYALTY_RATE"] = "0.1"
+
     # Fallback ensures test works even when CI environment doesn't set Chrome path
     env.setdefault("CHROME_BIN_PATH", "/usr/bin/chromium")
     env["PYTHONPATH"] = str(project_root) + ":" + env.get("PYTHONPATH", "")

process_report/tests/unit/processors/test_coldfront_fetch_processor.py

@@ -15,6 +15,7 @@ def _get_test_invoice(
         institute_code=None,
         cluster_name=None,
         is_course=None,
+        externally_funded=None,
     ):
         if not pi:
             pi = [""] * len(allocation_project_id)
@@ -31,6 +32,9 @@ def _get_test_invoice(
         if not is_course:
             is_course = [False] * len(allocation_project_id)
 
+        if not externally_funded:
+            externally_funded = [False] * len(allocation_project_id)
+
         return self.create_test_invoice(
             {
                 "Manager (PI)": pi,
@@ -39,6 +43,7 @@ def _get_test_invoice(
                 "Institution - Specific Code": institute_code,
                 "Cluster Name": cluster_name,
                 "Is Course": is_course,
+                "Is Externally Funded": externally_funded,
             }
         )
 
@@ -49,6 +54,7 @@ def _get_mock_allocation_data(
         institute_code_list,
         cluster_list,
         is_course_list=None,
+        externally_funded_list=None,
     ):
         mock_data = []
         for i, project in enumerate(project_id_list):
@@ -58,6 +64,7 @@ def _get_mock_allocation_data(
                 },
                 "project": {
                     "pi": pi_list[i],
+                    "attributes": {},
                 },
                 "attributes": {
                     "Allocated Project ID": project,
@@ -69,6 +76,11 @@ def _get_mock_allocation_data(
             if is_course_list:
                 mock_project_dict["attributes"]["Is Course?"] = is_course_list[i]
 
+            if externally_funded_list:
+                mock_project_dict["project"]["attributes"]["Is Externally Funded"] = (
+                    externally_funded_list[i]
+                )
+
             mock_data.append(mock_project_dict)
 
         return mock_data
@@ -93,6 +105,7 @@ def test_coldfront_fetch(self, mock_get_allocation_data):
             ["IC1", "IC1", "", "", "IC2"],
             ["stack"] * 5,
             is_course=[False] * 5,
+            externally_funded=[False] * 5,
         )
         test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
             data=test_invoice
@@ -231,6 +244,65 @@ def test_is_course_values(self, mock_get_allocation_data):
         output_invoice = test_coldfront_fetch_proc.data
         assert output_invoice.equals(answer_invoice)
 
+    @mock.patch(
+        "process_report.processors.coldfront_fetch_processor.ColdfrontFetchProcessor._fetch_coldfront_allocation_api",
+    )
+    def test_is_externally_funded_default(self, mock_get_allocation_data):
+        """If 'Is Externally Funded' is not set in the API data, default to False"""
+        mock_get_allocation_data.return_value = self._get_mock_allocation_data(
+            ["P1", "P2", "P3"],
+            ["PI1", "PI2", "PI3"],
+            ["IC1", "IC2", "IC3"],
+            ["stack", "stack", "stack"],
+        )
+        test_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"], cluster_name=["stack", "stack", "stack"]
+        )
+        answer_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"],
+            ["P1-name", "P2-name", "P3-name"],
+            ["PI1", "PI2", "PI3"],
+            ["IC1", "IC2", "IC3"],
+            ["stack", "stack", "stack"],
+            externally_funded=[False, False, False],
+        )
+        test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
+            data=test_invoice
+        )
+        test_coldfront_fetch_proc.process()
+        output_invoice = test_coldfront_fetch_proc.data
+        assert output_invoice.equals(answer_invoice)
+
+    @mock.patch(
+        "process_report.processors.coldfront_fetch_processor.ColdfrontFetchProcessor._fetch_coldfront_allocation_api",
+    )
+    def test_is_externally_funded_values(self, mock_get_allocation_data):
+        """If 'Is Externally Funded' is set in the API data, the output 'Is Externally Funded' column reflects True/False"""
+        mock_get_allocation_data.return_value = self._get_mock_allocation_data(
+            ["P1", "P2", "P3"],
+            ["PI1", "PI2", "PI3"],
+            ["IC1", "IC2", "IC3"],
+            ["stack", "stack", "stack"],
+            externally_funded_list=["Yes", "No", "yes"],
+        )
+        test_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"], cluster_name=["stack", "stack", "stack"]
+        )
+        answer_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"],
+            ["P1-name", "P2-name", "P3-name"],
+            ["PI1", "PI2", "PI3"],
+            ["IC1", "IC2", "IC3"],
+            ["stack", "stack", "stack"],
+            externally_funded=[True, False, True],
+        )
+        test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
+            data=test_invoice
+        )
+        test_coldfront_fetch_proc.process()
+        output_invoice = test_coldfront_fetch_proc.data
+        assert output_invoice.equals(answer_invoice)
+
     @mock.patch(
         "process_report.processors.coldfront_fetch_processor.ColdfrontFetchProcessor._fetch_coldfront_allocation_api",
     )