Add royalty mechanism

This involved several changes:
- New field `moc_member` in institution list
- New pipeline setting `royalty_rate`, default to fetching from nerc_rates
- Coldfront processor will now obtain info on whether a project is externally funded from Coldfront API
- New processor and invoice to calculate and export the royalties

Author: QuanMPhm

process_report/invoices/invoice.py

@@ -76,6 +76,7 @@ class InvoiceColumn:
 CREDIT_CODE_FIELD = "Credit Code"
 SUBSIDY_FIELD = "Subsidy"
 BALANCE_FIELD = "Balance"
+ROYALTY_FIELD = "Royalty"
 ###
 
 ### Internally used field names
@@ -86,6 +87,7 @@ class InvoiceColumn:
 GROUP_MANAGED_FIELD = "MGHPCC Managed"
 CLUSTER_NAME_FIELD = "Cluster Name"
 IS_COURSE_FIELD = "Is Course"
+IS_EXTERNALLY_FUNDED_FIELD = "Is Externally Funded"
 ###
 
 ### Initialized Column objects
@@ -142,6 +144,10 @@ class InvoiceColumn:
 IS_COURSE_COLUMN = InvoiceColumn(
     name=IS_COURSE_FIELD, dtype=BOOL_FIELD_TYPE, default_value=False
 )
+IS_EXTERNALLY_FUNDED_COLUMN = InvoiceColumn(
+    name=IS_EXTERNALLY_FUNDED_FIELD, dtype=BOOL_FIELD_TYPE, default_value=False
+)  # TODO: We are fine with this default?
+ROYALTY_COLUMN = InvoiceColumn(name=ROYALTY_FIELD, dtype=BALANCE_FIELD_TYPE)
 ###
 
 

process_report/invoices/royalty_invoice.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+
+import process_report.invoices.invoice as invoice
+
+
+@dataclass
+class RoyaltyInvoice(invoice.Invoice):
+    name: str = "Royalties"
+    export_columns_list = [  # TODO: Confirm list of information we want to include in royalty report
+        invoice.INVOICE_DATE_FIELD,
+        invoice.PROJECT_FIELD,
+        invoice.PI_FIELD,
+        invoice.CLUSTER_NAME_FIELD,
+        invoice.INSTITUTION_FIELD,
+        invoice.SU_TYPE_FIELD,
+        invoice.BALANCE_FIELD,
+        invoice.ROYALTY_FIELD,
+    ]
+
+    def _prepare_export(self):
+        self.export_data = self.data[~self.data[invoice.ROYALTY_FIELD].isna()]

process_report/loader.py

@@ -215,5 +215,10 @@ def get_nonbillable_timed_projects(self) -> list[tuple[str, str]]:
             ].itertuples(index=False, name=None)
         )
 
+    @functools.lru_cache
+    def get_royalty_exempt_institutions_list(self) -> tuple[str]:
+        with open(invoice_settings.royalty_exempt_institutions_filepath) as f:
+            return tuple(f.read().splitlines())
+
 
 loader = Loader()

process_report/process_report.py

@@ -18,6 +18,7 @@
     MOCA_prepaid_invoice,
     prepay_credits_snapshot,
     ocp_test_invoice,
+    royalty_invoice,
 )
 from process_report.processors import (
     coldfront_fetch_processor,
@@ -31,6 +32,7 @@
     bu_subsidy_processor,
     prepayment_processor,
     validate_cluster_name_processor,
+    royalty_processor,
 )
 
 PROCESSING_ORDER = [
@@ -45,6 +47,7 @@
     new_pi_credit_processor.NewPICreditProcessor,
     bu_subsidy_processor.BUSubsidyProcessor,
     prepayment_processor.PrepaymentProcessor,
+    royalty_processor.RoyaltyProcessor,
 ]
 
 
@@ -97,6 +100,7 @@ def main():
             MOCA_prepaid_invoice.MOCAPrepaidInvoice,
             prepay_credits_snapshot.PrepayCreditsSnapshot,
             ocp_test_invoice.OcpTestInvoice,
+            royalty_invoice.RoyaltyInvoice,
         ],
         invoice_settings.upload_to_s3,
     )

process_report/processors/coldfront_fetch_processor.py

@@ -25,6 +25,7 @@
 CF_ATTR_ALLOCATED_PROJECT_ID = "Allocated Project ID"
 CF_ATTR_INSTITUTION_SPECIFIC_CODE = "Institution-Specific Code"
 CF_ATTR_IS_COURSE = "Is Course?"
+CF_ATTR_IS_EXTERNALLY_FUNDED = "Is Externally Funded"
 
 
 @dataclass
@@ -34,7 +35,10 @@ class ColdfrontFetchProcessor(processor.Processor):
     )
     coldfront_data_filepath: str = invoice_settings.coldfront_api_filepath
 
-    initializes_columns = (invoice.IS_COURSE_COLUMN,)
+    initializes_columns = (
+        invoice.IS_COURSE_COLUMN,
+        invoice.IS_EXTERNALLY_FUNDED_COLUMN,
+    )
     operates_on_columns = (
         *initializes_columns,
         invoice.PROJECT_COLUMN,
@@ -125,12 +129,19 @@ def _get_allocation_data(self, coldfront_api_data):
                     project_dict["attributes"].get(CF_ATTR_IS_COURSE, "No").lower()
                     == "yes"
                 )
+                is_externally_funded = (
+                    project_dict["project"]["attributes"]
+                    .get(CF_ATTR_IS_EXTERNALLY_FUNDED, "No")
+                    .lower()
+                    == "yes"
+                )
                 allocation_data[(project_id, cluster_name)] = {
                     invoice.PROJECT_FIELD: project_name,
                     invoice.PI_FIELD: pi_name,
                     invoice.INSTITUTION_ID_FIELD: institute_code,
                     invoice.CLUSTER_NAME_FIELD: cluster_name,
                     invoice.IS_COURSE_FIELD: is_course,
+                    invoice.IS_EXTERNALLY_FUNDED_FIELD: is_externally_funded,
                 }
             except KeyError:
                 continue
@@ -164,6 +175,9 @@ def _apply_allocation_data(self, allocation_data):
                 invoice.INSTITUTION_ID_FIELD
             ]
             self.data.loc[mask, invoice.IS_COURSE_FIELD] = data[invoice.IS_COURSE_FIELD]
+            self.data.loc[mask, invoice.IS_EXTERNALLY_FUNDED_FIELD] = data[
+                invoice.IS_EXTERNALLY_FUNDED_FIELD
+            ]
 
     def _process(self):
         api_data = self._get_coldfront_api_data()

process_report/processors/prepayment_processor.py

@@ -17,7 +17,7 @@
 
 @dataclass
 class PrepaymentProcessor(discount_processor.DiscountProcessor):
-    IS_DISCOUNT_BY_NERC = True
+    IS_DISCOUNT_BY_NERC = False
     PREPAY_DEBITS_S3_FILEPATH = "Prepay/prepay_debits.csv"
 
     initializes_columns = (

process_report/processors/royalty_processor.py

@@ -0,0 +1,44 @@
+from decimal import Decimal
+import logging
+from dataclasses import dataclass, field
+
+from process_report.loader import loader
+from process_report.settings import invoice_settings
+from process_report.invoices import invoice
+from process_report.processors import processor
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+@dataclass
+class RoyaltyProcessor(processor.Processor):
+    """
+    Given a percentage royalty rate and list of exemept institutions, creates a new `Royalty` column equal to `Balance` * royalty_rate
+    """
+
+    royalty_rate: Decimal = invoice_settings.royalty_rate
+    royalty_exempt_institution_list: tuple[str] = field(
+        default_factory=loader.get_royalty_exempt_institutions_list
+    )
+
+    initializes_columns = (invoice.ROYALTY_COLUMN,)
+    operates_on_columns = (
+        *initializes_columns,
+        invoice.INSTITUTION_COLUMN,
+        invoice.IS_EXTERNALLY_FUNDED_COLUMN,
+        invoice.BALANCE_COLUMN,
+    )
+
+    def _process(self):
+        non_moc_member_mask = ~self.data[invoice.INSTITUTION_FIELD].isin(
+            self.royalty_exempt_institution_list
+        )
+        externally_funded_mask = self.data[invoice.INSTITUTION_FIELD].isin(
+            self.royalty_exempt_institution_list
+        ) & (self.data[invoice.IS_EXTERNALLY_FUNDED_FIELD] == True)  # noqa: E712
+
+        self.data[invoice.ROYALTY_FIELD] = (
+            self.data[invoice.BALANCE_FIELD] * self.royalty_rate
+        ).where(non_moc_member_mask | externally_funded_mask)

process_report/settings.py

@@ -30,6 +30,10 @@ class Settings(BaseSettings):
     prepay_credits_filepath: str = "prepaid_credits.csv"
     prepay_contacts_filepath: str = "prepaid_contacts.csv"
 
+    # Royalty configuration
+    royalty_rate: Decimal = Decimal("0.00")
+    royalty_exempt_institutions_filepath: str = "royalty_exempt_institutions.txt"
+
     # nerc_rates info
     new_pi_credit_amount: Decimal | None = None
     limit_new_pi_credit_to_partners: bool | None = None

process_report/tests/base.py

@@ -43,6 +43,8 @@
     "MGHPCC Managed": BOOL_FIELD_TYPE,
     "Cluster Name": STRING_FIELD_TYPE,
     "Is Course": BOOL_FIELD_TYPE,
+    "Is Externally Funded": BOOL_FIELD_TYPE,
+    "Royalty": BALANCE_FIELD_TYPE,
 }
 
 

process_report/tests/e2e/test_data/test_coldfront_api_data.json

@@ -2,7 +2,10 @@
     {
         "id": 1,
         "project": {
-            "pi": "pi1@bu.edu"
+            "pi": "pi1@bu.edu",
+            "attributes": {
+                "Is Externally Funded": "Yes"
+            }
         },
         "resource": {
             "name": "shift"
@@ -15,7 +18,8 @@
     {
         "id": 1,
         "project": {
-            "pi": "pi1@bu.edu"
+            "pi": "pi1@bu.edu",
+            "attributes": {}
         },
         "resource": {
             "name": "shift"
@@ -28,7 +32,8 @@
     {
         "id": 1,
         "project": {
-            "pi": "pi2@harvard.edu"
+            "pi": "pi2@harvard.edu",
+            "attributes": {}
         },
         "resource": {
             "name": "shift"