Merge pull request #197 from KelvinLinBU/clean-up

Removal of unused functions and variables

Author: QuanMPhm

process_report/process_report.py

@@ -1,6 +1,5 @@
 import argparse
 import sys
-import datetime
 import logging
 from decimal import Decimal
 import os
@@ -11,6 +10,7 @@
 
 from process_report import util
 from process_report.invoices import (
+    invoice,
     lenovo_invoice,
     nonbillable_invoice,
     billable_invoice,
@@ -33,33 +33,6 @@
     validate_cluster_name_processor,
 )
 
-### PI file field names
-PI_PI_FIELD = "PI"
-PI_FIRST_MONTH = "First Invoice Month"
-PI_INITIAL_CREDITS = "Initial Credits"
-PI_1ST_USED = "1st Month Used"
-PI_2ND_USED = "2nd Month Used"
-###
-
-
-### Invoice field names
-INVOICE_DATE_FIELD = "Invoice Month"
-PROJECT_FIELD = "Project - Allocation"
-PROJECT_ID_FIELD = "Project - Allocation ID"
-PI_FIELD = "Manager (PI)"
-INVOICE_EMAIL_FIELD = "Invoice Email"
-INVOICE_ADDRESS_FIELD = "Invoice Address"
-INSTITUTION_FIELD = "Institution"
-INSTITUTION_ID_FIELD = "Institution - Specific Code"
-SU_HOURS_FIELD = "SU Hours (GBhr or SUhr)"
-SU_TYPE_FIELD = "SU Type"
-RATE_FIELD = "Rate"
-COST_FIELD = "Cost"
-CREDIT_FIELD = "Credit"
-CREDIT_CODE_FIELD = "Credit Code"
-SUBSIDY_FIELD = "Subsidy"
-BALANCE_FIELD = "Balance"
-###
 
 PI_S3_FILEPATH = "PIs/PI.csv"
 ALIAS_S3_FILEPATH = "PIs/alias.csv"
@@ -96,10 +69,6 @@ def load_prepay_csv(prepay_credits_path, prepay_projects_path, prepay_contacts_p
     )
 
 
-def get_iso8601_time():
-    return datetime.datetime.now().strftime("%Y%m%dT%H%M%SZ")
-
-
 def validate_required_env_vars(required_env_vars):
     for required_env_var in required_env_vars:
         if required_env_var not in os.environ:
@@ -116,11 +85,6 @@ def main():
         nargs="*",
         help="One or more CSV files that need to be processed",
     )
-    parser.add_argument(
-        "--fetch-from-s3",
-        action="store_true",
-        help="If set, fetches invoices from S3 storage. Requires environment variables for S3 authentication to be set",
-    )
     parser.add_argument(
         "--upload-to-s3",
         action="store_true",
@@ -447,8 +411,8 @@ def merge_csv(files):
         dataframe = pandas.read_csv(
             file,
             dtype={
-                COST_FIELD: pandas.ArrowDtype(pyarrow.decimal128(12, 2)),
-                RATE_FIELD: str,
+                invoice.COST_FIELD: pandas.ArrowDtype(pyarrow.decimal128(12, 2)),
+                invoice.RATE_FIELD: str,
             },
         )
         dataframes.append(dataframe)
@@ -458,17 +422,6 @@ def merge_csv(files):
     return merged_dataframe
 
 
-def get_invoice_date(dataframe):
-    """Returns the invoice date as a pandas timestamp object
-
-    Note that it only checks the first entry because it should
-    be the same for every row.
-    """
-    invoice_date_str = dataframe[INVOICE_DATE_FIELD][0]
-    invoice_date = pandas.to_datetime(invoice_date_str, format="%Y-%m")
-    return invoice_date
-
-
 def timed_projects(timed_projects_file, invoice_date):
     """Returns list of projects that should be excluded based on dates"""
     dataframe = pandas.read_csv(timed_projects_file)
@@ -487,11 +440,9 @@ def timed_projects(timed_projects_file, invoice_date):
 
 def backup_to_s3_old_pi_file(old_pi_file):
     invoice_bucket = util.get_invoice_bucket()
-    invoice_bucket.upload_file(old_pi_file, f"PIs/Archive/PI {get_iso8601_time()}.csv")
-
-
-def export_billables(dataframe, output_file):
-    dataframe.to_csv(output_file, index=False)
+    invoice_bucket.upload_file(
+        old_pi_file, f"PIs/Archive/PI {util.get_iso8601_time()}.csv"
+    )
 
 
 def get_lenovo_su_charge_info(invoice_month, rates_info):

process_report/tests/unit/processors/test_validate_billable_pi_processor.py

@@ -8,9 +8,9 @@
 
 class TestValidateBillablePIProcessor(TestCase):
     def test_remove_nonbillables(self):
-        pis = [uuid.uuid4().hex for x in range(10)]
-        projects = [uuid.uuid4().hex for x in range(10)]
-        cluster_names = [uuid.uuid4().hex for x in range(10)]
+        pis = [uuid.uuid4().hex for _ in range(10)]
+        projects = [uuid.uuid4().hex for _ in range(10)]
+        cluster_names = [uuid.uuid4().hex for _ in range(10)]
         cluster_names[6:8] = ["ocp-test"] * 2  # Test that ocp-test is not billable
         nonbillable_pis = pis[:3]
         nonbillable_projects = [

process_report/tests/unit/test_util.py

@@ -59,21 +59,6 @@ def test_merge_csv(self):
         self.assertListEqual(merged_dataframe.columns.tolist(), self.header)
 
 
-class TestGetInvoiceDate(TestCase):
-    def test_get_invoice_date(self):
-        # The month in sample data is not the same
-        data = {"Invoice Month": ["2023-01", "2023-02", "2023-03"]}
-        dataframe = pandas.DataFrame(data)
-
-        invoice_date = process_report.get_invoice_date(dataframe)
-
-        self.assertIsInstance(invoice_date, pandas.Timestamp)
-
-        # Assert that the invoice_date is the first item
-        expected_date = pandas.Timestamp("2023-01")
-        self.assertEqual(invoice_date, expected_date)
-
-
 class TestTimedProjects(TestCase):
     def setUp(self):
         # Without the dedent method, our data will have leading spaces which

process_report/tests/util.py

@@ -2,15 +2,13 @@
 
 from process_report.invoices import (
     invoice,
-    billable_invoice,
     pi_specific_invoice,
     prepay_credits_snapshot,
     NERC_total_invoice,
 )
 
 from process_report.processors import (
     coldfront_fetch_processor,
-    add_institution_processor,
     validate_pi_alias_processor,
     lenovo_processor,
     validate_billable_pi_processor,
@@ -31,30 +29,6 @@ def new_base_invoice(
     return invoice.Invoice(name, invoice_month, data)
 
 
-def new_billable_invoice(
-    name="",
-    invoice_month="0000-00",
-    data=None,
-    nonbillable_pis=None,
-    nonbillable_projects=None,
-    old_pi_filepath="",
-    updated_old_pi_df=pandas.DataFrame(),
-):
-    if data is None:
-        data = pandas.DataFrame()
-    if nonbillable_pis is None:
-        nonbillable_pis = []
-    if nonbillable_projects is None:
-        nonbillable_projects = []
-    return billable_invoice.BillableInvoice(
-        name,
-        invoice_month,
-        data,
-        old_pi_filepath,
-        updated_old_pi_df,
-    )
-
-
 def new_pi_specific_invoice(
     name="",
     invoice_month="0000-00",
@@ -99,16 +73,6 @@ def new_coldfront_fetch_processor(
     )
 
 
-def new_add_institution_processor(
-    name="",
-    invoice_month="0000-00",
-    data=None,
-):
-    if data is None:
-        data = pandas.DataFrame()
-    return add_institution_processor.AddInstitutionProcessor(name, invoice_month, data)
-
-
 def new_validate_pi_alias_processor(
     name="", invoice_month="0000-00", data=None, alias_map=None
 ):

process_report/util.py

@@ -65,13 +65,6 @@ def get_iso8601_time():
     return datetime.datetime.now().strftime("%Y%m%dT%H%M%SZ")
 
 
-def compare_invoice_month(month_1, month_2):
-    """Returns True if 1st date is later than 2nd date"""
-    dt1 = datetime.datetime.strptime(month_1, "%Y-%m")
-    dt2 = datetime.datetime.strptime(month_2, "%Y-%m")
-    return dt1 > dt2
-
-
 def get_month_diff(month_1, month_2):
     """Returns a positive integer if month_1 is ahead in time of month_2"""
     dt1 = datetime.datetime.strptime(month_1, "%Y-%m")