Skip to content

Commit 8fb71dc

Browse files
QuanMPhmQuanMPhm
committed
Allow direct communication to Openshift Users API
A new invoice, `MOCAGroupInvoice`, has been added. Given the code overlap between it and `PIInvoice`. Some refactoring has been made. The list of different changes in this PR is: - The Prepay Group and PI invoices now subclasses from a new invoice class, `PDFInvoice`, which contains logic specific to exporting PDFs - Unused logger removed from `PIInvoice`
1 parent 05665d9 commit 8fb71dc

6 files changed

Lines changed: 461 additions & 53 deletions

File tree

‎process_report/invoices/MOCA_group_specific_invoice.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import os
2+
from dataclasses import dataclass
3+
import tempfile
4+
5+
import pandas
6+
7+
from process_report.invoices import invoice, pdf_invoice
8+
9+
10+
@dataclass
11+
class MOCAGroupInvoice(pdf_invoice.PDFInvoice):
12+
CREDIT_COLUMN_COPY_LIST = [
13+
invoice.INVOICE_DATE_FIELD,
14+
invoice.INVOICE_EMAIL_FIELD,
15+
invoice.GROUP_NAME_FIELD,
16+
invoice.GROUP_INSTITUTION_FIELD,
17+
]
18+
TOTAL_COLUMN_LIST = [
19+
invoice.COST_FIELD,
20+
invoice.GROUP_BALANCE_USED_FIELD,
21+
invoice.CREDIT_FIELD,
22+
invoice.BALANCE_FIELD,
23+
]
24+
25+
DOLLAR_COLUMN_LIST = [
26+
invoice.RATE_FIELD,
27+
invoice.GROUP_BALANCE_FIELD,
28+
invoice.COST_FIELD,
29+
invoice.GROUP_BALANCE_USED_FIELD,
30+
invoice.CREDIT_FIELD,
31+
invoice.BALANCE_FIELD,
32+
]
33+
34+
export_columns_list = [
35+
invoice.INVOICE_DATE_FIELD,
36+
invoice.PROJECT_FIELD,
37+
invoice.PROJECT_ID_FIELD,
38+
invoice.PI_FIELD,
39+
invoice.INVOICE_EMAIL_FIELD,
40+
invoice.INVOICE_ADDRESS_FIELD,
41+
invoice.INSTITUTION_FIELD,
42+
invoice.INSTITUTION_ID_FIELD,
43+
invoice.SU_HOURS_FIELD,
44+
invoice.SU_TYPE_FIELD,
45+
invoice.RATE_FIELD,
46+
invoice.GROUP_NAME_FIELD,
47+
invoice.GROUP_INSTITUTION_FIELD,
48+
invoice.GROUP_BALANCE_FIELD,
49+
invoice.COST_FIELD,
50+
invoice.GROUP_BALANCE_USED_FIELD,
51+
invoice.CREDIT_FIELD,
52+
invoice.CREDIT_CODE_FIELD,
53+
invoice.BALANCE_FIELD,
54+
]
55+
56+
prepay_credits: pandas.DataFrame
57+
58+
def _prepare(self):
59+
self.export_data = self.data[
60+
self.data[invoice.IS_BILLABLE_FIELD] & ~self.data[invoice.MISSING_PI_FIELD]
61+
]
62+
self.export_data = self.export_data[
63+
~self.export_data[invoice.GROUP_NAME_FIELD].isna()
64+
]
65+
self.group_list = self.export_data[invoice.GROUP_NAME_FIELD].unique()
66+
67+
def _get_group_dataframe(self, data, group):
68+
group_projects = (
69+
data[data[invoice.GROUP_NAME_FIELD] == group].copy().reset_index(drop=True)
70+
)
71+
72+
# Add row for each prepay credit for the group in the invoice month
73+
group_credit_mask = (
74+
self.prepay_credits[invoice.PREPAY_MONTH_FIELD] == self.invoice_month
75+
) & (self.prepay_credits[invoice.PREPAY_GROUP_NAME_FIELD] == group)
76+
group_credit_info = self.prepay_credits[group_credit_mask]
77+
for _, credit_info in group_credit_info.iterrows():
78+
group_credit = credit_info[invoice.PREPAY_CREDIT_FIELD]
79+
group_projects.loc[len(group_projects)] = None
80+
81+
# In this "credit row", certain values should be
82+
# the same for every columns (i.e Invoice Month, Group Name, etc.)
83+
for column_name in self.CREDIT_COLUMN_COPY_LIST:
84+
if column_name in group_projects.columns:
85+
group_projects.loc[group_projects.index[-1], column_name] = (
86+
group_projects.loc[0, column_name]
87+
)
88+
89+
# Group is billed for the credit amount
90+
group_projects.loc[
91+
group_projects.index[-1], [invoice.COST_FIELD, invoice.BALANCE_FIELD]
92+
] = [group_credit] * 2
93+
94+
# Add sum row
95+
column_sums = []
96+
sum_columns_list = []
97+
for column_name in self.TOTAL_COLUMN_LIST:
98+
if column_name in group_projects.columns:
99+
column_sums.append(group_projects[column_name].sum())
100+
sum_columns_list.append(column_name)
101+
group_projects.loc[len(group_projects)] = (
102+
None # Adds a new row to end of dataframe initialized with None
103+
)
104+
group_projects.loc[group_projects.index[-1], invoice.INVOICE_DATE_FIELD] = (
105+
"Total"
106+
)
107+
group_projects.loc[group_projects.index[-1], sum_columns_list] = column_sums
108+
109+
# Add dollar signs
110+
for column_name in self.DOLLAR_COLUMN_LIST:
111+
if column_name in group_projects.columns:
112+
group_projects[column_name] = group_projects[column_name].apply(
113+
lambda data: data if pandas.isna(data) else f"${data}"
114+
)
115+
116+
group_projects.fillna("", inplace=True)
117+
118+
return group_projects
119+
120+
def export(self):
121+
self._filter_columns()
122+
123+
if not os.path.exists(self.name):
124+
os.mkdir(self.name)
125+
126+
for group in self.group_list:
127+
group_dataframe = self._get_group_dataframe(self.export_data, group)
128+
group_instituition = group_dataframe[invoice.GROUP_INSTITUTION_FIELD].iat[0]
129+
group_contact_email = group_dataframe[invoice.INVOICE_EMAIL_FIELD].iat[0]
130+
group_invoice_path = f"{self.name}/{group_instituition}_{group_contact_email}_{self.invoice_month}.pdf"
131+
132+
with tempfile.NamedTemporaryFile(mode="w", suffix=".html") as temp_fd:
133+
self._create_html_invoice(temp_fd, group_dataframe, "pi_invoice.html")
134+
self._create_pdf_invoice(temp_fd.name, group_invoice_path)

‎process_report/invoices/pdf_invoice.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import os
2+
import sys
3+
from dataclasses import dataclass
4+
import subprocess
5+
6+
import pandas
7+
from jinja2 import Environment, FileSystemLoader
8+
9+
import process_report.invoices.invoice as invoice
10+
import process_report.util as util
11+
12+
13+
TEMPLATE_DIR_PATH = "process_report/templates"
14+
15+
16+
@dataclass
17+
class PDFInvoice(invoice.Invoice):
18+
@staticmethod
19+
def _create_html_invoice(temp_fd, data: pandas.DataFrame, template_filename: str):
20+
environment = Environment(loader=FileSystemLoader(TEMPLATE_DIR_PATH))
21+
template = environment.get_template(template_filename)
22+
content = template.render(
23+
data=data,
24+
)
25+
temp_fd.write(content)
26+
temp_fd.flush()
27+
28+
@staticmethod
29+
def _create_pdf_invoice(html_filepath: str, output_pdf_path: str):
30+
chrome_binary_location = os.environ.get("CHROME_BIN_PATH", "/usr/bin/chromium")
31+
if not os.path.exists(chrome_binary_location):
32+
sys.exit(
33+
f"Chrome binary does not exist at {chrome_binary_location}. Make sure the env var CHROME_BIN_PATH is set correctly or that Google Chrome is installed"
34+
)
35+
36+
subprocess.run(
37+
[
38+
chrome_binary_location,
39+
"--headless",
40+
"--no-sandbox",
41+
f"--print-to-pdf={output_pdf_path}",
42+
"--no-pdf-header-footer",
43+
"file://" + html_filepath,
44+
],
45+
capture_output=True,
46+
)
47+
48+
def export_s3(self, s3_bucket):
49+
def _export_s3_group_invoice(invoice):
50+
invoice_path = os.path.join(self.name, invoice)
51+
striped_invoice_path = os.path.splitext(invoice_path)[0]
52+
output_s3_path = f"Invoices/{self.invoice_month}/{striped_invoice_path}.pdf"
53+
output_s3_archive_path = f"Invoices/{self.invoice_month}/Archive/{striped_invoice_path} {util.get_iso8601_time()}.pdf"
54+
s3_bucket.upload_file(invoice_path, output_s3_path)
55+
s3_bucket.upload_file(invoice_path, output_s3_archive_path)
56+
57+
# self.name is name of folder storing PDF invoices
58+
for invoice_filename in os.listdir(self.name):
59+
_export_s3_group_invoice(invoice_filename)

‎process_report/invoices/pi_specific_invoice.py‎

Lines changed: 7 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,11 @@
11
import os
2-
import sys
32
from dataclasses import dataclass
4-
import subprocess
53
import tempfile
64
import logging
75

86
import pandas
9-
from jinja2 import Environment, FileSystemLoader
107

11-
import process_report.invoices.invoice as invoice
12-
import process_report.util as util
8+
from process_report.invoices import invoice, pdf_invoice
139

1410

1511
TEMPLATE_DIR_PATH = "process_report/templates"
@@ -20,7 +16,7 @@
2016

2117

2218
@dataclass
23-
class PIInvoice(invoice.Invoice):
19+
class PIInvoice(pdf_invoice.PDFInvoice):
2420
"""
2521
This invoice operates on data processed by these Processors:
2622
- ValidateBillablePIsProcessor
@@ -110,39 +106,6 @@ def _get_pi_dataframe(self, data, pi):
110106
return pi_projects
111107

112108
def export(self):
113-
def _create_html_invoice(temp_fd):
114-
environment = Environment(loader=FileSystemLoader(TEMPLATE_DIR_PATH))
115-
template = environment.get_template("pi_invoice.html")
116-
content = template.render(
117-
data=pi_dataframe,
118-
)
119-
temp_fd.write(content)
120-
temp_fd.flush()
121-
122-
def _create_pdf_invoice(temp_fd_name):
123-
chrome_binary_location = os.environ.get(
124-
"CHROME_BIN_PATH", "/usr/bin/chromium"
125-
)
126-
if not os.path.exists(chrome_binary_location):
127-
sys.exit(
128-
f"Chrome binary does not exist at {chrome_binary_location}. Make sure the env var CHROME_BIN_PATH is set correctly and that Google Chrome is installed"
129-
)
130-
131-
invoice_pdf_path = (
132-
f"{self.name}/{pi_instituition}_{pi}_{self.invoice_month}.pdf"
133-
)
134-
subprocess.run(
135-
[
136-
chrome_binary_location,
137-
"--headless",
138-
"--no-sandbox",
139-
f"--print-to-pdf={invoice_pdf_path}",
140-
"--no-pdf-header-footer",
141-
f"file://{temp_fd_name}",
142-
],
143-
capture_output=True,
144-
)
145-
146109
self._filter_columns()
147110

148111
# self.name is name of folder storing invoices
@@ -154,19 +117,10 @@ def _create_pdf_invoice(temp_fd_name):
154117

155118
pi_dataframe = self._get_pi_dataframe(self.export_data, pi)
156119
pi_instituition = pi_dataframe[invoice.INSTITUTION_FIELD].iat[0]
120+
invoice_pdf_path = (
121+
f"{self.name}/{pi_instituition}_{pi}_{self.invoice_month}.pdf"
122+
)
157123

158124
with tempfile.NamedTemporaryFile(mode="w", suffix=".html") as temp_fd:
159-
_create_html_invoice(temp_fd)
160-
_create_pdf_invoice(temp_fd.name)
161-
162-
def export_s3(self, s3_bucket):
163-
def _export_s3_pi_invoice(pi_invoice):
164-
pi_invoice_path = os.path.join(self.name, pi_invoice)
165-
striped_invoice_path = os.path.splitext(pi_invoice_path)[0]
166-
output_s3_path = f"Invoices/{self.invoice_month}/{striped_invoice_path}.pdf"
167-
output_s3_archive_path = f"Invoices/{self.invoice_month}/Archive/{striped_invoice_path} {util.get_iso8601_time()}.pdf"
168-
s3_bucket.upload_file(pi_invoice_path, output_s3_path)
169-
s3_bucket.upload_file(pi_invoice_path, output_s3_archive_path)
170-
171-
for pi_invoice in os.listdir(self.name):
172-
_export_s3_pi_invoice(pi_invoice)
125+
self._create_html_invoice(temp_fd, pi_dataframe, "pi_invoice.html")
126+
self._create_pdf_invoice(temp_fd.name, invoice_pdf_path)

‎process_report/process_report.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
MOCA_prepaid_invoice,
2121
prepay_credits_snapshot,
2222
ocp_test_invoice,
23+
MOCA_group_specific_invoice,
2324
)
2425
from process_report.processors import (
2526
coldfront_fetch_processor,
@@ -184,6 +185,12 @@ def main():
184185
default="pi_invoices",
185186
help="Name of output folder containing pi-specific invoice csvs",
186187
)
188+
parser.add_argument(
189+
"--prepay-groups-output-folder",
190+
required=False,
191+
default="group_invoices",
192+
help="Name of output folder containing prepay-group-specific invoice PDFs",
193+
)
187194
parser.add_argument(
188195
"--BU-invoice-file",
189196
required=False,
@@ -391,6 +398,13 @@ def main():
391398
prepay_contacts=prepay_info,
392399
)
393400

401+
moca_group_inv = MOCA_group_specific_invoice.MOCAGroupInvoice(
402+
name=args.prepay_groups_output_folder,
403+
invoice_month=invoice_month,
404+
data=processed_data,
405+
prepay_credits=prepay_credits,
406+
)
407+
394408
ocp_test_inv = ocp_test_invoice.OcpTestInvoice(
395409
name="", invoice_month=invoice_month, data=processed_data.copy()
396410
)
@@ -406,6 +420,7 @@ def main():
406420
moca_prepaid_inv,
407421
prepay_credits_snap,
408422
ocp_test_inv,
423+
moca_group_inv,
409424
],
410425
args.upload_to_s3,
411426
)

0 commit comments

Comments
 (0)