m

Author: jimmysway

process_report/data_tools/config.py

@@ -33,7 +33,7 @@ def iceberg_s3_properties(self) -> dict[str, str]:
         return {
             "s3.access-key-id": self.iceberg_s3_access_key_id,
             "s3.secret-access-key": self.iceberg_s3_secret_access_key,
-            "s3.endpoint": f"https://{self.iceberg_s3_endpoint}",
+            "s3.endpoint": self.iceberg_s3_endpoint,
             "s3.region": self.iceberg_s3_region,
         }
 

process_report/tests/unit/data_tools/test_data_tools.py

@@ -3,9 +3,10 @@
 
 from process_report.data_tools import costs
 
-PID = costs.invoice.PROJECT_ID_FIELD
-CLUSTER = costs.invoice.CLUSTER_NAME_FIELD
-BALANCE = costs.invoice.BALANCE_FIELD
+# These are the column names in the iceberg table using string literals instead of the invoice module to test column name correctness
+PID = "Project - Allocation ID"
+CLUSTER = "Cluster Name"
+BALANCE = "Balance"
 
 
 @pytest.fixture(autouse=True)

pyproject.toml

@@ -6,16 +6,14 @@ readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
     "nerc-rates>=1.0.1,<2.0.0",
-    "pandas>=3.0.0",
+    "pandas>=3.0.0,<4.0.0",
     "pyarrow",
     "pyiceberg[pyarrow]>=0.11.0",
     "boto3>=1.42.6,<2.0",
     "jinja2",
     "validators",
     "python-dateutil",
     "pydantic-settings",
-    "pyyaml>=6.0",
-    "pre-commit>=4.5.1",
 ]
 
 [build-system]