Merge branch 'main' into esebesto/pef_subs_resources

Author: lcouzens

Diff for: example_aws_static_data.yml

@@ -1,7 +1,7 @@
 ---
   generators:
     - EC2Generator:
-        start_date: 2020-03-01
+        start_date: today
         processor_arch: 32-bit
         resource_id: 55555555
         product_sku: VEAJHRNKTJZQ
@@ -52,6 +52,19 @@
           family: 'Memory Optimized'
           cost: 1.000
           rate: 0.500
+    - EBSGenerator:
+        start_date: today
+        product_sku: VEAJHRNBBBBD
+        resource_id: 12345674
+        amount: 8
+        rate: 0.1
+        tags:
+          resourceTags/user:storageclass: gamma
+          resourceTags/user:Mapping: d1
+          resourceTags/user:Map: d2
+        cost_category:
+          costCategory/env: ephemeral
+
 
 
   accounts:

Diff for: nise/__init__.py

@@ -1,3 +1,4 @@
-__version__ = "4.6.5"
+__version__ = "4.6.9"
+
 
 VERSION = __version__.split(".")

Diff for: nise/generators/aws/data_transfer_generator.py

@@ -45,6 +45,7 @@ def __init__(self, start_date, end_date, currency, payer_account, usage_accounts
         self._rate = float(self.attributes.get("rate", 0)) or None
         self._resource_id = f"i-{self.attributes.get('resource_id', self.fake.ean8())}"
         self._saving = float(self.attributes.get("saving", 0)) or None
+        self._negation = self.attributes.get("negation") or False
         self._tags = self.attributes.get("tags", self._tags)
 
     @property
@@ -81,7 +82,8 @@ def _update_data(self, row, start, end, **kwargs):
 
         resource_id = self._resource_id if self._resource_id else self.fake.ean8()
         rate = self._rate if self._rate else round(uniform(0.12, 0.19), 3)
-        saving = self._saving if self._saving else round(uniform(0.12, 0.19), 3)
+        saving = self._saving
+        negation = self._negation
         amount = self._amount if self._amount else uniform(0.000002, 0.09)
         cost = amount * rate
         trans_desc, operation, description, location1, location2, trans_type, aws_region = self._get_data_transfer(
@@ -114,12 +116,27 @@ def _update_data(self, row, start, end, **kwargs):
         row["pricing/term"] = "OnDemand"
         row["pricing/unit"] = "GB"
         row["savingsPlan/SavingsPlanEffectiveCost"] = str(saving)
+        row["savingsPlan/SavingsPlanRate"] = str(saving)
 
         # Overwrite lineItem/LineItemType for items with applied Savings plan
         if saving is not None:
             row["lineItem/LineItemType"] = "SavingsPlanCoveredUsage"
-        self._add_tag_data(row)
-        self._add_category_data(row)
+
+        if negation:
+            row["lineItem/LineItemType"] = "SavingsPlanNegation"
+            row["lineItem/UnblendedCost"] = -abs(cost)
+            row["lineItem/UnblendedRate"] = -abs(rate)
+            row["lineItem/BlendedCost"] = -abs(cost)
+            row["lineItem/BlendedRate"] = -abs(rate)
+            row[
+                "lineItem/LineItemDescription"
+            ] = f"SavingsPlanNegation used by AccountId : {self.payer_account} and UsageSku : {self._product_sku}"
+            row["lineItem/ResourceId"] = None
+            row["savingsPlan/SavingsPlanEffectiveCost"] = None
+            row["savingsPlan/SavingsPlanRate"] = None
+        else:
+            self._add_tag_data(row)
+            self._add_category_data(row)
         return row
 
     def generate_data(self, report_type=None):

Diff for: nise/generators/aws/ebs_generator.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 """Module for ebs data generation."""
+import calendar
 from random import choice
 from random import uniform
 
@@ -33,45 +34,50 @@ def __init__(self, start_date, end_date, currency, payer_account, usage_accounts
         """Initialize the EBS generator."""
         super().__init__(start_date, end_date, currency, payer_account, usage_accounts, attributes, tag_cols)
         self._resource_id = "vol-{}".format(self.fake.ean8())
-        self._amount = uniform(0.2, 300.99)
+        self._disk_size = choice([5, 10, 15, 20, 25])
         self._rate = round(uniform(0.02, 0.16), 3)
         self._product_sku = self.fake.pystr(min_chars=12, max_chars=12).upper()
 
         if self.attributes:
             if self.attributes.get("resource_id"):
                 self._resource_id = "vol-{}".format(self.attributes.get("resource_id"))
-            if self.attributes.get("amount"):
-                self._amount = float(self.attributes.get("amount"))
             if self.attributes.get("rate"):
                 self._rate = float(self.attributes.get("rate"))
             if self.attributes.get("product_sku"):
                 self._product_sku = self.attributes.get("product_sku")
             if self.attributes.get("tags"):
                 self._tags = self.attributes.get("tags")
+            if _disk_size := self.attributes.get("disk_size"):
+                self._disk_size = int(_disk_size)
 
     def _get_storage(self):
         """Get storage data."""
         return choice(self.STORAGE)
 
+    def _calculate_hourly_rate(self, start):
+        """Calculates the houly rate based of the provided monthly rate."""
+        num_days_in_month = calendar.monthrange(start.year, start.month)[1]
+        hours_in_month = num_days_in_month * 24
+        return self._rate / hours_in_month
+
     def _update_data(self, row, start, end, **kwargs):
         """Update data with generator specific data."""
         row = self._add_common_usage_info(row, start, end)
-
-        rate = self._rate
-        amount = self._amount
-        cost = amount * rate
+        hourly_rate = self._calculate_hourly_rate(start)
+        cost = round(self._disk_size * hourly_rate, 10)
+        amount = round(cost / self._rate, 10)
         location, aws_region, _, storage_region = self._get_location()
-        description = f"${rate} per GB-Month of snapshot data stored - {location}"
+        description = f"${self._rate} per GB-Month of snapshot data stored - {location}"
         burst, max_iops, max_thru, max_vol_size, vol_backed, vol_type = self._get_storage()
 
         row["lineItem/ProductCode"] = "AmazonEC2"
         row["lineItem/UsageType"] = f"{storage_region}:VolumeUsage"
         row["lineItem/Operation"] = "CreateVolume"
         row["lineItem/ResourceId"] = self._resource_id
         row["lineItem/UsageAmount"] = str(amount)
-        row["lineItem/UnblendedRate"] = str(rate)
+        row["lineItem/UnblendedRate"] = str(self._rate)
         row["lineItem/UnblendedCost"] = str(cost)
-        row["lineItem/BlendedRate"] = str(rate)
+        row["lineItem/BlendedRate"] = str(self._rate)
         row["lineItem/BlendedCost"] = str(cost)
         row["lineItem/LineItemDescription"] = description
         row["product/ProductName"] = "Amazon Elastic Compute Cloud"
@@ -89,7 +95,7 @@ def _update_data(self, row, start, end, **kwargs):
         row["product/usagetype"] = f"{storage_region}:VolumeUsage"
         row["product/volumeType"] = vol_type
         row["pricing/publicOnDemandCost"] = str(cost)
-        row["pricing/publicOnDemandRate"] = str(rate)
+        row["pricing/publicOnDemandRate"] = str(self._rate)
         row["pricing/term"] = "OnDemand"
         row["pricing/unit"] = "GB-Mo"
         self._add_tag_data(row)

Diff for: nise/generators/aws/ec2_generator.py

@@ -36,6 +36,8 @@ class EC2Generator(AWSGenerator):
             "0.096",
             "0.096",
             "0.045",
+            1,
+            False,
             "${cost} per On Demand Linux {inst_type} Instance Hour",
         ),
         (
@@ -48,6 +50,8 @@ class EC2Generator(AWSGenerator):
             "0.34",
             "0.34",
             "0.17",
+            1,
+            False,
             "${cost} per On Demand Linux {inst_type} Instance Hour",
         ),
         (
@@ -60,6 +64,8 @@ class EC2Generator(AWSGenerator):
             "0.199",
             "0.199",
             "0.099",
+            1,
+            False,
             "${cost} per On Demand Linux {inst_type} Instance Hour",
         ),
         (
@@ -72,6 +78,8 @@ class EC2Generator(AWSGenerator):
             "0.133",
             "0.133",
             "0.067",
+            1,
+            False,
             "${cost} per On Demand Linux {inst_type} Instance Hour",
         ),
     )
@@ -115,12 +123,27 @@ def __init__(self, start_date, end_date, currency, payer_account, usage_accounts
                 instance_type.get("cost"),
                 instance_type.get("rate"),
                 instance_type.get("saving"),
+                instance_type.get("amount", "1"),
+                instance_type.get("negation", False),
                 "${cost} per On Demand Linux {inst_type} Instance Hour",
             )
 
     def _update_data(self, row, start, end, **kwargs):
         """Update data with generator specific data."""
-        inst_type, physical_cores, vcpu, memory, storage, family, cost, rate, saving, description = self._instance_type
+        (
+            inst_type,
+            physical_cores,
+            vcpu,
+            memory,
+            storage,
+            family,
+            cost,
+            rate,
+            saving,
+            amount,
+            negation,
+            description,
+        ) = self._instance_type
 
         inst_description = description.format(cost=cost, inst_type=inst_type)
         product_name = "Amazon Elastic Compute Cloud"
@@ -137,7 +160,7 @@ def _update_data(self, row, start, end, **kwargs):
         row["lineItem/Operation"] = "RunInstances"
         row["lineItem/AvailabilityZone"] = avail_zone
         row["lineItem/ResourceId"] = self._resource_id
-        row["lineItem/UsageAmount"] = "1"
+        row["lineItem/UsageAmount"] = amount
         row["lineItem/UnblendedRate"] = rate
         row["lineItem/UnblendedCost"] = cost
         row["lineItem/BlendedRate"] = rate
@@ -175,13 +198,29 @@ def _update_data(self, row, start, end, **kwargs):
         row["pricing/term"] = "OnDemand"
         row["pricing/unit"] = "Hrs"
         row["savingsPlan/SavingsPlanEffectiveCost"] = saving
+        row["savingsPlan/SavingsPlanRate"] = saving
 
         # Overwrite lineItem/LineItemType for items with applied Savings plan
         if saving is not None:
             row["lineItem/LineItemType"] = "SavingsPlanCoveredUsage"
 
-        self._add_tag_data(row)
-        self._add_category_data(row)
+        if negation:
+            row["lineItem/LineItemType"] = "SavingsPlanNegation"
+            row["lineItem/UnblendedCost"] = -abs(cost)
+            row["lineItem/UnblendedRate"] = -abs(rate)
+            row["lineItem/BlendedCost"] = -abs(cost)
+            row["lineItem/BlendedRate"] = -abs(rate)
+            row[
+                "lineItem/LineItemDescription"
+            ] = f"SavingsPlanNegation used by AccountId : {self.payer_account} and UsageSku : {self._product_sku}"
+            row["lineItem/ResourceId"] = None
+            row["savingsPlan/SavingsPlanEffectiveCost"] = None
+            row["savingsPlan/SavingsPlanRate"] = None
+
+        else:
+            self._add_tag_data(row)
+            self._add_category_data(row)
+
         return row
 
     def generate_data(self, report_type=None):

Diff for: tests/test_aws_generator.py

@@ -209,6 +209,7 @@ def setUp(self):
         self.amount = 1
         self.rate = 0.1
         self.saving = 0.1
+        self.disk_size = 10
         self.currency = "USD"
         self.legal_entity = "Red Hat"
         self.attributes = {
@@ -219,6 +220,7 @@ def setUp(self):
             "product_name": self.product_name,
             "resource_id": self.resource_id,
             "amount": self.amount,
+            "disk_size": self.disk_size,
             "rate": self.rate,
             "saving": self.saving,
             "product_family": self.product_family,
@@ -327,6 +329,21 @@ def test_update_data(self):
         self.assertEqual(row["product/productFamily"], "Data Transfer")
         self.assertEqual(row[self.cost_category_key], self.cost_category_value)
 
+    def test_update_data_transfer_negation(self):
+        """Test DataTransfer specific update data with negation costs."""
+        self.attributes = {
+            "rate": 20,
+            "amount": 1,
+            "negation": True,
+        }
+        generator = DataTransferGenerator(
+            self.two_hours_ago, self.now, self.currency, self.payer_account, self.usage_accounts, self.attributes
+        )
+        start_row = {}
+        row = generator._update_data(start_row, self.two_hours_ago, self.now)
+
+        self.assertEqual(row["lineItem/LineItemType"], "SavingsPlanNegation")
+
 
 class TestEBSGenerator(AWSGeneratorTestCase):
     """Tests for the EBS Generator type."""
@@ -339,7 +356,7 @@ def test_init_with_attributes(self):
         self.assertEqual(generator._product_sku, self.product_sku)
         self.assertEqual(generator._tags, self.tags)
         self.assertEqual(generator._resource_id, "vol-" + self.resource_id)
-        self.assertEqual(generator._amount, self.amount)
+        self.assertEqual(generator._disk_size, self.disk_size)
         self.assertEqual(generator._rate, self.rate)
 
     def test_update_data(self):
@@ -377,6 +394,8 @@ def test_init_with_attributes(self):
             "cost": "1",
             "rate": "1",
             "saving": "1",
+            "amount": 1,
+            "negation": False,
         }
         self.attributes["instance_type"] = self.instance_type
 
@@ -388,6 +407,18 @@ def test_init_with_attributes(self):
         self.assertEqual(generator._resource_id, "i-" + self.resource_id)
         self.assertEqual(generator._instance_type[:-1], tuple(self.instance_type.values()))
 
+    def test_update_data_ec2_negation(self):
+        """Test EC2 specific update data with negation costs."""
+        self.instance_type = {"negation": True, "cost": 10, "rate": 1}
+        self.attributes["instance_type"] = self.instance_type
+        generator = EC2Generator(
+            self.two_hours_ago, self.now, self.currency, self.payer_account, self.usage_accounts, self.attributes
+        )
+        start_row = {}
+        row = generator._update_data(start_row, self.two_hours_ago, self.now)
+
+        self.assertEqual(row["lineItem/LineItemType"], "SavingsPlanNegation")
+
     def test_update_data(self):
         """Test EBS specific update data method."""
         generator = EC2Generator(