Skip to content

Add feature for multiple data_source secrets #79

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 1, 2024
Merged

Add feature for multiple data_source secrets #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 57 additions & 3 deletions src/cloudforet/cost_analysis/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ def data_source_init(params: dict) -> dict:
}
"""
data_source_mgr = DataSourceManager()
print(params)
return data_source_mgr.init_response(**params)


Expand Down Expand Up @@ -67,9 +66,27 @@ def job_get_tasks(params: dict) -> dict:
}

"""
tasks = {
"tasks": [],
"changed": [],
"synced_accounts": [],
}

job_mgr = JobManager()
params["schema"] = params.pop("schema_name", None)
return job_mgr.get_tasks(**params)
secret_data = params.pop("secret_data")

secrets = secret_data.get("secrets", [secret_data])
for _secret_data in secrets:
params["secret_data"] = _secret_data
job_tasks = job_mgr.get_tasks(**params)

tasks["tasks"].extend(job_tasks.get("tasks", []))
tasks["changed"].extend(job_tasks.get("changed", []))
tasks["synced_accounts"].extend(job_tasks.get("synced_accounts", []))

tasks["changed"] = __remove_duplicate_list_of_dict(tasks.get("changed", []))
return tasks


@app.route("Cost.get_linked_accounts")
Expand All @@ -89,9 +106,17 @@ def cost_get_linked_accounts(params: dict) -> dict:
'results': 'list'
}
"""
result = []
cost_mgr = CostManager()
params["schema"] = params.pop("schema_name", None)
return cost_mgr.get_linked_accounts(**params)
secret_data = params.pop("secret_data")

secrets = secret_data.get("secrets", [secret_data])
for _secret_data in secrets:
params["secret_data"] = _secret_data
result.extend(cost_mgr.get_linked_accounts(**params))

return {"results": result}


@app.route("Cost.get_data")
Expand Down Expand Up @@ -127,7 +152,12 @@ def cost_get_data(params: dict) -> Generator[dict, None, None]:
cost_mgr = CostManager()
options = params.get("options", {})
task_options = params.get("task_options", {})
secret_data = params.pop("secret_data")

params["schema"] = params.pop("schema_name", None)
params["secret_data"] = __get_secret_data_with_tenant_id(
secret_data, task_options.get("tenant_id")
)

if options.get("cost_metric") == "AmortizedCost" and task_options.get(
"is_benefit_job", False
Expand All @@ -137,3 +167,27 @@ def cost_get_data(params: dict) -> Generator[dict, None, None]:
else:
for cost_response in cost_mgr.get_data(**params):
yield {"results": cost_response}


def __remove_duplicate_list_of_dict(changed: list) -> list:
seen = set()
unique_list = []
for changed_info in changed:
# Convert dictionary to frozenset of tuples
frozenset_changed_info = frozenset(changed_info.items())
if frozenset_changed_info not in seen:
seen.add(frozenset_changed_info)
unique_list.append(changed_info)
return unique_list


def __get_secret_data_with_tenant_id(secret_data: dict, tenant_id: str = None) -> dict:
secrets = secret_data.get("secrets", [secret_data])
if len(secrets) == 1:
return secrets[0]

for _secret_data in secrets:
if _secret_data["tenant_id"] == tenant_id:
secret_data = _secret_data

return secret_data
114 changes: 57 additions & 57 deletions src/cloudforet/cost_analysis/manager/cost_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ def __init__(self, *args, **kwargs):
self.retail_price_map = {}

def get_linked_accounts(
self,
options: dict,
secret_data: dict,
schema: str,
domain_id: str,
) -> dict:
self,
options: dict,
secret_data: dict,
schema: str,
domain_id: str,
) -> list:
self.azure_cm_connector.create_session(options, secret_data, schema)
billing_account_info = self.azure_cm_connector.get_billing_account()
agreement_type = billing_account_info.get("agreement_type")
Expand All @@ -54,15 +54,15 @@ def get_linked_accounts(
_LOGGER.debug(
f"[get_linked_accounts] total accounts count: {len(accounts_info)}, domain_id: {domain_id}"
)
return {"results": accounts_info}
return accounts_info

def get_benefit_data(
self,
options: dict,
secret_data: dict,
schema: str,
task_options: dict,
domain_id: str,
self,
options: dict,
secret_data: dict,
schema: str,
task_options: dict,
domain_id: str,
):
self.azure_cm_connector.create_session(options, secret_data, schema)
start: datetime = self._get_first_date_of_month(task_options["start"])
Expand All @@ -85,12 +85,12 @@ def get_benefit_data(
)

def _make_benefit_cost_data(
self,
results: dict,
end: datetime,
options: dict,
tenant_id: str = None,
agreement_type: str = None,
self,
results: dict,
end: datetime,
options: dict,
tenant_id: str = None,
agreement_type: str = None,
) -> list:
benefit_costs_data = []
try:
Expand Down Expand Up @@ -143,12 +143,12 @@ def _make_benefit_cost_info(self, result: dict, billed_at: str) -> dict:
return data

def get_data(
self,
options: dict,
secret_data: dict,
schema: str,
task_options: dict,
domain_id: str,
self,
options: dict,
secret_data: dict,
schema: str,
task_options: dict,
domain_id: str,
) -> list:
self.azure_cm_connector.create_session(options, secret_data, schema)
self._check_task_options(task_options)
Expand Down Expand Up @@ -202,12 +202,12 @@ def get_data(
yield []

def _make_cost_data(
self,
results: list,
end: datetime,
options: dict,
tenant_id: str = None,
agreement_type: str = None,
self,
results: list,
end: datetime,
options: dict,
tenant_id: str = None,
agreement_type: str = None,
) -> list:
"""Source Data Model"""

Expand Down Expand Up @@ -248,14 +248,14 @@ def _make_transaction_cost_data(self, tenant_id: str, end: datetime) -> list:

try:
for (
reservation_transaction
reservation_transaction
) in self.azure_cm_connector.list_reservation_transactions_by_billing_profile_id(
query_filter
):

if (
reservation_transaction.invoice_section_id.split("/")[-1]
== invoice_section_id
reservation_transaction.invoice_section_id.split("/")[-1]
== invoice_section_id
):
reservation_transaction_info = (
self.azure_cm_connector.convert_nested_dictionary(
Expand Down Expand Up @@ -312,12 +312,12 @@ def _make_transaction_cost_data(self, tenant_id: str, end: datetime) -> list:
return transaction_cost_data

def _make_data_info(
self,
result: dict,
billed_date: str,
options: dict,
tenant_id: str = None,
agreement_type: str = None,
self,
result: dict,
billed_date: str,
options: dict,
tenant_id: str = None,
agreement_type: str = None,
):
additional_info: dict = self._get_additional_info(result, options, tenant_id)
cost: float = self._get_cost_from_result_with_options(result, options)
Expand Down Expand Up @@ -391,8 +391,8 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
additional_info["Benefit Name"] = benefit_name

if (
result.get("pricingmodel") == "Reservation"
and result["metercategory"] == ""
result.get("pricingmodel") == "Reservation"
and result["metercategory"] == ""
):
result["metercategory"] = self._set_product_from_benefit_name(
benefit_name
Expand All @@ -404,14 +404,14 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
if result.get("metersubcategory") != "" and result.get("metersubcategory"):
additional_info["Meter SubCategory"] = result.get("metersubcategory")
if (
result.get("pricingmodel") == "OnDemand"
and result.get("metercategory") == ""
result.get("pricingmodel") == "OnDemand"
and result.get("metercategory") == ""
):
result["metercategory"] = result.get("metercategory")

if result.get("customername") is None:
if result.get("invoicesectionname") != "" and result.get(
"invoicesectionname"
"invoicesectionname"
):
additional_info["Department Name"] = result.get("invoicesectionname")
elif result.get("departmentname") != "" and result.get("departmentname"):
Expand All @@ -420,7 +420,7 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
if result.get("accountname") != "" and result.get("accountname"):
additional_info["Enrollment Account Name"] = result["accountname"]
elif result.get("enrollmentaccountname") != "" and result.get(
"enrollmentaccountname"
"enrollmentaccountname"
):
additional_info["Enrollment Account Name"] = result["enrollmentaccountname"]

Expand All @@ -429,9 +429,9 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non

collect_resource_id = options.get("collect_resource_id", False)
if (
collect_resource_id
and result.get("resourceid") != ""
and result.get("resourceid")
collect_resource_id
and result.get("resourceid") != ""
and result.get("resourceid")
):
additional_info["Resource Id"] = result["resourceid"]
additional_info["Resource Name"] = result["resourceid"].split("/")[-1]
Expand Down Expand Up @@ -569,10 +569,10 @@ def _get_tenant_ids(task_options: dict, collect_scope: str) -> list:

@staticmethod
def _make_scope(
secret_data: dict,
task_options: dict,
collect_scope: str,
customer_tenant_id: str = None,
secret_data: dict,
task_options: dict,
collect_scope: str,
customer_tenant_id: str = None,
):
if collect_scope == "subscription_id":
subscription_id = task_options["subscription_id"]
Expand Down Expand Up @@ -673,7 +673,7 @@ def _convert_date_format_to_utc(date_format: str) -> datetime:
return datetime.strptime(date_format, "%Y-%m-%d").replace(tzinfo=timezone.utc)

def _make_monthly_time_period(
self, start_date: datetime, end_date: datetime
self, start_date: datetime, end_date: datetime
) -> list:
monthly_time_period = []
current_date = end_date
Expand Down Expand Up @@ -703,7 +703,7 @@ def _make_monthly_time_period(

@staticmethod
def _get_linked_customer_tenants(
secret_data: dict, billing_accounts_info: list
secret_data: dict, billing_accounts_info: list
) -> list:
customer_tenants = secret_data.get("customer_tenants", [])
if not customer_tenants:
Expand All @@ -716,7 +716,7 @@ def _get_linked_customer_tenants(

@staticmethod
def _make_accounts_info_from_customer_tenants(
billing_accounts_info: list, customer_tenants: list
billing_accounts_info: list, customer_tenants: list
) -> list:
accounts_info = []
for billing_account_info in billing_accounts_info:
Expand Down Expand Up @@ -754,7 +754,7 @@ def _exclude_cost_data_with_options(result: dict, options: dict) -> bool:

@staticmethod
def _set_network_traffic_cost(
additional_info: dict, product: str, usage_type: str
additional_info: dict, product: str, usage_type: str
) -> dict:
if product in ["Bandwidth", "Content Delivery Network"]:
additional_info["Usage Type Details"] = usage_type
Expand Down
6 changes: 3 additions & 3 deletions src/cloudforet/cost_analysis/manager/job_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def get_tasks(
"account_agreement_type": billing_account_agreement_type,
"collect_scope": "customer_tenant_id",
"customer_tenants": divided_customer_tenant_info,
"tenant_id": secret_data["tenant_id"],
}
}
)
Expand All @@ -81,6 +82,7 @@ def get_tasks(
"account_agreement_type": billing_account_agreement_type,
"collect_scope": "customer_tenant_id",
"customer_tenants": first_sync_tenants,
"tenant_id": secret_data["tenant_id"],
"is_sync": False,
}
}
Expand All @@ -104,6 +106,7 @@ def get_tasks(
"start": start_month,
"account_agreement_type": billing_account_agreement_type,
"collect_scope": "billing_account_id",
"tenant_id": secret_data["tenant_id"],
"is_benefit_job": True,
}
}
Expand Down Expand Up @@ -142,9 +145,6 @@ def get_tasks(
raise ERROR_INVALID_SECRET_TYPE(secret_type=options.get("secret_type"))

tasks = {"tasks": tasks, "changed": changed, "synced_accounts": synced_accounts}

print(tasks)

return tasks

def _get_tenants_from_billing_account(self):
Expand Down
Loading