From 1196a65490c928994f531efdadd03e1f3e2ce1c4 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Tue, 26 May 2026 13:49:27 +0530 Subject: [PATCH] Squash-Upgrade ElasticSearch/OpenSearch client libraries --- .github/workflows/Build.yml | 5 +- .github/workflows/PR.yml | 6 +- .../elasticsearch/elasticsearch_operations.py | 98 +++++++++---------- .../main/environment_variables.py | 1 + cloud_governance/main/es_uploader.py | 6 +- cloud_governance/main/main.py | 2 - .../cloud_resource_orchestration/Jenkinsfile | 2 + .../run_cloud_resource_orchestration.py | 8 +- .../aws/daily/cost_explorer/Jenkinsfile | 2 + .../aws/daily/cost_explorer/run_upload_es.py | 17 ++-- .../aws/daily/org_cost_explorer/Jenkinsfile | 2 + .../org_cost_explorer/run_org_upload_es.py | 12 ++- jenkins/clouds/aws/daily/policies/Jenkinsfile | 2 + .../clouds/aws/daily/policies/run_policies.py | 8 +- jenkins/clouds/aws/monthly/Jenkinsfile | 2 + .../aws/monthly/run_monthly_policies.py | 4 +- .../aws/weekly/cost_over_usage/Jenkinsfile | 2 + .../weekly/cost_over_usage/run_upload_es.py | 8 +- .../azure/daily/cost_reports/Jenkinsfile | 2 + .../azure/daily/cost_reports/run_policies.py | 5 +- .../clouds/azure/daily/policies/Jenkinsfile | 2 + .../azure/daily/policies/run_policies.py | 4 +- .../clouds/gcp/daily/cost_reports/Jenkinsfile | 2 + .../gcp/daily/cost_reports/run_reports.py | 5 +- .../ibm/daily/cost_billings/Jenkinsfile | 2 + .../cost_billings/run_ibm_cost_reports.py | 6 +- .../ibm/monthly/cost_invoice/Jenkinsfile | 2 + .../monthly/cost_invoice/ibm_invoice_to_es.py | 4 +- .../tenant/aws/common/run_cost_policies.py | 8 +- jenkins/tenant/aws/common/run_policies.py | 8 +- .../aws/ecoeng_01/PolicyJenkinsfileDaily | 1 + .../aws/ecoeng_02/PolicyJenkinsfileDaily | 1 + .../aws/ecoeng_03/PolicyJenkinsfileDaily | 1 + jenkins/tenant/aws/ovn/PolicyJenkinsfileDaily | 2 + jenkins/tenant/aws/ovn/run_policies.py | 4 +- .../aws/qe/qe_01/PolicyJenkinsfileDaily | 2 + .../aws/qe/qe_02/PolicyJenkinsfileDaily | 2 + requirements.txt | 6 +- setup.py | 8 +- .../mocks/elasticsearch/mock_elasticsearch.py | 5 +- tests_requirements.txt | 4 +- 41 files changed, 172 insertions(+), 101 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 1c508f7fe..a37d5bd42 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -125,10 +125,11 @@ jobs: matrix: python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 + opensearch: + image: opensearchproject/opensearch:3.2.0 env: discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" options: >- --health-cmd "curl http://localhost:9200/_cluster/health" --health-interval 10s diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 6e22ebb4b..ecb9c4680 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -103,17 +103,17 @@ jobs: matrix: python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 + opensearch: + image: opensearchproject/opensearch:3.2.0 env: discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" options: >- --health-cmd "curl http://localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 ports: - # : - 9200:9200 steps: - uses: actions/checkout@v6 diff --git a/cloud_governance/common/elasticsearch/elasticsearch_operations.py b/cloud_governance/common/elasticsearch/elasticsearch_operations.py index 9ef8abf93..af6faa830 100644 --- a/cloud_governance/common/elasticsearch/elasticsearch_operations.py +++ b/cloud_governance/common/elasticsearch/elasticsearch_operations.py @@ -1,12 +1,13 @@ from datetime import datetime, timezone import time import pandas as pd -from elasticsearch.helpers import bulk from cloud_governance.main.environment_variables import environment_variables -from elasticsearch_dsl import Search +from opensearchpy import OpenSearch +from opensearchpy.helpers import bulk as opensearch_bulk from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk as es_bulk from typeguard import typechecked from cloud_governance.common.elasticsearch.elasticsearch_exceptions import ElasticSearchDataNotUploaded @@ -46,69 +47,63 @@ def __init__(self, self.__environment_variables_dict.get('ES_TIMEOUT')) if self.__environment_variables_dict.get( 'ES_TIMEOUT') else timeout self.__account = self.__environment_variables_dict.get('account') + self.__server_type = self.__environment_variables_dict.get('ES_SERVER_TYPE', 'opensearch') try: - add_host = {'host': self.__es_host, 'port': self.__es_port, - 'http_auth': f'{self.__es_user}:{self.__es_password}'} - if int(self.__es_port) == 443: - add_host['use_ssl'] = True - self.__es = Elasticsearch([add_host], - timeout=self.__timeout, - max_retries=2) + if self.__server_type == 'elasticsearch': + scheme = 'https' if int(self.__es_port) == 443 else 'http' + hosts = [{'host': self.__es_host, 'port': int(self.__es_port), 'scheme': scheme}] + basic_auth = (self.__es_user, self.__es_password) if self.__es_user else None + self.__es = Elasticsearch(hosts, basic_auth=basic_auth, verify_certs=False, + request_timeout=self.__timeout, max_retries=2) + self.__bulk_fn = es_bulk + else: + add_host = {'host': self.__es_host, 'port': self.__es_port} + if self.__es_user: + add_host['http_auth'] = (self.__es_user, self.__es_password) + if int(self.__es_port) == 443: + add_host['use_ssl'] = True + add_host['verify_certs'] = False + self.__es = OpenSearch([add_host], timeout=self.__timeout, max_retries=2) + self.__bulk_fn = opensearch_bulk except Exception as err: + logger.error(f'Failed to connect to {self.__server_type} at {self.__es_host}:{self.__es_port}: {err}') self.__es = None - # Skip product check for OpenSearch compatibility (elasticsearch-py 7.14+ rejects non-Elasticsearch servers) - try: - if self.__es and hasattr(self.__es.transport, '_verified_elasticsearch'): - self.__es.transport._verified_elasticsearch = True - except AttributeError as err: - logger.warning(f"Could not bypass Elasticsearch product check: {err}") - def __elasticsearch_get_index_hits(self, index: str, uuid: str = '', workload: str = '', fast_check: bool = False, id: bool = False): """ - This method search for data per index in last 2 minutes and return the number of docs or zero + This method search for data per index in last 15 minutes and return the number of docs or zero :param index: - :param workload: need only if there is different timestamp parameter in Elasticsearch + :param workload: need only if there is different timestamp parameter in the server :param id: True to return the doc ids :param fast_check: return fast response :return: """ - """ - :return: - """ ids = [] - # https://github.com/elastic/elasticsearch-dsl-py/issues/49 self.__es.indices.refresh(index=index) - # timestamp name in Elasticsearch is different - search = Search(using=self.__es, index=index).filter('range', timestamp={ - 'gte': f'now-{self.ES_FETCH_MIN_TIME}m', 'lt': 'now'}) - # reduce the search result - if fast_check: - search = search[0:self.MIN_SEARCH_RESULTS] - else: - search = search[0:self.MAX_SEARCH_RESULTS] - search_response = search.execute() - if search_response.hits: + size = self.MIN_SEARCH_RESULTS if fast_check else self.MAX_SEARCH_RESULTS + query = {"query": {"range": {"timestamp": {"gte": f"now-{self.ES_FETCH_MIN_TIME}m", "lt": "now"}}}} + search_response = self.__es.search(index=index, body=query, size=size) + hits = search_response.get('hits', {}).get('hits', []) + if hits: if uuid: count_hits = 0 - for row in search_response: - if type(row['uuid']) == str: - # uperf return str - current_uuid = row['uuid'] - else: - current_uuid = row['uuid'][0] + for row in hits: + source = row['_source'] + current_uuid = source.get('uuid', '') + if isinstance(current_uuid, list): + current_uuid = current_uuid[0] if current_uuid == uuid: if fast_check: return 1 - ids.append(row.meta.id) + ids.append(row['_id']) count_hits += 1 if id: return ids else: return count_hits else: - return len(search_response.hits) + return len(hits) else: return 0 @@ -141,13 +136,12 @@ def verify_elasticsearch_data_uploaded(self, index: str, uuid: str = '', workloa raise ElasticSearchDataNotUploaded @typechecked() - def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc', es_add_items: dict = None, + def upload_to_elasticsearch(self, index: str, data: dict, es_add_items: dict = None, **kwargs): """ This method is upload json data into elasticsearch :param index: index name to be stored in elasticsearch :param data: data must be in dictionary i.e. {'key': 'value'} - :param doc_type: :param es_add_items: :return: """ @@ -178,10 +172,10 @@ def upload_to_elasticsearch(self, index: str, data: dict, doc_type: str = '_doc' kwargs['id'] = data.get('IndexId') try: if isinstance(data, dict): # JSON Object - self.__es.index(index=index, doc_type=doc_type, body=data, **kwargs) + self.__es.index(index=index, body=data, **kwargs) else: # JSON Array for record in data: - self.__es.index(index=index, doc_type=doc_type, body=record, **kwargs) + self.__es.index(index=index, body=record, **kwargs) return True except Exception as err: raise err @@ -217,12 +211,12 @@ def get_index_hits(self, days: int, index: str): @param index: @return: """ - search = Search(using=self.__es, index=index).filter('range', timestamp={'gte': f'now-{days}d', 'lt': 'now'}) - search = search[0:self.MAX_SEARCH_RESULTS] - search_response = search.execute() + query = {"query": {"range": {"timestamp": {"gte": f"now-{days}d", "lt": "now"}}}} + search_response = self.__es.search(index=index, body=query, size=self.MAX_SEARCH_RESULTS) + hits = search_response.get('hits', {}).get('hits', []) df = pd.DataFrame() - for row in search_response: - df = pd.concat([df, pd.DataFrame([row.to_dict()])], ignore_index=True).fillna({}) + for row in hits: + df = pd.concat([df, pd.DataFrame([row['_source']])], ignore_index=True).fillna({}) return df.to_dict('records') @typechecked() @@ -273,7 +267,7 @@ def fetch_data_by_es_query(self, es_index: str, query: dict = None, start_dateti if start_datetime and end_datetime: query = self.get_query_data_between_range(start_datetime=start_datetime, end_datetime=end_datetime) if query: - response = self.__es.search(index=es_index, body=query, doc_type='_doc', size=search_size, scroll='1h', + response = self.__es.search(index=es_index, body=query, size=search_size, scroll='1h', filter_path=filter_path) if result_agg: es_data.extend(response.get('aggregations').get(group_by).get('buckets')) @@ -363,7 +357,7 @@ def upload_data_in_bulk(self, data_items: list, index: str, **kwargs): if 'CleanUpDays' not in item: item['ExpireDays'] = self.__environment_variables_dict.get('DAYS_TO_TAKE_ACTION') item['policy'] = self.__environment_variables_dict.get('policy') - response = bulk(self.__es, bulk_items) + response = self.__bulk_fn(self.__es, bulk_items) if response: total_uploaded += len(bulk_items) else: @@ -399,7 +393,7 @@ def post_query(self, query: dict, es_index: str, result_agg: bool = False): if result_agg: return response.get('aggregations') else: - return response.get('hits', {}).get('hits', {}) + return response.get('hits', {}).get('hits', []) except Exception as err: logger.error(err) raise err diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index f4cd6b728..b43e6bbdb 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -210,6 +210,7 @@ def __init__(self): self._environment_variables_dict['es_index'] = EnvironmentVariables.get_env('es_index', es_index) self._environment_variables_dict['es_doc_type'] = EnvironmentVariables.get_env('es_doc_type', '') self._environment_variables_dict['ES_TIMEOUT'] = EnvironmentVariables.get_env('ES_TIMEOUT', 2000) + self._environment_variables_dict['ES_SERVER_TYPE'] = EnvironmentVariables.get_env('ES_SERVER_TYPE', 'opensearch') # GitHub credentials self._environment_variables_dict['git_access_token'] = EnvironmentVariables.get_env('git_access_token', '') diff --git a/cloud_governance/main/es_uploader.py b/cloud_governance/main/es_uploader.py index 396737624..e552a8b4c 100644 --- a/cloud_governance/main/es_uploader.py +++ b/cloud_governance/main/es_uploader.py @@ -17,7 +17,6 @@ def __init__(self, **kwargs): self.__es_host = kwargs.get('es_host') self.__es_port = kwargs.get('es_port') self.__es_index = kwargs.get('es_index') - self.__es_doc_type = kwargs.get('es_doc_type') self.__es_add_items = kwargs.get('es_add_items') self.__bucket_name = kwargs.get('bucket') self.__s3_file_name = kwargs.get('s3_file_name') @@ -95,14 +94,13 @@ def __get_user_cost(self, data): num += 1 return user_cost_results - def upload_last_policy_to_elasticsearch(self, policy: str, index: str, doc_type: str, s3_json_file: str, + def upload_last_policy_to_elasticsearch(self, policy: str, index: str, s3_json_file: str, es_add_items: dict = None): """ This method is upload json kubernetes cluster data into elasticsearch :param policy: :param s3_json_file: :param index: - :param doc_type: :param es_add_items: :return: """ @@ -210,5 +208,5 @@ def upload_to_es(self, account): """ self.__es_add_items.update({'policy': self.__policy_name, 'region': self.__region_name}) self.upload_last_policy_to_elasticsearch(policy=self.__policy_name, index=self.__es_index, - doc_type=self.__es_doc_type, s3_json_file=self.__s3_file_name, + s3_json_file=self.__s3_file_name, es_add_items=self.__es_add_items) diff --git a/cloud_governance/main/main.py b/cloud_governance/main/main.py index 96f6d98d8..9f875be8c 100644 --- a/cloud_governance/main/main.py +++ b/cloud_governance/main/main.py @@ -218,7 +218,6 @@ def main(): es_host = environment_variables_dict.get('es_host', '') es_port = environment_variables_dict.get('es_port', '') es_index = environment_variables_dict.get('es_index', '') - es_doc_type = environment_variables_dict.get('es_doc_type', '') bucket = environment_variables_dict.get('bucket', '') main_operations = MainOperations() response = main_operations.run() @@ -309,7 +308,6 @@ def run_gcp_policy_runner(): input_data = {'es_host': es_host, 'es_port': int(es_port), 'es_index': es_index, - 'es_doc_type': es_doc_type, 'es_add_items': {'account': account}, 'bucket': bucket, 'logs_bucket_key': 'logs', diff --git a/jenkins/cloud_resource_orchestration/Jenkinsfile b/jenkins/cloud_resource_orchestration/Jenkinsfile index b08babfae..570c3744e 100644 --- a/jenkins/cloud_resource_orchestration/Jenkinsfile +++ b/jenkins/cloud_resource_orchestration/Jenkinsfile @@ -16,6 +16,8 @@ pipeline { AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE = credentials('cloud-governance-aws-secret-access-key-delete-perf-scale') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') CLOUD_GOVERNANCE_SPECIAL_USER_MAILS = credentials('cloud-governance-special-user-mails') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') JIRA_URL = credentials('JIRA_URL') diff --git a/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py b/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py index 6e806a05a..a2f73acec 100644 --- a/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py +++ b/jenkins/cloud_resource_orchestration/run_cloud_resource_orchestration.py @@ -8,6 +8,8 @@ AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] JIRA_URL = os.environ['JIRA_URL'] JIRA_USERNAME = os.environ['JIRA_USERNAME'] @@ -30,7 +32,8 @@ es_index = CLOUD_RESOURCE_ORCHESTRATION_INDEX common_env_vars = { - 'es_host': ES_HOST, 'es_port': ES_PORT, 'CRO_ES_INDEX': CRO_ES_INDEX, 'log_level': 'INFO', + 'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'CRO_ES_INDEX': CRO_ES_INDEX, 'log_level': 'INFO', 'LDAP_HOST_NAME': LDAP_HOST_NAME, 'JIRA_QUEUE': JIRA_QUEUE, 'JIRA_TOKEN': JIRA_TOKEN, 'JIRA_USERNAME': JIRA_USERNAME, 'JIRA_URL': JIRA_URL, 'CRO_COST_OVER_USAGE': CRO_COST_OVER_USAGE, 'CRO_PORTAL': CRO_PORTAL, 'CRO_DEFAULT_ADMINS': CRO_DEFAULT_ADMINS, @@ -48,7 +51,8 @@ os.system('echo Run CloudResourceOrchestration in pre active region') -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'CRO_ES_INDEX': CRO_ES_INDEX, 'log_level': 'INFO', +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'CRO_ES_INDEX': CRO_ES_INDEX, 'log_level': 'INFO', 'LDAP_HOST_NAME': LDAP_HOST_NAME, 'JIRA_QUEUE': JIRA_QUEUE, 'JIRA_TOKEN': JIRA_TOKEN, 'JIRA_USERNAME': JIRA_USERNAME, 'JIRA_URL': JIRA_URL, diff --git a/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile b/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile index fa647cc6c..e068793ac 100644 --- a/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile +++ b/jenkins/clouds/aws/daily/cost_explorer/Jenkinsfile @@ -26,6 +26,8 @@ pipeline { BUCKET_PERF_SCALE = credentials('cloud-governance-bucket-perf_scale') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') GITHUB_TOKEN = credentials('cloud-governance-git-access-token') CLOUD_GOVERNANCE_SPECIAL_USER_MAILS = credentials('cloud-governance-special-user-mails') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') diff --git a/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py b/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py index bab428de2..b0ab547d5 100644 --- a/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py +++ b/jenkins/clouds/aws/daily/cost_explorer/run_upload_es.py @@ -18,6 +18,8 @@ BUCKET_PERF_SCALE = os.environ['BUCKET_PERF_SCALE'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] special_user_mails = os.environ['CLOUD_GOVERNANCE_SPECIAL_USER_MAILS'] COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] @@ -35,19 +37,19 @@ cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost granularity = 'DAILY' # DAILY/MONTHLY/HOURLY os.system( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_perf}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_psap}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_psap}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index={es_index_perf_scale} -e cost_explorer_tags="{cost_tags}" -e granularity={granularity} -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index={es_index_perf_scale} -e cost_explorer_tags="{cost_tags}" -e granularity={granularity} -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") es_index_global = 'cloud-governance-cost-explorer-perf-global-cost' os.system( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-dept" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="psap" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="perf-scale" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_global}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") input_vars_to_container = [{'account': 'perf-dept', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PERF, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PERF}, @@ -56,7 +58,8 @@ {'account': 'psap', 'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID_DELETE_PSAP, 'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY_DELETE_PSAP}] -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-global-cost-billing-reports', +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'es_index': 'cloud-governance-global-cost-billing-reports', 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS} combine_vars = lambda item: f'{item[0]}="{item[1]}"' common_envs = list(map(combine_vars, common_input_vars.items())) diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile b/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile index 7dc3f3294..45a914464 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile +++ b/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile @@ -12,6 +12,8 @@ pipeline { AWS_SECRET_ACCESS_KEY_DELETE_PERF = credentials('cloud-governance-aws-secret-access-key-delete-perf') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') COST_SPREADSHEET_ID = credentials('cloud-governance-cost-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') AWS_ACCOUNT_ROLE = credentials('cloud-governance-aws-account-role') diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py index d98dcc6d7..9c942b3bb 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py +++ b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py @@ -4,6 +4,8 @@ AWS_SECRET_ACCESS_KEY_DELETE_PERF = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PERF'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] AWS_ACCOUNT_ROLE = os.environ['AWS_ACCOUNT_ROLE'] @@ -33,7 +35,8 @@ cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost granularity = 'DAILY' # DAILY/MONTHLY/HOURLY -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-global-cost-billing-reports', +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'es_index': 'cloud-governance-global-cost-billing-reports', 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, 'COST_CENTER_OWNER': f"{COST_CENTER_OWNER}", 'REPLACE_ACCOUNT_NAME': REPLACE_ACCOUNT_NAME, 'PAYER_SUPPORT_FEE_CREDIT': PAYER_SUPPORT_FEE_CREDIT} @@ -48,7 +51,7 @@ os.system(f"""podman run --rm --net="host" --name cloud-governance -e policy="spot_savings_analysis" -e account="pnt-payer" \ -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_ATHIRUMA_BOT}" \ -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_ATHIRUMA_BOT}" \ --e es_host="{ES_HOST}" -e es_port="{ES_PORT}" \ +-e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" \ -e es_index="cloud-governance-clouds-billing-reports" \ -e S3_RESULTS_PATH="{S3_RESULTS_PATH}" \ -e ATHENA_DATABASE_NAME="{ATHENA_DATABASE_NAME}" \ @@ -65,6 +68,8 @@ -e account="{account}" \ -e es_host="{ES_HOST}" \ -e es_port="{ES_PORT}" \ +-e es_user="{ES_USER}" \ +-e es_password="{ES_PASSWORD}" \ -e es_index="cloud-governance-policy-es-index" \ -e log_level="INFO" \ {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") @@ -104,7 +109,8 @@ def generate_shell_cmd(policy: str, env_variables: dict, mounted_volumes: str = common_env_vars = { - 'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': COST_ES_INDEX, + 'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'es_index': COST_ES_INDEX, 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, 'SPREADSHEET_ID': COST_SPREADSHEET_ID, } diff --git a/jenkins/clouds/aws/daily/policies/Jenkinsfile b/jenkins/clouds/aws/daily/policies/Jenkinsfile index c4178a1a5..adaff5f64 100644 --- a/jenkins/clouds/aws/daily/policies/Jenkinsfile +++ b/jenkins/clouds/aws/daily/policies/Jenkinsfile @@ -18,6 +18,8 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') ES_INDEX = credentials('cloud-governance-es-index') GITHUB_TOKEN = credentials('cloud-governance-git-access-token') CLOUD_GOVERNANCE_SPECIAL_USER_MAILS = credentials('cloud-governance-special-user-mails') diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index c35d02899..e476f0837 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -14,6 +14,8 @@ account_admin = os.environ['ACCOUNT_ADMIN'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') ES_INDEX = os.environ.get('ES_INDEX') GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] @@ -88,7 +90,7 @@ def get_container_cmd(env_dict: dict): "account": account_name, "AWS_DEFAULT_REGION": "us-east-1", "PUBLIC_CLOUD_NAME": "AWS", "AWS_ACCESS_KEY_ID": access_key, "AWS_SECRET_ACCESS_KEY": secret_key, "dry_run": "yes", "LDAP_HOST_NAME": LDAP_HOST_NAME, "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource, - "es_host": ES_HOST, "es_port": ES_PORT, + "es_host": ES_HOST, "es_port": ES_PORT, "es_user": ES_USER, "es_password": ES_PASSWORD, "MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO", 'DAYS_TO_TAKE_ACTION': days_to_delete_resource, 'special_user_mails': f"{special_user_mails}", 'account_admin': f"{account_admin}", @@ -133,7 +135,7 @@ def run_policies(policies: list, dry_run: str = 'yes'): # Running the trust advisor reports, data dumped into default index - cloud-governance-policy-es-index run_cmd( - f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="optimize_resources_report" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") # Git-leaks run on GitHub not related to any aws account # run_cmd("echo Run Git-leaks") @@ -145,4 +147,4 @@ def run_policies(policies: list, dry_run: str = 'yes'): run_cmd("echo Run Aggregated Email Alert") run_cmd( - f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/aws/monthly/Jenkinsfile b/jenkins/clouds/aws/monthly/Jenkinsfile index 8dd4db75a..ac4366c6d 100644 --- a/jenkins/clouds/aws/monthly/Jenkinsfile +++ b/jenkins/clouds/aws/monthly/Jenkinsfile @@ -10,6 +10,8 @@ pipeline { QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') TO_MAIL = credentials('cloud-governance-to-mail') CC_MAIL = credentials('cloud-governance-cc-mail') diff --git a/jenkins/clouds/aws/monthly/run_monthly_policies.py b/jenkins/clouds/aws/monthly/run_monthly_policies.py index 26cdaa038..4aeef9493 100644 --- a/jenkins/clouds/aws/monthly/run_monthly_policies.py +++ b/jenkins/clouds/aws/monthly/run_monthly_policies.py @@ -2,6 +2,8 @@ ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') TO_MAIL = os.environ['TO_MAIL'] CC_MAIL = os.environ['CC_MAIL'] QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ['QUAY_CLOUD_GOVERNANCE_REPOSITORY'] @@ -11,4 +13,4 @@ # RUN AWS Monthly Policies print("Run AWS Monthly Policies") os.system( - f"""podman run --rm --name cloud-governance --net="host" -e policy="monthly_report" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance --net="host" -e policy="monthly_report" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile b/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile index 0aae75984..d5804af70 100644 --- a/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile +++ b/jenkins/clouds/aws/weekly/cost_over_usage/Jenkinsfile @@ -19,6 +19,8 @@ pipeline { BUCKET_PERF_SCALE = credentials('cloud-governance-bucket-perf_scale') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') GITHUB_TOKEN = credentials('cloud-governance-git-access-token') CLOUD_GOVERNANCE_SPECIAL_USER_MAILS = credentials('cloud-governance-special-user-mails') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') diff --git a/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py b/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py index 72ff7cdc1..e3d9eb996 100644 --- a/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py +++ b/jenkins/clouds/aws/weekly/cost_over_usage/run_upload_es.py @@ -11,6 +11,8 @@ BUCKET_PERF_SCALE = os.environ['BUCKET_PERF_SCALE'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME'] special_user_mails = os.environ['CLOUD_GOVERNANCE_SPECIAL_USER_MAILS'] IGNORE_MAILS = os.environ['IGNORE_MAILS'] @@ -21,8 +23,8 @@ es_index_perf_scale = 'cloud-governance-cost-explorer-perf-scale' os.system( - f"""podman run --rm --name cloud-governance --net="host" -e account="perf-dept" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PERF}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance --net="host" -e account="perf-dept" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_perf}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PERF}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --name cloud-governance --net="host" -e account="psap" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_psap}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PSAP}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance --net="host" -e account="psap" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PSAP}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PSAP}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_psap}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e log_level="INFO" -e policy_output="{BUCKET_PSAP}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system( - f"""podman run --rm --name cloud-governance --net="host" -e account="perf-scale" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{es_index_perf_scale}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e policy_output="{BUCKET_PERF_SCALE}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance --net="host" -e account="perf-scale" -e policy="cost_over_usage" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF_SCALE}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF_SCALE}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e es_index="{es_index_perf_scale}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e special_user_mails="{special_user_mails}" -e IGNORE_MAILS="{IGNORE_MAILS}" -e policy_output="{BUCKET_PERF_SCALE}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile b/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile index ecbe5dbac..b375623cc 100644 --- a/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile +++ b/jenkins/clouds/azure/daily/cost_reports/Jenkinsfile @@ -14,6 +14,8 @@ pipeline { AZURE_CLIENT_ID = credentials('cloud-governance-azure-client-id') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') COST_SPREADSHEET_ID = credentials('cloud-governance-cost-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') diff --git a/jenkins/clouds/azure/daily/cost_reports/run_policies.py b/jenkins/clouds/azure/daily/cost_reports/run_policies.py index 8a1a36763..13c3e0c2d 100644 --- a/jenkins/clouds/azure/daily/cost_reports/run_policies.py +++ b/jenkins/clouds/azure/daily/cost_reports/run_policies.py @@ -5,6 +5,8 @@ AZURE_CLIENT_ID = os.environ['AZURE_CLIENT_ID'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] AZURE_ACCOUNT_ID = os.environ['AZURE_ACCOUNT_ID'] @@ -15,7 +17,8 @@ 'AZURE_TENANT_ID': AZURE_TENANT_ID, 'AZURE_CLIENT_SECRET': AZURE_CLIENT_SECRET, 'AZURE_ACCOUNT_ID': AZURE_ACCOUNT_ID}] -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-clouds-billing-reports', +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'es_index': 'cloud-governance-clouds-billing-reports', 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS} combine_vars = lambda item: f'{item[0]}="{item[1]}"' common_envs = list(map(combine_vars, common_input_vars.items())) diff --git a/jenkins/clouds/azure/daily/policies/Jenkinsfile b/jenkins/clouds/azure/daily/policies/Jenkinsfile index df9cbcd7c..8fb60cd4e 100644 --- a/jenkins/clouds/azure/daily/policies/Jenkinsfile +++ b/jenkins/clouds/azure/daily/policies/Jenkinsfile @@ -16,6 +16,8 @@ pipeline { AZURE_CLIENT_ID = credentials('cloud-governance-azure-client-id') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') GLOBAL_TAGS = credentials('GLOBAL_TAGS') contact1 = "ebattat@redhat.com" diff --git a/jenkins/clouds/azure/daily/policies/run_policies.py b/jenkins/clouds/azure/daily/policies/run_policies.py index 214e0b0cd..3046ffdb0 100644 --- a/jenkins/clouds/azure/daily/policies/run_policies.py +++ b/jenkins/clouds/azure/daily/policies/run_policies.py @@ -13,6 +13,8 @@ LOGS = os.environ.get('LOGS', 'logs') ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ['QUAY_CLOUD_GOVERNANCE_REPOSITORY'] @@ -78,7 +80,7 @@ def get_container_cmd(env_dict: dict): "dry_run": "yes", "LDAP_HOST_NAME": LDAP_HOST_NAME, "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource, - "es_host": ES_HOST, "es_port": ES_PORT, + "es_host": ES_HOST, "es_port": ES_PORT, "es_user": ES_USER, "es_password": ES_PASSWORD, "MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO", 'DAYS_TO_TAKE_ACTION': days_to_delete_resource, 'GLOBAL_TAGS': GLOBAL_TAGS diff --git a/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile b/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile index 053de3977..29f9a30cf 100644 --- a/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile +++ b/jenkins/clouds/gcp/daily/cost_reports/Jenkinsfile @@ -12,6 +12,8 @@ pipeline { GCP_DATABASE_TABLE_NAME = credentials('cloud-governance-gcp-database-table-name') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') COST_SPREADSHEET_ID = credentials('cloud-governance-cost-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') diff --git a/jenkins/clouds/gcp/daily/cost_reports/run_reports.py b/jenkins/clouds/gcp/daily/cost_reports/run_reports.py index 8c187ee72..539d82152 100644 --- a/jenkins/clouds/gcp/daily/cost_reports/run_reports.py +++ b/jenkins/clouds/gcp/daily/cost_reports/run_reports.py @@ -4,13 +4,16 @@ GCP_DATABASE_TABLE_NAME = os.environ['GCP_DATABASE_TABLE_NAME'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') COST_SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ['QUAY_CLOUD_GOVERNANCE_REPOSITORY'] print('Running the GCP cost billing reports') -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-clouds-billing-reports', +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_user': ES_USER, 'es_password': ES_PASSWORD, + 'es_index': 'cloud-governance-clouds-billing-reports', 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, 'PUBLIC_CLOUD_NAME': 'GCP', 'SPREADSHEET_ID': COST_SPREADSHEET_ID, 'GCP_DATABASE_NAME': GCP_DATABASE_NAME, 'GCP_DATABASE_TABLE_NAME': GCP_DATABASE_TABLE_NAME} diff --git a/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile b/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile index ab06e7725..de1c14168 100644 --- a/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile +++ b/jenkins/clouds/ibm/daily/cost_billings/Jenkinsfile @@ -10,6 +10,8 @@ pipeline { QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') IBM_API_USERNAME_PERFORMANCE_SCALE = credentials('cloud-governance-ibm-api-username-performance-scale') IBM_API_KEY_PERFORMANCE_SCALE = credentials('cloud-governance-ibm-api-key-performance-scale') IBM_ACCOUNT_ID_PERFORMANCE_SCALE = credentials('cloud-governance-ibm-account-id-performance-scale') diff --git a/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py b/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py index 33e845644..56f8dc1d3 100644 --- a/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py +++ b/jenkins/clouds/ibm/daily/cost_billings/run_ibm_cost_reports.py @@ -13,6 +13,8 @@ SPREADSHEET_ID = os.environ['COST_SPREADSHEET_ID'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') LOGS = os.environ.get('LOGS', 'logs') QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ['QUAY_CLOUD_GOVERNANCE_REPOSITORY'] @@ -30,7 +32,7 @@ for keys in key_list: os.system( - f"""podman run --rm --net="host" --name cloud-governance -e account="{keys.get('account')}" -e COST_CENTER_OWNER="Shai" -e policy="cost_billing_reports" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e "IBM_API_USERNAME"="{keys.get('IBM_API_USERNAME')}" -e IBM_API_KEY="{keys.get('IBM_API_KEY')}" -e USAGE_REPORTS_APIKEY="{keys.get('USAGE_REPORTS_APIKEY')}" -e IBM_ACCOUNT_ID="{keys.get('IBM_ACCOUNT_ID')}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e account="{keys.get('account')}" -e COST_CENTER_OWNER="Shai" -e policy="cost_billing_reports" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e "IBM_API_USERNAME"="{keys.get('IBM_API_USERNAME')}" -e IBM_API_KEY="{keys.get('IBM_API_KEY')}" -e USAGE_REPORTS_APIKEY="{keys.get('USAGE_REPORTS_APIKEY')}" -e IBM_ACCOUNT_ID="{keys.get('IBM_ACCOUNT_ID')}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") # Cloudability reports # Cloudability env variables @@ -82,6 +84,8 @@ def run_shell_cmd(cmd: str): "IBM_ACCOUNT_ID": IBM_ACCOUNT_ID_PERFORMANCE_SCALE, "es_host": ES_HOST, "es_port": ES_PORT, + "es_user": ES_USER, + "es_password": ES_PASSWORD, "es_index": "cloudability-cloud-governance-ibm-cost-usage-reports", "PUBLIC_CLOUD_NAME": "IBM" } diff --git a/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile b/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile index 97acd760e..d718d13c5 100644 --- a/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile +++ b/jenkins/clouds/ibm/monthly/cost_invoice/Jenkinsfile @@ -10,6 +10,8 @@ pipeline { QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') IBM_API_USERNAME = credentials('cloud-governance-ibm-api-username-performance-scale') IBM_API_KEY = credentials('cloud-governance-ibm-api-key-performance-scale') IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') diff --git a/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py b/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py index 942c2a90a..7db4ef42c 100644 --- a/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py +++ b/jenkins/clouds/ibm/monthly/cost_invoice/ibm_invoice_to_es.py @@ -7,6 +7,8 @@ SPREADSHEET_ID = os.environ['IAM_USER_SPREADSHEET_ID'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') LOGS = os.environ.get('LOGS', 'logs') QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ['QUAY_CLOUD_GOVERNANCE_REPOSITORY'] @@ -15,4 +17,4 @@ es_index = 'cloud-governance-ibm-invoice-cost' os.system( - f"""podman run --rm --net="host" --name cloud-governance -e account="IBM-PERF" -e policy="ibm_cost_report" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --net="host" --name cloud-governance -e account="IBM-PERF" -e policy="ibm_cost_report" -e es_index="{es_index}" -e es_port="{ES_PORT}" -e es_host="{ES_HOST}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/common/run_cost_policies.py b/jenkins/tenant/aws/common/run_cost_policies.py index c96b65867..f5401df9d 100644 --- a/jenkins/tenant/aws/common/run_cost_policies.py +++ b/jenkins/tenant/aws/common/run_cost_policies.py @@ -5,6 +5,9 @@ account_name = os.environ['account_name'] ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_SERVER_TYPE = os.environ.get('ES_SERVER_TYPE', 'opensearch') +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') ES_INDEX = os.environ.get('ES_INDEX', None) QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ['QUAY_CLOUD_GOVERNANCE_REPOSITORY'] @@ -20,7 +23,7 @@ os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) os.system( - f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" {env_es_index} -e es_port="{ES_PORT}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" {env_es_index} -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e ES_SERVER_TYPE="{ES_SERVER_TYPE}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") os.system(f"""echo "Running yearly savings report for account {account_name}" """) os.system(f"""podman run --rm --net="host" --name cloud-governance -e policy="yearly_savings_report" \ @@ -28,6 +31,9 @@ -e account="{account_name}" \ -e es_host="{ES_HOST}" \ -e es_port="{ES_PORT}" \ +-e es_user="{ES_USER}" \ +-e es_password="{ES_PASSWORD}" \ +-e ES_SERVER_TYPE="{ES_SERVER_TYPE}" \ -e es_index="cloud-governance-policy-es-index" \ -e log_level="INFO" \ {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py index 0b7791101..cc0cee7f6 100644 --- a/jenkins/tenant/aws/common/run_policies.py +++ b/jenkins/tenant/aws/common/run_policies.py @@ -54,6 +54,9 @@ def run_cmd(cmd: str): ALERT_DRY_RUN = os.environ.get('ALERT_DRY_RUN', False) ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_SERVER_TYPE = os.environ.get('ES_SERVER_TYPE', 'opensearch') +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] ADMIN_MAIL_LIST = os.environ.get('ADMIN_MAIL_LIST', '') @@ -91,7 +94,8 @@ def get_container_cmd(env_dict: dict): "account": account_name, "AWS_DEFAULT_REGION": "us-east-1", "PUBLIC_CLOUD_NAME": "AWS", "AWS_ACCESS_KEY_ID": access_key, "AWS_SECRET_ACCESS_KEY": secret_key, "dry_run": "yes", "LDAP_HOST_NAME": LDAP_HOST_NAME, "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource, - "es_host": ES_HOST, "es_port": ES_PORT, + "es_host": ES_HOST, "es_port": ES_PORT, "es_user": ES_USER, "es_password": ES_PASSWORD, + "ES_SERVER_TYPE": ES_SERVER_TYPE, "MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO", 'DAYS_TO_TAKE_ACTION': days_to_delete_resource, 'ALERT_DRY_RUN': ALERT_DRY_RUN, 'CLUSTER_PREFIX': '["kubernetes.io/cluster","sigs.k8s.io/cluster-api-provider-aws/cluster"]' @@ -130,4 +134,4 @@ def run_policies(policies: list, dry_run: str = 'yes'): # Run the AggMail run_cmd( - f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e SKIP_POLICIES_ALERT='{SKIP_POLICIES_ALERT}' -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" {env_es_index} -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" -e ALERT_DRY_RUN="{ALERT_DRY_RUN}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + f"""podman run --rm --name cloud-governance-haim --net="host" -e account="{account_name}" -e policy="send_aggregated_alerts" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e SKIP_POLICIES_ALERT='{SKIP_POLICIES_ALERT}' -e log_level="INFO" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_user="{ES_USER}" -e es_password="{ES_PASSWORD}" -e ES_SERVER_TYPE="{ES_SERVER_TYPE}" {env_es_index} -e ADMIN_MAIL_LIST="{ADMIN_MAIL_LIST}" -e ALERT_DRY_RUN="{ALERT_DRY_RUN}" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily index 39d1626b7..2c159413c 100644 --- a/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_01/PolicyJenkinsfileDaily @@ -36,6 +36,7 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + ES_SERVER_TYPE = 'elasticsearch' ALERT_DRY_RUN = true contact1 = "ebattat@redhat.com" contact2 = "yinsong@redhat.com" diff --git a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily index a4f27758d..4705ff7d5 100644 --- a/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_02/PolicyJenkinsfileDaily @@ -21,6 +21,7 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + ES_SERVER_TYPE = 'elasticsearch' ALERT_DRY_RUN = true contact1 = "ebattat@redhat.com" contact2 = "yinsong@redhat.com" diff --git a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily index 65a1e3f52..24b233832 100644 --- a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily @@ -18,6 +18,7 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') + ES_SERVER_TYPE = 'elasticsearch' ALERT_DRY_RUN = true contact1 = "ebattat@redhat.com" contact2 = "yinsong@redhat.com" diff --git a/jenkins/tenant/aws/ovn/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ovn/PolicyJenkinsfileDaily index 74d88cf17..25c833113 100644 --- a/jenkins/tenant/aws/ovn/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ovn/PolicyJenkinsfileDaily @@ -15,6 +15,8 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') contact1 = "ebattat@redhat.com" contact2 = "yinsong@redhat.com" contact3 = "pragchau@redhat.com" diff --git a/jenkins/tenant/aws/ovn/run_policies.py b/jenkins/tenant/aws/ovn/run_policies.py index c15ec12db..6a426ae2e 100644 --- a/jenkins/tenant/aws/ovn/run_policies.py +++ b/jenkins/tenant/aws/ovn/run_policies.py @@ -28,6 +28,8 @@ def run_cmd(cmd: str): ALERT_DRY_RUN = os.environ.get('ALERT_DRY_RUN', False) ES_HOST = os.environ['ES_HOST'] ES_PORT = os.environ['ES_PORT'] +ES_USER = os.environ.get('ES_USER', '') +ES_PASSWORD = os.environ.get('ES_PASSWORD', '') ADMIN_MAIL_LIST = os.environ.get('ADMIN_MAIL_LIST', '') # Set es_index if given @@ -62,7 +64,7 @@ def get_container_cmd(env_dict: dict): "account": account_name, "AWS_DEFAULT_REGION": "us-east-1", "PUBLIC_CLOUD_NAME": "AWS", "AWS_ACCESS_KEY_ID": access_key, "AWS_SECRET_ACCESS_KEY": secret_key, "dry_run": "yes", "LDAP_HOST_NAME": LDAP_HOST_NAME, "DAYS_TO_DELETE_RESOURCE": days_to_delete_resource, - "es_host": ES_HOST, "es_port": ES_PORT, + "es_host": ES_HOST, "es_port": ES_PORT, "es_user": ES_USER, "es_password": ES_PASSWORD, "MANAGER_EMAIL_ALERT": "False", "EMAIL_ALERT": "False", "log_level": "INFO", 'DAYS_TO_TAKE_ACTION': days_to_delete_resource, 'ALERT_DRY_RUN': ALERT_DRY_RUN } diff --git a/jenkins/tenant/aws/qe/qe_01/PolicyJenkinsfileDaily b/jenkins/tenant/aws/qe/qe_01/PolicyJenkinsfileDaily index 05be7931f..0ebf379d9 100644 --- a/jenkins/tenant/aws/qe/qe_01/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/qe/qe_01/PolicyJenkinsfileDaily @@ -17,6 +17,8 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') ALERT_DRY_RUN = true contact1 = "ebattat@redhat.com" contact2 = "yinsong@redhat.com" diff --git a/jenkins/tenant/aws/qe/qe_02/PolicyJenkinsfileDaily b/jenkins/tenant/aws/qe/qe_02/PolicyJenkinsfileDaily index f96c1b151..6d8f89102 100644 --- a/jenkins/tenant/aws/qe/qe_02/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/qe/qe_02/PolicyJenkinsfileDaily @@ -21,6 +21,8 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') ES_HOST = credentials('cloud-governance-es-host') ES_PORT = credentials('cloud-governance-es-port') + ES_USER = credentials('cloud-governance-es-user') + ES_PASSWORD = credentials('cloud-governance-es-password') ALERT_DRY_RUN = true contact1 = "ebattat@redhat.com" contact2 = "yinsong@redhat.com" diff --git a/requirements.txt b/requirements.txt index 33133ccae..77dc94498 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,7 @@ azure-mgmt-resource==23.0.1 azure-mgmt-subscription==3.1.1 boto3==1.42.89 botocore==1.42.89 -elasticsearch==7.17.13 # opensearch 1.2.4 for elasticsearch -elasticsearch-dsl==7.4.0 +elasticsearch==8.17.0 # ecoeng ES 8.8.0 server google-api-python-client==2.57.0 google-auth-httplib2==0.1.0 google-auth-oauthlib==0.5.2 @@ -23,8 +22,9 @@ ibm-platform-services==0.75.0 ibm-schematics==1.0.1 ibm-vpc==0.33.0 myst-parser==1.0.0 -numpy<=1.26.4 # opensearch 1.2.4 for elasticsearch +numpy<=1.26.4 oauthlib~=3.1.1 +opensearch-py==3.2.0 # intlab OpenSearch 3.2.0 server (includes DSL) pandas protobuf==5.29.6 PyAthena[Pandas]==3.0.5 diff --git a/setup.py b/setup.py index 696c1fce3..6696dc8a4 100644 --- a/setup.py +++ b/setup.py @@ -51,8 +51,8 @@ 'azure-mgmt-monitor==6.0.2', 'boto3==1.42.89', 'botocore==1.42.89', - 'elasticsearch-dsl==7.4.0', - 'elasticsearch==7.17.13', # opensearch 1.2.4 for elasticsearch + 'elasticsearch==8.17.0', # ecoeng ES 8.8.0 server + 'opensearch-py==3.2.0', # intlab OpenSearch 3.2.0 server (includes DSL) 'google-api-python-client==2.57.0', # google drive 'google-auth-httplib2==0.1.0', # google drive 'google-auth-oauthlib==0.5.2', # google drive @@ -64,14 +64,14 @@ 'ibm-schematics==1.0.1', 'ibm-vpc==0.33.0', 'myst-parser==1.0.0', # readthedocs - 'numpy<=1.26.4', # opensearch 1.2.4 for elasticsearch + 'numpy<=1.26.4', 'oauthlib~=3.1.1', # required by jira 'pandas', # latest: aggregate ec2/ebs cluster data 'PyAthena[Pandas]==3.0.5', # AWS Athena package 'PyGitHub==1.55', # gitleaks 'python-ldap==3.4.2', # prerequisite: sudo dnf install -y python3-devel openldap-devel gcc 'protobuf==5.29.6', # google-cloud transitive dep - 'pytz', # timezone handling (azure cost management, elasticsearch) + 'pytz', # timezone handling (azure cost management, opensearch) 'requests==2.33.1', # rest api & lambda 'retry==0.9.2', 'setuptools', # CI: setuptools<82 for IBM sdist builds diff --git a/tests/unittest/mocks/elasticsearch/mock_elasticsearch.py b/tests/unittest/mocks/elasticsearch/mock_elasticsearch.py index 9f5b9cb3a..5a59501e9 100644 --- a/tests/unittest/mocks/elasticsearch/mock_elasticsearch.py +++ b/tests/unittest/mocks/elasticsearch/mock_elasticsearch.py @@ -2,6 +2,7 @@ from functools import wraps from unittest.mock import patch +from opensearchpy import OpenSearch from elasticsearch import Elasticsearch @@ -43,7 +44,9 @@ def method_wrapper(*args, **kwargs): @return: """ mock_class = MockElasticsearch() - with patch.object(Elasticsearch, 'index', mock_class.index), \ + with patch.object(OpenSearch, 'index', mock_class.index), \ + patch.object(OpenSearch, 'search', mock_class.search), \ + patch.object(Elasticsearch, 'index', mock_class.index), \ patch.object(Elasticsearch, 'search', mock_class.search): result = method(*args, **kwargs) return result diff --git a/tests_requirements.txt b/tests_requirements.txt index 262d54395..49323cdd4 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -5,8 +5,7 @@ azure-mgmt-costmanagement==3.0.0 azure-mgmt-monitor==6.0.2 azure-mgmt-subscription==3.1.1 boto3==1.42.89 -elasticsearch==7.17.13 -elasticsearch-dsl==7.4.0 +elasticsearch==8.17.0 freezegun==1.5.1 ibm-cloud-sdk-core==3.24.4 ibm-platform-services==0.75.0 @@ -15,6 +14,7 @@ ibm-vpc==0.33.0 moto[all]==5.1.22 numpy<=1.26.4 oauthlib~=3.1.1 +opensearch-py==3.2.0 pandas pre-commit==3.5.0 pytest