From 92839336c0c90cb8edf0edc71ae228d02e24f7a3 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Thu, 29 Jan 2026 15:52:38 +0530 Subject: [PATCH 1/3] Move yearly savings report to daily policies --- .../org_cost_explorer/run_org_upload_es.py | 20 ++++++- .../aws/monthly/run_monthly_policies.py | 14 ----- .../tenant/aws/common/run_cost_policies.py | 15 +++++- .../aws/ecoeng_03/PolicyJenkinsfileDaily | 52 +++++++++---------- 4 files changed, 57 insertions(+), 44 deletions(-) diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py index cb73e275e..0537af78b 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py +++ b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py @@ -15,8 +15,9 @@ S3_RESULTS_PATH = os.environ['S3_RESULTS_PATH'] ATHENA_DATABASE_NAME = os.environ['ATHENA_DATABASE_NAME'] ATHENA_TABLE_NAME = os.environ['ATHENA_TABLE_NAME'] -QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', - 'quay.io/cloud-governance/cloud-governance:latest') +# QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', +# 'quay.io/cloud-governance/cloud-governance:latest') +QUAY_CLOUD_GOVERNANCE_REPOSITORY = 'quay.io/rh-ee-pragchau/cloud-governance:latest' # Cloudability env variables @@ -56,6 +57,21 @@ -e ATHENA_TABLE_NAME="{ATHENA_TABLE_NAME}" \ {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") +os.system('echo "Running yearly savings report for all accounts"') +accounts = ['PSAP', 'PERFSCALE', 'PERF-DEPT'] + +for account in accounts: + os.system(f'echo "Running yearly savings report for account {account}"') + os.system(f"""podman run --rm --net="host" --name cloud-governance -e policy="yearly_savings_report" \ +-e PUBLIC_CLOUD_NAME="AWS" \ +-e account="{account}" \ +-e es_host="{ES_HOST}" \ +-e es_port="{ES_PORT}" \ +-e es_index="cloud-governance-policy-es-index" \ +-e log_level="INFO" \ +{QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + + CONTAINER_NAME = "cloud-governance" COST_ES_INDEX = "cloud-governance-clouds-billing-reports" CLOUDABILITY_POLICY = 'cloudability_cost_reports' diff --git a/jenkins/clouds/aws/monthly/run_monthly_policies.py b/jenkins/clouds/aws/monthly/run_monthly_policies.py index 51c483e34..ca6400f67 100644 --- a/jenkins/clouds/aws/monthly/run_monthly_policies.py +++ b/jenkins/clouds/aws/monthly/run_monthly_policies.py @@ -13,17 +13,3 @@ print("Run AWS Monthly Policies") os.system( f"""podman run --rm --name cloud-governance --net="host" -e policy="monthly_report" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e to_mail="{TO_MAIL}" -e cc_mail="{CC_MAIL}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") - -os.system('echo "Running yearly savings report for all accounts"') -accounts = ['PSAP', 'PERFSCALE', 'PERF-DEPT'] - -for account in accounts: - os.system(f'echo "Running yearly savings report for account {account}"') - os.system(f"""podman run --rm --net="host" --name cloud-governance -e policy="yearly_savings_report" \ --e PUBLIC_CLOUD_NAME="AWS" \ --e account="{account}" \ --e es_host="{ES_HOST}" \ --e es_port="{ES_PORT}" \ --e es_index="cloud-governance-policy-es-index" \ --e log_level="INFO" \ -{QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/common/run_cost_policies.py b/jenkins/tenant/aws/common/run_cost_policies.py index dba179ea5..4f9a03671 100644 --- a/jenkins/tenant/aws/common/run_cost_policies.py +++ b/jenkins/tenant/aws/common/run_cost_policies.py @@ -7,8 +7,9 @@ ES_PORT = os.environ['ES_PORT'] ES_INDEX = os.environ.get('ES_INDEX', None) -QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', - 'quay.io/cloud-governance/cloud-governance') +# QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', +# 'quay.io/cloud-governance/cloud-governance') +QUAY_CLOUD_GOVERNANCE_REPOSITORY = 'quay.io/rh-ee-pragchau/cloud-governance:latest' cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot', 'cluster_id'] @@ -22,3 +23,13 @@ os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """) os.system( f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" {env_es_index} -e es_port="{ES_PORT}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") + +os.system(f"""echo "Running yearly savings report for account {account_name}" """) +os.system(f"""podman run --rm --net="host" --name cloud-governance -e policy="yearly_savings_report" \ +-e PUBLIC_CLOUD_NAME="AWS" \ +-e account="{account_name}" \ +-e es_host="{ES_HOST}" \ +-e es_port="{ES_PORT}" \ +-e es_index="cloud-governance-policy-es-index" \ +-e log_level="INFO" \ +{QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") diff --git a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily index 1edff73c6..d2249c158 100644 --- a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily @@ -56,22 +56,22 @@ pipeline { } } } - stage('Run Daily Policies') { - steps { - script { - for (account in accounts_list.keySet()) { - echo "Running for account ${account.toUpperCase()}" - withCredentials([string(credentialsId: "${account}-aws-access-key-id", variable: 'access_key'), - string(credentialsId: "${account}-aws-secret-key-id", variable: 'secret_key'), - string(credentialsId: "${account}-s3-bucket", variable: 's3_bucket')]) { - env.account_name = "${account}" - env.ADMIN_MAIL_LIST = "${accounts_list[account]}" - sh 'python3 jenkins/tenant/aws/common/run_policies.py' - } - } - } - } - } + // stage('Run Daily Policies') { + // steps { + // script { + // for (account in accounts_list.keySet()) { + // echo "Running for account ${account.toUpperCase()}" + // withCredentials([string(credentialsId: "${account}-aws-access-key-id", variable: 'access_key'), + // string(credentialsId: "${account}-aws-secret-key-id", variable: 'secret_key'), + // string(credentialsId: "${account}-s3-bucket", variable: 's3_bucket')]) { + // env.account_name = "${account}" + // env.ADMIN_MAIL_LIST = "${accounts_list[account]}" + // sh 'python3 jenkins/tenant/aws/common/run_policies.py' + // } + // } + // } + // } + // } stage('Finalize Cleanup') { steps { sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' @@ -79,14 +79,14 @@ pipeline { } } } - post { - failure { - script { - msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - emailext body: """\ - Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}, ${contact5}" - } - } - } + // post { + // failure { + // script { + // msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + // emailext body: """\ + // Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + // """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}, ${contact5}" + // } + // } + // } } From 225566b59778072d9ce72300b8c968754f0f3176 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Thu, 29 Jan 2026 16:08:16 +0530 Subject: [PATCH 2/3] Undo debug code --- .../org_cost_explorer/run_org_upload_es.py | 5 +- .../tenant/aws/common/run_cost_policies.py | 5 +- .../aws/ecoeng_03/PolicyJenkinsfileDaily | 52 +++++++++---------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py index 0537af78b..9409d0a9b 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py +++ b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py @@ -15,9 +15,8 @@ S3_RESULTS_PATH = os.environ['S3_RESULTS_PATH'] ATHENA_DATABASE_NAME = os.environ['ATHENA_DATABASE_NAME'] ATHENA_TABLE_NAME = os.environ['ATHENA_TABLE_NAME'] -# QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', -# 'quay.io/cloud-governance/cloud-governance:latest') -QUAY_CLOUD_GOVERNANCE_REPOSITORY = 'quay.io/rh-ee-pragchau/cloud-governance:latest' +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance:latest') # Cloudability env variables diff --git a/jenkins/tenant/aws/common/run_cost_policies.py b/jenkins/tenant/aws/common/run_cost_policies.py index 4f9a03671..3473e6967 100644 --- a/jenkins/tenant/aws/common/run_cost_policies.py +++ b/jenkins/tenant/aws/common/run_cost_policies.py @@ -7,9 +7,8 @@ ES_PORT = os.environ['ES_PORT'] ES_INDEX = os.environ.get('ES_INDEX', None) -# QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', -# 'quay.io/cloud-governance/cloud-governance') -QUAY_CLOUD_GOVERNANCE_REPOSITORY = 'quay.io/rh-ee-pragchau/cloud-governance:latest' +QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', + 'quay.io/cloud-governance/cloud-governance') cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner', 'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot', 'cluster_id'] diff --git a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily index d2249c158..1edff73c6 100644 --- a/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily +++ b/jenkins/tenant/aws/ecoeng_03/PolicyJenkinsfileDaily @@ -56,22 +56,22 @@ pipeline { } } } - // stage('Run Daily Policies') { - // steps { - // script { - // for (account in accounts_list.keySet()) { - // echo "Running for account ${account.toUpperCase()}" - // withCredentials([string(credentialsId: "${account}-aws-access-key-id", variable: 'access_key'), - // string(credentialsId: "${account}-aws-secret-key-id", variable: 'secret_key'), - // string(credentialsId: "${account}-s3-bucket", variable: 's3_bucket')]) { - // env.account_name = "${account}" - // env.ADMIN_MAIL_LIST = "${accounts_list[account]}" - // sh 'python3 jenkins/tenant/aws/common/run_policies.py' - // } - // } - // } - // } - // } + stage('Run Daily Policies') { + steps { + script { + for (account in accounts_list.keySet()) { + echo "Running for account ${account.toUpperCase()}" + withCredentials([string(credentialsId: "${account}-aws-access-key-id", variable: 'access_key'), + string(credentialsId: "${account}-aws-secret-key-id", variable: 'secret_key'), + string(credentialsId: "${account}-s3-bucket", variable: 's3_bucket')]) { + env.account_name = "${account}" + env.ADMIN_MAIL_LIST = "${accounts_list[account]}" + sh 'python3 jenkins/tenant/aws/common/run_policies.py' + } + } + } + } + } stage('Finalize Cleanup') { steps { sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' @@ -79,14 +79,14 @@ pipeline { } } } - // post { - // failure { - // script { - // msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - // emailext body: """\ - // Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - // """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}, ${contact5}" - // } - // } - // } + post { + failure { + script { + msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" + emailext body: """\ + Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n + """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}, ${contact4}, ${contact5}" + } + } + } } From 17c125c5906fd83c9caf347468660d44eb5625f7 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Thu, 29 Jan 2026 16:10:34 +0530 Subject: [PATCH 3/3] Remove cloud governance monthly job --- .../common/monthly/PolicyJenkinsFileMonthly | 82 ------------------- .../common/monthly/run_monthly_policies.py | 18 ---- 2 files changed, 100 deletions(-) delete mode 100644 jenkins/tenant/aws/common/monthly/PolicyJenkinsFileMonthly delete mode 100644 jenkins/tenant/aws/common/monthly/run_monthly_policies.py diff --git a/jenkins/tenant/aws/common/monthly/PolicyJenkinsFileMonthly b/jenkins/tenant/aws/common/monthly/PolicyJenkinsFileMonthly deleted file mode 100644 index b8a2ff8e6..000000000 --- a/jenkins/tenant/aws/common/monthly/PolicyJenkinsFileMonthly +++ /dev/null @@ -1,82 +0,0 @@ -accounts_list = ['industry-partners': "hhalbfin@redhat.com", - 'certification-pipeline': "matt.dorn@redhat.com, mhillsma@redhat.com, babak@redhat.com, hhalbfin@redhat.com", - 'ecoengverticals-qe': "augol@redhat.com, hhalbfin@redhat.com", - 'emerge-partner': "jsalomon@redhat.com, ltomasbo@redhat.com, hhalbfin@redhat.com", - 'telco5g-ci': "yjoseph@redhat.com, hhalbfin@redhat.com, sshnaidm@redhat.com", - 'ecoeng-sap': "cbrune@redhat.com, ksatarin@redhat.com, babak@redhat.com, hhalbfin@redhat.com", - 'sysdeseng': "scollier@redhat.com, hhalbfin@redhat.com, mavazque@redhat.com", - 'verticals-ui': "brotman@redhat.com, hhalbfin@redhat.com", - 'special-projects': "chen.yosef@redhat.com, hhalbfin@redhat.com", - 'edgeinfra': "lgamliel@redhat.com, bthurber@redhat.com, oourfali@redhat.com, hhalbfin@redhat.com", - 'specialprojects-qe': "augol@redhat.com, hhalbfin@redhat.com", - 'partnerlab': "matt.dorn@redhat.com, jomckenz@redhat.com, babak@redhat.com, hhalbfin@redhat.com, mrhillsman@redhat.com", - 'blueprints': "abeekhof@redhat.com, hhalbfin@redhat.com", - 'coreos-training': "matt.dorn@redhat.com, jomckenz@redhat.com, babak@redhat.com, hhalbfin@redhat.com, mrhillsman@redhat.com", - 'edgeinfra-ci': "rfreiman@redhat.com, hhalbfin@redhat.com", - 'ecoeng-flightctl': "ayogev@redhat.com, hhalbfin@redhat.com", - 'partners-eng': "hhalbfin@redhat.com", - 'fusionaccess': "abeekhof@redhat.com, hhalbfin@redhat.com", - 'ecoeng-assistedci': "alkaplan@redhat.com, dmanor@redhat.com, hhalbfin@redhat.com", - 'medik8s-ci': "ushkalim@redhat.com, hhalbfin@redhat.com" - ] -pipeline { - options { - disableConcurrentBuilds() - } - triggers { - cron('H H 1 * *') - } - agent { - docker { - label 'haim-cloud-governance-worker' - image 'quay.io/cloud-governance/fedora38-podman:latest' - args '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged' - } - } - environment { - QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') - ES_HOST = credentials('haim-cloud-governance-elasticsearch-url') - ES_PORT = credentials('haim-cloud-governance-elasticsearch-port') - contact1 = "ebattat@redhat.com" - contact2 = "yinsong@redhat.com" - } - stages { - stage('Checkout') { - steps { - checkout scm - } - } - stage('Initial Cleanup') { - steps { - sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' - } - } - stage('Run Yearly Savings Report') { - steps { - script { - for (account in accounts_list.keySet()) { - echo "Running yearly savings report for account ${account.toUpperCase()}" - env.account_name = "${account}" - sh 'python3 jenkins/tenant/aws/common/monthly/run_monthly_policies.py' - } - } - } - } - stage('Finalize Cleanup') { - steps { - sh '''if [[ "$(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q ${QUAY_CLOUD_GOVERNANCE_REPOSITORY} 2> /dev/null); fi''' - deleteDir() - } - } - } - post { - failure { - script { - msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})" - emailext body: """\ - Jenkins job: ${env.BUILD_URL}\nSee the console output for more details: ${env.BUILD_URL}consoleFull\n\n - """,subject: msg, to: "${contact1}, ${contact2}" - } - } - } -} diff --git a/jenkins/tenant/aws/common/monthly/run_monthly_policies.py b/jenkins/tenant/aws/common/monthly/run_monthly_policies.py deleted file mode 100644 index 071f9a70a..000000000 --- a/jenkins/tenant/aws/common/monthly/run_monthly_policies.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -account_name = os.environ['account_name'] -ES_HOST = os.environ['ES_HOST'] -ES_PORT = os.environ['ES_PORT'] - -QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', - 'quay.io/cloud-governance/cloud-governance') - -os.system(f"""echo "Running yearly savings report for account {account_name}" """) -os.system(f"""podman run --rm --net="host" --name cloud-governance -e policy="yearly_savings_report" \ --e PUBLIC_CLOUD_NAME="AWS" \ --e account="{account_name}" \ --e es_host="{ES_HOST}" \ --e es_port="{ES_PORT}" \ --e es_index="cloud-governance-policy-es-index" \ --e log_level="INFO" \ -{QUAY_CLOUD_GOVERNANCE_REPOSITORY}""")