From 0b009cc0683899e6aa3fd05ec1da3d0f3d98a6d2 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 13:34:58 +0530 Subject: [PATCH 01/17] Enable unused_access_key policy in action --- cloud_governance/common/utils/configs.py | 2 +- jenkins/clouds/aws/daily/policies/Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index 2310efdf..19c6dc74 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -22,7 +22,7 @@ EC2_NAMESPACE = 'AWS/EC2' CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14 AWS_DEFAULT_GLOBAL_REGION = 'us-east-1' -UNUSED_ACCESS_KEY_DAYS = 90 +UNUSED_ACCESS_KEY_DAYS = 180 UNUSED_ACCESS_KEY_MAX_DAY = 1000 # X86 to Graviton diff --git a/jenkins/clouds/aws/daily/policies/Jenkinsfile b/jenkins/clouds/aws/daily/policies/Jenkinsfile index b6f677ad..20b1e924 100644 --- a/jenkins/clouds/aws/daily/policies/Jenkinsfile +++ b/jenkins/clouds/aws/daily/policies/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { } environment { QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') - POLICIES_IN_ACTION = '["instance_idle", "ec2_stop", "unattached_volume", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles", "zombie_cluster_resource"]' + POLICIES_IN_ACTION = '["instance_idle", "ec2_stop", "unattached_volume", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles", "zombie_cluster_resource", "unused_access_key"]' AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') From 182aaa368cd62041359ba5772c410a989ed21b90 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 14:03:50 +0530 Subject: [PATCH 02/17] Resolve pkg install error --- .github/workflows/Build.yml | 4 ++++ .github/workflows/PR.yml | 4 ++++ .github/workflows/PR_Approval.yml | 2 ++ 3 files changed, 10 insertions(+) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 429a346e..f385a6ad 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -31,6 +31,8 @@ jobs: sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y python -m pip install --upgrade pip pip install flake8 pytest pytest-cov + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Lint with flake8 @@ -146,6 +148,8 @@ jobs: sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y python -m pip install --upgrade pip pip install flake8 pytest pytest-cov + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Lint with flake8 diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 1593d602..abcd9be7 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -131,6 +131,8 @@ jobs: sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip pip install pytest pytest-cov + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Configure AWS credentials @@ -233,6 +235,8 @@ jobs: sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip pip install pytest pytest-cov + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Configure AWS credentials diff --git a/.github/workflows/PR_Approval.yml b/.github/workflows/PR_Approval.yml index 29bbbeda..66c50e98 100644 --- a/.github/workflows/PR_Approval.yml +++ b/.github/workflows/PR_Approval.yml @@ -31,6 +31,8 @@ jobs: sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip pip install flake8 pytest pytest-cov + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Lint with flake8 From e12415237ee75d5995b142bd6f785a75c5628ec8 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 14:15:44 +0530 Subject: [PATCH 03/17] Resolve pkg dependency issues --- .github/workflows/Build.yml | 4 ---- .github/workflows/PR.yml | 4 ---- .github/workflows/PR_Approval.yml | 2 -- requirements.txt | 12 ++++++------ setup.py | 22 +++++++++++----------- tests_requirements.txt | 10 +++++----- 6 files changed, 22 insertions(+), 32 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index f385a6ad..429a346e 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -31,8 +31,6 @@ jobs: sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y python -m pip install --upgrade pip pip install flake8 pytest pytest-cov - pip install "setuptools<82" - pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Lint with flake8 @@ -148,8 +146,6 @@ jobs: sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y python -m pip install --upgrade pip pip install flake8 pytest pytest-cov - pip install "setuptools<82" - pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Lint with flake8 diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index abcd9be7..1593d602 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -131,8 +131,6 @@ jobs: sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip pip install pytest pytest-cov - pip install "setuptools<82" - pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Configure AWS credentials @@ -235,8 +233,6 @@ jobs: sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip pip install pytest pytest-cov - pip install "setuptools<82" - pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Configure AWS credentials diff --git a/.github/workflows/PR_Approval.yml b/.github/workflows/PR_Approval.yml index 66c50e98..29bbbeda 100644 --- a/.github/workflows/PR_Approval.yml +++ b/.github/workflows/PR_Approval.yml @@ -31,8 +31,6 @@ jobs: sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip pip install flake8 pytest pytest-cov - pip install "setuptools<82" - pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi - name: Lint with flake8 diff --git a/requirements.txt b/requirements.txt index 50bc302e..7946870c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,18 +8,18 @@ azure-mgmt-monitor==6.0.2 azure-mgmt-network==25.0.0 azure-mgmt-resource==23.0.1 azure-mgmt-subscription==3.1.1 -boto3==1.33.1 -botocore==1.33.8 -elasticsearch==7.13.4 # opensearch 1.2.4 for elasticsearch +boto3==1.35.0 +botocore==1.35.0 +elasticsearch==7.17.13 # opensearch 1.2.4; 7.17.13+ has wheels, urllib3 2.x elasticsearch-dsl==7.4.0 google-api-python-client==2.57.0 google-auth-httplib2==0.1.0 google-auth-oauthlib==0.5.2 google-cloud-bigquery==3.5.0 google-cloud-billing==1.9.1 -ibm-cloud-sdk-core==3.18.0 +ibm-cloud-sdk-core==3.22.1 ibm-cos-sdk==2.13.6 -ibm-platform-services==0.27.0 +ibm-platform-services==0.60.0 ibm-schematics==1.1.0 ibm-vpc==0.21.0 myst-parser==1.0.0 @@ -37,4 +37,4 @@ sphinx==5.0.0 sphinx-rtd-theme==1.0.0 typeguard==2.13.3 typing==3.7.4.3 -urllib3==1.26.19 +urllib3>=2.1.0,<3.0.0 diff --git a/setup.py b/setup.py index e86fb35f..64ad604b 100644 --- a/setup.py +++ b/setup.py @@ -48,18 +48,18 @@ 'azure-mgmt-compute==30.1.0', 'azure-mgmt-network==25.0.0', 'azure-mgmt-monitor==6.0.2', - 'boto3==1.33.1', - 'botocore==1.33.8', + 'boto3==1.35.0', + 'botocore==1.35.0', 'elasticsearch-dsl==7.4.0', - 'elasticsearch==7.13.4', # opensearch 1.2.4 for elasticsearch - 'google-api-python-client==2.57.0', # google drive - 'google-auth-httplib2==0.1.0', # google drive - 'google-auth-oauthlib==0.5.2', # google drive - 'google-cloud-bigquery==3.5.0', # google cloud cost - 'google-cloud-billing==1.9.1', # google cloud cost - 'ibm-cloud-sdk-core==3.18.0', + 'elasticsearch==7.17.13', + 'google-api-python-client==2.57.0', + 'google-auth-httplib2==0.1.0', + 'google-auth-oauthlib==0.5.2', + 'google-cloud-bigquery==3.5.0', + 'google-cloud-billing==1.9.1', + 'ibm-cloud-sdk-core==3.22.1', 'ibm-cos-sdk==2.13.6', - 'ibm-platform-services==0.27.0', # IBM Usage reports + 'ibm-platform-services==0.60.0', 'ibm-schematics==1.1.0', 'ibm-vpc==0.21.0', 'myst-parser==1.0.0', # readthedocs @@ -77,7 +77,7 @@ 'sphinx==5.0.0', # readthedocs 'typeguard==2.13.3', # checking types 'typing==3.7.4.3', - 'urllib3==1.26.19' # required by jira + 'urllib3>=2.1.0,<3.0.0' ], setup_requires=['pytest', 'pytest-runner', 'wheel', 'coverage'], diff --git a/tests_requirements.txt b/tests_requirements.txt index 3525cc61..47cf3261 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -4,12 +4,12 @@ azure-mgmt-billing==6.0.0 azure-mgmt-costmanagement==3.0.0 azure-mgmt-monitor==6.0.2 azure-mgmt-subscription==3.1.1 -boto3==1.33.1 -elasticsearch==7.13.4 +boto3==1.35.0 +elasticsearch==7.17.13 elasticsearch-dsl==7.4.0 freezegun==1.5.1 -ibm-cloud-sdk-core==3.18.0 -ibm-platform-services==0.27.0 +ibm-cloud-sdk-core==3.22.1 +ibm-platform-services==0.60.0 ibm-schematics==1.1.0 ibm-vpc==0.21.0 moto==4.0.1 @@ -24,4 +24,4 @@ retry==0.9.2 setuptools SoftLayer==6.0.0 typeguard==2.13.3 -urllib3==1.26.19 +urllib3>=2.1.0,<3.0.0 From 03ce3789fdd0bb421a8f0d171d43d8359693fe6f Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 14:23:22 +0530 Subject: [PATCH 04/17] Resolve pkg dependency issues --- requirements.txt | 2 +- setup.py | 2 +- tests_requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7946870c..6ff81475 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ google-cloud-billing==1.9.1 ibm-cloud-sdk-core==3.22.1 ibm-cos-sdk==2.13.6 ibm-platform-services==0.60.0 -ibm-schematics==1.1.0 +ibm-schematics==1.0.1 ibm-vpc==0.21.0 myst-parser==1.0.0 numpy<=1.26.4 # opensearch 1.2.4 for elasticsearch diff --git a/setup.py b/setup.py index 64ad604b..bd7d8c3f 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ 'ibm-cloud-sdk-core==3.22.1', 'ibm-cos-sdk==2.13.6', 'ibm-platform-services==0.60.0', - 'ibm-schematics==1.1.0', + 'ibm-schematics==1.0.1', 'ibm-vpc==0.21.0', 'myst-parser==1.0.0', # readthedocs 'numpy<=1.26.4', # opensearch 1.2.4 for elasticsearch diff --git a/tests_requirements.txt b/tests_requirements.txt index 47cf3261..b1ce7c85 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -10,7 +10,7 @@ elasticsearch-dsl==7.4.0 freezegun==1.5.1 ibm-cloud-sdk-core==3.22.1 ibm-platform-services==0.60.0 -ibm-schematics==1.1.0 +ibm-schematics==1.0.1 ibm-vpc==0.21.0 moto==4.0.1 numpy<=1.26.4 From f847f5a0502754daeb5ff30152e4c5db1f196389 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 14:30:01 +0530 Subject: [PATCH 05/17] Resolve pkg dependency issues --- requirements.txt | 2 +- setup.py | 2 +- tests_requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6ff81475..62a4febf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ ibm-cloud-sdk-core==3.22.1 ibm-cos-sdk==2.13.6 ibm-platform-services==0.60.0 ibm-schematics==1.0.1 -ibm-vpc==0.21.0 +ibm-vpc==0.26.3 myst-parser==1.0.0 numpy<=1.26.4 # opensearch 1.2.4 for elasticsearch oauthlib~=3.1.1 diff --git a/setup.py b/setup.py index bd7d8c3f..e07d71c3 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ 'ibm-cos-sdk==2.13.6', 'ibm-platform-services==0.60.0', 'ibm-schematics==1.0.1', - 'ibm-vpc==0.21.0', + 'ibm-vpc==0.26.3', 'myst-parser==1.0.0', # readthedocs 'numpy<=1.26.4', # opensearch 1.2.4 for elasticsearch 'oauthlib~=3.1.1', # required by jira diff --git a/tests_requirements.txt b/tests_requirements.txt index b1ce7c85..e05aea7f 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -11,7 +11,7 @@ freezegun==1.5.1 ibm-cloud-sdk-core==3.22.1 ibm-platform-services==0.60.0 ibm-schematics==1.0.1 -ibm-vpc==0.21.0 +ibm-vpc==0.26.3 moto==4.0.1 numpy<=1.26.4 oauthlib~=3.1.1 From 4d348f73ac3a8a6dcfd0cd2b30080324c50ae756 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 14:38:23 +0530 Subject: [PATCH 06/17] Resolve pkg dependency issues --- .github/workflows/Build.yml | 4 ++++ .github/workflows/PR.yml | 4 ++++ .github/workflows/PR_Approval.yml | 2 ++ requirements.txt | 16 ++++++++-------- setup.py | 17 +++++++++-------- tests_requirements.txt | 12 ++++++------ 6 files changed, 33 insertions(+), 22 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 429a346e..bda89722 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -30,6 +30,8 @@ jobs: sudo apt update -y sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y python -m pip install --upgrade pip + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 ibm-platform-services==0.27.0 ibm-vpc==0.21.0 pip install flake8 pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi @@ -145,6 +147,8 @@ jobs: sudo apt update -y sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y python -m pip install --upgrade pip + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 ibm-platform-services==0.27.0 ibm-vpc==0.21.0 pip install flake8 pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 1593d602..aaebbc5f 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -130,6 +130,8 @@ jobs: sudo apt update -y sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 ibm-platform-services==0.27.0 ibm-vpc==0.21.0 pip install pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi @@ -232,6 +234,8 @@ jobs: sudo apt update -y sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 ibm-platform-services==0.27.0 ibm-vpc==0.21.0 pip install pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi diff --git a/.github/workflows/PR_Approval.yml b/.github/workflows/PR_Approval.yml index 29bbbeda..c2412f2e 100644 --- a/.github/workflows/PR_Approval.yml +++ b/.github/workflows/PR_Approval.yml @@ -30,6 +30,8 @@ jobs: sudo apt update -y sudo apt-get install -y build-essential python3-dev libldap2-dev libsasl2-dev python -m pip install --upgrade pip + pip install "setuptools<82" + pip install --no-build-isolation ibm-cloud-sdk-core==3.18.0 ibm-platform-services==0.27.0 ibm-vpc==0.21.0 pip install flake8 pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f tests_requirements.txt ]; then pip install -r tests_requirements.txt; fi diff --git a/requirements.txt b/requirements.txt index 62a4febf..5012161a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,22 +8,22 @@ azure-mgmt-monitor==6.0.2 azure-mgmt-network==25.0.0 azure-mgmt-resource==23.0.1 azure-mgmt-subscription==3.1.1 -boto3==1.35.0 -botocore==1.35.0 -elasticsearch==7.17.13 # opensearch 1.2.4; 7.17.13+ has wheels, urllib3 2.x +boto3==1.33.1 +botocore==1.33.8 +elasticsearch==7.13.4 elasticsearch-dsl==7.4.0 google-api-python-client==2.57.0 google-auth-httplib2==0.1.0 google-auth-oauthlib==0.5.2 google-cloud-bigquery==3.5.0 google-cloud-billing==1.9.1 -ibm-cloud-sdk-core==3.22.1 +ibm-cloud-sdk-core==3.18.0 ibm-cos-sdk==2.13.6 -ibm-platform-services==0.60.0 +ibm-platform-services==0.27.0 ibm-schematics==1.0.1 -ibm-vpc==0.26.3 +ibm-vpc==0.21.0 myst-parser==1.0.0 -numpy<=1.26.4 # opensearch 1.2.4 for elasticsearch +numpy<=1.26.4 oauthlib~=3.1.1 pandas PyAthena[Pandas]==3.0.5 @@ -37,4 +37,4 @@ sphinx==5.0.0 sphinx-rtd-theme==1.0.0 typeguard==2.13.3 typing==3.7.4.3 -urllib3>=2.1.0,<3.0.0 +urllib3==1.26.19 diff --git a/setup.py b/setup.py index e07d71c3..82d0efcf 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ author_email='ebattat@redhat.com, pragchau@redhat.com', url='https://github.com/redhat-performance/cloud-governance', license="Apache License 2.0", + python_requires='>=3.9', classifiers=[ 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 3', @@ -48,20 +49,20 @@ 'azure-mgmt-compute==30.1.0', 'azure-mgmt-network==25.0.0', 'azure-mgmt-monitor==6.0.2', - 'boto3==1.35.0', - 'botocore==1.35.0', + 'boto3==1.33.1', + 'botocore==1.33.8', 'elasticsearch-dsl==7.4.0', - 'elasticsearch==7.17.13', + 'elasticsearch==7.13.4', 'google-api-python-client==2.57.0', 'google-auth-httplib2==0.1.0', 'google-auth-oauthlib==0.5.2', 'google-cloud-bigquery==3.5.0', 'google-cloud-billing==1.9.1', - 'ibm-cloud-sdk-core==3.22.1', + 'ibm-cloud-sdk-core==3.18.0', 'ibm-cos-sdk==2.13.6', - 'ibm-platform-services==0.60.0', + 'ibm-platform-services==0.27.0', 'ibm-schematics==1.0.1', - 'ibm-vpc==0.26.3', + 'ibm-vpc==0.21.0', 'myst-parser==1.0.0', # readthedocs 'numpy<=1.26.4', # opensearch 1.2.4 for elasticsearch 'oauthlib~=3.1.1', # required by jira @@ -71,13 +72,13 @@ 'python-ldap==3.4.2', # prerequisite: sudo dnf install -y python39-devel openldap-devel gcc 'requests==2.32.2', # rest api & lambda 'retry==0.9.2', - 'setuptools', # Requires for python3.12 + 'setuptools', # CI: setuptools<82 for IBM sdist builds on 3.9 'SoftLayer==6.0.0', # IBM SoftLayer 'sphinx-rtd-theme==1.0.0', # readthedocs 'sphinx==5.0.0', # readthedocs 'typeguard==2.13.3', # checking types 'typing==3.7.4.3', - 'urllib3>=2.1.0,<3.0.0' + 'urllib3==1.26.19' ], setup_requires=['pytest', 'pytest-runner', 'wheel', 'coverage'], diff --git a/tests_requirements.txt b/tests_requirements.txt index e05aea7f..b39b5f59 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -4,14 +4,14 @@ azure-mgmt-billing==6.0.0 azure-mgmt-costmanagement==3.0.0 azure-mgmt-monitor==6.0.2 azure-mgmt-subscription==3.1.1 -boto3==1.35.0 -elasticsearch==7.17.13 +boto3==1.33.1 +elasticsearch==7.13.4 elasticsearch-dsl==7.4.0 freezegun==1.5.1 -ibm-cloud-sdk-core==3.22.1 -ibm-platform-services==0.60.0 +ibm-cloud-sdk-core==3.18.0 +ibm-platform-services==0.27.0 ibm-schematics==1.0.1 -ibm-vpc==0.26.3 +ibm-vpc==0.21.0 moto==4.0.1 numpy<=1.26.4 oauthlib~=3.1.1 @@ -24,4 +24,4 @@ retry==0.9.2 setuptools SoftLayer==6.0.0 typeguard==2.13.3 -urllib3>=2.1.0,<3.0.0 +urllib3==1.26.19 From 2b5df99a485656cda09e79e23bb338724286638d Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 15:40:36 +0530 Subject: [PATCH 07/17] Resolve pkg dependency issues --- requirements.txt | 4 ++-- setup.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5012161a..cd17c8ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ azure-mgmt-resource==23.0.1 azure-mgmt-subscription==3.1.1 boto3==1.33.1 botocore==1.33.8 -elasticsearch==7.13.4 +elasticsearch==7.13.4 # opensearch 1.2.4 for elasticsearch elasticsearch-dsl==7.4.0 google-api-python-client==2.57.0 google-auth-httplib2==0.1.0 @@ -23,7 +23,7 @@ ibm-platform-services==0.27.0 ibm-schematics==1.0.1 ibm-vpc==0.21.0 myst-parser==1.0.0 -numpy<=1.26.4 +numpy<=1.26.4 # opensearch 1.2.4 for elasticsearch oauthlib~=3.1.1 pandas PyAthena[Pandas]==3.0.5 diff --git a/setup.py b/setup.py index 82d0efcf..d3309af1 100644 --- a/setup.py +++ b/setup.py @@ -52,15 +52,15 @@ 'boto3==1.33.1', 'botocore==1.33.8', 'elasticsearch-dsl==7.4.0', - 'elasticsearch==7.13.4', - 'google-api-python-client==2.57.0', - 'google-auth-httplib2==0.1.0', - 'google-auth-oauthlib==0.5.2', - 'google-cloud-bigquery==3.5.0', - 'google-cloud-billing==1.9.1', + 'elasticsearch==7.13.4', # opensearch 1.2.4 for elasticsearch + 'google-api-python-client==2.57.0', # google drive + 'google-auth-httplib2==0.1.0', # google drive + 'google-auth-oauthlib==0.5.2', # google drive + 'google-cloud-bigquery==3.5.0', # google cloud cost + 'google-cloud-billing==1.9.1', # google cloud cost 'ibm-cloud-sdk-core==3.18.0', 'ibm-cos-sdk==2.13.6', - 'ibm-platform-services==0.27.0', + 'ibm-platform-services==0.27.0', # IBM Usage reports 'ibm-schematics==1.0.1', 'ibm-vpc==0.21.0', 'myst-parser==1.0.0', # readthedocs @@ -78,7 +78,7 @@ 'sphinx==5.0.0', # readthedocs 'typeguard==2.13.3', # checking types 'typing==3.7.4.3', - 'urllib3==1.26.19' + 'urllib3==1.26.19' # required by jira ], setup_requires=['pytest', 'pytest-runner', 'wheel', 'coverage'], From ee5b3c59730a002211b19fa61beaebc295dd9314 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 25 Feb 2026 17:08:33 +0530 Subject: [PATCH 08/17] Modify config value --- cloud_governance/common/utils/configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index 19c6dc74..2310efdf 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -22,7 +22,7 @@ EC2_NAMESPACE = 'AWS/EC2' CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14 AWS_DEFAULT_GLOBAL_REGION = 'us-east-1' -UNUSED_ACCESS_KEY_DAYS = 180 +UNUSED_ACCESS_KEY_DAYS = 90 UNUSED_ACCESS_KEY_MAX_DAY = 1000 # X86 to Graviton From a96df8a8f9be0d84af537edb8ed6846cb059dd16 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 4 Mar 2026 14:34:40 +0530 Subject: [PATCH 09/17] Unused access key policy changes --- cloud_governance/common/clouds/aws/.DS_Store | Bin 0 -> 6148 bytes .../common/clouds/aws/iam/iam_operations.py | 52 ++++++ cloud_governance/common/mails/mail_message.py | 23 +++ cloud_governance/common/utils/configs.py | 3 +- .../main/environment_variables.py | 1 + .../policy/aws/unused_access_key.py | 172 ++++++++++++++---- .../helpers/aws/aws_policy_operations.py | 8 + 7 files changed, 226 insertions(+), 33 deletions(-) create mode 100644 cloud_governance/common/clouds/aws/.DS_Store diff --git a/cloud_governance/common/clouds/aws/.DS_Store b/cloud_governance/common/clouds/aws/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..6e34d4a9cb0272e8039b718d520a43ef4a55144a GIT binary patch literal 6148 zcmeHKO>Wab6n-xWG--i2Z33kT$s!x1Zi>Vf)r2C55K2Uq%)z%cNyF(BUEEcoEU$oEdazkweoTU%c&3p3f= zx${f;!t%<6i$X(%U9N~Zd}{wT5CPmI~YcM687)$fJe1p!k!Hyzuy|W zo*xd|345Ki{A1R&R(}e8!ul^yOuI*8iYcEMWVr_sW1+0q7z|>Tv`^`su2qb^hzCKF z$A04Tu=70L=ZE!#jq2fm(~MJ}9X7_o74NYqCS1sAcibp)!y&63y$IYM3mRdKCkY?5 zeUFFgmltJO8a5*)N>;bOsb^Vr9?g`SO0l4-6xuh-Gfg8_KX|FscS9bsLe*(1Pl5K> zO}Gn>VGj=A7~a5Jcn9y{BYcLh@Ew=X!W+1WxA6{c<3p_DQ}l2jeT-Cr3~j+uu4~kQ zF)8{HPg_wK@Nh_$S@e9c8J!^p^j{#qa)#DymTDL<4Ez}bVt=ry33N233gy;8o%nnZ z{+vJ|q>H%;#Di!q=164ZNZ5`)mOcknhKshtj(KDmIp`e_d?6wRC zbQGH2Fkl#%XCSMFmN@_K|Nj0zpJYac0mHz5#Q^4ZT01pbl0I7(7AMYHo7yXCLPT7u nP^M6)k7KEbqj;T~6!O_ZA#^mR3durZJ_IBUrZWuuRR(?laJ`Wr literal 0 HcmV?d00001 diff --git a/cloud_governance/common/clouds/aws/iam/iam_operations.py b/cloud_governance/common/clouds/aws/iam/iam_operations.py index 351456f5..d9808434 100644 --- a/cloud_governance/common/clouds/aws/iam/iam_operations.py +++ b/cloud_governance/common/clouds/aws/iam/iam_operations.py @@ -175,6 +175,53 @@ def tag_user(self, user_name: str, tags: list): except Exception as err: raise err + def untag_user(self, user_name: str, tag_keys: list): + """ + Removes the given tag keys from the IAM user. + :param user_name: The name of the IAM user. + :param tag_keys: List of tag key names to remove (e.g. ['UnusedAccessKey1InactiveDate']). + """ + if not tag_keys: + return + try: + self.iam_client.untag_user(UserName=user_name, TagKeys=tag_keys) + logger.info(f"Untagged user '{user_name}': {tag_keys}") + except Exception as err: + logger.error(f"Failed to untag user '{user_name}': {err}") + raise err + + def delete_user_access_key(self, username: str, access_key_label: str): + """ + Deletes the specified access key for the given IAM user and removes the + UnusedAccessKeyNInactiveDate tag (so we only delete keys we had deactivated). + :param username: IAM user name. + :param access_key_label: "Access key 1" or "Access key 2" (case-insensitive). + """ + access_key_label_lower = access_key_label.lower() + if not access_key_label_lower or 'access key' not in access_key_label_lower: + logger.warning("Invalid access key label for deletion.") + return + try: + access_keys = self.iam_client.list_access_keys(UserName=username)['AccessKeyMetadata'] + except Exception as e: + logger.error(f"Failed to list access keys for user '{username}': {e}") + raise + access_keys.sort(key=lambda k: k['CreateDate']) + idx = self.ACCESS_KEY_LABEL_MAP.get(access_key_label_lower) + if idx is None or idx >= len(access_keys): + logger.warning(f"Access key label '{access_key_label}' not found for user '{username}'") + return + key_num = access_key_label_lower.split()[-1] + inactive_tag_key = f"UnusedAccessKey{key_num}InactiveDate" + access_key_id = access_keys[idx]['AccessKeyId'] + try: + self.iam_client.delete_access_key(UserName=username, AccessKeyId=access_key_id) + logger.info(f"Deleted access key '{access_key_id}' for user '{username}'") + except Exception as e: + logger.error(f"Failed to delete access key '{access_key_id}' for user '{username}': {e}") + raise + self.untag_user(username, [inactive_tag_key]) + def get_iam_users_access_keys(self): """ Retrieves IAM users and summarizes: @@ -312,6 +359,11 @@ def deactivate_user_access_key(self, username: str, **kwargs): Status='Inactive' ) logger.info(f"Access key '{access_key_id}' deactivated for user '{username}'") + # Tag the user so we only delete keys we deactivated (after DELETE_ACCESS_KEY_DAYS) + key_num = access_key_label.split()[-1] + inactive_tag_key = f"UnusedAccessKey{key_num}InactiveDate" + inactive_date = datetime.now(timezone.utc).strftime("%Y-%m-%d") + self.tag_user(username, [{'Key': inactive_tag_key, 'Value': inactive_date}]) except Exception as e: logger.error(f"Failed to deactivate access key '{access_key_id}' for user '{username}': {e}") else: diff --git a/cloud_governance/common/mails/mail_message.py b/cloud_governance/common/mails/mail_message.py index ec31e94a..d722daa6 100644 --- a/cloud_governance/common/mails/mail_message.py +++ b/cloud_governance/common/mails/mail_message.py @@ -102,6 +102,29 @@ def iam_user_add_tags(self, name: str, user: str, spreadsheet_id: str): {self.RESTRICTION} Best Regards +Cloud-governance Team""".strip() + return subject, body + + def unused_access_key_reminder(self, name: str, user: str, account: str, age_days: int, key_label: str, + reminder_number: int, deactivate_days: int = 90): + """ + Reminder mail for IAM access key rotation (key age > reminder_days and <= deactivate_days). + """ + subject = f'cloud-governance alert: Rotate AWS IAM access key ({key_label}) – reminder {reminder_number}/2' + body = f""" +Hi {name}, + +Your AWS IAM user "{user}" in account {account} has an access key ({key_label}) that is {age_days} days old. +Please rotate this access key before it is automatically deactivated. + +The key will be deactivated after {deactivate_days} days from creation if no action is taken. +This is reminder {reminder_number} of 2. + +To avoid deactivation, create a new access key and update your applications, then deactivate or delete the old key. + +{self.RESTRICTION} + +Best Regards, Cloud-governance Team""".strip() return subject, body diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index 2310efdf..3d7d5f36 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -23,7 +23,8 @@ CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14 AWS_DEFAULT_GLOBAL_REGION = 'us-east-1' UNUSED_ACCESS_KEY_DAYS = 90 -UNUSED_ACCESS_KEY_MAX_DAY = 1000 +UNUSED_ACCESS_KEY_REMINDER_DAYS = 80 +DELETE_ACCESS_KEY_DAYS = 120 # X86 to Graviton GRAVITON_MAPPINGS = { diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 43f07871..1202b347 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -121,6 +121,7 @@ def __init__(self): self._environment_variables_dict['user_tags'] = EnvironmentVariables.get_env('user_tags', '') self._environment_variables_dict['user_tag_operation'] = EnvironmentVariables.get_env('user_tag_operation', '') self._environment_variables_dict['username'] = EnvironmentVariables.get_env('username', '') + self._environment_variables_dict['DELETE_INACTIVE_KEYS_WITHOUT_TAG'] = EnvironmentVariables.get_env('DELETE_INACTIVE_KEYS_WITHOUT_TAG', 'false').lower() == 'true' self._environment_variables_dict['remove_tags'] = EnvironmentVariables.get_env('remove_tags', '') self._environment_variables_dict['resource'] = EnvironmentVariables.get_env('resource', '') self._environment_variables_dict['cluster_tag'] = EnvironmentVariables.get_env('cluster_tag', '') diff --git a/cloud_governance/policy/aws/unused_access_key.py b/cloud_governance/policy/aws/unused_access_key.py index e8803110..c0266669 100644 --- a/cloud_governance/policy/aws/unused_access_key.py +++ b/cloud_governance/policy/aws/unused_access_key.py @@ -1,5 +1,12 @@ -from cloud_governance.common.utils.configs import UNUSED_ACCESS_KEY_DAYS, UNUSED_ACCESS_KEY_MAX_DAY +from cloud_governance.common.utils.configs import ( + UNUSED_ACCESS_KEY_DAYS, + UNUSED_ACCESS_KEY_REMINDER_DAYS, + DELETE_ACCESS_KEY_DAYS, +) from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations +from cloud_governance.common.mails.mail_message import MailMessage +from cloud_governance.common.mails.postfix import Postfix +from cloud_governance.common.logger.init_logger import logger class UnusedAccessKey(AWSPolicyOperations): @@ -7,46 +14,147 @@ class UnusedAccessKey(AWSPolicyOperations): def __init__(self): super().__init__() + self._mail_message = MailMessage() + self._postfix_mail = Postfix() + + def _send_reminder_and_update_tag(self, user_name: str, tags: list, access_key_label: str, + age_days: int, reminder_count: int): + """Send one reminder email and set IAM user tag UnusedAccessKeyNReminderCount to reminder_count.""" + tag_key = f"UnusedAccessKey{access_key_label.split()[-1]}ReminderCount" + to_user = self.get_tag_name_from_tags(tags=tags, tag_name='User') or user_name + display_name = self._mail_message.get_user_ldap_details(user_name=to_user) or to_user + subject, body = self._mail_message.unused_access_key_reminder( + name=display_name, + user=user_name, + account=self.account or self._environment_variables_dict.get('account', ''), + age_days=age_days, + key_label=access_key_label, + reminder_number=reminder_count, + deactivate_days=UNUSED_ACCESS_KEY_DAYS, + ) + try: + self._postfix_mail.send_email_postfix(to=to_user, cc=[], subject=subject, content=body) + self._iam_operations.tag_user(user_name, [{'Key': tag_key, 'Value': str(reminder_count)}]) + logger.info(f"Sent access key rotation reminder {reminder_count}/2 to {to_user} for {user_name}") + except Exception as err: + logger.warning(f"Failed to send reminder or update tag for {user_name}: {err}") def run_policy_operations(self): """ - This method returns a list of users with at least one active access key whose last used date is greater than UNUSED_ACCESS_KEY_DAYS - :return: - :rtype: + For key age > 80 and <= 90 days: send up to two reminder emails to rotate the key. + For key age > 90 days: deactivate the access key after grace period. + For keys we previously deactivated (tagged with UnusedAccessKeyNInactiveDate): if key age + > DELETE_ACCESS_KEY_DAYS (120), delete the key (~30 days after deactivation). Only keys we tagged are deleted. """ unused_access_keys = [] iam_users_access_keys = self._get_iam_users_access_keys() for username, user_data in iam_users_access_keys.items(): + tags = user_data.get('tags', user_data.get('Tags', [])) + region = user_data['region'] + user_name = username + for access_key_label, access_key_data in user_data.items(): - if 'access key' in access_key_label.lower(): - last_activity_days = access_key_data['last_activity_days'] - age_days = access_key_data['age_days'] - region = user_data['region'] - user_name = username - tags = user_data.get('Tags', []) - cleanup_result = False - cleanup_days = 0 - if last_activity_days and int(last_activity_days) >= UNUSED_ACCESS_KEY_DAYS and self._has_active_access_keys(user_name, access_key_label) and self.get_skip_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'): - cleanup_days = self.get_clean_up_days_count(tags=tags) - cleanup_result = self.verify_and_delete_resource(resource_id=user_name, tags=tags, - clean_up_days=cleanup_days, access_key_label=access_key_label) - resource_data = self._get_es_schema(resource_id=user_name, - user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), - skip_policy=self.get_skip_policy_value(tags=tags), - cleanup_days=cleanup_days, - dry_run=self._dry_run, - name=user_name, - region=region, - cleanup_result=str(cleanup_result), - resource_action=self.RESOURCE_ACTION, - cloud_name=self._cloud_name, - resource_type='UnusedAccessKey', - resource_state='Active', - age_days=age_days, - last_activity_days=last_activity_days, - unit_price=0) + if 'access key' not in access_key_label.lower(): + continue + last_activity_days = access_key_data.get('last_activity_days') + age_days = access_key_data.get('age_days') + status = (access_key_data.get('status') or '').lower() + if age_days is None: + continue + age_days = int(age_days) + + # Inactive key: delete if (we tagged it and key age > 120 days) OR (DELETE_INACTIVE_KEYS_WITHOUT_TAG and key age > 120) + if status == 'inactive': + key_num = access_key_label.split()[-1] + inactive_tag_key = f"UnusedAccessKey{key_num}InactiveDate" + inactive_date_str = self.get_tag_name_from_tags(tags=tags, tag_name=inactive_tag_key) + delete_all_inactive = self._environment_variables_dict.get('DELETE_INACTIVE_KEYS_WITHOUT_TAG', False) + should_delete = age_days > DELETE_ACCESS_KEY_DAYS and (inactive_date_str or delete_all_inactive) + if should_delete: + if self._dry_run == 'no': + try: + self._delete_inactive_access_key(user_name, access_key_label) + cleanup_result = True + except Exception as err: + logger.warning(f"Failed to delete inactive key for {user_name}: {err}") + cleanup_result = False + else: + cleanup_result = False + resource_data = self._get_es_schema( + resource_id=user_name, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=DELETE_ACCESS_KEY_DAYS, + dry_run=self._dry_run, + name=user_name, + region=region, + cleanup_result=str(cleanup_result) if self._dry_run == 'no' else 'dry_run', + resource_action='Delete', + cloud_name=self._cloud_name, + resource_type='UnusedAccessKey', + resource_state='Inactive', + age_days=age_days, + last_activity_days=last_activity_days, + unit_price=0, + ) unused_access_keys.append(resource_data) + continue + + if not self._has_active_access_keys(user_name, access_key_label): + continue + if self.get_skip_policy_value(tags=tags) in ('NOTDELETE', 'SKIP'): + continue + + cleanup_result = False + cleanup_days = 0 + + # Reminder window: 80 < age <= 90 – send up to two reminders + if UNUSED_ACCESS_KEY_REMINDER_DAYS < age_days <= UNUSED_ACCESS_KEY_DAYS: + tag_key = f"UnusedAccessKey{access_key_label.split()[-1]}ReminderCount" + reminder_val = self.get_tag_name_from_tags(tags=tags, tag_name=tag_key) + reminder_count = 0 + if reminder_val in ('1', '2'): + reminder_count = int(reminder_val) + if reminder_count < 2 and self._dry_run == 'no': + self._send_reminder_and_update_tag( + user_name=user_name, + tags=tags, + access_key_label=access_key_label, + age_days=age_days, + reminder_count=reminder_count + 1, + ) + continue + + # Deactivate when age > 90 days + if age_days >= UNUSED_ACCESS_KEY_DAYS: + cleanup_days = self.get_clean_up_days_count(tags=tags) + cleanup_result = self.verify_and_delete_resource( + resource_id=user_name, + tags=tags, + clean_up_days=cleanup_days, + access_key_label=access_key_label, + ) + resource_data = self._get_es_schema( + resource_id=user_name, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=cleanup_days, + dry_run=self._dry_run, + name=user_name, + region=region, + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + resource_type='UnusedAccessKey', + resource_state='Active', + age_days=age_days, + last_activity_days=last_activity_days, + unit_price=0, + ) + unused_access_keys.append(resource_data) if not cleanup_result: - self.update_resource_day_count_tag(resource_id=user_name, cleanup_days=cleanup_days, tags=tags) + self.update_resource_day_count_tag( + resource_id=user_name, cleanup_days=cleanup_days, tags=tags + ) return unused_access_keys diff --git a/cloud_governance/policy/helpers/aws/aws_policy_operations.py b/cloud_governance/policy/helpers/aws/aws_policy_operations.py index 3d7af264..fd9564a9 100644 --- a/cloud_governance/policy/helpers/aws/aws_policy_operations.py +++ b/cloud_governance/policy/helpers/aws/aws_policy_operations.py @@ -81,6 +81,14 @@ def _delete_resource(self, resource_id: str, **kwargs): logger.error(f'Exception raised: {err}: {resource_id}') raise err + def _delete_inactive_access_key(self, user_name: str, access_key_label: str): + """ + Deletes an access key that was previously deactivated by this policy (and tagged with + UnusedAccessKeyNInactiveDate). Used when the key has been inactive for more than + DELETE_ACCESS_KEY_DAYS. + """ + self._iam_operations.delete_user_access_key(username=user_name, access_key_label=access_key_label) + def __remove_tag_key_aws(self, tags: list): """ This method returns the tags that does not contain key startswith aws: From 5dd6d3120640dbdde586f93e0cbd415220df5fb0 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 4 Mar 2026 17:27:29 +0530 Subject: [PATCH 10/17] Review comments --- .gitignore | 3 +++ cloud_governance/common/clouds/aws/.DS_Store | Bin 6148 -> 0 bytes cloud_governance/common/mails/mail_message.py | 2 ++ 3 files changed, 5 insertions(+) delete mode 100644 cloud_governance/common/clouds/aws/.DS_Store diff --git a/.gitignore b/.gitignore index d6a70a72..d075f63a 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,6 @@ empty_test_environment_variables.py cloudsensei/.env.txt .vscode env.yaml + +# macOS +.DS_Store diff --git a/cloud_governance/common/clouds/aws/.DS_Store b/cloud_governance/common/clouds/aws/.DS_Store deleted file mode 100644 index 6e34d4a9cb0272e8039b718d520a43ef4a55144a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKO>Wab6n-xWG--i2Z33kT$s!x1Zi>Vf)r2C55K2Uq%)z%cNyF(BUEEcoEU$oEdazkweoTU%c&3p3f= zx${f;!t%<6i$X(%U9N~Zd}{wT5CPmI~YcM687)$fJe1p!k!Hyzuy|W zo*xd|345Ki{A1R&R(}e8!ul^yOuI*8iYcEMWVr_sW1+0q7z|>Tv`^`su2qb^hzCKF z$A04Tu=70L=ZE!#jq2fm(~MJ}9X7_o74NYqCS1sAcibp)!y&63y$IYM3mRdKCkY?5 zeUFFgmltJO8a5*)N>;bOsb^Vr9?g`SO0l4-6xuh-Gfg8_KX|FscS9bsLe*(1Pl5K> zO}Gn>VGj=A7~a5Jcn9y{BYcLh@Ew=X!W+1WxA6{c<3p_DQ}l2jeT-Cr3~j+uu4~kQ zF)8{HPg_wK@Nh_$S@e9c8J!^p^j{#qa)#DymTDL<4Ez}bVt=ry33N233gy;8o%nnZ z{+vJ|q>H%;#Di!q=164ZNZ5`)mOcknhKshtj(KDmIp`e_d?6wRC zbQGH2Fkl#%XCSMFmN@_K|Nj0zpJYac0mHz5#Q^4ZT01pbl0I7(7AMYHo7yXCLPT7u nP^M6)k7KEbqj;T~6!O_ZA#^mR3durZJ_IBUrZWuuRR(?laJ`Wr diff --git a/cloud_governance/common/mails/mail_message.py b/cloud_governance/common/mails/mail_message.py index d722daa6..24d0fa74 100644 --- a/cloud_governance/common/mails/mail_message.py +++ b/cloud_governance/common/mails/mail_message.py @@ -3,6 +3,7 @@ from jinja2 import Environment, FileSystemLoader from cloud_governance.common.ldap.ldap_search import LdapSearch +from cloud_governance.common.utils.configs import DELETE_ACCESS_KEY_DAYS from cloud_governance.main.environment_variables import environment_variables @@ -118,6 +119,7 @@ def unused_access_key_reminder(self, name: str, user: str, account: str, age_day Please rotate this access key before it is automatically deactivated. The key will be deactivated after {deactivate_days} days from creation if no action is taken. +Keys older than {DELETE_ACCESS_KEY_DAYS} days (including deactivated ones) will be permanently deleted. This is reminder {reminder_number} of 2. To avoid deactivation, create a new access key and update your applications, then deactivate or delete the old key. From 00c2ca32c2103d454f73b117313163e77110584f Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Mon, 9 Mar 2026 14:00:11 +0530 Subject: [PATCH 11/17] Permissions for Policy changes --- .../aws/CloudGovernanceInfra/CloudGovernanceDeletePolicy.json | 2 ++ .../aws/CloudGovernanceInfra/CloudGovernanceReadPolicy.json | 1 + 2 files changed, 3 insertions(+) diff --git a/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceDeletePolicy.json b/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceDeletePolicy.json index a954dd3a..10122e2a 100644 --- a/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceDeletePolicy.json +++ b/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceDeletePolicy.json @@ -118,6 +118,7 @@ "Sid": "IAM", "Effect": "Allow", "Action": [ + "iam:DeleteAccessKey", "iam:DeleteInstanceProfile", "iam:DeletePolicy", "iam:DeleteRole", @@ -136,6 +137,7 @@ "iam:ListRoles", "iam:ListUserPolicies", "iam:ListUsers", + "iam:ListUserTags", "iam:RemoveRoleFromInstanceProfile", "iam:TagRole", "iam:TagUser", diff --git a/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceReadPolicy.json b/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceReadPolicy.json index 65d2f3ee..a732d617 100644 --- a/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceReadPolicy.json +++ b/iam/clouds/aws/CloudGovernanceInfra/CloudGovernanceReadPolicy.json @@ -95,6 +95,7 @@ "iam:ListRoles", "iam:ListUserPolicies", "iam:ListUsers", + "iam:ListUserTags", "iam:TagRole", "iam:TagUser", "iam:UntagRole", From 4b659a5b88d73a958e3971e447e86ebce867b4f1 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Mon, 9 Mar 2026 14:41:50 +0530 Subject: [PATCH 12/17] Add delete policy, modify unused key policy --- .../common/clouds/aws/iam/iam_operations.py | 60 ++++--- cloud_governance/common/mails/mail_message.py | 49 +++--- .../templates/policy_alert_agg_message.j2 | 6 + cloud_governance/common/utils/configs.py | 1 - .../main/environment_variables.py | 2 +- .../main/main_oerations/main_operations.py | 2 +- .../policy/aws/delete_access_key.py | 71 ++++++++ .../policy/aws/unused_access_key.py | 155 ++++-------------- .../common_policies/send_aggregated_alerts.py | 28 ++-- .../helpers/aws/aws_policy_operations.py | 30 +++- 10 files changed, 224 insertions(+), 180 deletions(-) create mode 100644 cloud_governance/policy/aws/delete_access_key.py diff --git a/cloud_governance/common/clouds/aws/iam/iam_operations.py b/cloud_governance/common/clouds/aws/iam/iam_operations.py index d9808434..d5685a56 100644 --- a/cloud_governance/common/clouds/aws/iam/iam_operations.py +++ b/cloud_governance/common/clouds/aws/iam/iam_operations.py @@ -190,37 +190,53 @@ def untag_user(self, user_name: str, tag_keys: list): logger.error(f"Failed to untag user '{user_name}': {err}") raise err - def delete_user_access_key(self, username: str, access_key_label: str): - """ - Deletes the specified access key for the given IAM user and removes the - UnusedAccessKeyNInactiveDate tag (so we only delete keys we had deactivated). + def delete_user_access_key( + self, + username: str, + access_key_label: str, + remove_inactive_tag: bool = True, + access_key_id: str = None, + ): + """ + Deletes the specified access key for the given IAM user. Optionally removes the + UnusedAccessKeyNInactiveDate tag (only when this policy had set it by deactivating the key). :param username: IAM user name. :param access_key_label: "Access key 1" or "Access key 2" (case-insensitive). + :param remove_inactive_tag: If True, remove UnusedAccessKeyNInactiveDate after delete. + Set False when the key was not deactivated by this policy (e.g. manually deactivated). + :param access_key_id: Optional. When provided, delete this key by ID instead of resolving by + label. """ - access_key_label_lower = access_key_label.lower() + access_key_label_lower = (access_key_label or '').lower() if not access_key_label_lower or 'access key' not in access_key_label_lower: logger.warning("Invalid access key label for deletion.") return - try: - access_keys = self.iam_client.list_access_keys(UserName=username)['AccessKeyMetadata'] - except Exception as e: - logger.error(f"Failed to list access keys for user '{username}': {e}") - raise - access_keys.sort(key=lambda k: k['CreateDate']) - idx = self.ACCESS_KEY_LABEL_MAP.get(access_key_label_lower) - if idx is None or idx >= len(access_keys): - logger.warning(f"Access key label '{access_key_label}' not found for user '{username}'") - return key_num = access_key_label_lower.split()[-1] inactive_tag_key = f"UnusedAccessKey{key_num}InactiveDate" - access_key_id = access_keys[idx]['AccessKeyId'] + + if not access_key_id: + try: + access_keys = self.iam_client.list_access_keys(UserName=username)['AccessKeyMetadata'] + except Exception as e: + logger.error(f"Failed to list access keys for user '{username}': {e}") + raise + access_keys.sort(key=lambda k: k['CreateDate']) + idx = self.ACCESS_KEY_LABEL_MAP.get(access_key_label_lower) + if idx is None or idx >= len(access_keys): + logger.warning(f"Access key label '{access_key_label}' not found for user '{username}'") + return + access_key_id = access_keys[idx]['AccessKeyId'] + try: self.iam_client.delete_access_key(UserName=username, AccessKeyId=access_key_id) logger.info(f"Deleted access key '{access_key_id}' for user '{username}'") except Exception as e: logger.error(f"Failed to delete access key '{access_key_id}' for user '{username}': {e}") raise - self.untag_user(username, [inactive_tag_key]) + tag_keys_to_remove = [access_key_id] + if remove_inactive_tag: + tag_keys_to_remove.append(inactive_tag_key) + self.untag_user(username, tag_keys_to_remove) def get_iam_users_access_keys(self): """ @@ -255,8 +271,8 @@ def get_iam_users_access_keys(self): for user in page['Users']: username = user['UserName'] result[username] = {} - # Access keys access_keys = self.iam_client.list_access_keys(UserName=username)['AccessKeyMetadata'] + access_keys = sorted(access_keys, key=lambda k: k['CreateDate']) for idx, key in enumerate(access_keys, start=1): label = f"Access key {idx}" status = key['Status'].lower() @@ -276,7 +292,13 @@ def get_iam_users_access_keys(self): logger.error(f"Failed to get last used date for access key") last_used_days = None - result[username][label] = {'label': label, 'status': status, 'age_days': age_days, 'last_activity_days': last_used_days} + result[username][label] = { + 'label': label, + 'status': status, + 'age_days': age_days, + 'last_activity_days': last_used_days, + 'access_key_id': key['AccessKeyId'], + } # Tags as list of dicts try: diff --git a/cloud_governance/common/mails/mail_message.py b/cloud_governance/common/mails/mail_message.py index 24d0fa74..251bd807 100644 --- a/cloud_governance/common/mails/mail_message.py +++ b/cloud_governance/common/mails/mail_message.py @@ -106,29 +106,26 @@ def iam_user_add_tags(self, name: str, user: str, spreadsheet_id: str): Cloud-governance Team""".strip() return subject, body - def unused_access_key_reminder(self, name: str, user: str, account: str, age_days: int, key_label: str, - reminder_number: int, deactivate_days: int = 90): + def _get_unused_access_key_alert_message(self): """ - Reminder mail for IAM access key rotation (key age > reminder_days and <= deactivate_days). + Returns the custom message for unused_access_key policy in aggregated alerts. """ - subject = f'cloud-governance alert: Rotate AWS IAM access key ({key_label}) – reminder {reminder_number}/2' - body = f""" -Hi {name}, - -Your AWS IAM user "{user}" in account {account} has an access key ({key_label}) that is {age_days} days old. -Please rotate this access key before it is automatically deactivated. - -The key will be deactivated after {deactivate_days} days from creation if no action is taken. -Keys older than {DELETE_ACCESS_KEY_DAYS} days (including deactivated ones) will be permanently deleted. -This is reminder {reminder_number} of 2. - -To avoid deactivation, create a new access key and update your applications, then deactivate or delete the old key. - -{self.RESTRICTION} + return ( + f"For the IAM access keys listed below: please rotate these access keys before they are " + f"automatically deactivated. Keys will be deactivated after the grace period if no action is taken. " + f"Keys older than {DELETE_ACCESS_KEY_DAYS} days (including deactivated ones) will be permanently deleted. " + "To avoid deactivation, create a new access key and update your applications, then deactivate or delete the old key." + ) -Best Regards, -Cloud-governance Team""".strip() - return subject, body + def _get_delete_access_key_alert_message(self): + """ + Returns the custom message for delete_access_key policy in aggregated alerts. + """ + return ( + f"Your IAM access key age has exceeded {DELETE_ACCESS_KEY_DAYS} days. " + "The key(s) listed below are in the deletion grace period and will be permanently deleted " + "if no action is taken. Please rotate or remove these keys before the grace period ends." + ) def aws_user_over_usage_cost(self, user: str, usage_cost: int, name: str, user_usage: int): """ @@ -542,5 +539,17 @@ def get_policy_alert_message(self, policy_data: list, user: str = ''): template_loader = self.env_loader.get_template('policy_alert_agg_message.j2') columns = ['User', 'PublicCloud', 'policy', 'RegionName', 'ResourceId', 'Name', 'DeleteDate'] context = {'records': policy_data, 'columns': columns, 'User': user, 'account': self.account, 'cloud_name': self.__public_cloud_name} + has_unused_access_key = any( + (r.get('policy') or r.get('Policy') or '').lower() == 'unused_access_key' + for r in (policy_data or []) + ) + has_delete_access_key = any( + (r.get('policy') or r.get('Policy') or '').lower() == 'delete_access_key' + for r in (policy_data or []) + ) + if has_unused_access_key: + context['unused_access_key_message'] = self._get_unused_access_key_alert_message() + if has_delete_access_key: + context['delete_access_key_message'] = self._get_delete_access_key_alert_message() body = template_loader.render(context) return subject, body diff --git a/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 b/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 index 2c749d5c..ccc7748c 100644 --- a/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 +++ b/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 @@ -44,6 +44,12 @@

You can find below your unused resources in the {{ cloud_name }} account ({{ account }}).

If you want to keep them, please add "Policy=Not_Delete" or "Policy=skip" tag for each resource

+ {% if unused_access_key_message is defined and unused_access_key_message %} +

Unused access keys: {{ unused_access_key_message }}

+ {% endif %} + {% if delete_access_key_message is defined and delete_access_key_message %} +

Access keys pending deletion (age > 120 days): {{ delete_access_key_message }}

+ {% endif %}
diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index 3d7d5f36..e5137a4b 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -23,7 +23,6 @@ CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14 AWS_DEFAULT_GLOBAL_REGION = 'us-east-1' UNUSED_ACCESS_KEY_DAYS = 90 -UNUSED_ACCESS_KEY_REMINDER_DAYS = 80 DELETE_ACCESS_KEY_DAYS = 120 # X86 to Graviton diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 1202b347..fd43e8d1 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -98,7 +98,7 @@ def __init__(self): 'ip_unattached', 'unused_nat_gateway', 'instance_idle', 'ec2_stop', 'ebs_in_use', 'database_idle', - 's3_inactive', 'unused_access_key', + 's3_inactive', 'unused_access_key', 'delete_access_key', 'empty_roles', 'zombie_snapshots', 'skipped_resources', 'monthly_report', 'optimize_resources_report'] diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py index b2ebbf48..bebc9d8d 100644 --- a/cloud_governance/main/main_oerations/main_operations.py +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -43,7 +43,7 @@ def run(self): if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run", "ip_unattached", "unused_nat_gateway", "instance_idle", "zombie_snapshots", "database_idle", "s3_inactive", "unused_access_key", - "empty_roles", "tag_resources", "cost_usage_reports"]: + "delete_access_key", "empty_roles", "tag_resources", "cost_usage_reports"]: source = policy_type if Utils.equal_ignore_case(policy_type, self._public_cloud_name): source = '' diff --git a/cloud_governance/policy/aws/delete_access_key.py b/cloud_governance/policy/aws/delete_access_key.py new file mode 100644 index 00000000..5c11165a --- /dev/null +++ b/cloud_governance/policy/aws/delete_access_key.py @@ -0,0 +1,71 @@ +""" +Policy to delete inactive IAM access keys older than DELETE_ACCESS_KEY_DAYS. +""" +from cloud_governance.common.utils.configs import DELETE_ACCESS_KEY_DAYS +from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations + + +class DeleteAccessKey(AWSPolicyOperations): + RESOURCE_ACTION = "Delete" + + def run_policy_operations(self): + """ + For inactive keys with age > DELETE_ACCESS_KEY_DAYS: apply a grace period + (deletion_grace_days = age_days - DELETE_ACCESS_KEY_DAYS, capped at DAYS_TO_TAKE_ACTION). During + grace period write to ES with cleanup_days 1..7 so send_aggregated_alerts sends + reminder emails. After grace period, delete the key. + """ + result = [] + days_to_take_action = int(self._days_to_take_action) + iam_users_access_keys = self._get_iam_users_access_keys() + + for username, user_data in iam_users_access_keys.items(): + tags = user_data.get('tags', user_data.get('Tags', [])) + region = user_data['region'] + user_name = username + + for access_key_label, access_key_data in user_data.items(): + if 'access key' not in access_key_label.lower(): + continue + age_days = access_key_data.get('age_days') + status = (access_key_data.get('status') or '').lower() + if age_days is None or status != 'inactive': + continue + age_days = int(age_days) + + key_num = access_key_label.split()[-1] + inactive_tag_key = f"UnusedAccessKey{key_num}InactiveDate" + inactive_date_str = self.get_tag_name_from_tags(tags=tags, tag_name=inactive_tag_key) + delete_all_inactive = self._environment_variables_dict.get('DELETE_INACTIVE_KEYS_WITHOUT_TAG', False) + if not (age_days > DELETE_ACCESS_KEY_DAYS and (inactive_date_str or delete_all_inactive)): + continue + + deletion_grace_days = min(age_days - DELETE_ACCESS_KEY_DAYS, days_to_take_action) + cleanup_result = self.verify_and_delete_resource( + resource_id=user_name, + tags=tags, + clean_up_days=deletion_grace_days, + access_key_label=access_key_label, + access_key_id=access_key_data.get('access_key_id'), + remove_inactive_tag=bool(inactive_date_str), + ) + resource_data = self._get_es_schema( + resource_id=user_name, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=deletion_grace_days, + dry_run=self._dry_run, + name=user_name, + region=region, + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + resource_type='UnusedAccessKey', + resource_state='Inactive', + age_days=age_days, + last_activity_days=access_key_data.get('last_activity_days'), + unit_price=0, + ) + result.append(resource_data) + + return result diff --git a/cloud_governance/policy/aws/unused_access_key.py b/cloud_governance/policy/aws/unused_access_key.py index c0266669..eb45ab72 100644 --- a/cloud_governance/policy/aws/unused_access_key.py +++ b/cloud_governance/policy/aws/unused_access_key.py @@ -1,52 +1,19 @@ -from cloud_governance.common.utils.configs import ( - UNUSED_ACCESS_KEY_DAYS, - UNUSED_ACCESS_KEY_REMINDER_DAYS, - DELETE_ACCESS_KEY_DAYS, -) +from cloud_governance.common.utils.configs import UNUSED_ACCESS_KEY_DAYS from cloud_governance.policy.helpers.aws.aws_policy_operations import AWSPolicyOperations -from cloud_governance.common.mails.mail_message import MailMessage -from cloud_governance.common.mails.postfix import Postfix -from cloud_governance.common.logger.init_logger import logger class UnusedAccessKey(AWSPolicyOperations): RESOURCE_ACTION = "DeActivate" - def __init__(self): - super().__init__() - self._mail_message = MailMessage() - self._postfix_mail = Postfix() - - def _send_reminder_and_update_tag(self, user_name: str, tags: list, access_key_label: str, - age_days: int, reminder_count: int): - """Send one reminder email and set IAM user tag UnusedAccessKeyNReminderCount to reminder_count.""" - tag_key = f"UnusedAccessKey{access_key_label.split()[-1]}ReminderCount" - to_user = self.get_tag_name_from_tags(tags=tags, tag_name='User') or user_name - display_name = self._mail_message.get_user_ldap_details(user_name=to_user) or to_user - subject, body = self._mail_message.unused_access_key_reminder( - name=display_name, - user=user_name, - account=self.account or self._environment_variables_dict.get('account', ''), - age_days=age_days, - key_label=access_key_label, - reminder_number=reminder_count, - deactivate_days=UNUSED_ACCESS_KEY_DAYS, - ) - try: - self._postfix_mail.send_email_postfix(to=to_user, cc=[], subject=subject, content=body) - self._iam_operations.tag_user(user_name, [{'Key': tag_key, 'Value': str(reminder_count)}]) - logger.info(f"Sent access key rotation reminder {reminder_count}/2 to {to_user} for {user_name}") - except Exception as err: - logger.warning(f"Failed to send reminder or update tag for {user_name}: {err}") - def run_policy_operations(self): """ - For key age > 80 and <= 90 days: send up to two reminder emails to rotate the key. - For key age > 90 days: deactivate the access key after grace period. - For keys we previously deactivated (tagged with UnusedAccessKeyNInactiveDate): if key age - > DELETE_ACCESS_KEY_DAYS (120), delete the key (~30 days after deactivation). Only keys we tagged are deleted. + For key age >= UNUSED_ACCESS_KEY_DAYS (e.g. 90): apply a grace period (deactivation_grace_days + = age_days - UNUSED_ACCESS_KEY_DAYS, capped at DAYS_TO_TAKE_ACTION). During grace period + write to ES with cleanup_days 1..7 so send_aggregated_alerts sends reminder emails. After + grace period, deactivate the key. """ unused_access_keys = [] + days_to_take_action = int(self._days_to_take_action) iam_users_access_keys = self._get_iam_users_access_keys() for username, user_data in iam_users_access_keys.items(): tags = user_data.get('tags', user_data.get('Tags', [])) @@ -63,41 +30,7 @@ def run_policy_operations(self): continue age_days = int(age_days) - # Inactive key: delete if (we tagged it and key age > 120 days) OR (DELETE_INACTIVE_KEYS_WITHOUT_TAG and key age > 120) if status == 'inactive': - key_num = access_key_label.split()[-1] - inactive_tag_key = f"UnusedAccessKey{key_num}InactiveDate" - inactive_date_str = self.get_tag_name_from_tags(tags=tags, tag_name=inactive_tag_key) - delete_all_inactive = self._environment_variables_dict.get('DELETE_INACTIVE_KEYS_WITHOUT_TAG', False) - should_delete = age_days > DELETE_ACCESS_KEY_DAYS and (inactive_date_str or delete_all_inactive) - if should_delete: - if self._dry_run == 'no': - try: - self._delete_inactive_access_key(user_name, access_key_label) - cleanup_result = True - except Exception as err: - logger.warning(f"Failed to delete inactive key for {user_name}: {err}") - cleanup_result = False - else: - cleanup_result = False - resource_data = self._get_es_schema( - resource_id=user_name, - user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), - skip_policy=self.get_skip_policy_value(tags=tags), - cleanup_days=DELETE_ACCESS_KEY_DAYS, - dry_run=self._dry_run, - name=user_name, - region=region, - cleanup_result=str(cleanup_result) if self._dry_run == 'no' else 'dry_run', - resource_action='Delete', - cloud_name=self._cloud_name, - resource_type='UnusedAccessKey', - resource_state='Inactive', - age_days=age_days, - last_activity_days=last_activity_days, - unit_price=0, - ) - unused_access_keys.append(resource_data) continue if not self._has_active_access_keys(user_name, access_key_label): @@ -105,56 +38,32 @@ def run_policy_operations(self): if self.get_skip_policy_value(tags=tags) in ('NOTDELETE', 'SKIP'): continue - cleanup_result = False - cleanup_days = 0 - - # Reminder window: 80 < age <= 90 – send up to two reminders - if UNUSED_ACCESS_KEY_REMINDER_DAYS < age_days <= UNUSED_ACCESS_KEY_DAYS: - tag_key = f"UnusedAccessKey{access_key_label.split()[-1]}ReminderCount" - reminder_val = self.get_tag_name_from_tags(tags=tags, tag_name=tag_key) - reminder_count = 0 - if reminder_val in ('1', '2'): - reminder_count = int(reminder_val) - if reminder_count < 2 and self._dry_run == 'no': - self._send_reminder_and_update_tag( - user_name=user_name, - tags=tags, - access_key_label=access_key_label, - age_days=age_days, - reminder_count=reminder_count + 1, - ) + if age_days < UNUSED_ACCESS_KEY_DAYS: continue - - # Deactivate when age > 90 days - if age_days >= UNUSED_ACCESS_KEY_DAYS: - cleanup_days = self.get_clean_up_days_count(tags=tags) - cleanup_result = self.verify_and_delete_resource( - resource_id=user_name, - tags=tags, - clean_up_days=cleanup_days, - access_key_label=access_key_label, - ) - resource_data = self._get_es_schema( - resource_id=user_name, - user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), - skip_policy=self.get_skip_policy_value(tags=tags), - cleanup_days=cleanup_days, - dry_run=self._dry_run, - name=user_name, - region=region, - cleanup_result=str(cleanup_result), - resource_action=self.RESOURCE_ACTION, - cloud_name=self._cloud_name, - resource_type='UnusedAccessKey', - resource_state='Active', - age_days=age_days, - last_activity_days=last_activity_days, - unit_price=0, - ) - unused_access_keys.append(resource_data) - if not cleanup_result: - self.update_resource_day_count_tag( - resource_id=user_name, cleanup_days=cleanup_days, tags=tags - ) + deactivation_grace_days = min(age_days - UNUSED_ACCESS_KEY_DAYS, days_to_take_action) + cleanup_result = self.verify_and_delete_resource( + resource_id=user_name, + tags=tags, + clean_up_days=deactivation_grace_days, + access_key_label=access_key_label, + ) + resource_data = self._get_es_schema( + resource_id=user_name, + user=self.get_tag_name_from_tags(tags=tags, tag_name='User'), + skip_policy=self.get_skip_policy_value(tags=tags), + cleanup_days=deactivation_grace_days, + dry_run=self._dry_run, + name=user_name, + region=region, + cleanup_result=str(cleanup_result), + resource_action=self.RESOURCE_ACTION, + cloud_name=self._cloud_name, + resource_type='UnusedAccessKey', + resource_state='Active', + age_days=age_days, + last_activity_days=last_activity_days, + unit_price=0, + ) + unused_access_keys.append(resource_data) return unused_access_keys diff --git a/cloud_governance/policy/common_policies/send_aggregated_alerts.py b/cloud_governance/policy/common_policies/send_aggregated_alerts.py index 3670b1a9..8fd9b991 100644 --- a/cloud_governance/policy/common_policies/send_aggregated_alerts.py +++ b/cloud_governance/policy/common_policies/send_aggregated_alerts.py @@ -80,17 +80,23 @@ def __get_es_data(self): def __remove_duplicates(self, policy_es_data: list): """ - This method removes the duplicate data - :return: - :rtype: + This method removes the duplicate data. + For unused_access_key and delete_access_key, one user can have two keys, so we do not + deduplicate by ResourceId (username); for all other policies we keep one record per ResourceId. """ - if policy_es_data: - df = pandas.DataFrame(policy_es_data) - df.sort_values(inplace=True, by=['policy']) - df.fillna(value='', inplace=True) - df.drop_duplicates(subset='ResourceId', inplace=True) - return df.to_dict(orient="records") - return policy_es_data + if not policy_es_data: + return policy_es_data + df = pandas.DataFrame(policy_es_data) + policy_col = 'policy' if 'policy' in df.columns else 'Policy' + sort_col = policy_col if policy_col in df.columns else df.columns[0] + df.sort_values(inplace=True, by=[sort_col]) + df.fillna(value='', inplace=True) + access_key_policies = ['unused_access_key', 'delete_access_key'] + mask = df[policy_col].str.lower().isin(access_key_policies) + df_other = df[~mask].drop_duplicates(subset='ResourceId', keep='first') + df_access_keys = df[mask] + df = pandas.concat([df_other, df_access_keys], ignore_index=True) + return df.to_dict(orient="records") def __group_by_policy(self, policy_data: list): """ @@ -165,7 +171,7 @@ def __update_delete_days(self, policy_es_data: list): delete_date = datetime.utcnow().date().__str__() alert_user = True # Cross region policies - if record.get('policy') in ['empty_roles', 's3_inactive', 'unused_access_key']: + if record.get('policy') in ['empty_roles', 's3_inactive', 'unused_access_key', 'delete_access_key']: record['RegionName'] = 'us-east-1' if Utils.equal_ignore_case(dry_run, 'yes'): record['DeleteDate'] = 'dry_run=yes' diff --git a/cloud_governance/policy/helpers/aws/aws_policy_operations.py b/cloud_governance/policy/helpers/aws/aws_policy_operations.py index fd9564a9..7b84ee9a 100644 --- a/cloud_governance/policy/helpers/aws/aws_policy_operations.py +++ b/cloud_governance/policy/helpers/aws/aws_policy_operations.py @@ -59,7 +59,16 @@ def _delete_resource(self, resource_id: str, **kwargs): if self._policy == 's3_inactive': self._s3_client.delete_bucket(Bucket=resource_id) elif self._policy == 'unused_access_key': + action = "deactivated" self._iam_operations.deactivate_user_access_key(username=resource_id, **kwargs) + elif self._policy == 'delete_access_key': + action = "deleted" + self._delete_inactive_access_key( + user_name=resource_id, + access_key_label=kwargs.get('access_key_label'), + remove_inactive_tag=kwargs.get('remove_inactive_tag', True), + access_key_id=kwargs.get('access_key_id'), + ) elif self._policy == 'empty_roles': response = self._iam_operations.delete_role(role_name=resource_id) elif self._policy == 'unattached_volume': @@ -81,13 +90,26 @@ def _delete_resource(self, resource_id: str, **kwargs): logger.error(f'Exception raised: {err}: {resource_id}') raise err - def _delete_inactive_access_key(self, user_name: str, access_key_label: str): + def _delete_inactive_access_key( + self, + user_name: str, + access_key_label: str, + remove_inactive_tag: bool = True, + access_key_id: str = None, + ): """ Deletes an access key that was previously deactivated by this policy (and tagged with UnusedAccessKeyNInactiveDate). Used when the key has been inactive for more than DELETE_ACCESS_KEY_DAYS. - """ - self._iam_operations.delete_user_access_key(username=user_name, access_key_label=access_key_label) + :param remove_inactive_tag: If True, remove the UnusedAccessKeyNInactiveDate tag after delete. + Pass False when the key was not deactivated by this policy (e.g. DELETE_INACTIVE_KEYS_WITHOUT_TAG). + """ + self._iam_operations.delete_user_access_key( + username=user_name, + access_key_label=access_key_label, + remove_inactive_tag=remove_inactive_tag, + access_key_id=access_key_id, + ) def __remove_tag_key_aws(self, tags: list): """ @@ -157,7 +179,7 @@ def update_resource_tags(self, tags: list, resource_id: str): try: if self._policy == 's3_inactive': self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags}) - elif self._policy == 'unused_access_key': + elif self._policy in ('unused_access_key', 'delete_access_key'): self._iam_operations.tag_user(user_name=resource_id, tags=tags) elif self._policy == 'empty_roles': self._iam_operations.tag_role(role_name=resource_id, tags=tags) From 71d3946d33ebf7d5f3c6d3f920ef5095b33d6b13 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Tue, 10 Mar 2026 12:38:12 +0530 Subject: [PATCH 13/17] Disable dry run no for unused_access_keys --- jenkins/clouds/aws/daily/policies/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/clouds/aws/daily/policies/Jenkinsfile b/jenkins/clouds/aws/daily/policies/Jenkinsfile index 20b1e924..b6f677ad 100644 --- a/jenkins/clouds/aws/daily/policies/Jenkinsfile +++ b/jenkins/clouds/aws/daily/policies/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { } environment { QUAY_CLOUD_GOVERNANCE_REPOSITORY = credentials('QUAY_CLOUD_GOVERNANCE_REPOSITORY') - POLICIES_IN_ACTION = '["instance_idle", "ec2_stop", "unattached_volume", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles", "zombie_cluster_resource", "unused_access_key"]' + POLICIES_IN_ACTION = '["instance_idle", "ec2_stop", "unattached_volume", "ip_unattached", "zombie_snapshots", "unused_nat_gateway", "s3_inactive", "empty_roles", "zombie_cluster_resource"]' AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') From dfe482a91aec61b9a85210301382a3e6a3379775 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Wed, 11 Mar 2026 17:57:01 +0530 Subject: [PATCH 14/17] Unittest files --- .../policy/aws/test_delete_access_key.py | 195 ++++++++++++++++++ .../policy/aws/test_unused_access_key.py | 166 +++++++++++++++ 2 files changed, 361 insertions(+) create mode 100644 tests/unittest/cloud_governance/policy/aws/test_delete_access_key.py create mode 100644 tests/unittest/cloud_governance/policy/aws/test_unused_access_key.py diff --git a/tests/unittest/cloud_governance/policy/aws/test_delete_access_key.py b/tests/unittest/cloud_governance/policy/aws/test_delete_access_key.py new file mode 100644 index 00000000..b2875abd --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/test_delete_access_key.py @@ -0,0 +1,195 @@ +""" +Unit tests for cloud_governance.policy.aws.delete_access_key.DeleteAccessKey. +""" +from unittest.mock import patch + +from moto import mock_ec2, mock_s3, mock_iam + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.aws.delete_access_key import DeleteAccessKey +from cloud_governance.common.utils.configs import DELETE_ACCESS_KEY_DAYS +from tests.unittest.configs import DRY_RUN_YES, DRY_RUN_NO, AWS_DEFAULT_REGION, TEST_USER_NAME + + +def _mock_iam_users_inactive_key( + age_days: int, + last_activity_days: int = 130, + with_inactive_tag: bool = True, + access_key_id: str = 'AKIAIOSFODNN7EXAMPLE', +): + """Build mock IAM user with one inactive access key, as returned by get_iam_users_access_keys().""" + tags = [{'Key': 'User', 'Value': TEST_USER_NAME}] + if with_inactive_tag: + tags.append({'Key': 'UnusedAccessKey1InactiveDate', 'Value': '2024-01-15'}) + return { + TEST_USER_NAME: { + 'Access key 1': { + 'label': 'Access key 1', + 'status': 'Inactive', + 'age_days': age_days, + 'last_activity_days': last_activity_days, + 'access_key_id': access_key_id, + }, + 'tags': tags, + 'region': AWS_DEFAULT_REGION, + 'ResourceId': 'AIDAEXAMPLE', + } + } + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_skips_active_keys(): + """Only inactive keys are considered; active keys are skipped.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_inactive_key(age_days=DELETE_ACCESS_KEY_DAYS + 10, with_inactive_tag=True) + mock_data[TEST_USER_NAME]['Access key 1']['status'] = 'Active' + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + policy = DeleteAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_skips_when_age_at_or_below_threshold(): + """Inactive keys with age_days <= DELETE_ACCESS_KEY_DAYS are skipped.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_inactive_key(age_days=DELETE_ACCESS_KEY_DAYS, with_inactive_tag=True) + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + policy = DeleteAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_includes_inactive_old_key_with_tag(): + """Inactive key with age > DELETE_ACCESS_KEY_DAYS and UnusedAccessKey1InactiveDate tag is included.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + age_days = DELETE_ACCESS_KEY_DAYS + 5 + mock_data = _mock_iam_users_inactive_key(age_days=age_days, with_inactive_tag=True) + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + policy = DeleteAccessKey() + result = policy.run_policy_operations() + assert len(result) == 1 + assert result[0]['ResourceId'] == TEST_USER_NAME + # ES schema stores cleanup_result in ResourceAction (dry_run=yes -> verify returns False) + assert result[0]['ResourceAction'] == 'False' + assert result[0]['ResourceType'] == 'UnusedAccessKey' + assert result[0]['ResourceState'] == 'Inactive' + assert result[0]['AgeDays'] == age_days + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_skips_inactive_old_key_without_tag_unless_flag(): + """Inactive key with age > threshold but no UnusedAccessKey1InactiveDate tag is skipped unless DELETE_INACTIVE_KEYS_WITHOUT_TAG.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict.pop('DELETE_INACTIVE_KEYS_WITHOUT_TAG', None) + + mock_data = _mock_iam_users_inactive_key(age_days=DELETE_ACCESS_KEY_DAYS + 10, with_inactive_tag=False) + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + policy = DeleteAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_includes_inactive_old_key_without_tag_when_flag_set(): + """When DELETE_INACTIVE_KEYS_WITHOUT_TAG is True, inactive keys over threshold are included even without tag.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict['DELETE_INACTIVE_KEYS_WITHOUT_TAG'] = True + + age_days = DELETE_ACCESS_KEY_DAYS + 10 + mock_data = _mock_iam_users_inactive_key(age_days=age_days, with_inactive_tag=False) + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + policy = DeleteAccessKey() + result = policy.run_policy_operations() + assert len(result) == 1 + assert result[0]['ResourceId'] == TEST_USER_NAME + # ES schema stores cleanup_result in ResourceAction (dry_run=yes -> verify returns False) + assert result[0]['ResourceAction'] == 'False' + assert result[0]['AgeDays'] == age_days + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_empty_when_no_users(): + """When no IAM users have access keys, result is empty.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value={}): + policy = DeleteAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_deletion_grace_days_capped(): + """Deletion grace days is min(age_days - DELETE_ACCESS_KEY_DAYS, DAYS_TO_TAKE_ACTION).""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + age_days = DELETE_ACCESS_KEY_DAYS + 20 + mock_data = _mock_iam_users_inactive_key(age_days=age_days, with_inactive_tag=True) + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(DeleteAccessKey, 'verify_and_delete_resource', return_value=False) as mock_verify: + policy = DeleteAccessKey() + policy.run_policy_operations() + call_kwargs = mock_verify.call_args[1] + assert call_kwargs['clean_up_days'] == 7 + assert call_kwargs['access_key_id'] == 'AKIAIOSFODNN7EXAMPLE' + assert call_kwargs['remove_inactive_tag'] is True + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_delete_access_key_remove_inactive_tag_false_when_no_tag(): + """When key has no UnusedAccessKey1InactiveDate tag, verify_and_delete_resource is called with remove_inactive_tag=False.""" + environment_variables.environment_variables_dict['policy'] = 'delete_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + environment_variables.environment_variables_dict['DELETE_INACTIVE_KEYS_WITHOUT_TAG'] = True + + mock_data = _mock_iam_users_inactive_key(age_days=DELETE_ACCESS_KEY_DAYS + 5, with_inactive_tag=False) + with patch.object(DeleteAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(DeleteAccessKey, 'verify_and_delete_resource', return_value=False) as mock_verify: + policy = DeleteAccessKey() + policy.run_policy_operations() + call_kwargs = mock_verify.call_args[1] + assert call_kwargs['remove_inactive_tag'] is False diff --git a/tests/unittest/cloud_governance/policy/aws/test_unused_access_key.py b/tests/unittest/cloud_governance/policy/aws/test_unused_access_key.py new file mode 100644 index 00000000..ee0c5333 --- /dev/null +++ b/tests/unittest/cloud_governance/policy/aws/test_unused_access_key.py @@ -0,0 +1,166 @@ +""" +Unit tests for cloud_governance.policy.aws.unused_access_key.UnusedAccessKey. +""" +from unittest.mock import patch + +from moto import mock_ec2, mock_s3, mock_iam + +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.aws.unused_access_key import UnusedAccessKey +from cloud_governance.common.utils.configs import UNUSED_ACCESS_KEY_DAYS +from tests.unittest.configs import DRY_RUN_YES, DRY_RUN_NO, AWS_DEFAULT_REGION, TEST_USER_NAME + + +def _mock_iam_users_access_keys(age_days: int, status: str = 'Active', last_activity_days: int = 100): + """Build mock IAM users access keys dict as returned by IAMOperations.get_iam_users_access_keys().""" + return { + TEST_USER_NAME: { + 'Access key 1': { + 'label': 'Access key 1', + 'status': status, + 'age_days': age_days, + 'last_activity_days': last_activity_days, + 'access_key_id': 'AKIAIOSFODNN7EXAMPLE', + }, + 'tags': [{'Key': 'User', 'Value': TEST_USER_NAME}], + 'region': AWS_DEFAULT_REGION, + 'ResourceId': 'AIDAEXAMPLE', + } + } + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_skips_when_age_below_threshold(): + """Keys with age_days < UNUSED_ACCESS_KEY_DAYS are skipped (no deactivation).""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_NO + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_access_keys(age_days=UNUSED_ACCESS_KEY_DAYS - 1, status='Active') + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(UnusedAccessKey, '_has_active_access_keys', return_value=True): + policy = UnusedAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_includes_when_age_at_or_above_threshold(): + """Keys with age_days >= UNUSED_ACCESS_KEY_DAYS are included for deactivation path.""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_access_keys(age_days=UNUSED_ACCESS_KEY_DAYS, status='Active') + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(UnusedAccessKey, '_has_active_access_keys', return_value=True): + policy = UnusedAccessKey() + result = policy.run_policy_operations() + assert len(result) == 1 + assert result[0]['ResourceId'] == TEST_USER_NAME + # ES schema stores cleanup_result in ResourceAction (dry_run=yes -> verify returns False) + assert result[0]['ResourceAction'] == 'False' + assert result[0]['ResourceType'] == 'UnusedAccessKey' + assert result[0]['AgeDays'] == UNUSED_ACCESS_KEY_DAYS + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_skips_inactive_keys(): + """Keys with status 'Inactive' are skipped.""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_access_keys(age_days=UNUSED_ACCESS_KEY_DAYS + 10, status='Inactive') + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(UnusedAccessKey, '_has_active_access_keys', return_value=False): + policy = UnusedAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_skips_when_skip_policy_tag(): + """Users with Policy=notdelete or skip are skipped.""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_access_keys(age_days=UNUSED_ACCESS_KEY_DAYS + 5, status='Active') + mock_data[TEST_USER_NAME]['tags'] = [ + {'Key': 'User', 'Value': TEST_USER_NAME}, + {'Key': 'Policy', 'Value': 'not-delete'}, + ] + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(UnusedAccessKey, '_has_active_access_keys', return_value=True): + policy = UnusedAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_skips_when_no_active_keys(): + """When _has_active_access_keys returns False for the key, it is skipped.""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + mock_data = _mock_iam_users_access_keys(age_days=UNUSED_ACCESS_KEY_DAYS + 5, status='Active') + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(UnusedAccessKey, '_has_active_access_keys', return_value=False): + policy = UnusedAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_empty_when_no_users(): + """When no IAM users have access keys, result is empty.""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value={}): + policy = UnusedAccessKey() + result = policy.run_policy_operations() + assert len(result) == 0 + + +@mock_ec2 +@mock_s3 +@mock_iam +def test_unused_access_key_deactivation_grace_days_capped(): + """Deactivation grace days is min(age_days - UNUSED_ACCESS_KEY_DAYS, DAYS_TO_TAKE_ACTION).""" + environment_variables.environment_variables_dict['policy'] = 'unused_access_key' + environment_variables.environment_variables_dict['dry_run'] = DRY_RUN_YES + environment_variables.environment_variables_dict['AWS_DEFAULT_REGION'] = AWS_DEFAULT_REGION + environment_variables.environment_variables_dict['DAYS_TO_TAKE_ACTION'] = 7 + + age_days = UNUSED_ACCESS_KEY_DAYS + 20 + mock_data = _mock_iam_users_access_keys(age_days=age_days, status='Active') + with patch.object(UnusedAccessKey, '_get_iam_users_access_keys', return_value=mock_data): + with patch.object(UnusedAccessKey, '_has_active_access_keys', return_value=True): + with patch.object(UnusedAccessKey, 'verify_and_delete_resource', return_value=False) as mock_verify: + policy = UnusedAccessKey() + policy.run_policy_operations() + call_kwargs = mock_verify.call_args[1] + # grace = min(20, 7) = 7 + assert call_kwargs['clean_up_days'] == 7 From 9e75a0c0da2045d3f94d044e669be166f85a6bff Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Mon, 16 Mar 2026 13:54:09 +0530 Subject: [PATCH 15/17] Enable delete without tag --- cloud_governance/main/environment_variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index fd43e8d1..f4cd6b72 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -121,7 +121,7 @@ def __init__(self): self._environment_variables_dict['user_tags'] = EnvironmentVariables.get_env('user_tags', '') self._environment_variables_dict['user_tag_operation'] = EnvironmentVariables.get_env('user_tag_operation', '') self._environment_variables_dict['username'] = EnvironmentVariables.get_env('username', '') - self._environment_variables_dict['DELETE_INACTIVE_KEYS_WITHOUT_TAG'] = EnvironmentVariables.get_env('DELETE_INACTIVE_KEYS_WITHOUT_TAG', 'false').lower() == 'true' + self._environment_variables_dict['DELETE_INACTIVE_KEYS_WITHOUT_TAG'] = EnvironmentVariables.get_env('DELETE_INACTIVE_KEYS_WITHOUT_TAG', 'true').lower() == 'true' self._environment_variables_dict['remove_tags'] = EnvironmentVariables.get_env('remove_tags', '') self._environment_variables_dict['resource'] = EnvironmentVariables.get_env('resource', '') self._environment_variables_dict['cluster_tag'] = EnvironmentVariables.get_env('cluster_tag', '') From a55610bbb74cf63205d45cf4fa4d57349d54f864 Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Tue, 31 Mar 2026 14:27:24 +0530 Subject: [PATCH 16/17] delete access key policy modifications --- cloud_governance/common/utils/configs.py | 2 +- jenkins/clouds/aws/daily/policies/Jenkinsfile | 2 +- jenkins/clouds/aws/daily/policies/run_policies.py | 6 +++--- jenkins/tenant/aws/common/run_policies.py | 5 ++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index e5137a4b..d982d85c 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -23,7 +23,7 @@ CLOUDWATCH_METRICS_AVAILABLE_DAYS = 14 AWS_DEFAULT_GLOBAL_REGION = 'us-east-1' UNUSED_ACCESS_KEY_DAYS = 90 -DELETE_ACCESS_KEY_DAYS = 120 +DELETE_ACCESS_KEY_DAYS = 365 # X86 to Graviton GRAVITON_MAPPINGS = { diff --git a/jenkins/clouds/aws/daily/policies/Jenkinsfile b/jenkins/clouds/aws/daily/policies/Jenkinsfile index b6f677ad..c475de47 100644 --- a/jenkins/clouds/aws/daily/policies/Jenkinsfile +++ b/jenkins/clouds/aws/daily/policies/Jenkinsfile @@ -1,4 +1,4 @@ -accounts_list = ['perf-dept' : "", 'perfscale': "", 'psap': ""] +accounts_list = ['perf-dept': '', 'perfscale': '', 'psap': ''] pipeline { options { disableConcurrentBuilds() diff --git a/jenkins/clouds/aws/daily/policies/run_policies.py b/jenkins/clouds/aws/daily/policies/run_policies.py index 1d968a3c..11c29522 100644 --- a/jenkins/clouds/aws/daily/policies/run_policies.py +++ b/jenkins/clouds/aws/daily/policies/run_policies.py @@ -47,7 +47,7 @@ def get_policies(file_type: str = '.py', exclude_policies: list = None): exclude_global_cost_policies = ['cost_explorer', 'optimize_resources_report', 'monthly_report', 'cost_over_usage', 'skipped_resources', 'cost_explorer_payer_billings', 'cost_billing_reports', 'spot_savings_analysis', 'yearly_savings_report'] -GLOBAL_POLICIES = ["s3_inactive", "empty_roles", "unused_access_key"] +GLOBAL_POLICIES = ["s3_inactive", "empty_roles", "unused_access_key", "delete_access_key"] available_policies = get_policies(exclude_policies=exclude_global_cost_policies) @@ -104,10 +104,10 @@ def run_policies(policies: list, dry_run: str = 'yes'): for policy in policies: container_env_dict.update({"AWS_DEFAULT_REGION": region, 'policy': policy}) container_cmd = '' - if policy in ('empty_roles', 's3_inactive', 'unused_access_key') and region == 'us-east-1': + if policy in ('empty_roles', 's3_inactive', 'unused_access_key', 'delete_access_key') and region == 'us-east-1': container_cmd = get_container_cmd(container_env_dict) else: - if policy not in ('empty_roles', 's3_inactive', 'unused_access_key'): + if policy not in ('empty_roles', 's3_inactive', 'unused_access_key', 'delete_access_key'): container_cmd = get_container_cmd(container_env_dict) if container_cmd: run_cmd(container_cmd) diff --git a/jenkins/tenant/aws/common/run_policies.py b/jenkins/tenant/aws/common/run_policies.py index ff60bf1c..39aa55f1 100644 --- a/jenkins/tenant/aws/common/run_policies.py +++ b/jenkins/tenant/aws/common/run_policies.py @@ -24,7 +24,8 @@ def get_policies(file_type: str = '.py', exclude_policies: list = None): exclude_policies = ['cost_explorer', 'optimize_resources_report', 'monthly_report', 'cost_over_usage', 'skipped_resources', 'cost_explorer_payer_billings', 'cost_billing_reports', - 'spot_savings_analysis', 'yearly_savings_report'] + 'spot_savings_analysis', 'yearly_savings_report', + 'delete_access_key'] available_policies = get_policies(exclude_policies=exclude_policies) QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', 'quay.io/cloud-governance/cloud-governance') @@ -77,6 +78,8 @@ def get_container_cmd(env_dict: dict): policies_in_action = os.environ.get('POLICIES_IN_ACTION', []) if isinstance(policies_in_action, str): policies_in_action = literal_eval(policies_in_action) +# POLICIES_IN_ACTION cannot run policies outside available_policies (e.g. tenant-excluded delete_access_key). +policies_in_action = [p for p in policies_in_action if p in available_policies] policies_not_action = list(set(available_policies) - set(policies_in_action)) regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ap-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', From 7ab778d888b4242cf7aca3b4ad97a713bd092ece Mon Sep 17 00:00:00 2001 From: Pragya Chaudhary Date: Tue, 31 Mar 2026 19:32:54 +0530 Subject: [PATCH 17/17] delete_access_key message changes --- cloud_governance/common/mails/mail_message.py | 1 + .../common/mails/templates/policy_alert_agg_message.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cloud_governance/common/mails/mail_message.py b/cloud_governance/common/mails/mail_message.py index 251bd807..0db503ab 100644 --- a/cloud_governance/common/mails/mail_message.py +++ b/cloud_governance/common/mails/mail_message.py @@ -550,6 +550,7 @@ def get_policy_alert_message(self, policy_data: list, user: str = ''): if has_unused_access_key: context['unused_access_key_message'] = self._get_unused_access_key_alert_message() if has_delete_access_key: + context['delete_access_key_days'] = DELETE_ACCESS_KEY_DAYS context['delete_access_key_message'] = self._get_delete_access_key_alert_message() body = template_loader.render(context) return subject, body diff --git a/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 b/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 index ccc7748c..ae962de7 100644 --- a/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 +++ b/cloud_governance/common/mails/templates/policy_alert_agg_message.j2 @@ -48,7 +48,7 @@

Unused access keys: {{ unused_access_key_message }}

{% endif %} {% if delete_access_key_message is defined and delete_access_key_message %} -

Access keys pending deletion (age > 120 days): {{ delete_access_key_message }}

+

Access keys pending deletion (inactive, key age > {{ delete_access_key_days }} days): {{ delete_access_key_message }}

{% endif %}