Skip to content

Commit fe09f3a

Browse files
BFD-4802: Initial Log alarm creation (#3212)
Co-authored-by: Mitchell Alessio <5306896+malessi@users.noreply.github.com>
1 parent 6061ab5 commit fe09f3a

13 files changed

Lines changed: 751 additions & 10 deletions

File tree

.github/workflows/deploy-platform-services.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ on:
1818
alerting,
1919
backup,
2020
ecr,
21-
network
21+
network,
22+
log-retention-alarms
2223
required: true
2324
per-service-vars-json:
2425
description: >-
@@ -60,7 +61,8 @@ on:
6061
alerting,
6162
backup,
6263
ecr,
63-
network
64+
network,
65+
log-retention-alarms
6466
required: false
6567
per-service-vars-json:
6668
type: string

ops/platform/01-config/values/platform.non-prod.sopsw.yaml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,20 @@
1010
/bfd/platform/network/sensitive/route53/zone/test/external_vpcs_list_json: ENC[AES256_GCM,data:ZtMWOGrMx4znKmUeHA3/DOhlEoGWZr4/o5W7nGg7l6TLeszEH1c=,iv:K7ailUBD7JN8LdpIELeC0vN5hOPy15N8If0IcHTJQXg=,tag:bBLlGwtgAPgY91th8r2YrA==,type:str]
1111
/bfd/platform/network/sensitive/route53/zone/test/records: ENC[AES256_GCM,data:elc=,iv:JaN3qPDgxcrAKVgbAd69k5pVtOR64TnlEyyl/VIIZm8=,tag:RCS21GML9uul4CfmGwlQqg==,type:str]
1212
/bfd/platform/sonar/sensitive/service_account_access_key: ENC[AES256_GCM,data:I7hQVx3Wry0IUvANFGyJtdj6rT8nJ3zfpPMIiovJ6MeOqBhJgTWx1KqKWEE=,iv:5CMjVqrByv1PlF27QoVCLsfU+fa3vzLHZf+HRGV3w/Y=,tag:dY8Zl1wz92qqhnkOHdqfSA==,type:str]
13+
/bfd/platform/log-retention-alarms/nonsensitive/sns_topics/logs/alert: bfd-platform-slack-bfd-warnings
1314
sops:
1415
kms:
1516
- arn: arn:aws:kms:us-east-1:${ACCOUNT_ID}:alias/bfd-mgmt-config-cmk
17+
aws_profile: ""
1618
created_at: "2025-05-30T12:44:10Z"
1719
enc: AQICAHje54t884PQ3I3HmEfbyeDxeth8IxFtXCHA9cZvNPo2hQGyP0QC8tu6Ih5lpqXY+rpOAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMYewoC+MbQztXIxKsAgEQgDvG6oBn0T50sN7lF4nqr1NKNXM481afDxSIeEaIRCa554zT6jLDSvnQb/sbg3pZXnDOk1e6OJOJtb72xA==
18-
aws_profile: ""
1920
- arn: arn:aws:kms:us-east-1:${ACCOUNT_ID}:alias/bfd-platform-cmk
21+
aws_profile: ""
2022
created_at: "2025-05-30T12:43:15Z"
2123
enc: AQICAHjxly34ZBDoNlpGiKxZCgGefcvFT75wuI0miwFqykz7tQGBmOF467T0Ol5QmUcAF/ZMAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMCP0Phd4DSMFY3WjJAgEQgDu6gBh8O6AOTBRoQ6ERcZJkEBmHwlnQqvNshDaU17Gb+igYqmbTcusMH0fAt88+P724+DxP/Np6H3R5eg==
22-
aws_profile: ""
2324
- arn: arn:aws:kms:us-west-2:${ACCOUNT_ID}:alias/bfd-platform-cmk
25+
aws_profile: ""
2426
created_at: "2025-06-16T19:01:55Z"
2527
enc: AQICAHg4lseGbAt9RybBF1VZvUEr9DQG36gKCucJoGCH1eiXJAFoo4+2/9yTYqi454+yQcdtAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMzyVNfm3pg/LaNZD1AgEQgDvx4lFyv/o9+OlNWek2ux3mvdxFmuextAhZ9HRjHohto6gSoNhJCea9CbvXT/qriGBlz7l+FdWBhPy7wA==
26-
aws_profile: ""
2728
unencrypted_regex: /nonsensitive/
28-
version: 3.10.2
29+
version: 3.13.2

ops/platform/01-config/values/platform.prod.sopsw.yaml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,20 @@
2525
#ENC[AES256_GCM,data:QOpMGpO+e8AXTBF7/pZ3dMNWC2LsSVmMCs92qqa/NsYE5zmIXy6FjoOS0W8ATxN4H7b/KxNtlfGU7uy1jAdse1w2ylIqtiXq/4MdPw==,iv:y9rW6d68E3AXsS/Pfwl5osA7jlUychtLLzM/SG0ABj0=,tag:dh8bJ24T/V4UK2qwCMWHBg==,type:comment]
2626
/bfd/platform/network/sensitive/route53/zone/sandbox/external_vpcs_list_json: ENC[AES256_GCM,data:Eoz98YToPPHtn5B2P5VdaQQ/bNWejNph+cv0JVfNVXgdmqu6Kak=,iv:zFyBFj+tIOiG0/U7tg2x+KHPTvUjJKuBBaUOeQLdmbs=,tag:XD88QUzmaaW68fkm2R5t6A==,type:str]
2727
/bfd/platform/network/sensitive/route53/zones/sandbox/records: ENC[AES256_GCM,data:eLQ=,iv:IAJmrV+6kvbxfBDQUCNFiIChEq3A2dsScyTyfJKz7MQ=,tag:kf/9oEE6Neej4MZmCSK6ug==,type:str]
28+
/bfd/platform/log-retention-alarms/nonsensitive/sns_topics/logs/alert: bfd-platform-slack-bfd-warnings
2829
sops:
2930
kms:
3031
- arn: arn:aws:kms:us-east-1:${ACCOUNT_ID}:alias/bfd-mgmt-config-cmk
32+
aws_profile: ""
3133
created_at: "2025-06-16T18:39:56Z"
3234
enc: AQICAHje54t884PQ3I3HmEfbyeDxeth8IxFtXCHA9cZvNPo2hQHPEXVyrJuDRtg1OtVF2KPuAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM4A8g7SmdVfWwo/XqAgEQgDsOzD6kQd7qxqTSdpQQiio+5JVaib8QfGrdXid+5UdvYvBExleHOBNYR9lpko2aou4AswKUws2Z+TGnfg==
33-
aws_profile: ""
3435
- arn: arn:aws:kms:us-east-1:${ACCOUNT_ID}:alias/bfd-platform-cmk
36+
aws_profile: ""
3537
created_at: "2025-06-16T18:26:40Z"
3638
enc: AQICAHhfOyfG21CaEsyppnyWH5WGgx5DhYc4Ue0JLa+mNx8TlwGFUKs60Gy5jQcgCwW6h+A8AAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMoGhgCaspO+TCOEYFAgEQgDse/nCj+sp+pSxqhPOEBGztJ3u1Q1zy0J2BOWpuYcl/uplPnrYQTnPOKdM8FpkJuGKXN6SJvfQ9skakGA==
37-
aws_profile: ""
3839
- arn: arn:aws:kms:us-west-2:${ACCOUNT_ID}:alias/bfd-platform-cmk
40+
aws_profile: ""
3941
created_at: "2025-06-16T18:31:22Z"
4042
enc: AQICAHiUQJFGKP8gsiLp3u7hZwa9ETwwJmmjpmZLPLYGrqKCcAFrfmHLdibQUUGIorbiLIHKAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMUpTUbz9JXXSk6CZSAgEQgDt7csVKhVSq5ZPjMVpsJeGxn6PuAphyvoBf6f0ts1XLQTdOdmlG9c6qpju7L5XxFTmAeTmC8E3IIRLxkw==
41-
aws_profile: ""
4243
unencrypted_regex: /nonsensitive/
43-
version: 3.10.2
44+
version: 3.13.2
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.zip
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.14
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# `log_retention_checker` Lambda Source
2+
3+
This subdirectory contains the Python source code for the `log_retention_checker` Lambda.
4+
5+
## Environment Setup
6+
7+
It is assumed you are using `pyright`/`pylance` for type-checking, `uv` for virtual environment and dependency management, and `ruff` for linting, formatting, and import sorting.
8+
9+
1. Install `uv`:
10+
11+
```bash
12+
brew install uv
13+
```
14+
15+
2. Setup Python 3.13 virtual environment:
16+
17+
```bash
18+
uv sync
19+
```
20+
21+
3. Your virtual environment is now setup! By default, it is available under `.venv`; using VS Code, this Virtual Environment can be chosen using the `Python: Select Interpreter` command
22+
23+
## Updating/managing dependencies
24+
25+
See [`Managing dependencies`](https://docs.astral.sh/uv/concepts/projects/dependencies/) and [`Locking and syncing`](https://docs.astral.sh/uv/concepts/projects/sync/) in the `uv` docs for more information.
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import json
2+
import os
3+
from typing import TYPE_CHECKING
4+
5+
import boto3
6+
from botocore.config import Config
7+
from aws_lambda_powertools import Logger
8+
from aws_lambda_powertools.utilities.typing import LambdaContext
9+
10+
if TYPE_CHECKING:
11+
from mypy_boto3_logs.client import CloudWatchLogsClient
12+
from mypy_boto3_sns.client import SNSClient
13+
else:
14+
CloudWatchLogsClient = object
15+
SNSClient = object
16+
17+
18+
logger = Logger()
19+
20+
REQUIRED_RETENTION_DAYS = int(os.getenv("REQUIRED_RETENTION_DAYS", "2557"))
21+
ALERT_SNS_TOPIC_ARN = os.getenv("ALERT_SNS_TOPIC_ARN")
22+
REGION = os.environ.get("AWS_CURRENT_REGION", default="us-east-1")
23+
BOTO_CONFIG = Config(
24+
region_name=REGION,
25+
# Instructs boto3 to retry upto 10 times using an exponential backoff
26+
retries={
27+
"total_max_attempts": 10,
28+
"mode": "adaptive",
29+
},
30+
# Double the read timeout for some extra safety when synchrnously invoking the run-locust Lambda
31+
read_timeout=120,
32+
)
33+
34+
35+
def _list_non_compliant_log_groups(logs_client: CloudWatchLogsClient) -> list[dict[str, object]]:
36+
"""Return log groups that are missing or not equal to required retention."""
37+
non_compliant = []
38+
paginator = logs_client.get_paginator("describe_log_groups")
39+
40+
for page in paginator.paginate():
41+
for group in page.get("logGroups", []):
42+
log_group_name = group.get("logGroupName")
43+
configured_retention = group.get("retentionInDays")
44+
if configured_retention is None or configured_retention < REQUIRED_RETENTION_DAYS:
45+
non_compliant.append(
46+
{
47+
"logGroupName": log_group_name,
48+
"retentionInDays": configured_retention,
49+
"requiredRetentionInDays": REQUIRED_RETENTION_DAYS,
50+
}
51+
)
52+
if log_group_name:
53+
logger.info(
54+
"Updating log retention for %s (%s days).",
55+
log_group_name,
56+
REQUIRED_RETENTION_DAYS,
57+
)
58+
logs_client.put_retention_policy(
59+
logGroupName=log_group_name,
60+
retentionInDays=REQUIRED_RETENTION_DAYS,
61+
)
62+
63+
return non_compliant
64+
65+
66+
def _publish_alert(sns_client: SNSClient, message: str) -> None:
67+
"""Send alert to SNS when topic ARN is configured."""
68+
if not ALERT_SNS_TOPIC_ARN:
69+
logger.warning("ALERT_SNS_TOPIC_ARN not configured. Alert only logged to CloudWatch.")
70+
return
71+
72+
sns_client.publish(
73+
TopicArn=ALERT_SNS_TOPIC_ARN,
74+
Message=message,
75+
)
76+
77+
78+
@logger.inject_lambda_context(clear_state=True, log_event=True)
79+
def lambda_handler(event: dict[str, Any], context: LambdaContext):
80+
81+
logger.info(
82+
"Environment: REQUIRED_RETENTION_DAYS=%r, ALERT_SNS_TOPIC_ARN=%r",
83+
os.getenv("REQUIRED_RETENTION_DAYS"),
84+
os.getenv("ALERT_SNS_TOPIC_ARN"),
85+
)
86+
87+
logs_client = boto3.client("logs", config=BOTO_CONFIG)
88+
sns_client = boto3.client("sns", config=BOTO_CONFIG)
89+
90+
non_compliant = _list_non_compliant_log_groups(logs_client)
91+
92+
if not non_compliant:
93+
logger.info(
94+
"All CloudWatch log groups are compliant with retention policy (%s days).",
95+
REQUIRED_RETENTION_DAYS,
96+
)
97+
return {
98+
"statusCode": 200,
99+
"body": json.dumps(
100+
{
101+
"message": "All log groups are compliant.",
102+
"requiredRetentionInDays": REQUIRED_RETENTION_DAYS,
103+
"nonCompliantCount": 0,
104+
}
105+
),
106+
}
107+
108+
payload = {
109+
"AlarmName": "log-retention-non-compliance",
110+
"AlarmDescription": f"Found {len(non_compliant)} log groups with non-compliant retention (required: {REQUIRED_RETENTION_DAYS} days)",
111+
"NewStateReason": json.dumps(non_compliant, indent=2),
112+
"Trigger": {"MetricName": None},
113+
}
114+
# log the non-compliant log groups and publish an alert to SNS
115+
logger.warning(
116+
"Invalid resource payload: %s",
117+
json.dumps(non_compliant, indent=2),
118+
)
119+
alert_message = json.dumps(payload)
120+
121+
logger.warning(alert_message)
122+
_publish_alert(sns_client, alert_message)
123+
124+
return {
125+
"statusCode": 200,
126+
"body": json.dumps(
127+
{
128+
"message": "Found log groups with non-compliant retention.",
129+
"requiredRetentionInDays": REQUIRED_RETENTION_DAYS,
130+
"nonCompliantCount": len(non_compliant),
131+
"nonCompliantLogGroups": non_compliant,
132+
}
133+
),
134+
}
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
[project]
2+
name = "log_retention_checker"
3+
version = "1.0.0"
4+
readme = "README.md"
5+
requires-python = ">=3.14"
6+
dependencies = [
7+
"boto3>=1.37.19",
8+
]
9+
10+
[dependency-groups]
11+
dev = [
12+
"boto3-stubs[logs, sns]",
13+
"ruff",
14+
"uv>=0.11.15",
15+
]
16+
17+
[tool.uv]
18+
exclude-newer = "7 days"
19+
20+
[tool.ruff]
21+
# Set the maximum line length to 100.
22+
line-length = 100
23+
24+
[tool.ruff.lint]
25+
select = [
26+
"D", # pydocstyle
27+
"E501",
28+
# pycodestyle
29+
"E",
30+
# Pyflakes
31+
"F",
32+
# pyupgrade
33+
"UP",
34+
# flake8-bugbear
35+
"B",
36+
# flake8-simplify
37+
"SIM",
38+
# isort
39+
"I",
40+
"ANN",
41+
"LOG",
42+
"G",
43+
"PT",
44+
"RSE",
45+
"PIE",
46+
"RET",
47+
"SLF",
48+
"ARG",
49+
"PTH",
50+
"PLE",
51+
"PLW",
52+
"PERF",
53+
"FURB",
54+
"RUF",
55+
]
56+
57+
[tool.ruff.lint.pydocstyle]
58+
convention = "pep257"
59+
60+
[tool.ruff.format]
61+
quote-style = "double"
62+
indent-style = "space"
63+
docstring-code-format = true
64+
65+
[tool.pyright]
66+
# ...
67+
typeCheckingMode = "strict"
68+
venvPath = "."
69+
venv = ".venv"
70+
reportMissingTypeStubs = "none"
71+

0 commit comments

Comments
 (0)