Skip to content

Commit 57f97e9

Browse files
zhongkechenzhongkechen
committed
ci: add log retention to example Lambdas
1 parent 075b65a commit 57f97e9

4 files changed

Lines changed: 229 additions & 5 deletions

File tree

‎.github/workflows/cloud-tests.yml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,18 @@ jobs:
104104
echo "Using custom Lambda endpoint override"
105105
fi
106106
107+
PYTEST_FUNCTION_NAME_MAP=$(
108+
python packages/aws-durable-execution-sdk-python-examples/scripts/generate_function_name_map.py \
109+
--function-name-prefix "$FUNCTION_NAME_PREFIX"
110+
)
111+
112+
# TODO: Remove this one-time cleanup after existing CI stacks adopt the
113+
# generated AWS::Logs::LogGroup resources.
114+
python packages/aws-durable-execution-sdk-python-examples/scripts/cleanup_unmanaged_log_groups.py \
115+
--stack-name "$STACK_NAME" \
116+
--function-name-prefix "$FUNCTION_NAME_PREFIX" \
117+
--region "$AWS_REGION"
118+
107119
echo "Deploying stack $STACK_NAME with function prefix $FUNCTION_NAME_PREFIX and runtime $PYTHON_RUNTIME"
108120
sam deploy \
109121
--template-file .aws-sam/build/template.yaml \
@@ -114,11 +126,6 @@ jobs:
114126
--region "$AWS_REGION" \
115127
--parameter-overrides "${PARAMETER_OVERRIDES[@]}"
116128
117-
PYTEST_FUNCTION_NAME_MAP=$(
118-
python packages/aws-durable-execution-sdk-python-examples/scripts/generate_function_name_map.py \
119-
--function-name-prefix "$FUNCTION_NAME_PREFIX"
120-
)
121-
122129
echo "PYTEST_FUNCTION_NAME_MAP=$PYTEST_FUNCTION_NAME_MAP" >> $GITHUB_ENV
123130
124131
- name: Run Integration Tests

‎packages/aws-durable-execution-sdk-python-examples/scripts/cleanup_unmanaged_log_groups.py‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import subprocess
5+
6+
from generate_sam_template import load_catalog, to_logical_id
7+
8+
9+
def log_group_resources(function_name_prefix: str) -> list[tuple[str, str]]:
10+
"""Return generated LogGroup logical ids and physical names."""
11+
logical_ids = {
12+
to_logical_id(example["handler"]) for example in load_catalog()["examples"]
13+
}
14+
return [
15+
(f"{logical_id}LogGroup", f"/aws/lambda/{function_name_prefix}{logical_id}")
16+
for logical_id in sorted(logical_ids)
17+
]
18+
19+
20+
def run_aws(args: list[str], region: str) -> subprocess.CompletedProcess[str]:
21+
"""Run an AWS CLI command and capture its output."""
22+
return subprocess.run(
23+
["aws", *args, "--region", region],
24+
capture_output=True,
25+
check=False,
26+
text=True,
27+
)
28+
29+
30+
def stack_owns_resource(stack_name: str, logical_resource_id: str, region: str) -> bool:
31+
"""Return true when the stack already owns the generated log group resource."""
32+
result = run_aws(
33+
[
34+
"cloudformation",
35+
"describe-stack-resource",
36+
"--stack-name",
37+
stack_name,
38+
"--logical-resource-id",
39+
logical_resource_id,
40+
],
41+
region,
42+
)
43+
if result.returncode == 0:
44+
return True
45+
46+
if "does not exist" in result.stderr:
47+
return False
48+
49+
msg = result.stderr.strip() or result.stdout.strip()
50+
raise RuntimeError(msg)
51+
52+
53+
def delete_log_group(log_group_name: str, region: str) -> bool:
54+
"""Delete a log group, ignoring log groups that do not exist."""
55+
result = run_aws(
56+
["logs", "delete-log-group", "--log-group-name", log_group_name],
57+
region,
58+
)
59+
if result.returncode == 0:
60+
print(f"Deleted unmanaged log group {log_group_name}")
61+
return True
62+
63+
if "ResourceNotFoundException" in result.stderr:
64+
print(f"No unmanaged log group found for {log_group_name}")
65+
return True
66+
67+
print(result.stderr, end="")
68+
return False
69+
70+
71+
def cleanup_unmanaged_log_groups(
72+
stack_name: str,
73+
function_name_prefix: str,
74+
region: str,
75+
) -> bool:
76+
"""Remove log groups that would block CloudFormation from owning them."""
77+
success = True
78+
resources = log_group_resources(function_name_prefix)
79+
for logical_resource_id, log_group_name in resources:
80+
if stack_owns_resource(stack_name, logical_resource_id, region):
81+
print(f"Stack already owns {logical_resource_id}; leaving it in place")
82+
continue
83+
84+
success = delete_log_group(log_group_name, region) and success
85+
86+
return success
87+
88+
89+
def main() -> int:
90+
parser = argparse.ArgumentParser(
91+
description="Delete unmanaged Lambda log groups before SAM owns them"
92+
)
93+
parser.add_argument("--stack-name", required=True)
94+
parser.add_argument("--function-name-prefix", required=True)
95+
parser.add_argument("--region", required=True)
96+
args = parser.parse_args()
97+
98+
if not cleanup_unmanaged_log_groups(
99+
stack_name=args.stack_name,
100+
function_name_prefix=args.function_name_prefix,
101+
region=args.region,
102+
):
103+
return 1
104+
105+
return 0
106+
107+
108+
if __name__ == "__main__":
109+
raise SystemExit(main())

‎packages/aws-durable-execution-sdk-python-examples/scripts/generate_sam_template.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88

99
DEFAULT_FUNCTION_NAME_PREFIX = "DurablePythonExample-"
10+
LOG_RETENTION_DAYS = 7
1011

1112

1213
def load_catalog() -> dict[str, Any]:
@@ -94,6 +95,15 @@ def build_template(examples: list[dict[str, Any]]) -> dict[str, Any]:
9495
"Type": "AWS::Serverless::Function",
9596
"Properties": properties,
9697
}
98+
template["Resources"][f"{logical_id}LogGroup"] = {
99+
"Type": "AWS::Logs::LogGroup",
100+
"Properties": {
101+
"LogGroupName": {
102+
"Fn::Sub": f"/aws/lambda/${{FunctionNamePrefix}}{logical_id}"
103+
},
104+
"RetentionInDays": LOG_RETENTION_DAYS,
105+
},
106+
}
97107

98108
return template
99109

‎packages/aws-durable-execution-sdk-python-examples/test/test_log_group_deployment.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import importlib.util
2+
import sys
3+
from pathlib import Path
4+
5+
6+
SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts"
7+
if str(SCRIPTS_DIR) not in sys.path:
8+
sys.path.insert(0, str(SCRIPTS_DIR))
9+
10+
11+
def load_script(name: str):
12+
path = SCRIPTS_DIR / f"{name}.py"
13+
spec = importlib.util.spec_from_file_location(name, path)
14+
assert spec is not None
15+
module = importlib.util.module_from_spec(spec)
16+
assert spec.loader is not None
17+
spec.loader.exec_module(module)
18+
return module
19+
20+
21+
generate_sam_template = load_script("generate_sam_template")
22+
cleanup_unmanaged_log_groups = load_script("cleanup_unmanaged_log_groups")
23+
24+
25+
def test_build_template_creates_lambda_log_group_with_seven_day_retention():
26+
template = generate_sam_template.build_template(
27+
[
28+
{
29+
"handler": "hello_world.handler",
30+
"description": "Example handler",
31+
"durableConfig": {
32+
"RetentionPeriodInDays": 7,
33+
"ExecutionTimeout": 300,
34+
},
35+
}
36+
]
37+
)
38+
39+
assert template["Resources"]["HelloWorldLogGroup"] == {
40+
"Type": "AWS::Logs::LogGroup",
41+
"Properties": {
42+
"LogGroupName": {"Fn::Sub": "/aws/lambda/${FunctionNamePrefix}HelloWorld"},
43+
"RetentionInDays": 7,
44+
},
45+
}
46+
47+
48+
def test_log_group_resources_match_generated_template_names(monkeypatch):
49+
monkeypatch.setattr(
50+
cleanup_unmanaged_log_groups,
51+
"load_catalog",
52+
lambda: {
53+
"examples": [
54+
{"handler": "hello_world.handler"},
55+
{"handler": "step_with_name.handler"},
56+
]
57+
},
58+
)
59+
60+
assert cleanup_unmanaged_log_groups.log_group_resources("Py313-") == [
61+
("HelloWorldLogGroup", "/aws/lambda/Py313-HelloWorld"),
62+
("StepWithNameLogGroup", "/aws/lambda/Py313-StepWithName"),
63+
]
64+
65+
66+
def test_cleanup_skips_stack_owned_log_groups_and_deletes_unmanaged(monkeypatch):
67+
deleted_log_groups = []
68+
69+
monkeypatch.setattr(
70+
cleanup_unmanaged_log_groups,
71+
"log_group_resources",
72+
lambda _prefix: [
73+
("OwnedLogGroup", "/aws/lambda/Py313-Owned"),
74+
("UnmanagedLogGroup", "/aws/lambda/Py313-Unmanaged"),
75+
],
76+
)
77+
monkeypatch.setattr(
78+
cleanup_unmanaged_log_groups,
79+
"stack_owns_resource",
80+
lambda _stack_name, logical_id, _region: logical_id == "OwnedLogGroup",
81+
)
82+
83+
def fake_delete_log_group(log_group_name, _region):
84+
deleted_log_groups.append(log_group_name)
85+
return True
86+
87+
monkeypatch.setattr(
88+
cleanup_unmanaged_log_groups,
89+
"delete_log_group",
90+
fake_delete_log_group,
91+
)
92+
93+
assert cleanup_unmanaged_log_groups.cleanup_unmanaged_log_groups(
94+
stack_name="Py313-python-examples",
95+
function_name_prefix="Py313-",
96+
region="us-west-2",
97+
)
98+
assert deleted_log_groups == ["/aws/lambda/Py313-Unmanaged"]

0 commit comments

Comments
 (0)