Skip to content

Commit 1cf24d1

Browse files
committed
test(listener): add ignore-field-drift e2e coverage for weight drift
Add end-to-end coverage for the services.k8s.aws/ignore-field-drift annotation (aws-controllers-k8s/runtime#256) on an ELBv2 Listener's forward-action target-group weights, mirroring "Scenario 2" of #85: a blue/green deploy tool shifts traffic weights across target groups on the live listener, and without ignore-field-drift the controller reconciles the weights back to the declared spec, breaking the deployment. TestListenerIgnoreFieldDrift.test_weight_drift_ignored declares a weighted forward action (90/10) annotated to ignore spec.defaultActions, shifts the weights out-of-band to 50/50 via ModifyListener, and asserts that the external distribution survives, the resource stays Synced, and a subsequent spec edit is retained in the CR but not pushed to AWS. The IgnoreFieldDrift feature gate is Alpha and disabled by default, so the test enables it on the deployed controller for the module and restores the prior value afterwards (mirrors the ec2-controller VPC coverage for the same runtime feature). The gate is only present once the controller's runtime dependency includes runtime#256, so this coverage runs green after the corresponding runtime bump. Two ip-type target groups back the weighted action so the fixture does not depend on registered targets. Signed-off-by: Hao Wang <rhaowang@amazon.com>
1 parent 2553025 commit 1cf24d1

3 files changed

Lines changed: 365 additions & 0 deletions

File tree

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: elbv2.services.k8s.aws/v1alpha1
2+
kind: Listener
3+
metadata:
4+
name: $LISTENER_NAME
5+
annotations:
6+
services.k8s.aws/ignore-field-drift: "$IGNORE_PATHS"
7+
spec:
8+
defaultActions:
9+
- type: "forward"
10+
forwardConfig:
11+
targetGroups:
12+
- targetGroupRef:
13+
from:
14+
name: $TARGET_GROUP_NAME_1
15+
weight: $WEIGHT_1
16+
- targetGroupRef:
17+
from:
18+
name: $TARGET_GROUP_NAME_2
19+
weight: $WEIGHT_2
20+
loadBalancerARN: $LOAD_BALANCER_ARN
21+
port: 80
22+
protocol: "HTTP"
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: elbv2.services.k8s.aws/v1alpha1
2+
kind: TargetGroup
3+
metadata:
4+
name: $TARGET_GROUP_NAME
5+
spec:
6+
name: $TARGET_GROUP_NAME
7+
targetType: ip
8+
protocol: HTTP
9+
port: 80
10+
vpcID: $VPC_ID

test/e2e/tests/test_listener.py

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import time
1919

2020
import pytest
21+
from acktest.k8s import condition
2122
from acktest.k8s import resource as k8s
2223
from acktest.resources import random_suffix_name
2324
from e2e import CRD_GROUP, CRD_VERSION, load_elbv2_resource, service_marker
@@ -28,9 +29,11 @@
2829
from .test_load_balancer import simple_load_balancer
2930

3031
RESOURCE_PLURAL = "listeners"
32+
TARGET_GROUP_PLURAL = "targetgroups"
3133

3234
CREATE_WAIT_AFTER_SECONDS = 10
3335
UPDATE_WAIT_AFTER_SECONDS = 10
36+
MODIFY_WAIT_AFTER_SECONDS = 20
3437
DELETE_WAIT_AFTER_SECONDS = 10
3538

3639
@pytest.fixture(scope="module")
@@ -98,3 +101,333 @@ def test_create_delete(self, elbv2_client, simple_listener):
98101
listener = validator.get_listener(listener_arn)
99102
assert listener is not None
100103
assert listener["Port"] == 9000
104+
105+
106+
# ---------------------------------------------------------------------------
107+
# services.k8s.aws/ignore-field-drift coverage
108+
# ---------------------------------------------------------------------------
109+
#
110+
# The IgnoreFieldDrift runtime feature (aws-controllers-k8s/runtime#256) lets a
111+
# resource opt specific spec paths out of drift reconciliation via the
112+
# services.k8s.aws/ignore-field-drift annotation. This mirrors the motivating
113+
# use case in aws-controllers-k8s/elbv2-controller#85 "Scenario 2": a blue/green
114+
# deploy tool shifts traffic weights across target groups on the live listener,
115+
# and without ignore-field-drift the controller reconciles the weights back to
116+
# the declared spec, breaking the deployment.
117+
#
118+
# The feature gate is Alpha and disabled by default, so the test enables it on
119+
# the deployed controller for the duration of the module and restores the prior
120+
# value afterwards. This requires a controller built from a runtime that carries
121+
# the gate; against a runtime without it, enabling an unknown gate is fatal, so
122+
# this coverage runs green only once the controller's runtime dependency
123+
# includes the feature.
124+
125+
# Controller deployment coordinates in the kind test cluster (see
126+
# test-infra/scripts/controller-setup.sh and the controller Helm chart, which
127+
# wires FEATURE_GATES into the --feature-gates flag).
128+
CONTROLLER_NAMESPACE = "ack-system"
129+
CONTROLLER_DEPLOYMENT = "ack-elbv2-controller"
130+
CONTROLLER_CONTAINER = "controller"
131+
FEATURE_GATE = "IgnoreFieldDrift"
132+
# Generous window for the new pod to roll out and take over reconciliation.
133+
ROLLOUT_WAIT_SECONDS = 120
134+
135+
# The declared (spec) weights and the externally-shifted weights. The two must
136+
# differ so a revert would be observable.
137+
DECLARED_WEIGHT_1 = 90
138+
DECLARED_WEIGHT_2 = 10
139+
EXTERNAL_WEIGHT_1 = 50
140+
EXTERNAL_WEIGHT_2 = 50
141+
142+
143+
def _apps_client():
144+
# Build the AppsV1Api against acktest's configured ApiClient (which points
145+
# at the kind cluster). A bare AppsV1Api() would default to localhost:80.
146+
from kubernetes import client as k8s_client
147+
return k8s_client.AppsV1Api(k8s._get_k8s_api_client())
148+
149+
150+
def _get_feature_gates_env() -> str:
151+
"""Returns the current value of the FEATURE_GATES env var on the controller
152+
container, or "" if it is unset."""
153+
dep = _apps_client().read_namespaced_deployment(
154+
CONTROLLER_DEPLOYMENT, CONTROLLER_NAMESPACE,
155+
)
156+
for c in dep.spec.template.spec.containers:
157+
if c.name != CONTROLLER_CONTAINER:
158+
continue
159+
for e in (c.env or []):
160+
if e.name == "FEATURE_GATES":
161+
return e.value or ""
162+
return ""
163+
164+
165+
def _set_feature_gates_env(value: str):
166+
"""Patches the FEATURE_GATES env var on the controller container and waits
167+
for the rollout to complete. The controller wires this env var into its
168+
--feature-gates flag (see the controller Helm chart)."""
169+
body = {
170+
"spec": {
171+
"template": {
172+
"spec": {
173+
"containers": [
174+
{"name": CONTROLLER_CONTAINER,
175+
"env": [{"name": "FEATURE_GATES", "value": value}]},
176+
]
177+
}
178+
}
179+
}
180+
}
181+
_apps_client().patch_namespaced_deployment(
182+
CONTROLLER_DEPLOYMENT, CONTROLLER_NAMESPACE, body,
183+
)
184+
_wait_for_rollout()
185+
186+
187+
def _merge_gate(existing: str, gate: str, enabled: bool) -> str:
188+
"""Returns a FEATURE_GATES string with `gate` set to `enabled`, preserving
189+
any other gates already present."""
190+
pairs = {}
191+
for part in filter(None, (p.strip() for p in existing.split(","))):
192+
if "=" in part:
193+
k, v = part.split("=", 1)
194+
pairs[k.strip()] = v.strip()
195+
pairs[gate] = "true" if enabled else "false"
196+
return ",".join(f"{k}={v}" for k, v in pairs.items())
197+
198+
199+
def _wait_for_rollout():
200+
"""Blocks until the controller deployment reports all replicas updated and
201+
available for the current generation."""
202+
client = _apps_client()
203+
deadline = time.time() + ROLLOUT_WAIT_SECONDS
204+
while time.time() < deadline:
205+
dep = client.read_namespaced_deployment(
206+
CONTROLLER_DEPLOYMENT, CONTROLLER_NAMESPACE,
207+
)
208+
spec_replicas = dep.spec.replicas or 1
209+
status = dep.status
210+
if (status.observed_generation is not None
211+
and status.observed_generation >= dep.metadata.generation
212+
and (status.updated_replicas or 0) >= spec_replicas
213+
and (status.available_replicas or 0) >= spec_replicas
214+
and (status.unavailable_replicas or 0) == 0):
215+
# Give the fresh pod a moment to acquire leadership / start reconciling.
216+
time.sleep(5)
217+
return
218+
time.sleep(3)
219+
raise AssertionError(
220+
f"controller deployment {CONTROLLER_DEPLOYMENT} did not roll out within "
221+
f"{ROLLOUT_WAIT_SECONDS}s after toggling the {FEATURE_GATE} feature gate"
222+
)
223+
224+
225+
def _weights_by_tg_arn(listener: dict) -> dict:
226+
"""Returns {target_group_arn: weight} from a describe_listeners entry's
227+
first default forward action."""
228+
actions = listener.get("DefaultActions", [])
229+
forward = next(
230+
(a for a in actions if a.get("Type") == "forward"), None,
231+
)
232+
assert forward is not None, "listener has no forward default action"
233+
tgs = forward["ForwardConfig"]["TargetGroups"]
234+
return {tg["TargetGroupArn"]: tg["Weight"] for tg in tgs}
235+
236+
237+
@pytest.fixture(scope="module")
238+
def ignore_field_drift_enabled():
239+
"""Enables the IgnoreFieldDrift feature gate on the controller for the
240+
duration of the module, then restores the prior FEATURE_GATES value."""
241+
original = _get_feature_gates_env()
242+
_set_feature_gates_env(_merge_gate(original, FEATURE_GATE, True))
243+
yield
244+
# Restore exactly what was there before (which may be "").
245+
_set_feature_gates_env(original)
246+
247+
248+
@pytest.fixture(scope="module")
249+
def two_target_groups(elbv2_client):
250+
"""Creates two ip-type target groups (in the bootstrapped VPC) for the
251+
weighted forward action. ip-type is used instead of lambda so the fixture
252+
does not depend on registered targets -- the weight-drift scenario needs
253+
only the target groups themselves."""
254+
refs = []
255+
names = []
256+
for i in range(2):
257+
name = random_suffix_name(f"tg-ifd-{i+1}", 24)
258+
replacements = REPLACEMENT_VALUES.copy()
259+
replacements["TARGET_GROUP_NAME"] = name
260+
data = load_elbv2_resource(
261+
"target_group_ip", additional_replacements=replacements,
262+
)
263+
ref = k8s.CustomResourceReference(
264+
CRD_GROUP, CRD_VERSION, TARGET_GROUP_PLURAL, name, namespace="default",
265+
)
266+
k8s.create_custom_resource(ref, data)
267+
refs.append(ref)
268+
names.append(name)
269+
270+
time.sleep(CREATE_WAIT_AFTER_SECONDS)
271+
for ref in refs:
272+
cr = k8s.wait_resource_consumed_by_controller(ref)
273+
assert cr is not None
274+
275+
yield names
276+
277+
for ref in refs:
278+
try:
279+
_, deleted = k8s.delete_custom_resource(ref, 3, DELETE_WAIT_AFTER_SECONDS)
280+
assert deleted
281+
except Exception:
282+
pass
283+
284+
285+
@pytest.fixture
286+
def ignore_field_drift_listener(request, elbv2_client, simple_load_balancer, two_target_groups):
287+
"""A Listener with a weighted forward action across two target groups,
288+
annotated to ignore drift on spec.defaultActions.
289+
290+
Parametrize the ignored paths via an indirect fixture param, e.g.:
291+
292+
@pytest.mark.parametrize(
293+
"ignore_field_drift_listener",
294+
[{"ignore_paths": "spec.defaultActions"}],
295+
indirect=True,
296+
)
297+
298+
Defaults to ignoring spec.defaultActions so callers that don't parametrize
299+
keep the weight-drift behaviour."""
300+
(lb_ref, lb_cr, _) = simple_load_balancer
301+
param = getattr(request, "param", None) or {}
302+
ignore_paths = param.get("ignore_paths", "spec.defaultActions")
303+
304+
resource_name = random_suffix_name("listener-ifd", 24)
305+
replacements = REPLACEMENT_VALUES.copy()
306+
replacements["LISTENER_NAME"] = resource_name
307+
replacements["LOAD_BALANCER_ARN"] = lb_cr["status"]["ackResourceMetadata"]["arn"]
308+
replacements["TARGET_GROUP_NAME_1"] = two_target_groups[0]
309+
replacements["TARGET_GROUP_NAME_2"] = two_target_groups[1]
310+
replacements["WEIGHT_1"] = str(DECLARED_WEIGHT_1)
311+
replacements["WEIGHT_2"] = str(DECLARED_WEIGHT_2)
312+
replacements["IGNORE_PATHS"] = ignore_paths
313+
314+
resource_data = load_elbv2_resource(
315+
"listener_ignore_field_drift",
316+
additional_replacements=replacements,
317+
)
318+
logging.debug(resource_data)
319+
320+
ref = k8s.CustomResourceReference(
321+
CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
322+
resource_name, namespace="default",
323+
)
324+
k8s.create_custom_resource(ref, resource_data)
325+
time.sleep(CREATE_WAIT_AFTER_SECONDS)
326+
327+
cr = k8s.wait_resource_consumed_by_controller(ref)
328+
assert cr is not None
329+
assert k8s.get_resource_exists(ref)
330+
331+
yield (ref, cr)
332+
333+
try:
334+
_, deleted = k8s.delete_custom_resource(ref, 3, DELETE_WAIT_AFTER_SECONDS)
335+
assert deleted
336+
except Exception:
337+
pass
338+
339+
340+
@service_marker
341+
class TestListenerIgnoreFieldDrift:
342+
"""Verifies the services.k8s.aws/ignore-field-drift annotation on an ELBv2
343+
Listener's forward-action target-group weights (elbv2#85 Scenario 2).
344+
345+
The controller still applies the declared weights at create but stops
346+
reconciling drift on the ignored spec.defaultActions path: an externally
347+
shifted weight distribution survives, the resource stays Synced, and an edit
348+
to the ignored field is retained in the spec but not pushed to AWS."""
349+
350+
def test_weight_drift_ignored(
351+
self, elbv2_client, ignore_field_drift_enabled, ignore_field_drift_listener,
352+
):
353+
(ref, cr) = ignore_field_drift_listener
354+
listener_arn = cr["status"]["ackResourceMetadata"]["arn"]
355+
validator = ELBValidator(elbv2_client)
356+
357+
# Baseline: the declared weights were applied at create, and the
358+
# resource is Synced.
359+
listener = validator.get_listener(listener_arn)
360+
assert listener is not None
361+
baseline = _weights_by_tg_arn(listener)
362+
assert sorted(baseline.values()) == sorted(
363+
[DECLARED_WEIGHT_1, DECLARED_WEIGHT_2]
364+
), f"unexpected baseline weights: {baseline}"
365+
condition.assert_synced(ref)
366+
367+
# Snapshot the live forward action, then flip the weights out-of-band
368+
# (the blue/green deploy tool shifting traffic).
369+
forward = next(
370+
a for a in listener["DefaultActions"] if a.get("Type") == "forward"
371+
)
372+
tgs = forward["ForwardConfig"]["TargetGroups"]
373+
assert len(tgs) == 2
374+
shifted_tgs = [
375+
{"TargetGroupArn": tgs[0]["TargetGroupArn"], "Weight": EXTERNAL_WEIGHT_1},
376+
{"TargetGroupArn": tgs[1]["TargetGroupArn"], "Weight": EXTERNAL_WEIGHT_2},
377+
]
378+
elbv2_client.modify_listener(
379+
ListenerArn=listener_arn,
380+
DefaultActions=[
381+
{
382+
"Type": "forward",
383+
"ForwardConfig": {"TargetGroups": shifted_tgs},
384+
}
385+
],
386+
)
387+
time.sleep(MODIFY_WAIT_AFTER_SECONDS)
388+
389+
# The externally-shifted weights must survive: the controller does not
390+
# reconcile drift on spec.defaultActions, so it does not call
391+
# ModifyListener to revert them.
392+
after = _weights_by_tg_arn(validator.get_listener(listener_arn))
393+
assert after == {
394+
tgs[0]["TargetGroupArn"]: EXTERNAL_WEIGHT_1,
395+
tgs[1]["TargetGroupArn"]: EXTERNAL_WEIGHT_2,
396+
}, (
397+
"controller reverted externally-shifted listener weights despite "
398+
f"ignore-field-drift on spec.defaultActions: {after}"
399+
)
400+
401+
# The resource stays Synced even though the live weights (50/50) differ
402+
# from the declared spec (90/10).
403+
assert k8s.wait_on_condition(
404+
ref, "ACK.ResourceSynced", "True",
405+
wait_periods=6, period_length=10,
406+
)
407+
408+
# Editing the ignored field in the spec is retained but NOT pushed to
409+
# AWS: patch the declared weights to a third value and confirm the live
410+
# weights are unchanged (still the external 50/50).
411+
latest = k8s.get_resource(ref)
412+
new_actions = latest["spec"]["defaultActions"]
413+
new_actions[0]["forwardConfig"]["targetGroups"][0]["weight"] = 70
414+
new_actions[0]["forwardConfig"]["targetGroups"][1]["weight"] = 30
415+
k8s.patch_custom_resource(ref, {"spec": {"defaultActions": new_actions}})
416+
time.sleep(MODIFY_WAIT_AFTER_SECONDS)
417+
418+
after_edit = _weights_by_tg_arn(validator.get_listener(listener_arn))
419+
assert after_edit == {
420+
tgs[0]["TargetGroupArn"]: EXTERNAL_WEIGHT_1,
421+
tgs[1]["TargetGroupArn"]: EXTERNAL_WEIGHT_2,
422+
}, (
423+
"controller pushed a spec edit on an ignored field to AWS: "
424+
f"{after_edit}"
425+
)
426+
427+
# The declared edit is retained in the CR spec (retain semantics).
428+
latest = k8s.get_resource(ref)
429+
spec_weights = sorted(
430+
tg["weight"]
431+
for tg in latest["spec"]["defaultActions"][0]["forwardConfig"]["targetGroups"]
432+
)
433+
assert spec_weights == [30, 70], f"spec did not retain the edit: {spec_weights}"

0 commit comments

Comments
 (0)