test(appprotect): tolerate connection drops during reload in watch-namespace-label test

The watch-namespace-label AppProtect e2e test toggles namespace labels, each
of which triggers an NGINX reload plus an App Protect WAF soft reset. Requests
that land during the reload window hit recycled workers and get a closed
connection (access log status 000), surfacing as an uncaught
requests.exceptions.ConnectionError (RemoteDisconnected) that fails the test.

Add a retry_get_until_body_contains helper that retries the request and
tolerates ConnectionError during reloads, mirroring the transient-handling
already done by ensure_response_from_backend and wait_for_reload. Use it at the
three request sites in test_app_protect_watch_namespace_label.py and drop the
now-unused requests import.

Also add FLAKY_RELOAD_REQUESTS.md documenting the pattern so the same fix can be
applied to other reload-sensitive e2e tests.

Test-only change: no product code, codegen, or snapshot updates.

Author: javorszky

tests/suite/test_app_protect_watch_namespace_label.py

@@ -1,7 +1,6 @@
 import time
 
 import pytest
-import requests
 from settings import TEST_DATA
 from suite.utils.ap_resources_utils import (
     create_ap_logconf_from_yaml,
@@ -20,6 +19,7 @@
     ensure_connection_to_public_endpoint,
     ensure_response_from_backend,
     patch_namespace_with_label,
+    retry_get_until_body_contains,
     wait_before_test,
     wait_until_all_pods_are_ready,
 )
@@ -156,8 +156,8 @@ def test_responses(self, request, kube_apis, crd_ingress_controller_with_ap, bac
         ensure_response_from_backend(backend_setup.req_url, backend_setup.ingress_host, check404=True)
 
         print("----------------------- Send request ----------------------")
-        resp = requests.get(
-            f"{backend_setup.req_url}/test.bat", headers={"host": backend_setup.ingress_host}, verify=False
+        resp = retry_get_until_body_contains(
+            f"{backend_setup.req_url}/test.bat", backend_setup.ingress_host, valid_resp_body
         )
 
         print(resp.text)
@@ -176,17 +176,9 @@ def test_responses(self, request, kube_apis, crd_ingress_controller_with_ap, bac
         ensure_response_from_backend(backend_setup.req_url, backend_setup.ingress_host, check404=True)
 
         print("----------------------- Send request ----------------------")
-        resp = requests.get(
-            f"{backend_setup.req_url}/test.bat", headers={"host": backend_setup.ingress_host}, verify=False
+        resp = retry_get_until_body_contains(
+            f"{backend_setup.req_url}/test.bat", backend_setup.ingress_host, invalid_resp_body
         )
-        retry = 0
-        while invalid_resp_body not in resp.text and retry <= 60:
-            resp = requests.get(
-                f"{backend_setup.req_url}/test.bat", headers={"host": backend_setup.ingress_host}, verify=False
-            )
-            retry += 1
-            wait_before_test(1)
-            print(f"Policy not yet enforced, retrying... #{retry}")
 
         assert invalid_resp_body in resp.text
         assert resp.status_code == 200
@@ -202,17 +194,9 @@ def test_responses(self, request, kube_apis, crd_ingress_controller_with_ap, bac
         ensure_response_from_backend(backend_setup.req_url, backend_setup.ingress_host, check404=True)
 
         print("----------------------- Send request ----------------------")
-        resp = requests.get(
-            f"{backend_setup.req_url}/test.bat", headers={"host": backend_setup.ingress_host}, verify=False
+        resp = retry_get_until_body_contains(
+            f"{backend_setup.req_url}/test.bat", backend_setup.ingress_host, valid_resp_body
         )
-        retry = 0
-        while valid_resp_body not in resp.text and retry <= 60:
-            resp = requests.get(
-                f"{backend_setup.req_url}/test.bat", headers={"host": backend_setup.ingress_host}, verify=False
-            )
-            retry += 1
-            wait_before_test(1)
-            print(f"Policy not yet removed, retrying... #{retry}")
 
         assert valid_resp_body in resp.text
         assert resp.status_code == 200

tests/suite/utils/resources_utils.py

@@ -1948,6 +1948,32 @@ def ensure_response_from_backend(req_url, host, additional_headers=None, check40
         pytest.fail(f"Keep getting 502|504 from {req_url} after 60 seconds. Exiting...")
 
 
+def retry_get_until_body_contains(req_url, host, expected_body, retries=60, verify=False):
+    """
+    Repeatedly GET req_url until expected_body appears in the response body.
+
+    Tolerates ConnectionError/RemoteDisconnected caused by NGINX reloads
+    (worker recycling during App Protect reconfiguration closes connections).
+
+    :param req_url: url to request
+    :param host: value for the Host header
+    :param expected_body: substring expected to appear in the response body
+    :param retries: number of retries at 1 second interval
+    :param verify: passed through to requests.get for TLS verification
+    :return: the final requests.Response (may not contain expected_body if exhausted)
+    """
+    resp = None
+    for i in range(retries + 1):
+        try:
+            resp = requests.get(req_url, headers={"host": host}, verify=verify)
+            if expected_body in resp.text:
+                return resp
+        except requests.exceptions.ConnectionError as e:
+            print(f"Attempt {i + 1}: connection dropped during reload ({e})")
+        wait_before_test(1)
+    return resp
+
+
 def get_service_endpoint(kube_apis, service_name, namespace) -> str:
     """
     Wait for endpoint resource to spin up.