Try another perf fix

Author: yhaliaw

docs/changelog.md

@@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 Each revision is versioned by the date of the revision.
 
+## 2026-06-22
+
+- Fix a performance issue in `reconcile` where the hook blocked for ~30 seconds while the
+  Wazuh API was still starting up. The Wazuh API client now uses a tight retry/backoff and
+  short timeouts, and treats "API not ready" responses (connection errors and transient
+  5xx) as `WazuhNotReadyError` so `reconcile` fails fast to maintenance status instead of
+  blocking. The charm now also re-runs `reconcile` on `update-status` and on the
+  `wazuh-server` pebble check recovering, so deferred API configuration completes on a
+  later event. The Wazuh API calls are now wrapped in OpenTelemetry spans for visibility.
+- Fix a performance issue in `reconcile`: `_reconcile_users` no longer retries API
+  authentication with a one-second backoff on `WazuhAuthenticationError` (HTTP 401).
+  A 401 is deterministic for a given password, so the retries could not succeed and
+  only stalled reconcile by up to 5 seconds per user.
+
 ## 2026-06-18
 
 - Add distributed tracing support via `ops[tracing]`. Relates the charm to a Tempo-compatible

src/charm.py

@@ -74,7 +74,9 @@ def __init__(self, *args: typing.Any):
 
         self.framework.observe(self.on.install, self._on_install)
         self.framework.observe(self.on.wazuh_server_pebble_ready, self.reconcile)
+        self.framework.observe(self.on.wazuh_server_pebble_check_recovered, self.reconcile)
         self.framework.observe(self.on.config_changed, self.reconcile)
+        self.framework.observe(self.on.update_status, self.reconcile)
         self.framework.observe(self.on[WAZUH_PEER_RELATION_NAME].relation_joined, self.reconcile)
         self.framework.observe(self.on[WAZUH_PEER_RELATION_NAME].relation_changed, self.reconcile)
         self.framework.observe(self.on[WAZUH_PEER_RELATION_NAME].relation_departed, self.reconcile)
@@ -289,16 +291,17 @@ def _reconcile_users(self) -> None:  # noqa: C901
         credentials = self.state.api_credentials
 
         for username, user_details in state.WAZUH_USERS.items():
-            # Try to authenticate with the current credentials. If it fails, password is invalid.
-            retries = 5
+            # Try to authenticate with the current credentials. A 401 (WazuhAuthenticationError)
+            # is deterministic for a given password, so a single attempt is enough; retrying the
+            # same credentials can never succeed and only stalls reconcile. Transient failures
+            # (Wazuh not ready) surface as WazuhNotReadyError and propagate to the caller.
             valid = False
-            while credentials[username] and not valid and retries > 0:
+            if credentials[username]:
                 try:
                     wazuh.authenticate_user(username, credentials[username])
                     valid = True
                 except wazuh.WazuhAuthenticationError:
-                    retries -= 1
-                    time.sleep(1)
+                    valid = False
 
             # Secret exists but users are the default. Force recreation.
             if not valid:
@@ -307,23 +310,15 @@ def _reconcile_users(self) -> None:  # noqa: C901
             # create user if it doesn't exist yet
             current_password = credentials[username]
             password_to_save = None
-            retries = 5
-            while not current_password and retries > 0:
+            if not current_password:
                 try:
                     token = wazuh.authenticate_user("wazuh", credentials["wazuh"])
                     new_password = wazuh.generate_api_password()
                     wazuh.create_api_user(username, new_password, token)
                     current_password = new_password
                     password_to_save = new_password
                 except wazuh.WazuhAuthenticationError as exc:
-                    retries -= 1
-                    logger.error(
-                        "Failed to create user %s: %s. %s retries remaining.",
-                        username,
-                        exc,
-                        retries,
-                    )
-                    time.sleep(1)
+                    logger.error("Failed to create user %s: %s.", username, exc)
 
             # change credentials if they've never been changed
             if current_password == user_details["default_password"]:

src/wazuh.py

@@ -13,6 +13,7 @@
 from enum import Enum
 from pathlib import Path
 
+import opentelemetry.trace
 import ops
 import requests
 import requests.adapters
@@ -28,6 +29,14 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
+tracer = opentelemetry.trace.get_tracer(__name__)
+
+# Wazuh API request tuning. Keep retries tight and timeouts short so that, while the
+# Wazuh API is still starting up, calls fail fast (and surface as WazuhNotReadyError)
+# instead of blocking the reconcile hook for tens of seconds with exponential backoff.
+API_REQUEST_TIMEOUT = 5
+API_NOT_READY_STATUSES = frozenset({500, 502, 503, 504})
+
 CONTAINER_NAME = "wazuh-server"
 INGEST_LOG_DIR = "/var/log/collectors/rsyslog"  # logs intended for ingestion
 REPOSITORY_PATH = "/root/repository"
@@ -847,6 +856,28 @@ def _generate_cluster_snippet(
     """
 
 
+def _api_session() -> requests.Session:
+    """Build a requests session with a tight retry policy for the Wazuh API.
+
+    The retries are intentionally minimal so that calls against a Wazuh API that is
+    still starting up fail fast instead of blocking the reconcile hook for tens of
+    seconds. Transient connection issues are surfaced by the callers as
+    WazuhNotReadyError so the charm waits for a later event rather than erroring.
+
+    Returns: a configured requests Session.
+    """
+    session = requests.Session()
+    retries = requests.adapters.Retry(
+        total=2,
+        connect=2,
+        read=2,
+        backoff_factor=0.1,
+    )
+    session.mount("https://", requests.adapters.HTTPAdapter(max_retries=retries))
+    return session
+
+
+@tracer.start_as_current_span("authenticate_user")
 def authenticate_user(username: str, password: str) -> str:
     """Authenticate an API user.
 
@@ -865,30 +896,36 @@ def authenticate_user(username: str, password: str) -> str:
     # certificates may be self-signed and there's no value in verifying them
     # as a compromised localhost service would indicate we're already compromised
     try:
-        session = requests.Session()
-        retries = requests.adapters.Retry(connect=10, backoff_factor=0.2, status_forcelist=[500])
-        session.mount("https://", requests.adapters.HTTPAdapter(max_retries=retries))
-        response = session.get(  # nosec
+        response = _api_session().get(  # nosec
             AUTH_ENDPOINT,
             auth=(username, password),
-            timeout=10,
+            timeout=API_REQUEST_TIMEOUT,
             verify=False,
         )
         if response.status_code == 401:
             raise WazuhAuthenticationError(f"The provided password for {username} is not valid.")
+        if response.status_code in API_NOT_READY_STATUSES:
+            raise WazuhNotReadyError(
+                f"Wazuh API not ready (status {response.status_code}) authenticating {username}."
+            )
         response.raise_for_status()
         token = response.json()["data"]["token"] if response.json()["data"] else None
         if token is None:
             raise WazuhInstallationError(f"Response for {username} does not contain token.")
         logger.debug("Got Wazuh API auth token for username %s", username)
         return token
-    except requests.exceptions.ConnectionError as exc:
-        logger.warning("Wazuh API authentication failed: %s", exc)
+    except (
+        requests.exceptions.ConnectionError,
+        requests.exceptions.Timeout,
+        requests.exceptions.RetryError,
+    ) as exc:
+        logger.warning("Wazuh API not ready while authenticating: %s", exc)
         raise WazuhNotReadyError from exc
     except requests.exceptions.RequestException as exc:
         raise WazuhInstallationError from exc
 
 
+@tracer.start_as_current_span("change_api_password")
 def change_api_password(username: str, password: str, token: str) -> None:
     """Change Wazuh's API password for a given user.
 
@@ -899,31 +936,40 @@ def change_api_password(username: str, password: str, token: str) -> None:
 
     Raises:
         WazuhInstallationError: if an error occurs while processing the requests.
+        WazuhNotReadyError: if wazuh is not yet ready to accept requests.
     """
     # certificates may be self-signed and there's no value in verifying them
     # as a compromised localhost service would indicate we're already compromised
+    session = _api_session()
     try:
         headers = {"Authorization": f"Bearer {token}"}
-        response = requests.get(  # nosec
+        response = session.get(  # nosec
             f"https://localhost:{API_PORT}/security/users",
             headers=headers,
-            timeout=10,
-            verify=False,  # nosec  # noqa: S501
+            timeout=API_REQUEST_TIMEOUT,
+            verify=False,  # nosec
         )
         response.raise_for_status()
         data = response.json()["data"]
         user_id = next(
             user["id"] for user in data["affected_items"] if data and user["username"] == username
         )
-        response = requests.put(  # nosec
+        response = session.put(  # nosec
             f"https://localhost:{API_PORT}/security/users/{user_id}",
             headers=headers,
             json={"password": password},
-            timeout=10,
-            verify=False,  # nosec  # noqa: S501
+            timeout=API_REQUEST_TIMEOUT,
+            verify=False,  # nosec
         )
         response.raise_for_status()
         logger.info("Changed API password for user %s", username)
+    except (
+        requests.exceptions.ConnectionError,
+        requests.exceptions.Timeout,
+        requests.exceptions.RetryError,
+    ) as exc:
+        logger.warning("Wazuh API not ready while changing password: %s", exc)
+        raise WazuhNotReadyError from exc
     except requests.exceptions.RequestException as exc:
         raise WazuhInstallationError("Error modifying the default password.") from exc
 
@@ -946,6 +992,7 @@ def generate_api_password() -> str:
     return "".join(password)
 
 
+@tracer.start_as_current_span("create_api_user")
 def create_api_user(username: str, password: str, token: str, rolename: str = "readonly") -> None:
     """Create a new readonly user for Wazuh's API.
 
@@ -958,55 +1005,64 @@ def create_api_user(username: str, password: str, token: str, rolename: str = "r
     Raises:
         WazuhAuthenticationError: if a 401 error occurs while processing the requests.
         WazuhInstallationError: if any non-401 error occurs while processing the requests.
+        WazuhNotReadyError: if wazuh is not yet ready to accept requests.
     """
     # certificates may be self-signed and there's no value in verifying them
     # as a compromised localhost service would indicate we're already compromised
     response = None
+    session = _api_session()
     try:
         headers = {"Authorization": f"Bearer {token}"}
-        response = requests.get(  # nosec
+        response = session.get(  # nosec
             f"https://localhost:{API_PORT}/security/users",
             headers=headers,
-            timeout=10,
-            verify=False,  # nosec  # noqa: S501
+            timeout=API_REQUEST_TIMEOUT,
+            verify=False,  # nosec
         )
         response.raise_for_status()
         data = response.json()["data"]
         user_id = [
             user["id"] for user in data["affected_items"] if data and user["username"] == username
         ]
         if not user_id:  # user has not been created yet
-            response = requests.post(  # nosec
+            response = session.post(  # nosec
                 f"https://localhost:{API_PORT}/security/users",
                 headers=headers,
                 json={"username": username, "password": password},
-                timeout=10,
-                verify=False,  # nosec  # noqa: S501
+                timeout=API_REQUEST_TIMEOUT,
+                verify=False,  # nosec
             )
             response.raise_for_status()
             data = response.json()["data"]
         user_id = next(
             user["id"] for user in data["affected_items"] if data and user["username"] == username
         )
-        response = requests.get(  # nosec
+        response = session.get(  # nosec
             f"https://localhost:{API_PORT}/security/roles",
             headers=headers,
-            timeout=10,
-            verify=False,  # nosec  # noqa: S501
+            timeout=API_REQUEST_TIMEOUT,
+            verify=False,  # nosec
         )
         response.raise_for_status()
         data = response.json()["data"]
         role_id = next(
             role["id"] for role in data["affected_items"] if data and role["name"] == rolename
         )
-        response = requests.post(  # nosec
+        response = session.post(  # nosec
             f"https://localhost:{API_PORT}/security/users/{user_id}/roles?role_ids={role_id}",
             headers=headers,
-            timeout=10,
-            verify=False,  # nosec  # noqa: S501
+            timeout=API_REQUEST_TIMEOUT,
+            verify=False,  # nosec
         )
         response.raise_for_status()
         logger.info("Created user %s", username)
+    except (
+        requests.exceptions.ConnectionError,
+        requests.exceptions.Timeout,
+        requests.exceptions.RetryError,
+    ) as exc:
+        logger.warning("Wazuh API not ready while creating user: %s", exc)
+        raise WazuhNotReadyError from exc
     except requests.exceptions.RequestException as exc:
         if isinstance(response, requests.Response) and response.status_code == 401:
             raise WazuhAuthenticationError("401 error creating an API user") from exc