fix(proxy_server): remove redundant decryption of already-decrypted env variables

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -1,7 +1,11 @@
 import asyncio
+import base64
 import copy
+import hashlib
+import json
 import logging
 import os
+import sys
 import time
 import traceback
 from datetime import datetime, timedelta
@@ -36,11 +40,112 @@
 from litellm.proxy.middleware.in_flight_requests_middleware import (
     get_in_flight_requests,
 )
-from litellm.secret_managers.main import get_secret
 
 #### Health ENDPOINTS ####
 
 
+def _str_to_bool(value: Optional[str]) -> Optional[bool]:
+    if value is None:
+        return None
+
+    normalized_value = value.strip().lower()
+    if normalized_value == "true":
+        return True
+    if normalized_value == "false":
+        return False
+    return None
+
+
+def get_secret(
+    secret_name: str, default_value: Optional[Union[str, bool]] = None
+) -> Optional[Union[str, bool]]:
+    if secret_name.startswith("os.environ/"):
+        secret_name = secret_name.replace("os.environ/", "")
+
+    secret_value = os.getenv(secret_name)
+    if secret_value is None:
+        return default_value
+
+    return secret_value
+
+
+def get_secret_bool(
+    secret_name: str, default_value: Optional[bool] = None
+) -> Optional[bool]:
+    secret_value = get_secret(secret_name=secret_name)
+    if secret_value is None:
+        return default_value
+
+    if isinstance(secret_value, bool):
+        return secret_value
+
+    return _str_to_bool(secret_value)
+
+
+def _get_proxy_signing_key() -> Optional[str]:
+    salt_key = os.getenv("LITELLM_SALT_KEY")
+    if salt_key is not None:
+        return salt_key
+
+    proxy_server_module = sys.modules.get("litellm.proxy.proxy_server")
+    if proxy_server_module is not None:
+        proxy_master_key = getattr(proxy_server_module, "master_key", None)
+        if isinstance(proxy_master_key, str):
+            return proxy_master_key
+
+    return os.getenv("LITELLM_MASTER_KEY")
+
+
+def _decrypt_value(value: bytes, signing_key: str) -> str:
+    import nacl.secret
+
+    hash_bytes = hashlib.sha256(signing_key.encode()).digest()
+    box = nacl.secret.SecretBox(hash_bytes)
+
+    if len(value) == 0:
+        return ""
+
+    plaintext = box.decrypt(value)
+    return plaintext.decode("utf-8")
+
+
+def decrypt_value_helper(
+    value: Any,
+    key: str,
+    exception_type: Literal["debug", "error"] = "error",
+    return_original_value: bool = False,
+) -> Any:
+    signing_key = _get_proxy_signing_key()
+
+    try:
+        if isinstance(value, str):
+            if signing_key is None:
+                raise ValueError("No signing key configured")
+
+            try:
+                decoded_b64 = base64.urlsafe_b64decode(value)
+            except Exception:
+                decoded_b64 = base64.b64decode(value)
+
+            return _decrypt_value(value=decoded_b64, signing_key=signing_key)
+
+        return value
+    except Exception as e:
+        error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {str(e)}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key"
+        if exception_type == "debug":
+            verbose_proxy_logger.debug(error_message)
+            return value if return_original_value else None
+
+        verbose_proxy_logger.debug(
+            f"Unable to decrypt value={value} for key: {key}, returning None"
+        )
+        if return_original_value:
+            return value
+
+        verbose_proxy_logger.exception(error_message)
+        return None
+
+
 def _resolve_os_environ_variables(params: dict) -> dict:
     """
     Resolve ``os.environ/`` environment variables in ``litellm_params``.
@@ -145,6 +250,90 @@ def get_callback_identifier(callback):
     return callback_name(callback)
 
 
+def _parse_config_row_param_value(param_value: Any) -> dict:
+    if param_value is None:
+        return {}
+
+    if isinstance(param_value, str):
+        try:
+            parsed_value = json.loads(param_value)
+        except json.JSONDecodeError:
+            return {}
+        return parsed_value if isinstance(parsed_value, dict) else {}
+
+    if isinstance(param_value, dict):
+        return dict(param_value)
+
+    try:
+        parsed_value = dict(param_value)
+    except (TypeError, ValueError):
+        return {}
+
+    return parsed_value if isinstance(parsed_value, dict) else {}
+
+
+def _is_truthy_config_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+
+    if isinstance(value, str):
+        return value.strip().lower() == "true"
+
+    if value is None:
+        return False
+
+    return bool(value)
+
+
+async def _resolve_test_email_address(prisma_client: Any) -> Optional[str]:
+    test_email_address = os.getenv("TEST_EMAIL_ADDRESS")
+
+    try:
+        store_model_in_db = (
+            get_secret_bool("STORE_MODEL_IN_DB", default_value=False) is True
+        )
+
+        if not store_model_in_db and prisma_client is not None:
+            general_settings_row = await prisma_client.db.litellm_config.find_unique(
+                where={"param_name": "general_settings"}
+            )
+            general_settings = _parse_config_row_param_value(
+                getattr(general_settings_row, "param_value", None)
+            )
+            store_model_in_db = _is_truthy_config_flag(
+                general_settings.get("store_model_in_db")
+            )
+
+        if not store_model_in_db or prisma_client is None:
+            return test_email_address
+
+        environment_variables_row = await prisma_client.db.litellm_config.find_unique(
+            where={"param_name": "environment_variables"}
+        )
+        environment_variables = _parse_config_row_param_value(
+            getattr(environment_variables_row, "param_value", None)
+        )
+        db_test_email_address = environment_variables.get("TEST_EMAIL_ADDRESS")
+
+        if db_test_email_address is None:
+            return test_email_address
+
+        decrypted_test_email_address = decrypt_value_helper(
+            value=db_test_email_address,
+            key="TEST_EMAIL_ADDRESS",
+            exception_type="debug",
+            return_original_value=True,
+        )
+
+        return decrypted_test_email_address or test_email_address
+    except Exception as e:
+        verbose_proxy_logger.debug(
+            "Falling back to TEST_EMAIL_ADDRESS from env after DB lookup failed: %s",
+            str(e),
+        )
+        return test_email_address
+
+
 router = APIRouter()
 services = Union[
     Literal[
@@ -447,7 +636,7 @@ async def health_services_endpoint(  # noqa: PLR0915
                 spend=0,
                 max_budget=0,
                 user_id=user_api_key_dict.user_id,
-                user_email=os.getenv("TEST_EMAIL_ADDRESS"),
+                user_email=await _resolve_test_email_address(prisma_client),
                 team_id=user_api_key_dict.team_id,
             )
 

litellm/proxy/proxy_server.py

@@ -12929,11 +12929,11 @@ def normalize_callback(callback):
                     _value = os.getenv("SLACK_WEBHOOK_URL", None)
                     _slack_env_vars[_var] = _value
                 else:
-                    # decode + decrypt the value
-                    _decrypted_value = decrypt_value_helper(
-                        value=env_variable, key=_var
+                    _slack_env_vars[_var] = decrypt_value_helper(
+                        value=env_variable,
+                        key=_var,
+                        return_original_value=True,
                     )
-                    _slack_env_vars[_var] = _decrypted_value
 
             _alerting_types = proxy_logging_obj.slack_alerting_instance.alert_types
             _all_alert_types = (
@@ -12967,8 +12967,15 @@ def normalize_callback(callback):
             if env_variable is None:
                 _email_env_vars[_var] = None
             else:
-                # decode + decrypt the value
-                _decrypted_value = decrypt_value_helper(value=env_variable, key=_var)
+                # Use return_original_value=True so this works for both:
+                # - DB mode: values already decrypted by _update_config_from_db → decryption
+                #   fails gracefully and returns the original plaintext value
+                # - YAML mode: values still encrypted in config file → decrypted here
+                _decrypted_value = decrypt_value_helper(
+                    value=env_variable,
+                    key=_var,
+                    return_original_value=True,
+                )
                 _email_env_vars[_var] = _decrypted_value
 
         alerting_data.append(

tests/proxy_unit_tests/test_proxy_server.py

@@ -2743,6 +2743,73 @@ async def test_get_config_callbacks_environment_variables(client_no_auth):
         assert otel_vars["OTEL_HEADERS"] == "key=value"
 
 
+@pytest.mark.asyncio
+async def test_get_config_callbacks_email_and_slack_values_are_not_decrypted_again(
+    client_no_auth,
+):
+    """
+    Test that /get/config/callbacks returns already-decrypted email/slack values as-is.
+
+    decrypt_value_helper is called with return_original_value=True, so for already-plaintext
+    values (DB mode: decrypted by _update_config_from_db) it returns the original value
+    unchanged. For encrypted values (YAML mode) it properly decrypts them.
+    """
+    mock_config_data = {
+        "litellm_settings": {},
+        "environment_variables": {
+            "SLACK_WEBHOOK_URL": "https://hooks.slack.com/services/test/webhook",
+            "SMTP_HOST": "10.16.68.20",
+            "SMTP_PORT": "587",
+            "SMTP_USERNAME": "smtp-user",
+            "SMTP_PASSWORD": "smtp-password",
+            "SMTP_SENDER_EMAIL": "alerts@example.com",
+            "TEST_EMAIL_ADDRESS": "ops@example.com",
+            "EMAIL_LOGO_URL": "https://example.com/logo.png",
+            "EMAIL_SUPPORT_CONTACT": "support@example.com",
+        },
+        "general_settings": {"alerting": ["slack"]},
+    }
+
+    proxy_config = getattr(litellm.proxy.proxy_server, "proxy_config")
+
+    # Simulate return_original_value=True behaviour: return the value as-is (already plaintext)
+    def fake_decrypt(value, key, return_original_value=False, **kwargs):
+        return value
+
+    with patch.object(
+        proxy_config, "get_config", new=AsyncMock(return_value=mock_config_data)
+    ), patch(
+        "litellm.proxy.proxy_server.decrypt_value_helper",
+        side_effect=fake_decrypt,
+    ) as decrypt_mock:
+        response = client_no_auth.get("/get/config/callbacks")
+
+    assert response.status_code == 200
+    result = response.json()
+    alerts = result["alerts"]
+
+    slack_alert = next((alert for alert in alerts if alert["name"] == "slack"), None)
+    assert slack_alert is not None
+    assert slack_alert["variables"] == {
+        "SLACK_WEBHOOK_URL": "https://hooks.slack.com/services/test/webhook"
+    }
+
+    email_alert = next((alert for alert in alerts if alert["name"] == "email"), None)
+    assert email_alert is not None
+    assert email_alert["variables"] == {
+        "SMTP_HOST": "10.16.68.20",
+        "SMTP_PORT": "587",
+        "SMTP_USERNAME": "smtp-user",
+        "SMTP_PASSWORD": "smtp-password",
+        "SMTP_SENDER_EMAIL": "alerts@example.com",
+        "TEST_EMAIL_ADDRESS": "ops@example.com",
+        "EMAIL_LOGO_URL": "https://example.com/logo.png",
+        "EMAIL_SUPPORT_CONTACT": "support@example.com",
+    }
+    # decrypt_value_helper is called once per SMTP var + once for SLACK_WEBHOOK_URL
+    assert decrypt_mock.call_count == len(mock_config_data["environment_variables"])
+
+
 @pytest.mark.asyncio
 async def test_update_config_success_callback_normalization():
     """