[LEADS-443] OPENAI_API_KEY environmental variable failure

bsatapat-jpg · bsatapat-jpg · commit f5895137763c · 2026-06-24T15:50:32.000+05:30
diff --git a/src/lightspeed_evaluation/core/embedding/manager.py b/src/lightspeed_evaluation/core/embedding/manager.py
@@ -1,8 +1,12 @@
 """Embedding Manager - Generic embedding configuration, validation, and parameter provider."""
 
+import logging
+
 from lightspeed_evaluation.core.models import EmbeddingConfig, SystemConfig
 from lightspeed_evaluation.core.system.env_validator import validate_provider_env
 
+logger = logging.getLogger(__name__)
+
 
 class EmbeddingError(Exception):
     """Embedding config errors."""
@@ -31,16 +35,36 @@ def check_huggingface_available() -> None:
         ) from e
 
 
-class EmbeddingManager:  # pylint: disable=too-few-public-methods
-    """Generic Embedding Manager."""
+class EmbeddingManager:
+    """Generic Embedding Manager with lazy validation.
+
+    Validation of provider environment variables is deferred until first actual
+    use. This allows the evaluation pipeline to start without requiring embedding
+    provider credentials when no metric relying on embeddings is configured.
+    """
 
     def __init__(self, config: EmbeddingConfig):
-        """Initialize with validated environment and constructed model name."""
+        """Initialize with config. Validation is deferred until ensure_ready() is called."""
         self.config = config
+        self._validated = False
+
+    def ensure_ready(self) -> None:
+        """Validate provider config and environment variables on first use.
+
+        This method is idempotent - subsequent calls after successful validation
+        are no-ops. Should be called before accessing embedding functionality.
+
+        Raises:
+            EmbeddingError: If provider is unsupported or env vars are missing.
+        """
+        if self._validated:
+            return
         self._validate_config()
-        print(
-            f"""
-✅ Embedding Manager: {self.config.provider} -- {self.config.model} {self.config.provider_kwargs}"""
+        self._validated = True
+        logger.info(
+            "Embedding Manager ready: %s -- %s",
+            self.config.provider,
+            self.config.model,
         )
 
     def _validate_config(self) -> None:
diff --git a/src/lightspeed_evaluation/core/embedding/ragas.py b/src/lightspeed_evaluation/core/embedding/ragas.py
@@ -19,7 +19,12 @@ def __init__(self, embedding_manager: EmbeddingManager):
 
         Args:
             embedding_manager: Pre-configured EmbeddingManager with validated parameters
+
+        Raises:
+            EmbeddingError: If provider environment variables are not configured.
+            ConfigurationError: If embedding provider is unknown.
         """
+        embedding_manager.ensure_ready()
         self.config = embedding_manager.config
 
         # Map provider names to litellm format
diff --git a/src/lightspeed_evaluation/core/metrics/ragas.py b/src/lightspeed_evaluation/core/metrics/ragas.py
@@ -16,12 +16,16 @@
     Faithfulness,
 )
 
-from lightspeed_evaluation.core.embedding.manager import EmbeddingManager
+from lightspeed_evaluation.core.embedding.manager import (
+    EmbeddingError,
+    EmbeddingManager,
+)
 from lightspeed_evaluation.core.embedding.ragas import RagasEmbeddingManager
 from lightspeed_evaluation.core.llm.litellm_patch import litellm_state_lock
 from lightspeed_evaluation.core.llm.manager import LLMManager
 from lightspeed_evaluation.core.llm.ragas import RagasLLMManager
 from lightspeed_evaluation.core.models import EvaluationScope, TurnData
+from lightspeed_evaluation.core.system.exceptions import EvaluationError
 
 
 def _clamp_score(score: float) -> float:
@@ -70,7 +74,9 @@ def __init__(self, llm_manager: LLMManager, embedding_manager: EmbeddingManager)
 
         # Create Ragas LLM Manager for metric configuration
         self.llm_manager = RagasLLMManager(llm_manager)
-        self.embedding_manager = RagasEmbeddingManager(embedding_manager)
+        # Store base embedding manager for lazy initialization of RagasEmbeddingManager
+        self._embedding_manager = embedding_manager
+        self._ragas_embedding_manager: Optional[RagasEmbeddingManager] = None
 
         self.supported_metrics = {
             # Response evaluation metrics
@@ -85,6 +91,19 @@ def __init__(self, llm_manager: LLMManager, embedding_manager: EmbeddingManager)
             ),
         }
 
+    @property
+    def embedding_manager(self) -> RagasEmbeddingManager:
+        """Lazily initialize RagasEmbeddingManager on first access.
+
+        This defers embedding provider validation until a metric that actually
+        requires embeddings is evaluated (e.g., response_relevancy).
+        """
+        if self._ragas_embedding_manager is None:
+            self._ragas_embedding_manager = RagasEmbeddingManager(
+                self._embedding_manager
+            )
+        return self._ragas_embedding_manager
+
     def _extract_turn_data(
         self, turn_data: Optional[TurnData]
     ) -> tuple[str, str, list[str]]:
@@ -128,7 +147,14 @@ def evaluate(
                     f"(network/LLM timeout): {str(e)}"
                 )
             return None, err_msg
-        except (RuntimeError, ValueError, TypeError, ImportError) as e:
+        except (
+            EvaluationError,
+            EmbeddingError,
+            RuntimeError,
+            ValueError,
+            TypeError,
+            ImportError,
+        ) as e:
             return None, f"Ragas {metric_name} evaluation failed: {str(e)}"
 
         if result[0] is not None and math.isnan(result[0]):
diff --git a/tests/unit/core/embedding/test_manager.py b/tests/unit/core/embedding/test_manager.py
@@ -0,0 +1,85 @@
+# pylint: disable=protected-access
+
+"""Unit tests for EmbeddingManager."""
+
+import pytest
+from pytest_mock import MockerFixture
+
+from lightspeed_evaluation.core.embedding.manager import (
+    EmbeddingError,
+    EmbeddingManager,
+)
+from lightspeed_evaluation.core.models import EmbeddingConfig
+from lightspeed_evaluation.core.system.exceptions import LLMError
+
+
+class TestEnsureReadyIdempotency:
+    """Test that ensure_ready() is idempotent."""
+
+    def test_second_call_is_noop(self, mocker: MockerFixture) -> None:
+        """Calling ensure_ready() twice should only validate once."""
+        mocker.patch(
+            "lightspeed_evaluation.core.embedding.manager.validate_provider_env"
+        )
+        manager = EmbeddingManager(EmbeddingConfig(provider="openai"))
+
+        manager.ensure_ready()
+        manager.ensure_ready()
+
+        validate_mock = mocker.patch.object(manager, "_validate_config")
+        manager.ensure_ready()
+        validate_mock.assert_not_called()
+
+    def test_not_validated_after_failure(self, mocker: MockerFixture) -> None:
+        """_validated stays False when validation raises."""
+        mocker.patch(
+            "lightspeed_evaluation.core.embedding.manager.validate_provider_env",
+            side_effect=EmbeddingError("env var missing"),
+        )
+        manager = EmbeddingManager(EmbeddingConfig(provider="openai"))
+
+        with pytest.raises(EmbeddingError):
+            manager.ensure_ready()
+
+        assert not manager._validated
+
+
+class TestEnsureReadyErrors:
+    """Test ensure_ready() error cases."""
+
+    def test_unsupported_provider_raises_embedding_error(
+        self, mocker: MockerFixture
+    ) -> None:
+        """Unsupported provider should raise EmbeddingError.
+
+        Pydantic validates allowed providers at construction, so we bypass
+        it to exercise the defensive guard in _validate_config().
+        """
+        config = mocker.MagicMock()
+        config.provider = "unsupported_provider"
+        manager = EmbeddingManager(config)
+
+        with pytest.raises(EmbeddingError, match="Unsupported embedding provider"):
+            manager.ensure_ready()
+
+    def test_missing_env_vars_raises(self, mocker: MockerFixture) -> None:
+        """Missing env vars should propagate the error from validate_provider_env."""
+        mocker.patch(
+            "lightspeed_evaluation.core.embedding.manager.validate_provider_env",
+            side_effect=LLMError("OPENAI_API_KEY not set"),
+        )
+        manager = EmbeddingManager(EmbeddingConfig(provider="openai"))
+
+        with pytest.raises(LLMError, match="OPENAI_API_KEY not set"):
+            manager.ensure_ready()
+
+    def test_huggingface_missing_deps_raises(self, mocker: MockerFixture) -> None:
+        """Missing sentence-transformers should raise EmbeddingError."""
+        mocker.patch(
+            "lightspeed_evaluation.core.embedding.manager.check_huggingface_available",
+            side_effect=EmbeddingError("requires sentence-transformers"),
+        )
+        manager = EmbeddingManager(EmbeddingConfig(provider="huggingface"))
+
+        with pytest.raises(EmbeddingError, match="sentence-transformers"):
+            manager.ensure_ready()
diff --git a/tests/unit/core/metrics/test_ragas.py b/tests/unit/core/metrics/test_ragas.py
@@ -0,0 +1,143 @@
+# pylint: disable=redefined-outer-name, too-many-arguments, too-many-positional-arguments
+
+"""Unit tests for RagasMetrics."""
+
+from typing import Any
+
+import pytest
+from pytest_mock import MockerFixture
+
+from lightspeed_evaluation.core.embedding.manager import (
+    EmbeddingError,
+    EmbeddingManager,
+)
+from lightspeed_evaluation.core.metrics.ragas import RagasMetrics
+from lightspeed_evaluation.core.models import EmbeddingConfig, EvaluationScope, TurnData
+from lightspeed_evaluation.core.system.exceptions import (
+    ConfigurationError,
+    EvaluationError,
+)
+
+
+@pytest.fixture
+def mock_ragas_deps(mocker: MockerFixture) -> dict[str, Any]:
+    """Mock all heavy dependencies needed to construct RagasMetrics."""
+    mock_llm_manager = mocker.MagicMock()
+    mock_llm_config = mocker.MagicMock()
+    mock_llm_config.cache_enabled = False
+    mock_llm_manager.get_config.return_value = mock_llm_config
+
+    mock_embedding_manager = mocker.MagicMock(spec=EmbeddingManager)
+    mock_embedding_manager.config = EmbeddingConfig(
+        provider="openai", model="text-embedding-3-small", cache_enabled=False
+    )
+
+    mocker.patch("lightspeed_evaluation.core.metrics.ragas.RagasLLMManager")
+
+    return {
+        "llm_manager": mock_llm_manager,
+        "embedding_manager": mock_embedding_manager,
+    }
+
+
+@pytest.fixture
+def ragas_metrics(mock_ragas_deps: dict[str, Any]) -> RagasMetrics:
+    """Create RagasMetrics with mocked dependencies."""
+    return RagasMetrics(**mock_ragas_deps)
+
+
+@pytest.fixture
+def turn_scope() -> EvaluationScope:
+    """Create a turn-level evaluation scope."""
+    return EvaluationScope(
+        turn_idx=0,
+        turn_data=TurnData(
+            turn_id="t1",
+            query="What is Python?",
+            response="A programming language.",
+            expected_response="A programming language.",
+        ),
+        is_conversation=False,
+    )
+
+
+class TestLazyEmbeddingManagerProperty:
+    """Test lazy initialization of the embedding_manager property."""
+
+    def test_initialized_on_first_access(
+        self, ragas_metrics: RagasMetrics, mocker: MockerFixture
+    ) -> None:
+        """First property access should create and return RagasEmbeddingManager."""
+        mock_cls = mocker.patch(
+            "lightspeed_evaluation.core.metrics.ragas.RagasEmbeddingManager",
+        )
+
+        result = ragas_metrics.embedding_manager
+
+        assert result is mock_cls.return_value
+        mock_cls.assert_called_once()
+
+    def test_cached_after_first_access(
+        self, ragas_metrics: RagasMetrics, mocker: MockerFixture
+    ) -> None:
+        """Subsequent accesses should return the cached instance."""
+        mock_cls = mocker.patch(
+            "lightspeed_evaluation.core.metrics.ragas.RagasEmbeddingManager",
+        )
+
+        first = ragas_metrics.embedding_manager
+        second = ragas_metrics.embedding_manager
+
+        assert first is second
+        mock_cls.assert_called_once()
+
+
+class TestEvaluateExceptionHandling:
+    """Test that evaluate() catches the expected exception types."""
+
+    @pytest.mark.parametrize(
+        "exception_class,exception_msg",
+        [
+            (EvaluationError, "base evaluation error"),
+            (ConfigurationError, "unknown provider xyz"),
+            (EmbeddingError, "unsupported embedding provider"),
+            (RuntimeError, "unexpected runtime failure"),
+            (ValueError, "invalid value"),
+            (TypeError, "type mismatch"),
+            (ImportError, "missing module"),
+        ],
+    )
+    def test_catches_exception_gracefully(
+        self,
+        ragas_metrics: RagasMetrics,
+        turn_scope: EvaluationScope,
+        mocker: MockerFixture,
+        exception_class: type,
+        exception_msg: str,
+    ) -> None:
+        """evaluate() should return (None, error_message) for caught exceptions."""
+        ragas_metrics.supported_metrics["faithfulness"] = mocker.MagicMock(
+            side_effect=exception_class(exception_msg)
+        )
+
+        score, reason = ragas_metrics.evaluate(
+            "faithfulness", mocker.MagicMock(), turn_scope
+        )
+
+        assert score is None
+        assert exception_msg in reason
+        assert "evaluation failed" in reason
+
+    def test_unsupported_metric_returns_none(
+        self,
+        ragas_metrics: RagasMetrics,
+        turn_scope: EvaluationScope,
+        mocker: MockerFixture,
+    ) -> None:
+        """Unsupported metric name should return (None, message) without raising."""
+        score, reason = ragas_metrics.evaluate(
+            "nonexistent_metric", mocker.MagicMock(), turn_scope
+        )
+
+        assert score is None
+        assert "Unsupported Ragas metric" in reason
