prompt tuner changes

Author: Siba Rajendran

src/fmcore/experimental/adapters/deepeval_adapter.py

@@ -0,0 +1,93 @@
+from deepeval.models import DeepEvalBaseLLM
+from langchain_core.messages import HumanMessage
+from pydantic import BaseModel
+from fmcore.experimental.llm.base_llm import BaseLLM
+from langchain_core.messages import BaseMessage
+
+
+class DeepEvalLLMAdapter(DeepEvalBaseLLM):
+    """
+    Adapter class that bridges BaseLLM implementations with DeepEval's evaluation framework.
+
+    This adapter implements DeepEval's base LLM interface, allowing any BaseLLM instance
+    to be used with DeepEval's evaluation tools. It handles the conversion between
+    different message formats and provides both synchronous and asynchronous generation capabilities.
+
+    Note: This adapter currently supports only text generation metrics. Multimodal metrics support will be added soon.
+
+    Attributes:
+        llm (BaseLLM): The underlying language model implementation to be adapted.
+    """
+
+    llm: BaseLLM
+
+    def __init__(self, llm: BaseLLM):
+        """
+        Initialize the adapter with a BaseLLM instance.
+
+        Args:
+            llm (BaseLLM): The language model implementation to be wrapped.
+        """
+        self.llm = llm
+
+    def load_model(self):
+        """
+        Provide access to the underlying LLM instance.
+
+        This method is required by the DeepEval interface to access the model implementation.
+
+        Returns:
+            BaseLLM: The wrapped language model instance.
+        """
+        return self.llm
+
+    def generate(self, prompt: str, schema: BaseModel, **kwargs) -> BaseModel:
+        """
+        Synchronously generate a response for the given prompt.
+
+        This method converts the string prompt into a response by invoking the underlying
+        LLM and extracting the content from the returned message.
+
+        Args:
+            prompt (str): The input text to generate a response for.
+
+        Returns:
+            str: The generated response text.
+
+        Note:
+            This method handles the conversion from DeepEval's string-based interface
+            to BaseLLM's message-based interface.
+        """
+        messages = [HumanMessage(content=prompt)]
+        response: BaseMessage = self.llm.invoke(messages=messages)
+        return schema.model_validate_json(response.content)
+
+    async def a_generate(self, prompt: str, schema: BaseModel, **kwargs) -> BaseModel:
+        """
+        Asynchronously generate a response for the given prompt.
+
+        This method provides an asynchronous interface for generating responses,
+        useful for high-throughput or I/O-bound evaluation scenarios.
+
+        Args:
+            prompt (str): The input text to generate a response for.
+
+        Returns:
+            str: The generated response text.
+
+        Note:
+            This method handles the conversion from DeepEval's string-based interface
+            to BaseLLM's message-based interface in an asynchronous context.
+        """
+        messages = [HumanMessage(content=prompt)]
+        response: BaseMessage = await self.llm.ainvoke(messages=messages)
+        return schema.model_validate_json(response.content)
+
+    def get_model_name(self):
+        """
+        Retrieve the identifier of the underlying language model.
+
+        Returns:
+            str: The model identifier as specified in the LLM's configuration.
+        """
+        return self.llm.config.model_id

src/fmcore/experimental/adapters/dspy_adapter.py

@@ -0,0 +1,58 @@
+from typing import List, Optional
+import dspy
+from fmcore.experimental.llm.base_llm import BaseLLM
+from fmcore.experimental.types.llm_types import LLMConfig
+from langchain_core.messages import BaseMessage
+
+
+class DSPyLLMAdapter(dspy.LM):
+
+    def __init__(
+        self,
+        llm_config: LLMConfig,
+        **kwargs,
+    ):
+        super().__init__(model=llm_config.model_id, **kwargs)
+        self.llm: BaseLLM = BaseLLM.of(llm_config=llm_config)
+        self.history = []
+
+    def __call__(
+        self, prompt: Optional[str] = None, messages: Optional[List[BaseMessage]] = None, **kwargs
+    ) -> List[str]:
+        """
+        Executes inference with either a text prompt or predefined list of messages.
+
+        If a prompt is provided, it is converted into a list of HumanMessage objects.
+
+        Args:
+            prompt (str, optional): The input prompt to generate messages for.
+            messages (List[BaseMessage], optional): Predefined list of messages for inference.
+
+        Returns:
+            List[str]: The generated responses from the model.
+
+        Raises:
+            ValueError: If both prompt and messages are provided.
+
+        Example:
+            predictions = claude_model(prompt="the sky is blue")
+            print(predictions)
+        """
+        if prompt and messages:
+            raise ValueError("You can only provide either a 'prompt' or 'messages', not both.")
+
+        if prompt:
+            messages = [{"role": "user", "content": prompt}]
+
+        response = self.llm.invoke(messages)
+        result = [response.content]
+
+        # Updating LMs history using DSPy constructs, which currently support only dictionaries
+        entry = {
+            "messages": messages,
+            "outputs": result,
+            "kwargs": kwargs,
+        }
+        self.history.append(entry)
+        self.update_global_history(entry)
+        return result

src/fmcore/experimental/factory/bedrock_factory.py

@@ -8,26 +8,61 @@
 from fmcore.experimental.proxy.rate_limit_proxy import RateLimitedProxy
 from fmcore.experimental.types.provider_types import BedrockAccountConfig
 
-# Clearer alias
 BedrockClientProxy: TypeAlias = RateLimitedProxy[ChatBedrockConverse]
 
 
 class BedrockFactory:
-    """Factory class for creating Bedrock clients with additional functionalities like rate limiting."""
+    """Factory class for creating Bedrock clients with additional functionalities like rate limiting.
+
+    This class provides static methods to create and configure Amazon Bedrock clients
+    with built-in rate limiting capabilities. It handles the creation of both single
+    and multiple clients based on provided configurations.
+
+    The factory supports multiple AWS accounts and automatically configures rate limiting
+    based on account-specific parameters.
+    """
 
     @staticmethod
-    def create_bedrock_clients(*, llm_config: LLMConfig) -> List[BedrockClientProxy]:
-        """Creates multiple Bedrock clients based on the provided configuration."""
+    def create_bedrock_clients(llm_config: LLMConfig) -> List[BedrockClientProxy]:
+        """Creates multiple Bedrock clients based on the provided configuration.
+
+        Args:
+            llm_config (LLMConfig): Configuration object containing LLM settings and provider parameters,
+                                  including account configurations and model parameters.
+
+        Returns:
+            List[BedrockClientProxy]: A list of rate-limited Bedrock client proxies, one for each
+                                    account specified in the configuration.
+
+        Example:
+            llm_config = LLMConfig(...)
+            clients = BedrockFactory.create_bedrock_clients(llm_config)
+        """
         return [
-            BedrockFactory._create_bedrock_client_with_converse(account, llm_config)
+            BedrockFactory._create_bedrock_client_with_converse(
+                account_config=account, llm_config=llm_config
+            )
             for account in llm_config.provider_params.accounts
         ]
 
     @staticmethod
     def _create_bedrock_client_with_converse(
-            account_config: BedrockAccountConfig, llm_config: LLMConfig
+        account_config: BedrockAccountConfig, llm_config: LLMConfig
     ) -> BedrockClientProxy:
-        """Helper method to create a single Bedrock client with rate limiting."""
+        """Creates a single Bedrock client with rate limiting capabilities.
+
+        Args:
+            account_config (BedrockAccountConfig): Configuration for a specific AWS account,
+                                                 including region, role ARN, and rate limits.
+            llm_config (LLMConfig): Configuration containing model settings and parameters.
+
+        Returns:
+            BedrockClientProxy: A rate-limited proxy wrapper around the Bedrock client.
+
+        Note:
+            The method configures rate limiting based on the account's specified rate limit
+            and wraps the ChatBedrockConverse client in a proxy for controlled access.
+        """
         boto_client = BotoFactory.get_client(
             service_name="bedrock-runtime",
             region=account_config.region,
@@ -37,11 +72,11 @@ def _create_bedrock_client_with_converse(
         converse_client = ChatBedrockConverse(
             model_id=llm_config.model_id,
             client=boto_client,
-            **llm_config.model_params.dict(exclude_none=True),
+            **llm_config.model_params.model_dump(exclude_none=True),
         )
 
-        # Currently, We are using the off the shelf rate limiters provided by aiolimiter
-        # TODO Implement custom rate limiters
+        # Create rate limiter based on account config
         rate_limiter = AsyncLimiter(max_rate=account_config.rate_limit)
 
+        # Create proxy without weight
         return BedrockClientProxy(client=converse_client, rate_limiter=rate_limiter)

src/fmcore/experimental/factory/boto_factory.py

@@ -33,23 +33,15 @@ def __get_refreshable_session(
 
         def refresh() -> dict:
             """Refreshes credentials by assuming the specified role."""
-            sts_client = boto3.client(
-                AWSConstants.AWS_SERVICE_STS, region_name=region_name
-            )
-            response = sts_client.assume_role(
-                RoleArn=role_arn, RoleSessionName=session_name
-            )
+            sts_client = boto3.client(AWSConstants.AWS_SERVICE_STS, region_name=region_name)
+            response = sts_client.assume_role(RoleArn=role_arn, RoleSessionName=session_name)
             credentials = response[AWSConstants.CREDENTIALS]
             return {
-                AWSConstants.AWS_CREDENTIALS_ACCESS_KEY: credentials[
-                    AWSConstants.ACCESS_KEY_ID
-                ],
+                AWSConstants.AWS_CREDENTIALS_ACCESS_KEY: credentials[AWSConstants.ACCESS_KEY_ID],
                 AWSConstants.AWS_CREDENTIALS_SECRET_KEY: credentials[
                     AWSConstants.SECRET_ACCESS_KEY
                 ],
-                AWSConstants.AWS_CREDENTIALS_TOKEN: credentials[
-                    AWSConstants.SESSION_TOKEN
-                ],
+                AWSConstants.AWS_CREDENTIALS_TOKEN: credentials[AWSConstants.SESSION_TOKEN],
                 AWSConstants.AWS_CREDENTIALS_EXPIRY_TIME: credentials[
                     AWSConstants.EXPIRATION
                 ].isoformat(),
@@ -82,17 +74,13 @@ def __create_session(cls, *, region: str, role_arn: str) -> boto3.Session:
             boto3.Session: A configured Boto3 session.
         """
         return (
-            cls.__get_refreshable_session(
-                role_arn=role_arn, session_name="BedrockRealtime"
-            )
+            cls.__get_refreshable_session(role_arn=role_arn, session_name="BedrockRealtime")
             if role_arn
             else boto3.Session(region_name=region)
         )
 
     @classmethod
-    def get_client(
-        cls, *, service_name: str, region: str, role_arn: str
-    ) -> boto3.client:
+    def get_client(cls, *, service_name: str, region: str, role_arn: str) -> boto3.client:
         """
         Retrieves a cached Boto3 client or creates a new one.
 

src/fmcore/experimental/llm/__init__.py

@@ -1,2 +1,2 @@
 from fmcore.experimental.llm.base_llm import BaseLLM
-from fmcore.experimental.llm.bedrock_llm import BedrockLLM
+from fmcore.experimental.llm.bedrock_llm import BedrockLLM

src/fmcore/experimental/llm/base_llm.py

@@ -73,7 +73,7 @@ def of(cls, llm_config: LLMConfig):
         - Acts as a factory by retrieving the necessary constructor parameters and instantiating the subclass.
 
         Args:
-            config (LLMConfig): The configuration containing provider details.
+            llm_config (LLMConfig): The configuration containing provider details.
 
         Returns:
             BaseLLM: An instance of the corresponding LLM subclass.
@@ -133,31 +133,3 @@ def astream(self, messages: List[BaseMessage]) -> Iterator[BaseMessageChunk]:
             Iterator[BaseMessageChunk]: A stream of LLM response chunks.
         """
         pass
-
-    @abstractmethod
-    def batch(self, message_batches: List[List[BaseMessage]]) -> List[BaseMessage]:
-        """
-        Synchronously processes multiple batches of messages.
-
-        Args:
-            message_batches (List[List[BaseMessage]]): A list of message batches.
-
-        Returns:
-            List[BaseMessage]: The responses for each batch.
-        """
-        pass
-
-    @abstractmethod
-    async def abatch(
-        self, message_batches: List[List[BaseMessage]]
-    ) -> List[BaseMessage]:
-        """
-        Asynchronously processes multiple batches of messages.
-
-        Args:
-            message_batches (List[List[BaseMessage]]): A list of message batches.
-
-        Returns:
-            List[BaseMessage]: The responses for each batch.
-        """
-        pass