Litellm staging 05 10 2025 - openai pdf url support + sagemaker chat content length error fix (#10724)

* Support pdf url's to openai (#10640)

* fix(gpt_transformation.py): support pdf url input to openai

pass as base64 as openai doesn't support image url's

* fix(openai.py): support async message transformation

allows async get request to convert url to base64

* fix(gpt_transformation.py): fix linting errrors and use common components across sync + async flows

* fix: fix linting errors

* fix(openai.py): pop correct var

* Fix sagemaker chat calls - content length error  (#10607)

* fix(sagemaker_chat/): support passing dynamic aws params

previously being ignored

* refactor(sagemaker/chat): more refactoring

* fix(sagemaker_chat/): make sure streaming is correctly handled post-refactor

* refactor: more refactoring to support using signed json str

* fix(sagemaker/chat): working sync streaming post refactor

* fix(sagemaker/chat): support async streaming post refactor

* fix(llm_http_handler.py): await async function

* fix: remove print statements

* test: update test

* test: update test

* fix(llm_http_handler.py): retain passing in data as json str

* test: update test

* fix(base_model_iterator.py): fix linting error

* test: test auth

* fix: fix linting error

* test: update test

* test: update translation test

* fix(gpt_transformation.py): handle awaitable/non-awaitable object

* fix: handle async flow for message transformation on openai compatible api's

* test: cleanup testing

* test: update test

* test(test_router.py): use model with higher quota

* test: simplify test

* test: update test

Author: krrishdholakia

litellm/llms/base_llm/anthropic_messages/transformation.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple
 
 import httpx
 
@@ -87,15 +87,15 @@ def sign_request(
         model: Optional[str] = None,
         stream: Optional[bool] = None,
         fake_stream: Optional[bool] = None,
-    ) -> dict:
+    ) -> Tuple[dict, Optional[bytes]]:
         """
         OPTIONAL
 
         Sign the request, providers like Bedrock need to sign the request before sending it to the API
 
         For all other providers, this is a no-op and we just return the headers
         """
-        return headers
+        return headers, None
 
     def get_async_streaming_response_iterator(
         self,

litellm/llms/base_llm/base_model_iterator.py

@@ -41,13 +41,13 @@ def _handle_string_chunk(
         self, str_line: str
     ) -> Union[GenericStreamingChunk, ModelResponseStream]:
         # chunk is a str at this point
-
+        stripped_json_chunk: Optional[dict] = None
         stripped_chunk = litellm.CustomStreamWrapper._strip_sse_data_from_chunk(
             str_line
         )
         try:
             if stripped_chunk is not None:
-                stripped_json_chunk: Optional[dict] = json.loads(stripped_chunk)
+                stripped_json_chunk = json.loads(stripped_chunk)
             else:
                 stripped_json_chunk = None
         except json.JSONDecodeError:

litellm/llms/base_llm/chat/transformation.py

@@ -11,6 +11,7 @@
     Iterator,
     List,
     Optional,
+    Tuple,
     Type,
     Union,
     cast,
@@ -277,7 +278,7 @@ def sign_request(
         model: Optional[str] = None,
         stream: Optional[bool] = None,
         fake_stream: Optional[bool] = None,
-    ) -> dict:
+    ) -> Tuple[dict, Optional[bytes]]:
         """
         Some providers like Bedrock require signing the request. The sign request funtion needs access to `request_data` and `complete_url`
         Args:
@@ -290,7 +291,7 @@ def sign_request(
 
         Update the headers with the signed headers in this function. The return values will be sent as headers in the http request.
         """
-        return headers
+        return headers, None
 
     def get_complete_url(
         self,
@@ -323,6 +324,27 @@ def transform_request(
     ) -> dict:
         pass
 
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Override to allow for http requests on async calls - e.g. converting url to base64
+
+        Currently only used by openai.py
+        """
+        return self.transform_request(
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     @abstractmethod
     def transform_response(
         self,
@@ -354,7 +376,7 @@ def get_model_response_iterator(
     ) -> Any:
         pass
 
-    def get_async_custom_stream_wrapper(
+    async def get_async_custom_stream_wrapper(
         self,
         model: str,
         custom_llm_provider: str,
@@ -365,6 +387,7 @@ def get_async_custom_stream_wrapper(
         messages: list,
         client: Optional[AsyncHTTPHandler] = None,
         json_mode: Optional[bool] = None,
+        signed_json_body: Optional[bytes] = None,
     ) -> CustomStreamWrapper:
         raise NotImplementedError
 
@@ -379,6 +402,7 @@ def get_sync_custom_stream_wrapper(
         messages: list,
         client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
         json_mode: Optional[bool] = None,
+        signed_json_body: Optional[bytes] = None,
     ) -> CustomStreamWrapper:
         raise NotImplementedError
 

litellm/llms/bedrock/base_aws_llm.py

@@ -2,7 +2,17 @@
 import json
 import os
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast, get_args
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+    cast,
+    get_args,
+)
 
 import httpx
 from pydantic import BaseModel
@@ -625,3 +635,74 @@ def get_request_headers(
         prepped = request.prepare()
 
         return prepped
+
+    def _sign_request(
+        self,
+        service_name: Literal["bedrock", "sagemaker"],
+        headers: dict,
+        optional_params: dict,
+        request_data: dict,
+        api_base: str,
+        model: Optional[str] = None,
+        stream: Optional[bool] = None,
+        fake_stream: Optional[bool] = None,
+    ) -> Tuple[dict, Optional[bytes]]:
+        """
+        Sign a request for Bedrock or Sagemaker
+
+        Returns:
+            Tuple[dict, Optional[str]]: A tuple containing the headers and the json str body of the request
+        """
+        try:
+            from botocore.auth import SigV4Auth
+            from botocore.awsrequest import AWSRequest
+            from botocore.credentials import Credentials
+        except ImportError:
+            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+
+        ## CREDENTIALS ##
+        # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
+        aws_secret_access_key = optional_params.get("aws_secret_access_key", None)
+        aws_access_key_id = optional_params.get("aws_access_key_id", None)
+        aws_session_token = optional_params.get("aws_session_token", None)
+        aws_role_name = optional_params.get("aws_role_name", None)
+        aws_session_name = optional_params.get("aws_session_name", None)
+        aws_profile_name = optional_params.get("aws_profile_name", None)
+        aws_web_identity_token = optional_params.get("aws_web_identity_token", None)
+        aws_sts_endpoint = optional_params.get("aws_sts_endpoint", None)
+        aws_region_name = self._get_aws_region_name(
+            optional_params=optional_params, model=model
+        )
+
+        credentials: Credentials = self.get_credentials(
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            aws_session_token=aws_session_token,
+            aws_region_name=aws_region_name,
+            aws_session_name=aws_session_name,
+            aws_profile_name=aws_profile_name,
+            aws_role_name=aws_role_name,
+            aws_web_identity_token=aws_web_identity_token,
+            aws_sts_endpoint=aws_sts_endpoint,
+        )
+
+        sigv4 = SigV4Auth(credentials, service_name, aws_region_name)
+        if headers is not None:
+            headers = {"Content-Type": "application/json", **headers}
+        else:
+            headers = {"Content-Type": "application/json"}
+
+        request = AWSRequest(
+            method="POST",
+            url=api_base,
+            data=json.dumps(request_data),
+            headers=headers,
+        )
+        sigv4.add_auth(request)
+
+        request_headers_dict = dict(request.headers)
+        if (
+            headers is not None and "Authorization" in headers
+        ):  # prevent sigv4 from overwriting the auth header
+            request_headers_dict["Authorization"] = headers["Authorization"]
+        return request_headers_dict, request.body

litellm/llms/bedrock/chat/invoke_handler.py

@@ -272,6 +272,7 @@ def make_sync_call(
     api_base: str,
     headers: dict,
     data: str,
+    signed_json_body: Optional[bytes],
     model: str,
     messages: list,
     logging_obj: Logging,
@@ -286,7 +287,7 @@ def make_sync_call(
         response = client.post(
             api_base,
             headers=headers,
-            data=data,
+            data=signed_json_body if signed_json_body is not None else data,
             stream=not fake_stream,
             logging_obj=logging_obj,
         )

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -121,60 +121,17 @@ def sign_request(
         model: Optional[str] = None,
         stream: Optional[bool] = None,
         fake_stream: Optional[bool] = None,
-    ) -> dict:
-        try:
-            from botocore.auth import SigV4Auth
-            from botocore.awsrequest import AWSRequest
-            from botocore.credentials import Credentials
-        except ImportError:
-            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
-
-        ## CREDENTIALS ##
-        # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
-        aws_secret_access_key = optional_params.get("aws_secret_access_key", None)
-        aws_access_key_id = optional_params.get("aws_access_key_id", None)
-        aws_session_token = optional_params.get("aws_session_token", None)
-        aws_role_name = optional_params.get("aws_role_name", None)
-        aws_session_name = optional_params.get("aws_session_name", None)
-        aws_profile_name = optional_params.get("aws_profile_name", None)
-        aws_web_identity_token = optional_params.get("aws_web_identity_token", None)
-        aws_sts_endpoint = optional_params.get("aws_sts_endpoint", None)
-        aws_region_name = self._get_aws_region_name(
-            optional_params=optional_params, model=model
-        )
-
-        credentials: Credentials = self.get_credentials(
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
-            aws_session_token=aws_session_token,
-            aws_region_name=aws_region_name,
-            aws_session_name=aws_session_name,
-            aws_profile_name=aws_profile_name,
-            aws_role_name=aws_role_name,
-            aws_web_identity_token=aws_web_identity_token,
-            aws_sts_endpoint=aws_sts_endpoint,
-        )
-
-        sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
-        if headers is not None:
-            headers = {"Content-Type": "application/json", **headers}
-        else:
-            headers = {"Content-Type": "application/json"}
-
-        request = AWSRequest(
-            method="POST",
-            url=api_base,
-            data=json.dumps(request_data),
+    ) -> Tuple[dict, Optional[bytes]]:
+        return self._sign_request(
+            service_name="bedrock",
             headers=headers,
+            optional_params=optional_params,
+            request_data=request_data,
+            api_base=api_base,
+            model=model,
+            stream=stream,
+            fake_stream=fake_stream,
         )
-        sigv4.add_auth(request)
-
-        request_headers_dict = dict(request.headers)
-        if (
-            headers is not None and "Authorization" in headers
-        ):  # prevent sigv4 from overwriting the auth header
-            request_headers_dict["Authorization"] = headers["Authorization"]
-        return request_headers_dict
 
     def transform_request(
         self,
@@ -454,7 +411,7 @@ def get_error_class(
         return BedrockError(status_code=status_code, message=error_message)
 
     @track_llm_api_timing()
-    def get_async_custom_stream_wrapper(
+    async def get_async_custom_stream_wrapper(
         self,
         model: str,
         custom_llm_provider: str,
@@ -465,6 +422,7 @@ def get_async_custom_stream_wrapper(
         messages: list,
         client: Optional[AsyncHTTPHandler] = None,
         json_mode: Optional[bool] = None,
+        signed_json_body: Optional[bytes] = None,
     ) -> CustomStreamWrapper:
         streaming_response = CustomStreamWrapper(
             completion_stream=None,
@@ -499,6 +457,7 @@ def get_sync_custom_stream_wrapper(
         messages: list,
         client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
         json_mode: Optional[bool] = None,
+        signed_json_body: Optional[bytes] = None,
     ) -> CustomStreamWrapper:
         if client is None or isinstance(client, AsyncHTTPHandler):
             client = _get_httpx_client(params={})
@@ -510,6 +469,7 @@ def get_sync_custom_stream_wrapper(
                 api_base=api_base,
                 headers=headers,
                 data=json.dumps(data),
+                signed_json_body=signed_json_body,
                 model=model,
                 messages=messages,
                 logging_obj=logging_obj,

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple, Union
 
 import httpx
 
@@ -47,7 +47,7 @@ def sign_request(
         model: Optional[str] = None,
         stream: Optional[bool] = None,
         fake_stream: Optional[bool] = None,
-    ) -> dict:
+    ) -> Tuple[dict, Optional[bytes]]:
         return AmazonInvokeConfig.sign_request(
             self=self,
             headers=headers,