Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions src/api/models/bedrock.py
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,8 @@ async def chat_stream(self, chat_request: ChatRequest) -> AsyncIterable[bytes]:
message_id = self.generate_message_id()
stream = response.get("stream")
self.think_emitted = False
self._stream_tool_index_by_content_block = {}
self._next_stream_tool_index = 0
reasoning_tokens = 0
async for chunk in self._async_iterate(stream):
# Accumulate reasoning tokens from delta chunks before processing
Expand Down Expand Up @@ -457,6 +459,8 @@ async def chat_stream(self, chat_request: ChatRequest) -> AsyncIterable[bytes]:
# return an [DONE] message at the end.
yield self.stream_response_to_bytes()
self.think_emitted = False # Cleanup
self._stream_tool_index_by_content_block = {}
self._next_stream_tool_index = 0
except Exception as e:
logger.error("Stream error for model %s: %s", chat_request.model, str(e))
error_event = Error(error=ErrorMessage(message=str(e)))
Expand Down Expand Up @@ -1026,8 +1030,13 @@ def _create_response_stream(
# tool call start
delta = chunk["contentBlockStart"]["start"]
if "toolUse" in delta:
# first index is content
index = chunk["contentBlockStart"]["contentBlockIndex"] - 1
content_block_index = chunk["contentBlockStart"]["contentBlockIndex"]
tool_index_map = getattr(self, "_stream_tool_index_by_content_block", {})
if content_block_index not in tool_index_map:
tool_index_map[content_block_index] = getattr(self, "_next_stream_tool_index", 0)
self._next_stream_tool_index = tool_index_map[content_block_index] + 1
self._stream_tool_index_by_content_block = tool_index_map
index = tool_index_map[content_block_index]
message = ChatResponseMessage(
tool_calls=[
ToolCall(
Expand Down Expand Up @@ -1069,7 +1078,13 @@ def _create_response_stream(
return None # Ignore signature if no <think> started
else:
# tool use
index = chunk["contentBlockDelta"]["contentBlockIndex"] - 1
content_block_index = chunk["contentBlockDelta"]["contentBlockIndex"]
tool_index_map = getattr(self, "_stream_tool_index_by_content_block", {})
if content_block_index not in tool_index_map:
tool_index_map[content_block_index] = getattr(self, "_next_stream_tool_index", 0)
self._next_stream_tool_index = tool_index_map[content_block_index] + 1
self._stream_tool_index_by_content_block = tool_index_map
index = tool_index_map[content_block_index]
message = ChatResponseMessage(
tool_calls=[
ToolCall(
Expand Down
90 changes: 90 additions & 0 deletions tests/test_bedrock_stream_tool_call_index.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import unittest

from api.models.bedrock import BedrockModel


class BedrockStreamToolCallIndexTest(unittest.TestCase):
def setUp(self):
self.model = BedrockModel()
self.model.think_emitted = False
self.model._stream_tool_index_by_content_block = {}
self.model._next_stream_tool_index = 0

def test_first_streamed_tool_call_uses_non_negative_index(self):
start_chunk = {
"contentBlockStart": {
"contentBlockIndex": 0,
"start": {
"toolUse": {
"toolUseId": "tooluse_123",
"name": "get_weather",
}
},
}
}
delta_chunk = {
"contentBlockDelta": {
"contentBlockIndex": 0,
"delta": {
"toolUse": {
"input": "{\"city\":\"Boston\"}",
}
},
}
}

start_response = self.model._create_response_stream(
model_id="us.anthropic.claude-sonnet-4-6",
message_id="chatcmpl-test",
chunk=start_chunk,
)
delta_response = self.model._create_response_stream(
model_id="us.anthropic.claude-sonnet-4-6",
message_id="chatcmpl-test",
chunk=delta_chunk,
)

self.assertEqual(start_response.choices[0].delta.tool_calls[0].index, 0)
self.assertEqual(delta_response.choices[0].delta.tool_calls[0].index, 0)

def test_multiple_streamed_tool_calls_are_indexed_by_tool_ordinal(self):
tool_one_start = {
"contentBlockStart": {
"contentBlockIndex": 0,
"start": {
"toolUse": {
"toolUseId": "tooluse_1",
"name": "get_weather",
}
},
}
}
tool_two_start = {
"contentBlockStart": {
"contentBlockIndex": 2,
"start": {
"toolUse": {
"toolUseId": "tooluse_2",
"name": "lookup_news",
}
},
}
}

tool_one_response = self.model._create_response_stream(
model_id="us.anthropic.claude-sonnet-4-6",
message_id="chatcmpl-test",
chunk=tool_one_start,
)
tool_two_response = self.model._create_response_stream(
model_id="us.anthropic.claude-sonnet-4-6",
message_id="chatcmpl-test",
chunk=tool_two_start,
)

self.assertEqual(tool_one_response.choices[0].delta.tool_calls[0].index, 0)
self.assertEqual(tool_two_response.choices[0].delta.tool_calls[0].index, 1)


if __name__ == "__main__":
unittest.main()