feat(responses_api/): fix missing streaming events on responses api <-> chat completion bridge

ensure we are passing the required events when streaming non-openai models via responses api
This commit is contained in:
Krrish Dholakia 2025-10-10 14:53:15 -07:00
parent d944717d4b
commit 15b5e6f5d9
20 changed files with 361 additions and 42 deletions

View File

@ -19904,6 +19904,39 @@
"supports_parallel_function_calling": true,
"supports_tool_choice": true
},
"together_ai/moonshotai/Kimi-K2-Instruct-0905": {
"input_cost_per_token": 1e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://www.together.ai/models/kimi-k2-0905",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true
},
"tts-1": {
"input_cost_per_character": 1.5e-05,
"litellm_provider": "openai",

File diff suppressed because one or more lines are too long

View File

@ -1,29 +1,22 @@
model_list:
- model_name: gpt-5-mini
- model_name: openai/gpt-4o-mini
litellm_params:
model: openai/gpt-4o-mini
api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5"
api_key: dummy
- model_name: "byok-wildcard/*"
api_key: os.environ/OPENAI_API_KEY
- model_name: vertex/gemini-2.5-flash
litellm_params:
model: openai/*
- model_name: xai-grok-3
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
- model_name: anthropic/claude-sonnet-4-5
litellm_params:
model: xai/grok-3
- model_name: hosted_vllm/whisper-v3
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY
guardrails:
- guardrail_name: "bedrock-guardrail"
litellm_params:
model: hosted_vllm/whisper-v3
api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5"
api_key: dummy
mcp_servers:
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/Users/krrishdholakia/Documents/temp_py_folder/example_openapi.json"
auth_type: none
allowed_tools: ["getpetbyid", "my_api_mcp-findpetsbystatus"]
litellm_settings:
callbacks: ["prometheus"]
custom_prometheus_metadata_labels: ["metadata.initiative", "metadata.business-unit"]
guardrail: bedrock
mode: "post_call"
guardrailIdentifier: gf3sc1mzinjw
guardrailVersion: "DRAFT"
disable_exception_on_block: true # Prevents exceptions when content is blocked

View File

@ -58,7 +58,7 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=responses_api_request,
**kwargs,
)
completion_args = {}
completion_args.update(kwargs)
completion_args.update(litellm_completion_request)
@ -83,6 +83,7 @@ class LiteLLMCompletionTransformationHandler:
elif isinstance(litellm_completion_response, litellm.CustomStreamWrapper):
return LiteLLMCompletionStreamingIterator(
model=model,
litellm_custom_stream_wrapper=litellm_completion_response,
request_input=input,
responses_api_request=responses_api_request,
@ -106,7 +107,7 @@ class LiteLLMCompletionTransformationHandler:
previous_response_id=previous_response_id,
litellm_completion_request=litellm_completion_request,
)
acompletion_args = {}
acompletion_args.update(kwargs)
acompletion_args.update(litellm_completion_request)
@ -130,9 +131,12 @@ class LiteLLMCompletionTransformationHandler:
elif isinstance(litellm_completion_response, litellm.CustomStreamWrapper):
return LiteLLMCompletionStreamingIterator(
model=litellm_completion_request.get("model") or "",
litellm_custom_stream_wrapper=litellm_completion_response,
request_input=request_input,
responses_api_request=responses_api_request,
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
custom_llm_provider=litellm_completion_request.get(
"custom_llm_provider"
),
litellm_metadata=kwargs.get("litellm_metadata", {}),
)

View File

@ -1,3 +1,5 @@
import time
import uuid
from typing import List, Optional, Union
import litellm
@ -8,11 +10,17 @@ from litellm.responses.litellm_completion_transformation.transformation import (
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
BaseLiteLLMOpenAIResponseObject,
ContentPartAddedEvent,
OutputItemAddedEvent,
OutputTextDeltaEvent,
ReasoningSummaryTextDeltaEvent,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseInProgressEvent,
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
)
@ -32,12 +40,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def __init__(
self,
model: str,
litellm_custom_stream_wrapper: litellm.CustomStreamWrapper,
request_input: Union[str, ResponseInputParam],
responses_api_request: ResponsesAPIOptionalRequestParams,
custom_llm_provider: Optional[str] = None,
litellm_metadata: Optional[dict] = None,
):
self.model: str = model
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = (
litellm_custom_stream_wrapper
)
@ -50,14 +60,139 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.collected_chat_completion_chunks: List[ModelResponseStream] = []
self.finished: bool = False
self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj
self.sent_response_created_event: bool = False
self.sent_response_in_progress_event: bool = False
self.sent_output_item_added_event: bool = False
self.sent_content_part_added_event: bool = False
def _default_response_created_event_data(self) -> dict:
response_created_event_data = {
"id": f"resp_{str(uuid.uuid4())}",
"object": "response",
"created_at": int(time.time()),
"status": "in_progress",
"error": None,
"incomplete_details": None,
"instructions": self.request_input,
"max_output_tokens": None,
"model": self.model,
"output": [],
"parallel_tool_calls": True,
"previous_response_id": None,
"reasoning": {"effort": None, "summary": None},
"store": True,
}
if "temperature" in self.responses_api_request:
response_created_event_data["temperature"] = self.responses_api_request[
"temperature"
]
if "text" in self.responses_api_request:
response_created_event_data["text"] = self.responses_api_request["text"]
if "tool_choice" in self.responses_api_request:
response_created_event_data["tool_choice"] = self.responses_api_request[
"tool_choice"
]
else:
response_created_event_data["tool_choice"] = "auto"
if "tools" in self.responses_api_request:
response_created_event_data["tools"] = self.responses_api_request["tools"]
else:
response_created_event_data["tools"] = []
if "top_p" in self.responses_api_request:
response_created_event_data["top_p"] = self.responses_api_request["top_p"]
else:
response_created_event_data["top_p"] = 1.0
if "truncation" in self.responses_api_request:
response_created_event_data["truncation"] = self.responses_api_request[
"truncation"
]
if "usage" in self.responses_api_request:
response_created_event_data["usage"] = self.responses_api_request["usage"]
if "user" in self.responses_api_request:
response_created_event_data["user"] = self.responses_api_request["user"]
if "metadata" in self.responses_api_request:
response_created_event_data["metadata"] = self.responses_api_request[
"metadata"
]
return response_created_event_data
def create_response_created_event(self) -> ResponseCreatedEvent:
"""
data: {"type":"response.created","response":{"id":"resp_67c9fdcecf488190bdd9a0409de3a1ec07b8b0ad4e5eb654","object":"response","created_at":1741290958,"status":"in_progress","error":null,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}}
"""
response_created_event_data = self._default_response_created_event_data()
return ResponseCreatedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_CREATED,
response=ResponsesAPIResponse(**response_created_event_data),
)
def create_response_in_progress_event(self) -> ResponseInProgressEvent:
response_in_progress_event_data = self._default_response_created_event_data()
response_in_progress_event_data["status"] = "in_progress"
return ResponseInProgressEvent(
type=ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS,
response=ResponsesAPIResponse(**response_in_progress_event_data),
)
def create_output_item_added_event(self) -> OutputItemAddedEvent:
return OutputItemAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
output_index=0,
item=BaseLiteLLMOpenAIResponseObject(
**{
"id": f"msg_{str(uuid.uuid4())}",
"type": "message",
"status": "in_progress",
"role": "assistant",
"content": [],
}
),
)
def create_content_part_added_event(self) -> ContentPartAddedEvent:
return ContentPartAddedEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
item_id=f"msg_{str(uuid.uuid4())}",
output_index=0,
content_index=0,
part=BaseLiteLLMOpenAIResponseObject(
**{"type": "output_text", "text": "", "annotations": []}
),
)
def return_default_initial_events(
self,
) -> Optional[BaseLiteLLMOpenAIResponseObject]:
if self.sent_response_created_event is False:
self.sent_response_created_event = True
return self.create_response_created_event()
elif self.sent_response_in_progress_event is False:
self.sent_response_in_progress_event = True
return self.create_response_in_progress_event()
elif self.sent_output_item_added_event is False:
self.sent_output_item_added_event = True
return self.create_output_item_added_event()
elif self.sent_content_part_added_event is False:
self.sent_content_part_added_event = True
return self.create_content_part_added_event()
return None
async def __anext__(
self,
) -> Union[ResponsesAPIStreamingResponse, ResponseCompletedEvent]:
) -> Union[
ResponsesAPIStreamingResponse,
ResponseCompletedEvent,
BaseLiteLLMOpenAIResponseObject,
]:
try:
while True:
if self.finished is True:
raise StopAsyncIteration
result = self.return_default_initial_events()
if result:
return result
# Get the next chunk from the stream
try:
chunk = await self.litellm_custom_stream_wrapper.__anext__()
@ -87,12 +222,20 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def __next__(
self,
) -> Union[ResponsesAPIStreamingResponse, ResponseCompletedEvent]:
) -> Union[
ResponsesAPIStreamingResponse,
ResponseCompletedEvent,
BaseLiteLLMOpenAIResponseObject,
]:
try:
while True:
if self.finished is True:
raise StopIteration
# Get the next chunk from the stream
result = self.return_default_initial_events()
if result:
return result
try:
chunk = self.litellm_custom_stream_wrapper.__next__()
self.collected_chat_completion_chunks.append(chunk)
@ -168,23 +311,33 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def _emit_response_completed_event(self) -> Optional[ResponseCompletedEvent]:
litellm_model_response: Optional[
Union[ModelResponse, TextCompletionResponse]
] = stream_chunk_builder(chunks=self.collected_chat_completion_chunks, logging_obj=self.litellm_logging_obj)
] = stream_chunk_builder(
chunks=self.collected_chat_completion_chunks,
logging_obj=self.litellm_logging_obj,
)
if litellm_model_response and isinstance(litellm_model_response, ModelResponse):
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None:
if (
litellm.include_cost_in_streaming_usage
and self.litellm_logging_obj is not None
):
usage = getattr(litellm_model_response, "usage", None)
if usage is not None:
setattr(
usage, "cost", self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response)
usage,
"cost",
self.litellm_logging_obj._response_cost_calculator(
result=litellm_model_response
),
)
# Transform the response
responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
request_input=self.request_input,
chat_completion_response=litellm_model_response,
responses_api_request=self.responses_api_request,
)
# Encode the response ID to match non-streaming behavior
encoded_response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id(
responses_api_response=responses_api_response,

View File

@ -1,3 +1,4 @@
import uuid
from enum import Enum
from os import PathLike
from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union
@ -44,7 +45,7 @@ from openai.types.responses.response import (
# Handle OpenAI SDK version compatibility for Text type
try:
# fmt: off
from openai.types.responses.response_create_params import ( # type: ignore[attr-defined]
from openai.types.responses.response_create_params import (
Text as ResponseText, # type: ignore[attr-defined]
)
@ -992,7 +993,9 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
prompt_cache_key: Optional[str]
stream_options: Optional[dict]
top_logprobs: Optional[int]
partial_images: Optional[int] # Number of partial images to generate (1-3) for streaming image generation
partial_images: Optional[
int
] # Number of partial images to generate (1-3) for streaming image generation
class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False):
@ -1180,13 +1183,13 @@ class ReasoningSummaryTextDeltaEvent(BaseLiteLLMOpenAIResponseObject):
class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject):
type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
output_index: int
item: Optional[dict]
item: Optional[BaseLiteLLMOpenAIResponseObject]
class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject):
type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE]
output_index: int
item: dict
item: BaseLiteLLMOpenAIResponseObject
class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
@ -1194,7 +1197,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
item_id: str
output_index: int
content_index: int
part: dict
part: BaseLiteLLMOpenAIResponseObject
class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject):
@ -1202,7 +1205,7 @@ class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject):
item_id: str
output_index: int
content_index: int
part: dict
part: BaseLiteLLMOpenAIResponseObject
class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject):
@ -1413,6 +1416,7 @@ ResponsesAPIStreamingResponse = Annotated[
ImageGenerationPartialImageEvent,
ErrorEvent,
GenericEvent,
BaseLiteLLMOpenAIResponseObject,
],
Discriminator("type"),
]

View File

@ -361,6 +361,7 @@ def test_process_anthropic_headers_with_no_matching_headers():
def test_anthropic_tool_use(tool_type, tool_config, message_content):
"""Test Anthropic tool use with computer use and web fetch tools."""
from litellm import completion
litellm._turn_on_debug()
tools = [tool_config]
@ -1518,3 +1519,126 @@ def test_anthropic_streaming():
role_set_count += 1
assert role_set_count == 1
def test_anthropic_via_responses_api():
from litellm.types.llms.openai import ResponsesAPIStreamEvents
response = litellm.responses(
model="anthropic/claude-sonnet-4-5",
input="Who won the World Cup in 2022?",
max_output_tokens=100,
stream=True,
)
assert response is not None
# Expected event sequence
expected_events = [
ResponsesAPIStreamEvents.RESPONSE_CREATED,
ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS,
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, # Can occur multiple times
ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE,
ResponsesAPIStreamEvents.CONTENT_PART_DONE,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
]
events_seen = []
text_delta_count = 0
for chunk in response:
print(f"chunk: {chunk}")
# Each chunk should have a type attribute
assert hasattr(chunk, "type"), f"Chunk missing 'type' attribute: {chunk}"
event_type = chunk.type
# Track events seen
if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA:
text_delta_count += 1
if ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA not in events_seen:
events_seen.append(event_type)
else:
events_seen.append(event_type)
# Assert specific structures for each event type
if event_type == ResponsesAPIStreamEvents.RESPONSE_CREATED:
assert chunk.type == ResponsesAPIStreamEvents.RESPONSE_CREATED
assert hasattr(chunk, "response")
assert chunk.response.status == "in_progress"
assert hasattr(chunk.response, "id")
assert hasattr(chunk.response, "model")
elif event_type == ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS:
assert chunk.type == ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS
assert hasattr(chunk, "response")
assert chunk.response.status == "in_progress"
elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED:
assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
assert hasattr(chunk, "output_index")
assert hasattr(chunk, "item")
assert chunk.item.type == "message"
assert chunk.item.role == "assistant"
elif event_type == ResponsesAPIStreamEvents.CONTENT_PART_ADDED:
assert chunk.type == ResponsesAPIStreamEvents.CONTENT_PART_ADDED
assert hasattr(chunk, "item_id")
assert hasattr(chunk, "output_index")
assert hasattr(chunk, "content_index")
assert hasattr(chunk, "part")
assert chunk.part.type == "output_text"
elif event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA:
assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA
assert hasattr(chunk, "item_id")
assert hasattr(chunk, "output_index")
assert hasattr(chunk, "content_index")
assert hasattr(chunk, "delta")
assert isinstance(chunk.delta, str)
elif event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE:
assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE
assert hasattr(chunk, "item_id")
assert hasattr(chunk, "output_index")
assert hasattr(chunk, "content_index")
assert hasattr(chunk, "text")
elif event_type == ResponsesAPIStreamEvents.CONTENT_PART_DONE:
assert chunk.type == ResponsesAPIStreamEvents.CONTENT_PART_DONE
assert hasattr(chunk, "item_id")
assert hasattr(chunk, "output_index")
assert hasattr(chunk, "content_index")
assert hasattr(chunk, "part")
assert chunk.part.type == "output_text"
elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE
assert hasattr(chunk, "output_index")
assert hasattr(chunk, "item")
assert chunk.item.status == "completed"
elif event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
assert chunk.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
assert hasattr(chunk, "response")
assert chunk.response.status == "completed"
assert hasattr(chunk.response, "usage")
assert hasattr(chunk.response, "output")
# Assert we saw all expected events
print(f"Events seen: {events_seen}")
assert (
events_seen == expected_events
), f"Event sequence mismatch. Expected: {expected_events}, Got: {events_seen}"
# Assert we saw at least one text delta
assert (
text_delta_count > 0
), f"Expected at least one response.output_text.delta event, got {text_delta_count}"
print(f"✓ All {len(events_seen)} events matched expected structure")
print(f"✓ Received {text_delta_count} text delta chunks")

View File

@ -4,14 +4,20 @@ Test reasoning content preservation in Responses API transformation
from unittest.mock import AsyncMock
from litellm.types.utils import ModelResponseStream, StreamingChoices, Delta
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.utils import ModelResponse, Choices, Message
from litellm.types.utils import (
Choices,
Delta,
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
class TestReasoningContentStreaming:
@ -41,6 +47,7 @@ class TestReasoningContentStreaming:
mock_stream = AsyncMock()
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=mock_stream,
request_input="Test input",
responses_api_request={},
@ -78,6 +85,7 @@ class TestReasoningContentStreaming:
mock_stream = AsyncMock()
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=mock_stream,
request_input="Test input",
responses_api_request={},
@ -114,6 +122,7 @@ class TestReasoningContentStreaming:
mock_stream = AsyncMock()
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",
litellm_custom_stream_wrapper=mock_stream,
request_input="Test input",
responses_api_request={},