diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index cf69654d15d..210ab5dd97e 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -310,10 +310,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) self._pending_tool_events.append(item_done_event) + def _build_encoded_response_id(self) -> str: + """Build the provider/deployment-encoded id shared by every event of this stream.""" + metadata: dict[str, object] = cast(dict[str, object], self.litellm_metadata or {}) + model_info = metadata.get("model_info") + raw_model_id = cast(dict[str, object], model_info).get("id") if isinstance(model_info, dict) else None + model_id = raw_model_id if isinstance(raw_model_id, str) else None + return ResponsesAPIRequestUtils.build_responses_api_response_id( + custom_llm_provider=self.custom_llm_provider, + model_id=model_id, + response_id=f"resp_{uuid.uuid4()}", + ) + def _default_response_created_event_data(self) -> dict: # Use cached response ID if available, otherwise generate a new one if self._cached_response_id is None: - self._cached_response_id = f"resp_{str(uuid.uuid4())}" + self._cached_response_id = self._build_encoded_response_id() response_created_event_data = { "id": self._cached_response_id, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 234eb777aca..fc0dcfcddd0 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -207,7 +207,7 @@ class ResponsesAPIRequestUtils: if ResponsesAPIRequestUtils._is_litellm_encoded_response_id(response_id): return responses_api_response - updated_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + updated_id = ResponsesAPIRequestUtils.build_responses_api_response_id( model_id=model_id, custom_llm_provider=custom_llm_provider, response_id=response_id, @@ -408,7 +408,7 @@ class ResponsesAPIRequestUtils: return request_input @staticmethod - def _build_responses_api_response_id( + def build_responses_api_response_id( custom_llm_provider: Optional[str], model_id: Optional[str], response_id: str, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_response_id.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_response_id.py new file mode 100644 index 00000000000..4f969dfec45 --- /dev/null +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_response_id.py @@ -0,0 +1,54 @@ +""" +Tests for the response id emitted by the chat-completions -> Responses API streaming bridge. + +Every streaming event has to carry the same litellm-encoded id (provider + deployment id), so a +client that reads the id off `response.created` can send it back as `previous_response_id` and +still get routed to the deployment that served the session. +""" + +from unittest.mock import AsyncMock + +from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, +) +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.utils import Choices, Message, ModelResponse + + +def _build_iterator() -> LiteLLMCompletionStreamingIterator: + return LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="Test input", + responses_api_request={}, + custom_llm_provider="anthropic", + litellm_metadata={"model_info": {"id": "deployment-123"}}, + ) + + +def test_response_created_event_id_is_encoded_with_the_deployment_id(): + iterator = _build_iterator() + + created_event = iterator.create_response_created_event() + + decoded = ResponsesAPIRequestUtils._decode_responses_api_response_id(created_event.response.id) + assert decoded["model_id"] == "deployment-123" + assert decoded["custom_llm_provider"] == "anthropic" + + +def test_streaming_events_all_share_the_completed_event_id(): + iterator = _build_iterator() + + created_event = iterator.create_response_created_event() + in_progress_event = iterator.create_response_in_progress_event() + completed_event = iterator._emit_response_completed_event( + ModelResponse( + id="chatcmpl-1", + choices=[Choices(finish_reason="stop", index=0, message=Message(role="assistant", content="hi"))], + model="test-model", + ) + ) + + assert completed_event is not None + assert created_event.response.id == completed_event.response.id + assert in_progress_event.response.id == completed_event.response.id diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index bbc137b959f..869f6e3a6c8 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -102,7 +102,7 @@ class TestResponsesAPIRequestUtils: original_response_id = "resp_abc123" # Use the helper method to build an encoded response ID - encoded_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + encoded_id = ResponsesAPIRequestUtils.build_responses_api_response_id( custom_llm_provider=test_provider, model_id=test_model_id, response_id=original_response_id,