mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge d85172250f into dc81cf57f7
This commit is contained in:
commit
1ef541b07a
2 changed files with 64 additions and 3 deletions
|
|
@ -13,6 +13,9 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.base_llm.base_model_iterator import (
|
||||
convert_model_response_to_streaming,
|
||||
)
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
||||
decrypt_value_helper,
|
||||
|
|
@ -22,7 +25,13 @@ from litellm.types.llms.openai import (
|
|||
BaseLiteLLMOpenAIResponseObject,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.utils import CallTypesLiteral, LLMResponseTypes, SpecialEnums
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
LLMResponseTypes,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
SpecialEnums,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.caching.caching import DualCache
|
||||
|
|
@ -328,12 +337,16 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict
|
||||
) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject, None]:
|
||||
) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject | ModelResponseStream, None]:
|
||||
general_settings: Final = self._general_settings_reader()
|
||||
|
||||
# Create a request-scoped cache for consistent encryption across streaming chunks.
|
||||
request_encryption_cache: Final[dict[str, str]] = {}
|
||||
|
||||
if isinstance(response, ModelResponse):
|
||||
yield convert_model_response_to_streaming(response)
|
||||
return
|
||||
|
||||
async for chunk in response:
|
||||
if (
|
||||
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,13 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import SpecialEnums
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
Message,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
SpecialEnums,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -692,6 +698,48 @@ class TestAsyncPostCallStreamingIteratorHook:
|
|||
assert streamed_id == "resp_rawprovider123"
|
||||
assert not responses_id_security._is_encrypted_response_id(streamed_id)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_response_converted_to_streaming_chunk(
|
||||
self, responses_id_security, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij")
|
||||
|
||||
mock_auth = MagicMock()
|
||||
mock_auth.user_id = "user-a"
|
||||
mock_auth.team_id = "team-a"
|
||||
mock_auth.request_route = "/chat/completions"
|
||||
|
||||
model_response = ModelResponse(
|
||||
id="chatcmpl-test12345",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content="Final search result", role="assistant"),
|
||||
)
|
||||
],
|
||||
created=1677652288,
|
||||
model="gpt-4o",
|
||||
object="chat.completion",
|
||||
)
|
||||
|
||||
collected = [
|
||||
out
|
||||
async for out in responses_id_security.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=mock_auth,
|
||||
response=model_response,
|
||||
request_data={},
|
||||
)
|
||||
]
|
||||
|
||||
assert len(collected) == 1
|
||||
chunk = collected[0]
|
||||
assert isinstance(chunk, ModelResponseStream)
|
||||
assert chunk.id == "chatcmpl-test12345"
|
||||
assert chunk.object == "chat.completion.chunk"
|
||||
assert chunk.choices[0].delta.content == "Final search result"
|
||||
assert chunk.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
class TestStreamedGenericEventIdEncryption:
|
||||
"""A background stream carries event types with no typed model, which arrive as
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue