diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index bf038a57211..b64aeeb7271 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -260,7 +260,7 @@ async def convert_to_streaming_response_async( index=idx, delta=delta, logprobs=logprobs, - **_choice_provider_extras(choice, _CACHED_CHOICE_RESERVED_KEYS), + **_choice_provider_extras(_FIELD_MAP.validate_python(choice), _CACHED_CHOICE_RESERVED_KEYS), ) choice_list.append(choice) @@ -353,7 +353,7 @@ def convert_to_streaming_response( delta=delta, logprobs=logprobs, enhancements=enhancements, - **_choice_provider_extras(choice, _CACHED_CHOICE_RESERVED_KEYS), + **_choice_provider_extras(_FIELD_MAP.validate_python(choice), _CACHED_CHOICE_RESERVED_KEYS), ) choice_list.append(choice) @@ -858,7 +858,9 @@ def convert_to_model_response_object( logprobs=logprobs, enhancements=enhancements, provider_specific_fields=provider_specific_fields, - **_choice_provider_extras(provider_specific_fields, _CHOICES_EXPLICIT_KWARGS), + **_choice_provider_extras( + _FIELD_MAP.validate_python(provider_specific_fields), _CHOICES_EXPLICIT_KWARGS + ), ) if provider_sent_logprobs and logprobs is None: choice.logprobs = None diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index eccab7beacb..8c00e0d43b2 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -17,7 +17,7 @@ import httpx from openai.types.chat import ChatCompletionChunk from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import NotRequired, TypedDict import litellm @@ -71,6 +71,7 @@ TOOL_CALLS_ATTRIBUTE: Final = "tool_calls" FUNCTION_CALL_ATTRIBUTE: Final = "function_call" _SYNC_ITER_EXHAUSTED: Final = object() +_STREAM_OPTIONS: Final = TypeAdapter(Mapping[str, object] | None) _GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__) _CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None @@ -375,8 +376,8 @@ class CustomStreamWrapper: self.received_model_id: str | None = None self.logging_loop = None self.rules = Rules() - self.stream_options: Mapping[str, object] | None = stream_options or getattr( - logging_obj, "stream_options", None + self.stream_options: Mapping[str, object] | None = _STREAM_OPTIONS.validate_python( + stream_options or getattr(logging_obj, "stream_options", None) ) self.messages = getattr(logging_obj, "messages", None) self.sent_stream_usage = False diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 28ae51baf79..f0c110bd008 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -57,6 +57,7 @@ from litellm.litellm_core_utils.bug_report import ( strip_bug_report_notice, ) from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, @@ -568,6 +569,7 @@ def stream_usage_tracking_updates( return scrub always_include: Final = general_settings.get("always_include_stream_usage") stream_options: Final = data.get("stream_options") + stream_options_fields: Final = as_str_mapping(stream_options) if always_include is True: if "stream_options" not in data: return {**scrub, "stream_options": {"include_usage": True}, STREAM_USAGE_INJECTED_FIELD: True} @@ -587,8 +589,8 @@ def stream_usage_tracking_updates( if not supports_stream_options(): return scrub merged_stream_options: Final = ( - {k: v for k, v in stream_options.items() if k != "continuous_usage_stats"} - if isinstance(stream_options, dict) + {key: value for key, value in stream_options_fields.items() if key != "continuous_usage_stats"} + if isinstance(stream_options, dict) and stream_options_fields is not None else {} ) return { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5ead86da401..40507cb48fc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10086,7 +10086,7 @@ def _fast_serialize_simple_model_response_stream( choice: Final = choices[0] if getattr(choice, "logprobs", None) is not None or getattr(choice, "enhancements", None) is not None: return None - if _streaming_choice_carries_provider_extras(choice): + if _streaming_choice_carries_provider_extras(chunk.choices[0]): return None delta: Final[object] = getattr(choice, "delta", None)