scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

fix: Resolve vLLM response typing regressions

Browse source
This commit is contained in:
Alberto Perdomo 2026-10-27 11:09:18 +00:00
parent 6a7a7fcddb
commit aafdce1bd4
No known key found for this signature in database
4 changed files with 14 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -260,7 +260,7 @@ async def convert_to_streaming_response_async(
index=idx,
delta=delta,
logprobs=logprobs,
**_choice_provider_extras(choice, _CACHED_CHOICE_RESERVED_KEYS),
**_choice_provider_extras(_FIELD_MAP.validate_python(choice), _CACHED_CHOICE_RESERVED_KEYS),
)
choice_list.append(choice)
@ -353,7 +353,7 @@ def convert_to_streaming_response(
delta=delta,
logprobs=logprobs,
enhancements=enhancements,
**_choice_provider_extras(choice, _CACHED_CHOICE_RESERVED_KEYS),
**_choice_provider_extras(_FIELD_MAP.validate_python(choice), _CACHED_CHOICE_RESERVED_KEYS),
)
choice_list.append(choice)
@ -858,7 +858,9 @@ def convert_to_model_response_object(
logprobs=logprobs,
enhancements=enhancements,
provider_specific_fields=provider_specific_fields,
**_choice_provider_extras(provider_specific_fields, _CHOICES_EXPLICIT_KWARGS),
**_choice_provider_extras(
_FIELD_MAP.validate_python(provider_specific_fields), _CHOICES_EXPLICIT_KWARGS
),
)
if provider_sent_logprobs and logprobs is None:
choice.logprobs = None

7
litellm/litellm_core_utils/streaming_handler.py
View file

@ -17,7 +17,7 @@ import httpx
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import BaseModel, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import NotRequired, TypedDict
import litellm
@ -71,6 +71,7 @@ TOOL_CALLS_ATTRIBUTE: Final = "tool_calls"
FUNCTION_CALL_ATTRIBUTE: Final = "function_call"
_SYNC_ITER_EXHAUSTED: Final = object()
_STREAM_OPTIONS: Final = TypeAdapter(Mapping[str, object] | None)
_GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__)
_CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None
@ -375,8 +376,8 @@ class CustomStreamWrapper:
self.received_model_id: str | None = None
self.logging_loop = None
self.rules = Rules()
self.stream_options: Mapping[str, object] | None = stream_options or getattr(
logging_obj, "stream_options", None
self.stream_options: Mapping[str, object] | None = _STREAM_OPTIONS.validate_python(
stream_options or getattr(logging_obj, "stream_options", None)
)
self.messages = getattr(logging_obj, "messages", None)
self.sent_stream_usage = False

6
litellm/proxy/common_request_processing.py
View file

@ -57,6 +57,7 @@ from litellm.litellm_core_utils.bug_report import (
strip_bug_report_notice,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
get_or_create_metadata_bucket,
independent_snapshot,
is_expected_client_error,
@ -568,6 +569,7 @@ def stream_usage_tracking_updates(
return scrub
always_include: Final = general_settings.get("always_include_stream_usage")
stream_options: Final = data.get("stream_options")
stream_options_fields: Final = as_str_mapping(stream_options)
if always_include is True:
if "stream_options" not in data:
return {**scrub, "stream_options": {"include_usage": True}, STREAM_USAGE_INJECTED_FIELD: True}
@ -587,8 +589,8 @@ def stream_usage_tracking_updates(
if not supports_stream_options():
return scrub
merged_stream_options: Final = (
{k: v for k, v in stream_options.items() if k != "continuous_usage_stats"}
if isinstance(stream_options, dict)
{key: value for key, value in stream_options_fields.items() if key != "continuous_usage_stats"}
if isinstance(stream_options, dict) and stream_options_fields is not None
else {}
)
return {

2
litellm/proxy/proxy_server.py
View file

@ -10086,7 +10086,7 @@ def _fast_serialize_simple_model_response_stream(
choice: Final = choices[0]
if getattr(choice, "logprobs", None) is not None or getattr(choice, "enhancements", None) is not None:
return None
if _streaming_choice_carries_provider_extras(choice):
if _streaming_choice_carries_provider_extras(chunk.choices[0]):
return None
delta: Final[object] = getattr(choice, "delta", None)