mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge e2edb28a79 into 372c096388
This commit is contained in:
commit
a99618031f
5 changed files with 168 additions and 10 deletions
|
|
@ -680,19 +680,32 @@ def process_response_headers(
|
|||
return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})}
|
||||
|
||||
|
||||
def _upstream_non_openai_keys(response_type: type[BaseModel], original_chunk: BaseModel) -> tuple[str, ...]:
|
||||
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
|
||||
expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"})
|
||||
return tuple(
|
||||
key
|
||||
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
|
||||
if key not in expected_keys
|
||||
)
|
||||
|
||||
|
||||
def _dump_keys(original_chunk: BaseModel, keys: tuple[str, ...]) -> Mapping[str, object]:
|
||||
return original_chunk.model_dump(include=MappingProxyType(dict.fromkeys(keys, True)))
|
||||
|
||||
|
||||
def upstream_non_openai_attributes(response_type: type[BaseModel], original_chunk: BaseModel) -> Mapping[str, object]:
|
||||
keys: Final = _upstream_non_openai_keys(response_type, original_chunk)
|
||||
return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {})
|
||||
|
||||
|
||||
def preserve_upstream_non_openai_attributes(
|
||||
model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
|
||||
):
|
||||
"""
|
||||
Preserve non-OpenAI attributes from the original chunk.
|
||||
"""
|
||||
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
|
||||
expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"})
|
||||
candidate_keys: Final = tuple(
|
||||
key
|
||||
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
|
||||
if key not in expected_keys
|
||||
)
|
||||
candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk)
|
||||
if not candidate_keys:
|
||||
return
|
||||
candidate_values: Final[tuple[tuple[str, object], ...]] = tuple(
|
||||
|
|
@ -708,7 +721,7 @@ def preserve_upstream_non_openai_attributes(
|
|||
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
|
||||
_assign_attributes(model_response, candidate_values)
|
||||
return
|
||||
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
|
||||
_assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items())
|
||||
|
||||
|
||||
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ from litellm.types.utils import (
|
|||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
|
||||
from ..exceptions import OpenAIError, Timeout
|
||||
from .core_helpers import map_finish_reason, process_response_headers
|
||||
from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes
|
||||
from .exception_mapping_utils import exception_type, with_deployment_api_base
|
||||
from .llm_response_utils.get_api_base import get_api_base
|
||||
from .rules import Rules
|
||||
|
|
@ -152,6 +152,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
|
|||
return status_code is not None and 400 <= status_code < 500 and status_code != 429
|
||||
|
||||
|
||||
def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None:
|
||||
for key, value in attributes.items():
|
||||
if value is not None:
|
||||
setattr(response, key, value)
|
||||
|
||||
|
||||
def is_async_iterable(obj: object) -> bool:
|
||||
"""
|
||||
Check if an object is an async iterable (can be used with 'async for').
|
||||
|
|
@ -390,6 +396,7 @@ class CustomStreamWrapper:
|
|||
self.stream_options = stream_options or getattr(logging_obj, "stream_options", None)
|
||||
self.messages = getattr(logging_obj, "messages", None)
|
||||
self.sent_stream_usage = False
|
||||
self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({})
|
||||
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
|
||||
self.caller_requested_stream_usage: Final = (
|
||||
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
|
||||
|
|
@ -1718,6 +1725,10 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta()
|
||||
plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta
|
||||
else:
|
||||
if isinstance(original_chunk, BaseModel):
|
||||
self._usage_chunk_upstream_attributes = upstream_non_openai_attributes(
|
||||
ModelResponseStream, original_chunk
|
||||
)
|
||||
if self.send_stream_usage is True:
|
||||
model_response.choices = []
|
||||
return model_response
|
||||
|
|
@ -2208,6 +2219,7 @@ class CustomStreamWrapper:
|
|||
|
||||
if self.sent_stream_usage is False and self.send_stream_usage is True:
|
||||
self.sent_stream_usage = True
|
||||
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
|
||||
return response
|
||||
self.close()
|
||||
raise # Re-raise StopIteration
|
||||
|
|
@ -2409,6 +2421,7 @@ class CustomStreamWrapper:
|
|||
|
||||
if self.sent_stream_usage is False and self.send_stream_usage is True:
|
||||
self.sent_stream_usage = True
|
||||
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
|
||||
return response
|
||||
|
||||
_deferred_cb: Final = getattr(
|
||||
|
|
|
|||
|
|
@ -10171,6 +10171,7 @@ def _fast_serialize_simple_model_response_stream(
|
|||
getattr(chunk, "provider_specific_fields", None) is not None
|
||||
or getattr(chunk, "system_fingerprint", None) is not None
|
||||
or getattr(chunk, "usage", None) is not None
|
||||
or _has_provider_extras(chunk)
|
||||
):
|
||||
return None
|
||||
|
||||
|
|
@ -10246,6 +10247,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
|
|||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
|
||||
|
||||
|
||||
_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"})
|
||||
|
||||
|
||||
def _has_provider_extras(chunk: ModelResponseStream) -> bool:
|
||||
extras: Final = chunk.model_extra or {}
|
||||
return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS)
|
||||
|
||||
|
||||
def _without_injected_usage(chunk: object) -> ModelResponseStream | None:
|
||||
if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk):
|
||||
return None
|
||||
return chunk.model_copy(update={"usage": None})
|
||||
|
||||
|
||||
async def _apply_streaming_chunk_hooks(
|
||||
*,
|
||||
chunk: Any,
|
||||
|
|
@ -10632,7 +10647,9 @@ async def async_data_generator(
|
|||
fallback_errors=fallback_errors,
|
||||
)
|
||||
fallback_metadata_event_sent = True
|
||||
continue
|
||||
if (kept_chunk := _without_injected_usage(chunk)) is None:
|
||||
continue
|
||||
chunk = kept_chunk
|
||||
|
||||
if error_state is not None:
|
||||
error_state.observe_chunk(chunk)
|
||||
|
|
|
|||
|
|
@ -994,6 +994,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes(
|
|||
assert getattr(returned_chunk, "custom_field", None) == {"key": "value"}
|
||||
|
||||
|
||||
def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]:
|
||||
def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl-usage-fields",
|
||||
created=1742056047,
|
||||
model=None,
|
||||
object="chat.completion.chunk",
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30))
|
||||
setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5})
|
||||
return [
|
||||
chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]),
|
||||
chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool):
|
||||
response: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()),
|
||||
model="gpt-5.5",
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=Logging(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "Hey"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="usage-fields",
|
||||
function_id="usage-fields",
|
||||
),
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
chunks: Final = list(response) if sync_mode else [chunk async for chunk in response]
|
||||
|
||||
assert chunks[-1].usage is not None
|
||||
assert chunks[-1].usage.total_tokens == 30
|
||||
assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5}
|
||||
|
||||
|
||||
def test_set_response_id_propagation_empty_to_valid(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv
|
|||
assert _fast_serialize_simple_model_response_stream(chunk) is None
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_keeps_provider_fields():
|
||||
chunk = _simple_chunk()
|
||||
setattr(chunk, "provider_metrics", {"energy_wh": 0.5})
|
||||
|
||||
payload = json.loads(_serialize_streaming_chunk(chunk))
|
||||
|
||||
assert payload["provider_metrics"] == {"energy_wh": 0.5}
|
||||
assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"}
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path():
|
||||
chunk = _simple_chunk()
|
||||
setattr(chunk, "citations", None)
|
||||
|
||||
assert isinstance(_serialize_streaming_chunk(chunk), bytes)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _serialize_streaming_chunk
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False):
|
|||
)
|
||||
|
||||
|
||||
def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream:
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-test",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion.chunk",
|
||||
usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
|
||||
)
|
||||
for key, value in provider_fields.items():
|
||||
setattr(chunk, key, value)
|
||||
return chunk
|
||||
|
||||
|
||||
def _sse_payload(line: str | bytes) -> dict[str, object]:
|
||||
text: Final = line.decode() if isinstance(line, bytes) else line
|
||||
return json.loads(text.removeprefix("data: ").removesuffix("\n\n"))
|
||||
|
||||
|
||||
async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]:
|
||||
out: Final = [
|
||||
line
|
||||
async for line in async_data_generator(
|
||||
response=_async_iter([_simple_chunk(content="hello"), usage_chunk]),
|
||||
user_api_key_dict=_user_auth(),
|
||||
request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True},
|
||||
)
|
||||
]
|
||||
assert out[-1] == "data: [DONE]\n\n"
|
||||
return [_sse_payload(line) for line in out[:-1]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
||||
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5}))
|
||||
|
||||
assert len(payloads) == 2
|
||||
assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5}
|
||||
assert "usage" not in payloads[-1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
||||
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None))
|
||||
|
||||
assert len(payloads) == 1
|
||||
assert "usage" not in payloads[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue