mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix: preserve hidden parameter models when stamping response costs
Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com>
This commit is contained in:
parent
a161780dd3
commit
381c22c9f0
5 changed files with 35 additions and 8 deletions
|
|
@ -7,6 +7,7 @@ import httpx
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import LITELLM_DETAILED_TIMING
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject
|
||||
|
|
@ -18,8 +19,6 @@ from litellm.types.utils import (
|
|||
TranscriptionResponse,
|
||||
)
|
||||
|
||||
_HIDDEN_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _timing_window_start(
|
||||
start_time: datetime.datetime, logging_obj: LiteLLMLoggingObject
|
||||
|
|
@ -273,7 +272,7 @@ class ResponseMetadata:
|
|||
"""Apply metadata to the response object"""
|
||||
if hasattr(self.result, "_hidden_params"):
|
||||
self.result._hidden_params = self._hidden_params
|
||||
hidden: Final = _HIDDEN_PARAMS.validate_python(getattr(self, "_hidden_params", {}))
|
||||
hidden: Final = get_hidden_params_dict(self.result)
|
||||
stamp_usage_cost(self.result, validated_usage_cost(hidden.get("response_cost")))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -470,9 +470,9 @@ class UsageCostMessagesStream:
|
|||
|
||||
def close(self) -> None:
|
||||
self.sync_frames.close()
|
||||
close: Final = getattr(self.stream, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
close_upstream: Final = getattr(self.stream, "close", None)
|
||||
if callable(close_upstream):
|
||||
close_upstream()
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
return getattr(self.stream, name)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,22 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
|||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
from litellm.types.utils import EmbeddingResponse, HiddenParams, ModelResponse, TextCompletionResponse, Usage
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_type", [EmbeddingResponse, TextCompletionResponse])
|
||||
def test_metadata_apply_preserves_model_hidden_params(
|
||||
response_type: type[EmbeddingResponse] | type[TextCompletionResponse],
|
||||
) -> None:
|
||||
response: Final = response_type()
|
||||
hidden: Final = HiddenParams(response_cost=0.125)
|
||||
response._hidden_params = hidden
|
||||
before: Final = response.model_dump()
|
||||
|
||||
ResponseMetadata(response).apply()
|
||||
|
||||
assert response._hidden_params is hidden
|
||||
assert response.model_dump() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ["chat", "responses", "messages"])
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import patch
|
||||
|
||||
|
|
@ -71,6 +72,13 @@ def test_usage_cost_does_not_add_frames_to_an_empty_messages_stream() -> None:
|
|||
assert tuple(UsageCostMessagesStream(iter(()), _usage_cost_logging_obj())) == ()
|
||||
|
||||
|
||||
def test_usage_cost_stream_close_closes_the_upstream() -> None:
|
||||
upstream: Final = BytesIO()
|
||||
stream: Final = UsageCostMessagesStream(upstream, _usage_cost_logging_obj())
|
||||
stream.close()
|
||||
assert upstream.closed
|
||||
|
||||
|
||||
class _RecordingLoggingIterator(BaseAnthropicMessagesStreamingIterator):
|
||||
def __init__(self, litellm_logging_obj: LiteLLMLoggingObj, request_body: dict):
|
||||
super().__init__(litellm_logging_obj=litellm_logging_obj, request_body=request_body)
|
||||
|
|
|
|||
|
|
@ -714,7 +714,12 @@ def test_completion_defaults_missing_message_role_and_preserves_omitted_fields()
|
|||
"created": 1,
|
||||
"model": "gpt-5.4",
|
||||
"object": "chat.completion",
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
"usage": {
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
"cost": response._hidden_params["response_cost"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue