fix: preserve hidden parameter models when stamping response costs

Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-11-09 13:22:36 +00:00
parent a161780dd3
commit 381c22c9f0
5 changed files with 35 additions and 8 deletions

View file

@ -7,6 +7,7 @@ import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject
@ -18,8 +19,6 @@ from litellm.types.utils import (
TranscriptionResponse,
)
_HIDDEN_PARAMS: Final = TypeAdapter(Mapping[str, object])
def _timing_window_start(
start_time: datetime.datetime, logging_obj: LiteLLMLoggingObject
@ -273,7 +272,7 @@ class ResponseMetadata:
"""Apply metadata to the response object"""
if hasattr(self.result, "_hidden_params"):
self.result._hidden_params = self._hidden_params
hidden: Final = _HIDDEN_PARAMS.validate_python(getattr(self, "_hidden_params", {}))
hidden: Final = get_hidden_params_dict(self.result)
stamp_usage_cost(self.result, validated_usage_cost(hidden.get("response_cost")))

View file

@ -470,9 +470,9 @@ class UsageCostMessagesStream:
def close(self) -> None:
self.sync_frames.close()
close: Final = getattr(self.stream, "close", None)
if callable(close):
close()
close_upstream: Final = getattr(self.stream, "close", None)
if callable(close_upstream):
close_upstream()
def __getattr__(self, name: str) -> object:
return getattr(self.stream, name)

View file

@ -28,7 +28,22 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.utils import ModelResponse, Usage
from litellm.types.utils import EmbeddingResponse, HiddenParams, ModelResponse, TextCompletionResponse, Usage
@pytest.mark.parametrize("response_type", [EmbeddingResponse, TextCompletionResponse])
def test_metadata_apply_preserves_model_hidden_params(
response_type: type[EmbeddingResponse] | type[TextCompletionResponse],
) -> None:
response: Final = response_type()
hidden: Final = HiddenParams(response_cost=0.125)
response._hidden_params = hidden
before: Final = response.model_dump()
ResponseMetadata(response).apply()
assert response._hidden_params is hidden
assert response.model_dump() == before
@pytest.mark.parametrize("shape", ["chat", "responses", "messages"])

View file

@ -2,6 +2,7 @@ import asyncio
import json
from collections.abc import AsyncIterator, Mapping, Sequence
from datetime import datetime
from io import BytesIO
from typing import Final, Literal
from unittest.mock import patch
@ -71,6 +72,13 @@ def test_usage_cost_does_not_add_frames_to_an_empty_messages_stream() -> None:
assert tuple(UsageCostMessagesStream(iter(()), _usage_cost_logging_obj())) == ()
def test_usage_cost_stream_close_closes_the_upstream() -> None:
upstream: Final = BytesIO()
stream: Final = UsageCostMessagesStream(upstream, _usage_cost_logging_obj())
stream.close()
assert upstream.closed
class _RecordingLoggingIterator(BaseAnthropicMessagesStreamingIterator):
def __init__(self, litellm_logging_obj: LiteLLMLoggingObj, request_body: dict):
super().__init__(litellm_logging_obj=litellm_logging_obj, request_body=request_body)

View file

@ -714,7 +714,12 @@ def test_completion_defaults_missing_message_role_and_preserves_omitted_fields()
"created": 1,
"model": "gpt-5.4",
"object": "chat.completion",
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
"cost": response._hidden_params["response_cost"],
},
}