scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

fix: preserve hidden params for dynamic responses

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-06 01:17:06 +00:00
parent 1a990af652
commit aac16d4d97
29 changed files with 354 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -31,6 +31,7 @@ from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import MAX_FILE_LIST_LIMIT
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_metadata,
)
@ -1294,7 +1295,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings: Dict[str, str] = {}
for file_object in responses:
model_file_id_mapping = file_object._hidden_params.get("model_file_id_mapping")
file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
model_mappings.update(model_file_id_mapping)
@ -1344,7 +1348,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
_, file_type = extract_file_metadata(create_file_request["file"])
output_file_id = file_objects[0].id
model_id = file_objects[0]._hidden_params.get("model_id")
file_hidden_params: Final = cast(dict[str, object], getattr(file_objects[0], HIDDEN_PARAMS_ATTR))
model_id = file_hidden_params.get("model_id")
unified_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
file_type,
@ -1410,11 +1415,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if decoded_batch_id and is_litellm_executed_batch(decoded_batch_id):
return response
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id
is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
response_hidden_params: Final = cast(dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR))
unified_file_id = response_hidden_params.get("unified_file_id")
unified_batch_id = response_hidden_params.get("unified_batch_id")
is_batch_create: Final = response_hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
model_id = cast(Optional[str], response_hidden_params.get("model_id"))
model_name = cast(Optional[str], response_hidden_params.get("model_name"))
resolved_model_name = resolve_managed_output_file_model_name(
unified_input_file_id=unified_file_id if isinstance(unified_file_id, str) else response.input_file_id,
@ -1525,12 +1531,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
elif isinstance(response, LiteLLMFineTuningJob):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
unified_finetuning_job_id = response._hidden_params.get(
"unified_finetuning_job_id"
) # managed finetuning job id
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
finetuning_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
unified_file_id = finetuning_response_hidden_params.get("unified_file_id")
unified_finetuning_job_id = finetuning_response_hidden_params.get("unified_finetuning_job_id")
model_id = cast(Optional[str], finetuning_response_hidden_params.get("model_id"))
model_name = cast(Optional[str], finetuning_response_hidden_params.get("model_name"))
original_response_id = response.id
if (unified_file_id or unified_finetuning_job_id) and model_id:
response.id = self.get_unified_generic_response_id(model_id=model_id, generic_response_id=response.id)

5
litellm/caching/caching_handler.py
View file

@ -29,7 +29,7 @@ from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache, response_cache_phase
from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
update_response_metadata,
)
@ -710,6 +710,7 @@ class LLMCachingHandler:
if cached.usage is not None and embedding_response.usage is not None
else cached.usage
)
cached_hidden_params: Final = getattr(cached, HIDDEN_PARAMS_ATTR)
merged: Final = EmbeddingResponse(
model=cached.model,
data=[
@ -720,7 +721,7 @@ class LLMCachingHandler:
],
usage=merged_usage,
hidden_params={
**cached.hidden_params,
**cached_hidden_params,
"cache_hit": True,
},
_response_headers=cached._response_headers,

8
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -1002,8 +1002,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if key == "additional_headers" and key in model_response_hidden_params:
existing_additional_headers = model_response_hidden_params.get("additional_headers", {})
merged_headers = {
**cast("dict[str, object]", value),
**cast("dict[str, object]", existing_additional_headers),
**cast( # cast-ok: preserve mapping operations on dynamic response metadata
"dict[str, object]", value
),
**cast( # cast-ok: preserve mapping operations on dynamic response metadata
"dict[str, object]", existing_additional_headers
),
}
model_response_hidden_params[key] = merged_headers
else:

2
litellm/cost_calculator.py
View file

@ -2025,7 +2025,7 @@ def response_cost_calculator(
else:
if isinstance(response_object, BaseModel):
if hasattr(response_object, HIDDEN_PARAMS_ATTR):
hidden_params: Final = cast(
hidden_params: Final = cast( # cast-ok: cost metadata supports dict and Pydantic storage
dict[str, object] | BaseModel,
getattr(response_object, HIDDEN_PARAMS_ATTR),
)

30
litellm/litellm_core_utils/litellm_logging.py
View file

@ -78,6 +78,7 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_or_create_hidden_params
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
is_unbilled_non_inference_call,
@ -1851,7 +1852,7 @@ class Logging(LiteLLMLoggingBaseClass):
else result
)
result_hidden_params: Final = getattr(priced_result, "_hidden_params", None) or MappingProxyType({})
result_hidden_params: Final = getattr(priced_result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
if isinstance(priced_result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(
priced_result, "_hidden_params"
):
@ -2041,7 +2042,7 @@ class Logging(LiteLLMLoggingBaseClass):
def _custom_pricing_for(self, result: object) -> bool:
litellm_params: Final = getattr(self, "litellm_params", None)
result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({})
result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
additional_headers: Final = (
result_hidden_params.get("additional_headers")
if isinstance(result_hidden_params, dict)
@ -2377,7 +2378,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
if logging_result is None:
return
hidden_params: Final = getattr(logging_result, "_hidden_params", None)
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, None)
if not hidden_params:
return
if self.model_call_details.get("litellm_params") is None:
@ -2401,7 +2402,7 @@ class Logging(LiteLLMLoggingBaseClass):
):
"""Resolve hidden params, compute response cost, and emit the standard logging payload."""
self._surface_response_headers_from_result(logging_result)
hidden_params: Final = getattr(logging_result, "_hidden_params", {})
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {})
if hidden_params:
if self.model_call_details.get("litellm_params") is not None:
self.model_call_details["litellm_params"].setdefault("metadata", {})
@ -2676,7 +2677,7 @@ class Logging(LiteLLMLoggingBaseClass):
Left in place they overwrite the create's real deployment with the
poll's empty one in the payload every logging integration reads.
"""
settled_hidden_params: Final = getattr(result, "_hidden_params", None)
settled_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None)
if isinstance(settled_hidden_params, dict):
for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
settled_hidden_params.pop(poll_scoped_key, None)
@ -3250,13 +3251,14 @@ class Logging(LiteLLMLoggingBaseClass):
batch_successful_requests: Final = kwargs.get("batch_successful_requests", None)
batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
result_hidden_params: Final = get_or_create_hidden_params(result)
should_compute_batch_data: Final = not has_explicit_batch_data and batch_cost_is_final(result)
if has_explicit_batch_data:
result.hidden_params["response_cost"] = batch_cost
result.hidden_params["batch_models"] = batch_models
result._hidden_params["batch_successful_requests"] = batch_successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above
result._hidden_params["batch_failed_requests"] = batch_failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
result_hidden_params["response_cost"] = batch_cost
result_hidden_params["batch_models"] = batch_models
result_hidden_params["batch_successful_requests"] = batch_successful_requests
result_hidden_params["batch_failed_requests"] = batch_failed_requests
result.usage = batch_usage
batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None)
batch_completion_cost: Final = kwargs.get("batch_completion_cost", None)
@ -3281,10 +3283,10 @@ class Logging(LiteLLMLoggingBaseClass):
model_info=self.get_router_deployment_model_info(),
)
result.hidden_params["response_cost"] = batch_result.cost
result.hidden_params["batch_models"] = batch_result.models
result._hidden_params["batch_successful_requests"] = batch_result.successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
result._hidden_params["batch_failed_requests"] = batch_result.failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
result_hidden_params["response_cost"] = batch_result.cost
result_hidden_params["batch_models"] = batch_result.models
result_hidden_params["batch_successful_requests"] = batch_result.successful_requests
result_hidden_params["batch_failed_requests"] = batch_result.failed_requests
result.usage = batch_result.usage
self.set_cost_breakdown(
input_cost=batch_result.prompt_cost,
@ -6430,7 +6432,7 @@ def _extract_response_obj_and_hidden_params(
) -> tuple[dict, dict | None]:
"""Extract response_obj and hidden_params from init_response_obj."""
hidden_params: dict | None = (
getattr(init_response_obj, "_hidden_params", None)
getattr(init_response_obj, HIDDEN_PARAMS_ATTR, None)
if isinstance(init_response_obj, BaseModel | HttpxBinaryResponseContent)
else None
)

4
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -9,6 +9,7 @@ from typing import Final, Literal, cast
import litellm
from litellm._logging import verbose_logger
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_extract_reasoning_content,
)
@ -512,7 +513,8 @@ class LiteLLMResponseObjectHandler:
text_completion_response["choices"] = choices_list
text_completion_response["usage"] = response.get("usage", None)
text_completion_response.hidden_params = HiddenParams(**response.hidden_params)
response_hidden_params: Final = getattr(response, HIDDEN_PARAMS_ATTR)
setattr(text_completion_response, HIDDEN_PARAMS_ATTR, HiddenParams(**response_hidden_params))
return text_completion_response
@staticmethod

7
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypedDict, Union, cast
from typing_extensions import ReadOnly, Required
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.types.llms.openai import (
ChatCompletionAssistantContentValue,
ChatCompletionAudioDelta,
@ -265,7 +266,11 @@ class ChunkProcessor:
return model_response
# set hidden params from chunk to model_response
if model_response is not None and hasattr(model_response, "_hidden_params"):
model_response.hidden_params = chunk.get("_hidden_params", {})
chunk_hidden_params: Final = chunk.get("_hidden_params", {})
if isinstance(chunk_hidden_params, dict):
set_hidden_params(model_response, chunk_hidden_params)
else:
setattr(model_response, HIDDEN_PARAMS_ATTR, chunk_hidden_params)
return model_response
@staticmethod

37
litellm/litellm_core_utils/streaming_handler.py
View file

@ -20,6 +20,7 @@ import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
@ -1830,14 +1831,22 @@ class CustomStreamWrapper:
if getattr(response, "usage", None) is not None:
usage_to_preserve = response.usage
if usage_to_preserve:
response.hidden_params["usage"] = usage_to_preserve
response_hidden_params_for_usage = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
response_hidden_params_for_usage["usage"] = usage_to_preserve
obj_dict = response.model_dump()
if "usage" in obj_dict:
del obj_dict["usage"]
response = self.model_response_creator(chunk=obj_dict, hidden_params=response.hidden_params)
response_hidden_params_for_model = cast( # cast-ok: preserve dynamic mapping behavior
Mapping[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
response = self.model_response_creator(
chunk=obj_dict, hidden_params=response_hidden_params_for_model
)
## check if empty
is_empty = is_model_response_stream_empty(model_response=cast(ModelResponseStream, response))
@ -1846,8 +1855,11 @@ class CustomStreamWrapper:
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
response.hidden_params["usage"] = usage
self._last_returned_hidden_params = response.hidden_params
response_hidden_params_for_final_chunk = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
response_hidden_params_for_final_chunk["usage"] = usage
self._last_returned_hidden_params = response_hidden_params_for_final_chunk
# Add MCP metadata to final chunk if present
response = self._add_mcp_metadata_to_final_chunk(response)
# RETURN RESULT
@ -2042,7 +2054,10 @@ class CustomStreamWrapper:
if "usage" in obj_dict:
del obj_dict["usage"]
processed_chunk = self.model_response_creator(
chunk=obj_dict, hidden_params=processed_chunk.hidden_params
chunk=obj_dict,
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
),
)
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, processed_chunk)
@ -2056,8 +2071,11 @@ class CustomStreamWrapper:
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
processed_chunk.hidden_params["usage"] = usage
self._last_returned_hidden_params = processed_chunk.hidden_params
processed_chunk_hidden_params = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
)
processed_chunk_hidden_params["usage"] = usage
self._last_returned_hidden_params = processed_chunk_hidden_params
# Call post-call streaming deployment hook for final chunk
if self.sent_last_chunk is True:
@ -2212,7 +2230,10 @@ class CustomStreamWrapper:
self.chunks.append(processed_chunk)
if self.stream_options is None:
usage: Final = calculate_total_usage(chunks=self.chunks)
processed_chunk._hidden_params["usage"] = usage # pyright: ignore[reportPrivateUsage] # sync parity
processed_chunk_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR)
)
processed_chunk_hidden_params["usage"] = usage
# see sync __next__'s sibling branch: deliberately do NOT restore
# here - this chunk is still this call's own data, and restoring
# before returning it would corrupt the caller's own log

3
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -20,6 +20,7 @@ from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import MidStreamFallbackError
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
AppliedEdit,
@ -165,7 +166,7 @@ class _CombinedChunkSplitter:
chunk.usage = None
hidden_params: Final = getattr(chunk, "_hidden_params", None)
if isinstance(hidden_params, dict) and "usage" in hidden_params:
chunk.hidden_params = {key: value for key, value in hidden_params.items() if key != "usage"}
set_hidden_params(chunk, {key: value for key, value in hidden_params.items() if key != "usage"})
@staticmethod
def _split_by_payload_kind(chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":

14
litellm/llms/chatgpt/responses/transformation.py
View file

@ -1,10 +1,11 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@ -237,10 +238,13 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
) -> None:
raw_headers: Final = dict(raw_response.headers)
processed_headers: Final = process_response_headers(raw_headers)
if not hasattr(completed_response, "_hidden_params"):
setattr(completed_response, "_hidden_params", {})
completed_response.hidden_params["additional_headers"] = processed_headers
completed_response.hidden_params["headers"] = raw_headers
if not hasattr(completed_response, HIDDEN_PARAMS_ATTR):
set_hidden_params(completed_response, {})
hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(completed_response, HIDDEN_PARAMS_ATTR)
)
hidden_params["additional_headers"] = processed_headers
hidden_params["headers"] = raw_headers
def get_complete_url(
self,

17
litellm/llms/e2b/sandbox/transformation.py
View file

@ -9,11 +9,12 @@ Talks to e2b's REST API directly over httpx (no e2b SDK dependency):
import json
from collections.abc import Mapping
from typing import Final
from typing import Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.llms.base_llm.sandbox.transformation import (
SANDBOX_MAX_OUTPUT_BYTES,
BaseSandboxConfig,
@ -109,8 +110,11 @@ class E2BSandboxConfig(BaseSandboxConfig):
**kwargs,
) -> CodeExecutionResult:
handle: Final = self._as_handle(container)
hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle
dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR)
)
token: Final = handle.hidden_params.get("envd_access_token")
token: Final = hidden_params.get("envd_access_token")
if not token:
raise ValueError(
"Cannot run code from a sandbox id alone. e2b secure sandboxes "
@ -119,7 +123,7 @@ class E2BSandboxConfig(BaseSandboxConfig):
)
headers: Final = {"Content-Type": "application/json", "X-Access-Token": token}
traffic_token: Final = handle.hidden_params.get("traffic_access_token")
traffic_token: Final = hidden_params.get("traffic_access_token")
if traffic_token:
headers["E2B-Traffic-Access-Token"] = traffic_token
@ -143,8 +147,11 @@ class E2BSandboxConfig(BaseSandboxConfig):
**kwargs,
) -> bool:
handle: Final = self._as_handle(container)
key: Final = api_key or handle.hidden_params.get("api_key") or self.validate_environment()
base: Final = api_base or handle.hidden_params.get("api_base") or E2B_API_BASE
hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle
dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR)
)
key: Final = api_key or hidden_params.get("api_key") or self.validate_environment()
base: Final = api_base or hidden_params.get("api_base") or E2B_API_BASE
try:
response: Final = await self._http(client).delete(
url=f"{base}/sandboxes/{handle.id}",

14
litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py
View file

@ -1,9 +1,10 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageResponse
@ -237,12 +238,15 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig):
model_response.data.extend(fal_images_to_image_objects(images))
# Add additional metadata from Flux Pro response
if hasattr(model_response, "_hidden_params"):
if hasattr(model_response, HIDDEN_PARAMS_ATTR):
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
if "seed" in response_object:
model_response.hidden_params["seed"] = response_object["seed"]
hidden_params["seed"] = response_object["seed"]
if "timings" in response_object:
model_response.hidden_params["timings"] = response_object["timings"]
hidden_params["timings"] = response_object["timings"]
if "has_nsfw_concepts" in response_object:
model_response.hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"]
hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"]
return model_response

10
litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py
View file

@ -1,9 +1,10 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageObject, ImageResponse
@ -189,7 +190,10 @@ class FalAIIdeogramV3Config(FalAIBaseConfig):
)
)
if hasattr(model_response, "_hidden_params") and "seed" in response_object:
model_response.hidden_params["seed"] = response_object["seed"]
if hasattr(model_response, HIDDEN_PARAMS_ATTR) and "seed" in response_object:
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
hidden_params["seed"] = response_object["seed"]
return model_response

10
litellm/llms/fal_ai/image_generation/imagen4_transformation.py
View file

@ -1,7 +1,8 @@
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageObject, ImageResponse
@ -234,8 +235,11 @@ class FalAIImagen4Config(FalAIBaseConfig):
)
# Add seed metadata from Imagen4 response
if hasattr(model_response, "_hidden_params"):
if hasattr(model_response, HIDDEN_PARAMS_ATTR):
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
if "seed" in response_data:
model_response.hidden_params["seed"] = response_data["seed"]
hidden_params["seed"] = response_data["seed"]
return model_response

14
litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py
View file

@ -1,9 +1,10 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageObject, ImageResponse
@ -268,12 +269,15 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig):
)
# Add additional metadata from Stable Diffusion response
if hasattr(model_response, "_hidden_params"):
if hasattr(model_response, HIDDEN_PARAMS_ATTR):
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
if "seed" in response_object:
model_response.hidden_params["seed"] = response_object["seed"]
hidden_params["seed"] = response_object["seed"]
if "timings" in response_object:
model_response.hidden_params["timings"] = response_object["timings"]
hidden_params["timings"] = response_object["timings"]
if "has_nsfw_concepts" in response_object:
model_response.hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"]
hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"]
return model_response

3
litellm/llms/openai/completion/transformation.py
View file

@ -4,6 +4,7 @@ Support for gpt model family
from typing import Final
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params
from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig
from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionResponse
@ -111,7 +112,7 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
if "model" in response_object:
model_response_object.model = response_object["model"]
model_response_object.hidden_params["original_response"] = (
get_or_create_hidden_params(model_response_object)["original_response"] = (
response_object # track original response, if users make a litellm.text_completion() request, we can return the original response
)
return model_response_object

4
litellm/llms/openai/containers/transformation.py
View file

@ -168,7 +168,9 @@ class OpenAIContainerConfig(BaseContainerConfig):
container_hidden_params: Final = get_or_create_hidden_params(container_obj)
container_hidden_params.setdefault("additional_headers", {})
container_additional_headers: Final = cast("dict[str, object]", container_hidden_params["additional_headers"])
container_additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
"dict[str, object]", container_hidden_params["additional_headers"]
)
container_additional_headers["llm_provider-x-litellm-response-cost"] = container_cost
return container_obj

17
litellm/llms/openrouter/chat/transformation.py
View file

@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
import litellm
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@ -216,13 +217,17 @@ class OpenrouterConfig(OpenAIGPTConfig):
response_cost: Final = response_json["usage"].get("cost")
if response_cost is not None:
# Store cost in hidden params for the cost calculator to use
if not hasattr(model_response, "_hidden_params"):
model_response.hidden_params = {}
if "additional_headers" not in model_response.hidden_params:
model_response.hidden_params["additional_headers"] = {}
model_response.hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(
response_cost
if not hasattr(model_response, HIDDEN_PARAMS_ATTR):
set_hidden_params(model_response, {})
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
if "additional_headers" not in hidden_params:
hidden_params["additional_headers"] = {}
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
dict[str, object], hidden_params["additional_headers"]
)
additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
except Exception:
# If we can't extract cost, continue without it - don't fail the response
pass

31
litellm/llms/openrouter/image_generation/transformation.py
View file

@ -28,12 +28,13 @@ Response format:
"""
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
@ -226,17 +227,29 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
cost: Final = usage_data.get("cost")
if cost is not None:
if not hasattr(model_response, "_hidden_params"):
model_response.hidden_params = {}
if "additional_headers" not in model_response.hidden_params:
model_response.hidden_params["additional_headers"] = {}
model_response.hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(cost)
if not hasattr(model_response, HIDDEN_PARAMS_ATTR):
set_hidden_params(model_response, {})
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
if "additional_headers" not in hidden_params:
hidden_params["additional_headers"] = {}
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
dict[str, object], hidden_params["additional_headers"]
)
additional_headers["llm_provider-x-litellm-response-cost"] = float(cost)
cost_details: Final = usage_data.get("cost_details", {})
if cost_details:
if "response_cost_details" not in model_response.hidden_params:
model_response.hidden_params["response_cost_details"] = {}
model_response.hidden_params["response_cost_details"].update(cost_details)
cost_details_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
if "response_cost_details" not in cost_details_hidden_params:
cost_details_hidden_params["response_cost_details"] = {}
response_cost_details: Final = cast( # cast-ok: preserve mapping operations on response metadata
dict[str, object], cost_details_hidden_params["response_cost_details"]
)
response_cost_details.update(cost_details)
model_response.hidden_params["model"] = response_json.get("model", model)

52
litellm/llms/opensandbox/sandbox/transformation.py
View file

@ -1,7 +1,7 @@
import asyncio
import json
import time
from typing import Final
from typing import Final, cast
import httpx
@ -18,6 +18,7 @@ from litellm.constants import (
OPEN_SANDBOX_POLL_INTERVAL,
OPEN_SANDBOX_READY_TIMEOUT,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.sandbox.transformation import (
SANDBOX_MAX_OUTPUT_BYTES,
BaseSandboxConfig,
@ -147,9 +148,12 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
poll_interval=(float(poll_interval) if poll_interval is not None else DEFAULT_POLL_INTERVAL),
client=client,
)
endpoint: Final = str(handle.hidden_params["execd_endpoint"])
endpoint_headers: Final = self._as_str_dict(handle.hidden_params.get("execd_headers"))
base: Final = str(handle.hidden_params.get("api_base") or handle.domain or self._api_base(api_base))
hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle
dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR)
)
endpoint: Final = str(hidden_params["execd_endpoint"])
endpoint_headers: Final = self._as_str_dict(hidden_params.get("execd_headers"))
base: Final = str(hidden_params.get("api_base") or handle.domain or self._api_base(api_base))
lines: Final = await self._post_code(
url=f"{self._endpoint_base_url(endpoint, base)}/code",
headers={
@ -176,7 +180,10 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
**kwargs,
) -> bool:
handle: Final = self._as_handle(container, api_base=api_base)
base: Final = str(handle.hidden_params.get("api_base") or self._api_base(api_base))
hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle
dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR)
)
base: Final = str(hidden_params.get("api_base") or self._api_base(api_base))
key: Final = self._api_key(api_key=api_key, handle=handle)
try:
response: Final = await self._http(client).delete(
@ -201,12 +208,15 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
client: AsyncHTTPHandler | None,
) -> ContainerHandle:
handle: Final = self._as_handle(container, api_base=api_base)
if handle.hidden_params.get("execd_endpoint"):
hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle
dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR)
)
if hidden_params.get("execd_endpoint"):
return handle
base: Final = str(handle.hidden_params.get("api_base") or self._api_base(api_base))
base: Final = str(hidden_params.get("api_base") or self._api_base(api_base))
key: Final = self._api_key(api_key=api_key, handle=handle)
resolved_use_server_proxy: Final = bool(handle.hidden_params.get("use_server_proxy", use_server_proxy))
resolved_use_server_proxy: Final = bool(hidden_params.get("use_server_proxy", use_server_proxy))
endpoint, endpoint_headers = await self._wait_for_execd_endpoint(
sandbox_id=handle.id,
api_base=base,
@ -217,14 +227,17 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
poll_interval=poll_interval,
)
handle.domain = base
handle.hidden_params = {
**handle.hidden_params,
"api_base": base,
"api_key": key,
"execd_endpoint": endpoint,
"execd_headers": endpoint_headers,
"use_server_proxy": resolved_use_server_proxy,
}
set_hidden_params(
handle,
{
**hidden_params,
"api_base": base,
"api_key": key,
"execd_endpoint": endpoint,
"execd_headers": endpoint_headers,
"use_server_proxy": resolved_use_server_proxy,
},
)
return handle
async def _wait_until_running(
@ -329,8 +342,11 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig):
def _api_key(self, *, api_key: str | None, handle: ContainerHandle) -> str:
if api_key is not None:
return api_key
if "api_key" in handle.hidden_params:
return str(handle.hidden_params["api_key"])
hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle
dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR)
)
if "api_key" in hidden_params:
return str(hidden_params["api_key"])
return self.validate_environment()
@staticmethod

29
litellm/main.py
View file

@ -95,6 +95,7 @@ from litellm.litellm_core_utils.health_check_utils import (
_create_health_check_response,
_filter_model_params,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.mock_functions import (
mock_embedding,
@ -5515,9 +5516,12 @@ def completion(
)
)
if model_response is not None and hasattr(model_response, "_hidden_params"):
model_response.hidden_params["custom_llm_provider"] = custom_llm_provider
model_response.hidden_params["region_name"] = kwargs.get(
if model_response is not None and hasattr(model_response, HIDDEN_PARAMS_ATTR):
model_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
)
model_response_hidden_params["custom_llm_provider"] = custom_llm_provider
model_response_hidden_params["region_name"] = kwargs.get(
"aws_region_name", None
) # support region-based pricing for bedrock
@ -6243,7 +6247,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
elif asyncio.iscoroutine(init_response):
response = await init_response
if response is not None and isinstance(response, EmbeddingResponse) and hasattr(response, "_hidden_params"):
response.hidden_params["custom_llm_provider"] = custom_llm_provider
response_hidden_params: Final = get_hidden_params(response)
if response_hidden_params is not None:
response_hidden_params["custom_llm_provider"] = custom_llm_provider
if response is None:
raise ValueError("Unable to get Embedding Response. Please pass a valid llm_provider.")
@ -7360,7 +7366,9 @@ def embedding(
else:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
if response is not None and hasattr(response, "_hidden_params") and isinstance(response, EmbeddingResponse):
response.hidden_params["custom_llm_provider"] = custom_llm_provider
response_hidden_params: Final = get_hidden_params(response)
if response_hidden_params is not None:
response_hidden_params["custom_llm_provider"] = custom_llm_provider
if response is None:
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
@ -8198,7 +8206,10 @@ def transcription(
if existing_duration is None:
calculated_duration: Final = calculate_request_duration(file)
if calculated_duration is not None:
response.hidden_params["audio_transcription_duration"] = calculated_duration
response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
response_hidden_params["audio_transcription_duration"] = calculated_duration
if response is None:
raise ValueError("Unmapped provider passed in. Unable to get the response.")
@ -8813,7 +8824,7 @@ async def ahealth_check(
if mode in mode_handlers:
_response: Final = await mode_handlers[mode]()
# Only process headers for chat mode
_response_headers: Final[dict] = getattr(_response, "_hidden_params", {}).get("headers", {}) or {}
_response_headers: Final[dict] = getattr(_response, HIDDEN_PARAMS_ATTR, {}).get("headers", {}) or {}
return _create_health_check_response(_response_headers)
else:
raise Exception(f"Mode {mode} not supported. See modes here: https://docs.litellm.ai/docs/proxy/health")
@ -9072,7 +9083,7 @@ def stream_chunk_builder(
if isinstance(chunk, dict):
hidden = chunk.get("_hidden_params")
else:
hidden = getattr(chunk, "_hidden_params", None)
hidden = getattr(chunk, HIDDEN_PARAMS_ATTR, None)
if isinstance(hidden, dict) and "provider_specific_fields" in hidden:
response.hidden_params.setdefault("provider_specific_fields", {}).update(
hidden["provider_specific_fields"]
@ -9251,7 +9262,7 @@ def stream_chunk_builder(
if isinstance(chunk, dict):
hidden = chunk.get("_hidden_params")
else:
hidden = getattr(chunk, "_hidden_params", None)
hidden = getattr(chunk, HIDDEN_PARAMS_ATTR, None)
if isinstance(hidden, dict) and "provider_specific_fields" in hidden:
response.hidden_params.setdefault("provider_specific_fields", {}).update(
hidden["provider_specific_fields"]

17
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -17,6 +17,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
@ -211,7 +212,7 @@ async def _create_provider_batch_for_managed_file(
}
response: Final = await llm_router.acreate_batch(**request)
response.input_file_id = input_file_id
response.hidden_params["unified_file_id"] = unified_file_id
get_or_create_hidden_params(response)["unified_file_id"] = unified_file_id
return response
@ -484,7 +485,7 @@ async def create_batch(
**_create_batch_data,
)
response.hidden_params[BATCH_CREATE_HIDDEN_PARAM] = True
get_or_create_hidden_params(response)[BATCH_CREATE_HIDDEN_PARAM] = True
### CALL HOOKS ### - modify outgoing data
response = await proxy_logging_obj.post_call_success_hook(
@ -736,11 +737,12 @@ async def retrieve_batch(
)
response = await llm_router.aretrieve_batch(**data)
response.hidden_params["unified_batch_id"] = unified_batch_id
response_hidden_params: Final = get_or_create_hidden_params(response)
response_hidden_params["unified_batch_id"] = unified_batch_id
if unified_batch_id:
model_id_from_batch: Final = get_model_id_from_unified_batch_id(unified_batch_id)
if model_id_from_batch:
response.hidden_params["model_id"] = model_id_from_batch
response_hidden_params["model_id"] = model_id_from_batch
# SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
else:
@ -1168,10 +1170,11 @@ async def cancel_batch(
data["model"] = model_id_from_batch
data["batch_id"] = get_batch_id_from_unified_batch_id(unified_batch_id)
response = await llm_router.acancel_batch(**data)
response.hidden_params["unified_batch_id"] = unified_batch_id
response_hidden_params: Final = get_or_create_hidden_params(response)
response_hidden_params["unified_batch_id"] = unified_batch_id
if not response.hidden_params.get("model_id") and data.get("model"):
response.hidden_params["model_id"] = data["model"]
if not response_hidden_params.get("model_id") and data.get("model"):
response_hidden_params["model_id"] = data["model"]
# SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
else:

7
litellm/proxy/fine_tuning_endpoints/endpoints.py
View file

@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
@ -161,7 +162,7 @@ async def create_fine_tuning_job(
response = cast(LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data))
response.training_file = unified_file_id
response.hidden_params["unified_file_id"] = unified_file_id
get_or_create_hidden_params(response)["unified_file_id"] = unified_file_id
## ELSE, Route based on custom_llm_provider
elif fine_tuning_request.custom_llm_provider:
# get configs for custom_llm_provider
@ -304,7 +305,7 @@ async def retrieve_fine_tuning_job(
**data,
),
)
response.hidden_params["unified_finetuning_job_id"] = unified_finetuning_job_id
get_or_create_hidden_params(response)["unified_finetuning_job_id"] = unified_finetuning_job_id
elif custom_llm_provider:
# get configs for custom_llm_provider
llm_provider_config: Final = get_fine_tuning_provider_config(custom_llm_provider=custom_llm_provider)
@ -577,7 +578,7 @@ async def cancel_fine_tuning_job(
**data,
),
)
response.hidden_params["unified_finetuning_job_id"] = unified_finetuning_job_id
get_or_create_hidden_params(response)["unified_finetuning_job_id"] = unified_finetuning_job_id
else:
# get configs for custom_llm_provider
llm_provider_config: Final = get_fine_tuning_provider_config(custom_llm_provider=custom_llm_provider)

8
litellm/proxy/hooks/dynamic_rate_limiter.py
View file

@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.exceptions import RateLimitType
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import (
@ -243,9 +244,10 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger):
async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response):
try:
if isinstance(response, ModelResponse):
model_info: Final = self.llm_router.get_model_info(id=response.hidden_params["model_id"])
response_hidden_params: Final = get_or_create_hidden_params(response)
model_info: Final = self.llm_router.get_model_info(id=response_hidden_params["model_id"])
assert model_info is not None, "Model info for model with id={} is None".format(
response.hidden_params["model_id"]
response_hidden_params["model_id"]
)
key_priority: Final[str | None] = user_api_key_dict.metadata.get("priority", None)
(
@ -255,7 +257,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger):
model_rpm,
active_projects,
) = await self.check_available_usage(model=model_info["model_name"], priority=key_priority)
response.hidden_params["additional_headers"] = { # Add additional response headers - easier debugging
response_hidden_params["additional_headers"] = { # Add additional response headers - easier debugging
"x-litellm-model_group": model_info["model_name"],
"x-ratelimit-remaining-litellm-project-tokens": available_tpm,
"x-ratelimit-remaining-litellm-project-requests": available_rpm,

10
litellm/responses/mcp/chat_completions_handler.py
View file

@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Final, cast
from typing_extensions import TypedDict, Unpack
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
LiteLLM_Proxy_MCP_Handler,
)
@ -42,8 +43,8 @@ def _add_mcp_metadata_to_response(
# For streaming, store MCP metadata in _hidden_params
# CustomStreamWrapper._add_mcp_metadata_to_final_chunk() will automatically
# add it to the final chunk's delta.provider_specific_fields
if not hasattr(response, "_hidden_params"):
response.hidden_params = {}
if not hasattr(response, HIDDEN_PARAMS_ATTR):
set_hidden_params(response, {})
mcp_metadata: Final = {}
if openai_tools:
@ -54,7 +55,10 @@ def _add_mcp_metadata_to_response(
mcp_metadata["mcp_call_results"] = tool_results
if mcp_metadata:
response.hidden_params["mcp_metadata"] = mcp_metadata
hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
hidden_params["mcp_metadata"] = mcp_metadata
return
if not isinstance(response, ModelResponse):

7
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -48,6 +48,7 @@ from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
is_codex_user_agent,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata
from litellm.litellm_core_utils.prompt_templates.common_utils import (
as_openai_image_part,
@ -409,9 +410,9 @@ def _parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mappi
return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None}
def _response_cost_or_none(response: ModelResponse | ResponsesAPIResponse) -> float | None:
hidden_params: Final = response.hidden_params
if not isinstance(hidden_params, dict):
def _response_cost_or_none(response: object) -> float | None:
hidden_params: Final = get_hidden_params(response)
if hidden_params is None:
return None
cost: Final = hidden_params.get("response_cost")
if isinstance(cost, bool) or not isinstance(cost, (int, float)):

11
tests/unit/litellm_core_utils/test_hidden_params.py
View file

@ -6,6 +6,7 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.types.decisions import DecisionsResponse
from litellm.types.llms.base import HiddenParams
from litellm.types.utils import ModelResponse
@ -42,6 +43,16 @@ def test_get_hidden_params_preserves_model_response_identity() -> None:
assert get_hidden_params(response) is response.hidden_params
def test_get_hidden_params_returns_none_for_non_dict_storage() -> None:
class PlainResponse:
def __init__(self) -> None:
self._hidden_params = HiddenParams(response_cost=0.25)
response: Final = PlainResponse()
assert get_hidden_params(response) is None
def test_set_hidden_params_replaces_frozen_decisions_response_private_attr() -> None:
response: Final = DecisionsResponse(model="decider", answers={}, usage=None)
replacement: Final = {"replacement": True}

33
tests/unit/litellm_core_utils/test_litellm_logging.py
View file

@ -17,6 +17,7 @@ import pytest
from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent
from openai import AsyncOpenAI
from openai._legacy_response import HttpxBinaryResponseContent
from pydantic import BaseModel
import litellm
from litellm._internal_context import in_post_response_phase
@ -503,6 +504,38 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata():
litellm.model_cost.pop(custom_model_id, None)
def test_logging_success_path_reads_custom_pydantic_hidden_params() -> None:
class CustomLLMResponse(BaseModel):
_hidden_params = {"response_cost": 0.25, "custom_field": "preserved"}
response: Final = CustomLLMResponse()
logging_obj: Final = _make_dict_logging_obj()
metadata: Final[dict[str, object]] = {"request_tag": "preserved"}
logging_obj.model_call_details["litellm_params"] = {"metadata": metadata}
with (
patch.object(
logging_obj,
"_build_standard_logging_payload",
return_value={"response_cost": 0.25},
),
patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(logging_obj, "_is_recognized_call_type_for_logging", return_value=True),
patch.object(logging_obj, "_transform_usage_objects", side_effect=lambda result: result),
):
logging_obj.success_handler(
result=response,
start_time=time.time(),
end_time=time.time(),
)
assert logging_obj.model_call_details["response_cost"] == 0.25
assert metadata == {
"request_tag": "preserved",
"hidden_params": {"response_cost": 0.25, "custom_field": "preserved"},
}
class TestZeroCostDiagnostic:
DEPLOYMENT_ID: Final = "lit7898-query-only-priced-deployment"
MODEL_GROUP: Final = "query-only-priced-chat"

49
tests/unit/router_utils/test_add_retry_fallback_headers.py
View file

@ -153,6 +153,32 @@ def test_add_fallback_headers_to_streaming_wrapper():
}
def test_add_fallback_headers_updates_plain_duck_backing_storage() -> None:
class PlainDuckResponse:
def __init__(self) -> None:
self._hidden_params: dict[str, object] = {
"model_id": "deployment-1",
"custom_metadata": {"keep": True},
"additional_headers": {"x-existing": "keep"},
}
response: Final = PlainDuckResponse()
original_hidden_params: Final = response._hidden_params
result: Final = add_fallback_headers_to_response(response=response, attempted_fallbacks=2)
assert result is response
assert response._hidden_params is original_hidden_params
assert response._hidden_params == {
"model_id": "deployment-1",
"custom_metadata": {"keep": True},
"additional_headers": {
"x-existing": "keep",
"x-litellm-attempted-fallbacks": 2,
},
}
def test_add_fallback_headers_serializes_fallback_errors():
response = StreamingWrapper()
fallback_errors = [
@ -250,6 +276,29 @@ def test_ensure_response_additional_headers_updates_frozen_decisions_response()
assert response.hidden_params["additional_headers"] is additional_headers
def test_ensure_response_additional_headers_preserves_plain_duck_storage() -> None:
class PlainDuckResponse:
def __init__(self) -> None:
self._hidden_params: dict[str, object] = {
"model_id": "deployment-1",
"custom_metadata": {"keep": True},
"additional_headers": {"x-existing": "keep"},
}
response: Final = PlainDuckResponse()
original_hidden_params: Final = response._hidden_params
additional_headers: Final = ensure_response_additional_headers(response)
additional_headers["x-added"] = "value"
assert response._hidden_params is original_hidden_params
assert additional_headers is original_hidden_params["additional_headers"]
assert response._hidden_params == {
"model_id": "deployment-1",
"custom_metadata": {"keep": True},
"additional_headers": {"x-existing": "keep", "x-added": "value"},
}
def test_add_fallback_headers_returns_none_when_response_is_none():
result = add_fallback_headers_to_response(response=None, attempted_fallbacks=1)
assert result is None