diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 011bf95defd..ee3fcfa441c 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -31,6 +31,7 @@ from litellm._uuid import uuid from litellm.caching.caching import DualCache from litellm.constants import MAX_FILE_LIST_LIMIT from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_metadata, ) @@ -1294,7 +1295,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_mappings: Dict[str, str] = {} for file_object in responses: - model_file_id_mapping = file_object._hidden_params.get("model_file_id_mapping") + file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata + dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR) + ) + model_file_id_mapping = file_hidden_params.get("model_file_id_mapping") if model_file_id_mapping and isinstance(model_file_id_mapping, dict): model_mappings.update(model_file_id_mapping) @@ -1344,7 +1348,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): _, file_type = extract_file_metadata(create_file_request["file"]) output_file_id = file_objects[0].id - model_id = file_objects[0]._hidden_params.get("model_id") + file_hidden_params: Final = cast(dict[str, object], getattr(file_objects[0], HIDDEN_PARAMS_ATTR)) + model_id = file_hidden_params.get("model_id") unified_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( file_type, @@ -1410,11 +1415,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if decoded_batch_id and is_litellm_executed_batch(decoded_batch_id): return response ## Check if unified_file_id is in the response - unified_file_id = response._hidden_params.get("unified_file_id") # managed file id - unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id - is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True - model_id = cast(Optional[str], response._hidden_params.get("model_id")) - model_name = cast(Optional[str], response._hidden_params.get("model_name")) + response_hidden_params: Final = cast(dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)) + unified_file_id = response_hidden_params.get("unified_file_id") + unified_batch_id = response_hidden_params.get("unified_batch_id") + is_batch_create: Final = response_hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True + model_id = cast(Optional[str], response_hidden_params.get("model_id")) + model_name = cast(Optional[str], response_hidden_params.get("model_name")) resolved_model_name = resolve_managed_output_file_model_name( unified_input_file_id=unified_file_id if isinstance(unified_file_id, str) else response.input_file_id, @@ -1525,12 +1531,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): elif isinstance(response, LiteLLMFineTuningJob): ## Check if unified_file_id is in the response - unified_file_id = response._hidden_params.get("unified_file_id") # managed file id - unified_finetuning_job_id = response._hidden_params.get( - "unified_finetuning_job_id" - ) # managed finetuning job id - model_id = cast(Optional[str], response._hidden_params.get("model_id")) - model_name = cast(Optional[str], response._hidden_params.get("model_name")) + finetuning_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + unified_file_id = finetuning_response_hidden_params.get("unified_file_id") + unified_finetuning_job_id = finetuning_response_hidden_params.get("unified_finetuning_job_id") + model_id = cast(Optional[str], finetuning_response_hidden_params.get("model_id")) + model_name = cast(Optional[str], finetuning_response_hidden_params.get("model_name")) original_response_id = response.id if (unified_file_id or unified_finetuning_job_id) and model_id: response.id = self.get_unified_generic_response_id(model_id=model_id, generic_response_id=response.id) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 94e18bb8512..5f0bd085624 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -29,7 +29,7 @@ from litellm._logging import print_verbose, verbose_logger from litellm.caching import InMemoryCache from litellm.caching.caching import S3Cache, response_cache_phase from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS -from litellm.litellm_core_utils.hidden_params import get_hidden_params +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( update_response_metadata, ) @@ -710,6 +710,7 @@ class LLMCachingHandler: if cached.usage is not None and embedding_response.usage is not None else cached.usage ) + cached_hidden_params: Final = getattr(cached, HIDDEN_PARAMS_ATTR) merged: Final = EmbeddingResponse( model=cached.model, data=[ @@ -720,7 +721,7 @@ class LLMCachingHandler: ], usage=merged_usage, hidden_params={ - **cached.hidden_params, + **cached_hidden_params, "cache_hit": True, }, _response_headers=cached._response_headers, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index b2bb648c852..e3321198fbf 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1002,8 +1002,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if key == "additional_headers" and key in model_response_hidden_params: existing_additional_headers = model_response_hidden_params.get("additional_headers", {}) merged_headers = { - **cast("dict[str, object]", value), - **cast("dict[str, object]", existing_additional_headers), + **cast( # cast-ok: preserve mapping operations on dynamic response metadata + "dict[str, object]", value + ), + **cast( # cast-ok: preserve mapping operations on dynamic response metadata + "dict[str, object]", existing_additional_headers + ), } model_response_hidden_params[key] = merged_headers else: diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cbdf2c77542..0290b36b3af 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2025,7 +2025,7 @@ def response_cost_calculator( else: if isinstance(response_object, BaseModel): if hasattr(response_object, HIDDEN_PARAMS_ATTR): - hidden_params: Final = cast( + hidden_params: Final = cast( # cast-ok: cost metadata supports dict and Pydantic storage dict[str, object] | BaseModel, getattr(response_object, HIDDEN_PARAMS_ATTR), ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a4946dae5ae..09a603b6d0e 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -78,6 +78,7 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_or_create_hidden_params from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, is_unbilled_non_inference_call, @@ -1851,7 +1852,7 @@ class Logging(LiteLLMLoggingBaseClass): else result ) - result_hidden_params: Final = getattr(priced_result, "_hidden_params", None) or MappingProxyType({}) + result_hidden_params: Final = getattr(priced_result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({}) if isinstance(priced_result, (BaseModel, HttpxBinaryResponseContent)) and hasattr( priced_result, "_hidden_params" ): @@ -2041,7 +2042,7 @@ class Logging(LiteLLMLoggingBaseClass): def _custom_pricing_for(self, result: object) -> bool: litellm_params: Final = getattr(self, "litellm_params", None) - result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) + result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({}) additional_headers: Final = ( result_hidden_params.get("additional_headers") if isinstance(result_hidden_params, dict) @@ -2377,7 +2378,7 @@ class Logging(LiteLLMLoggingBaseClass): """ if logging_result is None: return - hidden_params: Final = getattr(logging_result, "_hidden_params", None) + hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, None) if not hidden_params: return if self.model_call_details.get("litellm_params") is None: @@ -2401,7 +2402,7 @@ class Logging(LiteLLMLoggingBaseClass): ): """Resolve hidden params, compute response cost, and emit the standard logging payload.""" self._surface_response_headers_from_result(logging_result) - hidden_params: Final = getattr(logging_result, "_hidden_params", {}) + hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {}) if hidden_params: if self.model_call_details.get("litellm_params") is not None: self.model_call_details["litellm_params"].setdefault("metadata", {}) @@ -2676,7 +2677,7 @@ class Logging(LiteLLMLoggingBaseClass): Left in place they overwrite the create's real deployment with the poll's empty one in the payload every logging integration reads. """ - settled_hidden_params: Final = getattr(result, "_hidden_params", None) + settled_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) if isinstance(settled_hidden_params, dict): for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"): settled_hidden_params.pop(poll_scoped_key, None) @@ -3250,13 +3251,14 @@ class Logging(LiteLLMLoggingBaseClass): batch_successful_requests: Final = kwargs.get("batch_successful_requests", None) batch_failed_requests: Final = kwargs.get("batch_failed_requests", None) has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models)) + result_hidden_params: Final = get_or_create_hidden_params(result) should_compute_batch_data: Final = not has_explicit_batch_data and batch_cost_is_final(result) if has_explicit_batch_data: - result.hidden_params["response_cost"] = batch_cost - result.hidden_params["batch_models"] = batch_models - result._hidden_params["batch_successful_requests"] = batch_successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above - result._hidden_params["batch_failed_requests"] = batch_failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above + result_hidden_params["response_cost"] = batch_cost + result_hidden_params["batch_models"] = batch_models + result_hidden_params["batch_successful_requests"] = batch_successful_requests + result_hidden_params["batch_failed_requests"] = batch_failed_requests result.usage = batch_usage batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None) batch_completion_cost: Final = kwargs.get("batch_completion_cost", None) @@ -3281,10 +3283,10 @@ class Logging(LiteLLMLoggingBaseClass): model_info=self.get_router_deployment_model_info(), ) - result.hidden_params["response_cost"] = batch_result.cost - result.hidden_params["batch_models"] = batch_result.models - result._hidden_params["batch_successful_requests"] = batch_result.successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above - result._hidden_params["batch_failed_requests"] = batch_result.failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above + result_hidden_params["response_cost"] = batch_result.cost + result_hidden_params["batch_models"] = batch_result.models + result_hidden_params["batch_successful_requests"] = batch_result.successful_requests + result_hidden_params["batch_failed_requests"] = batch_result.failed_requests result.usage = batch_result.usage self.set_cost_breakdown( input_cost=batch_result.prompt_cost, @@ -6430,7 +6432,7 @@ def _extract_response_obj_and_hidden_params( ) -> tuple[dict, dict | None]: """Extract response_obj and hidden_params from init_response_obj.""" hidden_params: dict | None = ( - getattr(init_response_obj, "_hidden_params", None) + getattr(init_response_obj, HIDDEN_PARAMS_ATTR, None) if isinstance(init_response_obj, BaseModel | HttpxBinaryResponseContent) else None ) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 22bff04590d..3c002f203bf 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -9,6 +9,7 @@ from typing import Final, Literal, cast import litellm from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.prompt_templates.common_utils import ( _extract_reasoning_content, ) @@ -512,7 +513,8 @@ class LiteLLMResponseObjectHandler: text_completion_response["choices"] = choices_list text_completion_response["usage"] = response.get("usage", None) - text_completion_response.hidden_params = HiddenParams(**response.hidden_params) + response_hidden_params: Final = getattr(response, HIDDEN_PARAMS_ATTR) + setattr(text_completion_response, HIDDEN_PARAMS_ATTR, HiddenParams(**response_hidden_params)) return text_completion_response @staticmethod diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 9f27a52439e..0b2a8722354 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypedDict, Union, cast from typing_extensions import ReadOnly, Required from litellm._logging import verbose_logger +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.types.llms.openai import ( ChatCompletionAssistantContentValue, ChatCompletionAudioDelta, @@ -265,7 +266,11 @@ class ChunkProcessor: return model_response # set hidden params from chunk to model_response if model_response is not None and hasattr(model_response, "_hidden_params"): - model_response.hidden_params = chunk.get("_hidden_params", {}) + chunk_hidden_params: Final = chunk.get("_hidden_params", {}) + if isinstance(chunk_hidden_params, dict): + set_hidden_params(model_response, chunk_hidden_params) + else: + setattr(model_response, HIDDEN_PARAMS_ATTR, chunk_hidden_params) return model_response @staticmethod diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 028401c6c11..e8a742fd17f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -20,6 +20,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, ) @@ -1830,14 +1831,22 @@ class CustomStreamWrapper: if getattr(response, "usage", None) is not None: usage_to_preserve = response.usage if usage_to_preserve: - response.hidden_params["usage"] = usage_to_preserve + response_hidden_params_for_usage = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + response_hidden_params_for_usage["usage"] = usage_to_preserve obj_dict = response.model_dump() if "usage" in obj_dict: del obj_dict["usage"] - response = self.model_response_creator(chunk=obj_dict, hidden_params=response.hidden_params) + response_hidden_params_for_model = cast( # cast-ok: preserve dynamic mapping behavior + Mapping[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + response = self.model_response_creator( + chunk=obj_dict, hidden_params=response_hidden_params_for_model + ) ## check if empty is_empty = is_model_response_stream_empty(model_response=cast(ModelResponseStream, response)) @@ -1846,8 +1855,11 @@ class CustomStreamWrapper: # add usage as hidden param if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) - response.hidden_params["usage"] = usage - self._last_returned_hidden_params = response.hidden_params + response_hidden_params_for_final_chunk = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + response_hidden_params_for_final_chunk["usage"] = usage + self._last_returned_hidden_params = response_hidden_params_for_final_chunk # Add MCP metadata to final chunk if present response = self._add_mcp_metadata_to_final_chunk(response) # RETURN RESULT @@ -2042,7 +2054,10 @@ class CustomStreamWrapper: if "usage" in obj_dict: del obj_dict["usage"] processed_chunk = self.model_response_creator( - chunk=obj_dict, hidden_params=processed_chunk.hidden_params + chunk=obj_dict, + hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage + Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR) + ), ) is_empty = is_model_response_stream_empty( model_response=cast(ModelResponseStream, processed_chunk) @@ -2056,8 +2071,11 @@ class CustomStreamWrapper: # add usage as hidden param if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) - processed_chunk.hidden_params["usage"] = usage - self._last_returned_hidden_params = processed_chunk.hidden_params + processed_chunk_hidden_params = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR) + ) + processed_chunk_hidden_params["usage"] = usage + self._last_returned_hidden_params = processed_chunk_hidden_params # Call post-call streaming deployment hook for final chunk if self.sent_last_chunk is True: @@ -2212,7 +2230,10 @@ class CustomStreamWrapper: self.chunks.append(processed_chunk) if self.stream_options is None: usage: Final = calculate_total_usage(chunks=self.chunks) - processed_chunk._hidden_params["usage"] = usage # pyright: ignore[reportPrivateUsage] # sync parity + processed_chunk_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR) + ) + processed_chunk_hidden_params["usage"] = usage # see sync __next__'s sibling branch: deliberately do NOT restore # here - this chunk is still this call's own data, and restoring # before returning it would corrupt the caller's own log diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index 24bc6b6f4a3..6432469366c 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -20,6 +20,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( AppliedEdit, @@ -165,7 +166,7 @@ class _CombinedChunkSplitter: chunk.usage = None hidden_params: Final = getattr(chunk, "_hidden_params", None) if isinstance(hidden_params, dict) and "usage" in hidden_params: - chunk.hidden_params = {key: value for key, value in hidden_params.items() if key != "usage"} + set_hidden_params(chunk, {key: value for key, value in hidden_params.items() if key != "usage"}) @staticmethod def _split_by_payload_kind(chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]": diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index a0ee907dd63..20df02d9cc0 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -1,10 +1,11 @@ from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) @@ -237,10 +238,13 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): ) -> None: raw_headers: Final = dict(raw_response.headers) processed_headers: Final = process_response_headers(raw_headers) - if not hasattr(completed_response, "_hidden_params"): - setattr(completed_response, "_hidden_params", {}) - completed_response.hidden_params["additional_headers"] = processed_headers - completed_response.hidden_params["headers"] = raw_headers + if not hasattr(completed_response, HIDDEN_PARAMS_ATTR): + set_hidden_params(completed_response, {}) + hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(completed_response, HIDDEN_PARAMS_ATTR) + ) + hidden_params["additional_headers"] = processed_headers + hidden_params["headers"] = raw_headers def get_complete_url( self, diff --git a/litellm/llms/e2b/sandbox/transformation.py b/litellm/llms/e2b/sandbox/transformation.py index 64fd1857de2..dbf3996248a 100644 --- a/litellm/llms/e2b/sandbox/transformation.py +++ b/litellm/llms/e2b/sandbox/transformation.py @@ -9,11 +9,12 @@ Talks to e2b's REST API directly over httpx (no e2b SDK dependency): import json from collections.abc import Mapping -from typing import Final +from typing import Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.llms.base_llm.sandbox.transformation import ( SANDBOX_MAX_OUTPUT_BYTES, BaseSandboxConfig, @@ -109,8 +110,11 @@ class E2BSandboxConfig(BaseSandboxConfig): **kwargs, ) -> CodeExecutionResult: handle: Final = self._as_handle(container) + hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle + dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) + ) - token: Final = handle.hidden_params.get("envd_access_token") + token: Final = hidden_params.get("envd_access_token") if not token: raise ValueError( "Cannot run code from a sandbox id alone. e2b secure sandboxes " @@ -119,7 +123,7 @@ class E2BSandboxConfig(BaseSandboxConfig): ) headers: Final = {"Content-Type": "application/json", "X-Access-Token": token} - traffic_token: Final = handle.hidden_params.get("traffic_access_token") + traffic_token: Final = hidden_params.get("traffic_access_token") if traffic_token: headers["E2B-Traffic-Access-Token"] = traffic_token @@ -143,8 +147,11 @@ class E2BSandboxConfig(BaseSandboxConfig): **kwargs, ) -> bool: handle: Final = self._as_handle(container) - key: Final = api_key or handle.hidden_params.get("api_key") or self.validate_environment() - base: Final = api_base or handle.hidden_params.get("api_base") or E2B_API_BASE + hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle + dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) + ) + key: Final = api_key or hidden_params.get("api_key") or self.validate_environment() + base: Final = api_base or hidden_params.get("api_base") or E2B_API_BASE try: response: Final = await self._http(client).delete( url=f"{base}/sandboxes/{handle.id}", diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 81a1c4a8e72..0fdd89aec91 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -1,9 +1,10 @@ from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from litellm.types.utils import ImageResponse @@ -237,12 +238,15 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): model_response.data.extend(fal_images_to_image_objects(images)) # Add additional metadata from Flux Pro response - if hasattr(model_response, "_hidden_params"): + if hasattr(model_response, HIDDEN_PARAMS_ATTR): + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) if "seed" in response_object: - model_response.hidden_params["seed"] = response_object["seed"] + hidden_params["seed"] = response_object["seed"] if "timings" in response_object: - model_response.hidden_params["timings"] = response_object["timings"] + hidden_params["timings"] = response_object["timings"] if "has_nsfw_concepts" in response_object: - model_response.hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"] + hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"] return model_response diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index 26406b3a2c5..aa80fa34f6a 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -1,9 +1,10 @@ from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from litellm.types.utils import ImageObject, ImageResponse @@ -189,7 +190,10 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): ) ) - if hasattr(model_response, "_hidden_params") and "seed" in response_object: - model_response.hidden_params["seed"] = response_object["seed"] + if hasattr(model_response, HIDDEN_PARAMS_ATTR) and "seed" in response_object: + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) + hidden_params["seed"] = response_object["seed"] return model_response diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py index a6d170de614..c5a5f0112ca 100644 --- a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -1,7 +1,8 @@ -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from litellm.types.utils import ImageObject, ImageResponse @@ -234,8 +235,11 @@ class FalAIImagen4Config(FalAIBaseConfig): ) # Add seed metadata from Imagen4 response - if hasattr(model_response, "_hidden_params"): + if hasattr(model_response, HIDDEN_PARAMS_ATTR): + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) if "seed" in response_data: - model_response.hidden_params["seed"] = response_data["seed"] + hidden_params["seed"] = response_data["seed"] return model_response diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index d850f337d45..30d51cc8084 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -1,9 +1,10 @@ from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from litellm.types.utils import ImageObject, ImageResponse @@ -268,12 +269,15 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): ) # Add additional metadata from Stable Diffusion response - if hasattr(model_response, "_hidden_params"): + if hasattr(model_response, HIDDEN_PARAMS_ATTR): + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) if "seed" in response_object: - model_response.hidden_params["seed"] = response_object["seed"] + hidden_params["seed"] = response_object["seed"] if "timings" in response_object: - model_response.hidden_params["timings"] = response_object["timings"] + hidden_params["timings"] = response_object["timings"] if "has_nsfw_concepts" in response_object: - model_response.hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"] + hidden_params["has_nsfw_concepts"] = response_object["has_nsfw_concepts"] return model_response diff --git a/litellm/llms/openai/completion/transformation.py b/litellm/llms/openai/completion/transformation.py index c638a88d5ca..0cde767b919 100644 --- a/litellm/llms/openai/completion/transformation.py +++ b/litellm/llms/openai/completion/transformation.py @@ -4,6 +4,7 @@ Support for gpt model family from typing import Final +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionResponse @@ -111,7 +112,7 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig): if "model" in response_object: model_response_object.model = response_object["model"] - model_response_object.hidden_params["original_response"] = ( + get_or_create_hidden_params(model_response_object)["original_response"] = ( response_object # track original response, if users make a litellm.text_completion() request, we can return the original response ) return model_response_object diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py index f665082a8e2..bff5e0cea10 100644 --- a/litellm/llms/openai/containers/transformation.py +++ b/litellm/llms/openai/containers/transformation.py @@ -168,7 +168,9 @@ class OpenAIContainerConfig(BaseContainerConfig): container_hidden_params: Final = get_or_create_hidden_params(container_obj) container_hidden_params.setdefault("additional_headers", {}) - container_additional_headers: Final = cast("dict[str, object]", container_hidden_params["additional_headers"]) + container_additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata + "dict[str, object]", container_hidden_params["additional_headers"] + ) container_additional_headers["llm_provider-x-litellm-response-cost"] = container_cost return container_obj diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index f8a551cd640..e97962a1c48 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx import litellm +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -216,13 +217,17 @@ class OpenrouterConfig(OpenAIGPTConfig): response_cost: Final = response_json["usage"].get("cost") if response_cost is not None: # Store cost in hidden params for the cost calculator to use - if not hasattr(model_response, "_hidden_params"): - model_response.hidden_params = {} - if "additional_headers" not in model_response.hidden_params: - model_response.hidden_params["additional_headers"] = {} - model_response.hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float( - response_cost + if not hasattr(model_response, HIDDEN_PARAMS_ATTR): + set_hidden_params(model_response, {}) + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) ) + if "additional_headers" not in hidden_params: + hidden_params["additional_headers"] = {} + additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata + dict[str, object], hidden_params["additional_headers"] + ) + additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost) except Exception: # If we can't extract cost, continue without it - don't fail the response pass diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index e18cfcc153d..355df81b87a 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -28,12 +28,13 @@ Response format: """ from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter import litellm +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -226,17 +227,29 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): cost: Final = usage_data.get("cost") if cost is not None: - if not hasattr(model_response, "_hidden_params"): - model_response.hidden_params = {} - if "additional_headers" not in model_response.hidden_params: - model_response.hidden_params["additional_headers"] = {} - model_response.hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(cost) + if not hasattr(model_response, HIDDEN_PARAMS_ATTR): + set_hidden_params(model_response, {}) + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) + if "additional_headers" not in hidden_params: + hidden_params["additional_headers"] = {} + additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata + dict[str, object], hidden_params["additional_headers"] + ) + additional_headers["llm_provider-x-litellm-response-cost"] = float(cost) cost_details: Final = usage_data.get("cost_details", {}) if cost_details: - if "response_cost_details" not in model_response.hidden_params: - model_response.hidden_params["response_cost_details"] = {} - model_response.hidden_params["response_cost_details"].update(cost_details) + cost_details_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) + if "response_cost_details" not in cost_details_hidden_params: + cost_details_hidden_params["response_cost_details"] = {} + response_cost_details: Final = cast( # cast-ok: preserve mapping operations on response metadata + dict[str, object], cost_details_hidden_params["response_cost_details"] + ) + response_cost_details.update(cost_details) model_response.hidden_params["model"] = response_json.get("model", model) diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py index bf1c3314c0d..d2c558241f0 100644 --- a/litellm/llms/opensandbox/sandbox/transformation.py +++ b/litellm/llms/opensandbox/sandbox/transformation.py @@ -1,7 +1,7 @@ import asyncio import json import time -from typing import Final +from typing import Final, cast import httpx @@ -18,6 +18,7 @@ from litellm.constants import ( OPEN_SANDBOX_POLL_INTERVAL, OPEN_SANDBOX_READY_TIMEOUT, ) +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.sandbox.transformation import ( SANDBOX_MAX_OUTPUT_BYTES, BaseSandboxConfig, @@ -147,9 +148,12 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): poll_interval=(float(poll_interval) if poll_interval is not None else DEFAULT_POLL_INTERVAL), client=client, ) - endpoint: Final = str(handle.hidden_params["execd_endpoint"]) - endpoint_headers: Final = self._as_str_dict(handle.hidden_params.get("execd_headers")) - base: Final = str(handle.hidden_params.get("api_base") or handle.domain or self._api_base(api_base)) + hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle + dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) + ) + endpoint: Final = str(hidden_params["execd_endpoint"]) + endpoint_headers: Final = self._as_str_dict(hidden_params.get("execd_headers")) + base: Final = str(hidden_params.get("api_base") or handle.domain or self._api_base(api_base)) lines: Final = await self._post_code( url=f"{self._endpoint_base_url(endpoint, base)}/code", headers={ @@ -176,7 +180,10 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): **kwargs, ) -> bool: handle: Final = self._as_handle(container, api_base=api_base) - base: Final = str(handle.hidden_params.get("api_base") or self._api_base(api_base)) + hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle + dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) + ) + base: Final = str(hidden_params.get("api_base") or self._api_base(api_base)) key: Final = self._api_key(api_key=api_key, handle=handle) try: response: Final = await self._http(client).delete( @@ -201,12 +208,15 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): client: AsyncHTTPHandler | None, ) -> ContainerHandle: handle: Final = self._as_handle(container, api_base=api_base) - if handle.hidden_params.get("execd_endpoint"): + hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle + dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) + ) + if hidden_params.get("execd_endpoint"): return handle - base: Final = str(handle.hidden_params.get("api_base") or self._api_base(api_base)) + base: Final = str(hidden_params.get("api_base") or self._api_base(api_base)) key: Final = self._api_key(api_key=api_key, handle=handle) - resolved_use_server_proxy: Final = bool(handle.hidden_params.get("use_server_proxy", use_server_proxy)) + resolved_use_server_proxy: Final = bool(hidden_params.get("use_server_proxy", use_server_proxy)) endpoint, endpoint_headers = await self._wait_for_execd_endpoint( sandbox_id=handle.id, api_base=base, @@ -217,14 +227,17 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): poll_interval=poll_interval, ) handle.domain = base - handle.hidden_params = { - **handle.hidden_params, - "api_base": base, - "api_key": key, - "execd_endpoint": endpoint, - "execd_headers": endpoint_headers, - "use_server_proxy": resolved_use_server_proxy, - } + set_hidden_params( + handle, + { + **hidden_params, + "api_base": base, + "api_key": key, + "execd_endpoint": endpoint, + "execd_headers": endpoint_headers, + "use_server_proxy": resolved_use_server_proxy, + }, + ) return handle async def _wait_until_running( @@ -329,8 +342,11 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): def _api_key(self, *, api_key: str | None, handle: ContainerHandle) -> str: if api_key is not None: return api_key - if "api_key" in handle.hidden_params: - return str(handle.hidden_params["api_key"]) + hidden_params: Final = cast( # cast-ok: preserve mapping operations on the validated handle + dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) + ) + if "api_key" in hidden_params: + return str(hidden_params["api_key"]) return self.validate_environment() @staticmethod diff --git a/litellm/main.py b/litellm/main.py index b2dbcf98edc..b2d9429fe3e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -95,6 +95,7 @@ from litellm.litellm_core_utils.health_check_utils import ( _create_health_check_response, _filter_model_params, ) +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.mock_functions import ( mock_embedding, @@ -5515,9 +5516,12 @@ def completion( ) ) - if model_response is not None and hasattr(model_response, "_hidden_params"): - model_response.hidden_params["custom_llm_provider"] = custom_llm_provider - model_response.hidden_params["region_name"] = kwargs.get( + if model_response is not None and hasattr(model_response, HIDDEN_PARAMS_ATTR): + model_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) + model_response_hidden_params["custom_llm_provider"] = custom_llm_provider + model_response_hidden_params["region_name"] = kwargs.get( "aws_region_name", None ) # support region-based pricing for bedrock @@ -6243,7 +6247,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: elif asyncio.iscoroutine(init_response): response = await init_response if response is not None and isinstance(response, EmbeddingResponse) and hasattr(response, "_hidden_params"): - response.hidden_params["custom_llm_provider"] = custom_llm_provider + response_hidden_params: Final = get_hidden_params(response) + if response_hidden_params is not None: + response_hidden_params["custom_llm_provider"] = custom_llm_provider if response is None: raise ValueError("Unable to get Embedding Response. Please pass a valid llm_provider.") @@ -7360,7 +7366,9 @@ def embedding( else: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) if response is not None and hasattr(response, "_hidden_params") and isinstance(response, EmbeddingResponse): - response.hidden_params["custom_llm_provider"] = custom_llm_provider + response_hidden_params: Final = get_hidden_params(response) + if response_hidden_params is not None: + response_hidden_params["custom_llm_provider"] = custom_llm_provider if response is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -8198,7 +8206,10 @@ def transcription( if existing_duration is None: calculated_duration: Final = calculate_request_duration(file) if calculated_duration is not None: - response.hidden_params["audio_transcription_duration"] = calculated_duration + response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + response_hidden_params["audio_transcription_duration"] = calculated_duration if response is None: raise ValueError("Unmapped provider passed in. Unable to get the response.") @@ -8813,7 +8824,7 @@ async def ahealth_check( if mode in mode_handlers: _response: Final = await mode_handlers[mode]() # Only process headers for chat mode - _response_headers: Final[dict] = getattr(_response, "_hidden_params", {}).get("headers", {}) or {} + _response_headers: Final[dict] = getattr(_response, HIDDEN_PARAMS_ATTR, {}).get("headers", {}) or {} return _create_health_check_response(_response_headers) else: raise Exception(f"Mode {mode} not supported. See modes here: https://docs.litellm.ai/docs/proxy/health") @@ -9072,7 +9083,7 @@ def stream_chunk_builder( if isinstance(chunk, dict): hidden = chunk.get("_hidden_params") else: - hidden = getattr(chunk, "_hidden_params", None) + hidden = getattr(chunk, HIDDEN_PARAMS_ATTR, None) if isinstance(hidden, dict) and "provider_specific_fields" in hidden: response.hidden_params.setdefault("provider_specific_fields", {}).update( hidden["provider_specific_fields"] @@ -9251,7 +9262,7 @@ def stream_chunk_builder( if isinstance(chunk, dict): hidden = chunk.get("_hidden_params") else: - hidden = getattr(chunk, "_hidden_params", None) + hidden = getattr(chunk, HIDDEN_PARAMS_ATTR, None) if isinstance(hidden, dict) and "provider_specific_fields" in hidden: response.hidden_params.setdefault("provider_specific_fields", {}).update( hidden["provider_specific_fields"] diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index c5c8ea78426..558304f17bb 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -17,6 +17,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit @@ -211,7 +212,7 @@ async def _create_provider_batch_for_managed_file( } response: Final = await llm_router.acreate_batch(**request) response.input_file_id = input_file_id - response.hidden_params["unified_file_id"] = unified_file_id + get_or_create_hidden_params(response)["unified_file_id"] = unified_file_id return response @@ -484,7 +485,7 @@ async def create_batch( **_create_batch_data, ) - response.hidden_params[BATCH_CREATE_HIDDEN_PARAM] = True + get_or_create_hidden_params(response)[BATCH_CREATE_HIDDEN_PARAM] = True ### CALL HOOKS ### - modify outgoing data response = await proxy_logging_obj.post_call_success_hook( @@ -736,11 +737,12 @@ async def retrieve_batch( ) response = await llm_router.aretrieve_batch(**data) - response.hidden_params["unified_batch_id"] = unified_batch_id + response_hidden_params: Final = get_or_create_hidden_params(response) + response_hidden_params["unified_batch_id"] = unified_batch_id if unified_batch_id: model_id_from_batch: Final = get_model_id_from_unified_batch_id(unified_batch_id) if model_id_from_batch: - response.hidden_params["model_id"] = model_id_from_batch + response_hidden_params["model_id"] = model_id_from_batch # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) else: @@ -1168,10 +1170,11 @@ async def cancel_batch( data["model"] = model_id_from_batch data["batch_id"] = get_batch_id_from_unified_batch_id(unified_batch_id) response = await llm_router.acancel_batch(**data) - response.hidden_params["unified_batch_id"] = unified_batch_id + response_hidden_params: Final = get_or_create_hidden_params(response) + response_hidden_params["unified_batch_id"] = unified_batch_id - if not response.hidden_params.get("model_id") and data.get("model"): - response.hidden_params["model_id"] = data["model"] + if not response_hidden_params.get("model_id") and data.get("model"): + response_hidden_params["model_id"] = data["model"] # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) else: diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index a22c2db084b..0f7ef4ae4cd 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -161,7 +162,7 @@ async def create_fine_tuning_job( response = cast(LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data)) response.training_file = unified_file_id - response.hidden_params["unified_file_id"] = unified_file_id + get_or_create_hidden_params(response)["unified_file_id"] = unified_file_id ## ELSE, Route based on custom_llm_provider elif fine_tuning_request.custom_llm_provider: # get configs for custom_llm_provider @@ -304,7 +305,7 @@ async def retrieve_fine_tuning_job( **data, ), ) - response.hidden_params["unified_finetuning_job_id"] = unified_finetuning_job_id + get_or_create_hidden_params(response)["unified_finetuning_job_id"] = unified_finetuning_job_id elif custom_llm_provider: # get configs for custom_llm_provider llm_provider_config: Final = get_fine_tuning_provider_config(custom_llm_provider=custom_llm_provider) @@ -577,7 +578,7 @@ async def cancel_fine_tuning_job( **data, ), ) - response.hidden_params["unified_finetuning_job_id"] = unified_finetuning_job_id + get_or_create_hidden_params(response)["unified_finetuning_job_id"] = unified_finetuning_job_id else: # get configs for custom_llm_provider llm_provider_config: Final = get_fine_tuning_provider_config(custom_llm_provider=custom_llm_provider) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter.py b/litellm/proxy/hooks/dynamic_rate_limiter.py index e7b2e989213..11cbd3940f3 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.exceptions import RateLimitType from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.rate_limiter_utils import ( @@ -243,9 +244,10 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): try: if isinstance(response, ModelResponse): - model_info: Final = self.llm_router.get_model_info(id=response.hidden_params["model_id"]) + response_hidden_params: Final = get_or_create_hidden_params(response) + model_info: Final = self.llm_router.get_model_info(id=response_hidden_params["model_id"]) assert model_info is not None, "Model info for model with id={} is None".format( - response.hidden_params["model_id"] + response_hidden_params["model_id"] ) key_priority: Final[str | None] = user_api_key_dict.metadata.get("priority", None) ( @@ -255,7 +257,7 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): model_rpm, active_projects, ) = await self.check_available_usage(model=model_info["model_name"], priority=key_priority) - response.hidden_params["additional_headers"] = { # Add additional response headers - easier debugging + response_hidden_params["additional_headers"] = { # Add additional response headers - easier debugging "x-litellm-model_group": model_info["model_name"], "x-ratelimit-remaining-litellm-project-tokens": available_tpm, "x-ratelimit-remaining-litellm-project-requests": available_rpm, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 5e40b887397..aa9c4603b96 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Final, cast from typing_extensions import TypedDict, Unpack +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) @@ -42,8 +43,8 @@ def _add_mcp_metadata_to_response( # For streaming, store MCP metadata in _hidden_params # CustomStreamWrapper._add_mcp_metadata_to_final_chunk() will automatically # add it to the final chunk's delta.provider_specific_fields - if not hasattr(response, "_hidden_params"): - response.hidden_params = {} + if not hasattr(response, HIDDEN_PARAMS_ATTR): + set_hidden_params(response, {}) mcp_metadata: Final = {} if openai_tools: @@ -54,7 +55,10 @@ def _add_mcp_metadata_to_response( mcp_metadata["mcp_call_results"] = tool_results if mcp_metadata: - response.hidden_params["mcp_metadata"] = mcp_metadata + hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + hidden_params["mcp_metadata"] = mcp_metadata return if not isinstance(response, ModelResponse): diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index cae2b70bfaa..ad42555f9fa 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -48,6 +48,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, is_codex_user_agent, ) +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import ( as_openai_image_part, @@ -409,9 +410,9 @@ def _parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mappi return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} -def _response_cost_or_none(response: ModelResponse | ResponsesAPIResponse) -> float | None: - hidden_params: Final = response.hidden_params - if not isinstance(hidden_params, dict): +def _response_cost_or_none(response: object) -> float | None: + hidden_params: Final = get_hidden_params(response) + if hidden_params is None: return None cost: Final = hidden_params.get("response_cost") if isinstance(cost, bool) or not isinstance(cost, (int, float)): diff --git a/tests/unit/litellm_core_utils/test_hidden_params.py b/tests/unit/litellm_core_utils/test_hidden_params.py index c1c82752528..3f4ef50fca6 100644 --- a/tests/unit/litellm_core_utils/test_hidden_params.py +++ b/tests/unit/litellm_core_utils/test_hidden_params.py @@ -6,6 +6,7 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.types.decisions import DecisionsResponse +from litellm.types.llms.base import HiddenParams from litellm.types.utils import ModelResponse @@ -42,6 +43,16 @@ def test_get_hidden_params_preserves_model_response_identity() -> None: assert get_hidden_params(response) is response.hidden_params +def test_get_hidden_params_returns_none_for_non_dict_storage() -> None: + class PlainResponse: + def __init__(self) -> None: + self._hidden_params = HiddenParams(response_cost=0.25) + + response: Final = PlainResponse() + + assert get_hidden_params(response) is None + + def test_set_hidden_params_replaces_frozen_decisions_response_private_attr() -> None: response: Final = DecisionsResponse(model="decider", answers={}, usage=None) replacement: Final = {"replacement": True} diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index c726127f892..a0a0c1f6ba7 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -17,6 +17,7 @@ import pytest from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent from openai import AsyncOpenAI from openai._legacy_response import HttpxBinaryResponseContent +from pydantic import BaseModel import litellm from litellm._internal_context import in_post_response_phase @@ -503,6 +504,38 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): litellm.model_cost.pop(custom_model_id, None) +def test_logging_success_path_reads_custom_pydantic_hidden_params() -> None: + class CustomLLMResponse(BaseModel): + _hidden_params = {"response_cost": 0.25, "custom_field": "preserved"} + + response: Final = CustomLLMResponse() + logging_obj: Final = _make_dict_logging_obj() + metadata: Final[dict[str, object]] = {"request_tag": "preserved"} + logging_obj.model_call_details["litellm_params"] = {"metadata": metadata} + + with ( + patch.object( + logging_obj, + "_build_standard_logging_payload", + return_value={"response_cost": 0.25}, + ), + patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"), + patch.object(logging_obj, "_is_recognized_call_type_for_logging", return_value=True), + patch.object(logging_obj, "_transform_usage_objects", side_effect=lambda result: result), + ): + logging_obj.success_handler( + result=response, + start_time=time.time(), + end_time=time.time(), + ) + + assert logging_obj.model_call_details["response_cost"] == 0.25 + assert metadata == { + "request_tag": "preserved", + "hidden_params": {"response_cost": 0.25, "custom_field": "preserved"}, + } + + class TestZeroCostDiagnostic: DEPLOYMENT_ID: Final = "lit7898-query-only-priced-deployment" MODEL_GROUP: Final = "query-only-priced-chat" diff --git a/tests/unit/router_utils/test_add_retry_fallback_headers.py b/tests/unit/router_utils/test_add_retry_fallback_headers.py index d6b09e766fb..1f4ac3be6ac 100644 --- a/tests/unit/router_utils/test_add_retry_fallback_headers.py +++ b/tests/unit/router_utils/test_add_retry_fallback_headers.py @@ -153,6 +153,32 @@ def test_add_fallback_headers_to_streaming_wrapper(): } +def test_add_fallback_headers_updates_plain_duck_backing_storage() -> None: + class PlainDuckResponse: + def __init__(self) -> None: + self._hidden_params: dict[str, object] = { + "model_id": "deployment-1", + "custom_metadata": {"keep": True}, + "additional_headers": {"x-existing": "keep"}, + } + + response: Final = PlainDuckResponse() + original_hidden_params: Final = response._hidden_params + + result: Final = add_fallback_headers_to_response(response=response, attempted_fallbacks=2) + + assert result is response + assert response._hidden_params is original_hidden_params + assert response._hidden_params == { + "model_id": "deployment-1", + "custom_metadata": {"keep": True}, + "additional_headers": { + "x-existing": "keep", + "x-litellm-attempted-fallbacks": 2, + }, + } + + def test_add_fallback_headers_serializes_fallback_errors(): response = StreamingWrapper() fallback_errors = [ @@ -250,6 +276,29 @@ def test_ensure_response_additional_headers_updates_frozen_decisions_response() assert response.hidden_params["additional_headers"] is additional_headers +def test_ensure_response_additional_headers_preserves_plain_duck_storage() -> None: + class PlainDuckResponse: + def __init__(self) -> None: + self._hidden_params: dict[str, object] = { + "model_id": "deployment-1", + "custom_metadata": {"keep": True}, + "additional_headers": {"x-existing": "keep"}, + } + + response: Final = PlainDuckResponse() + original_hidden_params: Final = response._hidden_params + additional_headers: Final = ensure_response_additional_headers(response) + additional_headers["x-added"] = "value" + + assert response._hidden_params is original_hidden_params + assert additional_headers is original_hidden_params["additional_headers"] + assert response._hidden_params == { + "model_id": "deployment-1", + "custom_metadata": {"keep": True}, + "additional_headers": {"x-existing": "keep", "x-added": "value"}, + } + + def test_add_fallback_headers_returns_none_when_response_is_none(): result = add_fallback_headers_to_response(response=None, attempted_fallbacks=1) assert result is None