From b46587cb35a2c87677b462d875448d5405c97741 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 5 Oct 2026 23:16:26 +0000 Subject: [PATCH] refactor: move hidden params helpers to core utils Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching_handler.py | 2 +- .../transformation.py | 2 +- litellm/litellm_core_utils/hidden_params.py | 27 ++++++++++ .../pass_through/adapters/transformation.py | 2 +- litellm/llms/azure_ai/agents/handler.py | 2 +- .../azure_model_router/transformation.py | 21 ++++---- .../llms/openai/containers/transformation.py | 2 +- .../storage_backend_service.py | 2 +- .../cohere_passthrough_logging_handler.py | 2 +- .../openai_passthrough_logging_handler.py | 2 +- .../streaming_iterator.py | 2 +- litellm/router.py | 15 +++--- .../add_retry_fallback_headers.py | 31 +++-------- .../litellm_core_utils/test_hidden_params.py | 52 +++++++++++++++++++ .../test_add_retry_fallback_headers.py | 35 ------------- ...test_router_streaming_fallback_metadata.py | 28 ++++++++++ 16 files changed, 144 insertions(+), 83 deletions(-) create mode 100644 litellm/litellm_core_utils/hidden_params.py create mode 100644 tests/unit/litellm_core_utils/test_hidden_params.py diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 04101cb3602..94e18bb8512 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -29,13 +29,13 @@ from litellm._logging import print_verbose, verbose_logger from litellm.caching import InMemoryCache from litellm.caching.caching import S3Cache, response_cache_phase from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( update_response_metadata, ) from litellm.litellm_core_utils.logging_utils import ( _assemble_complete_response_from_streaming_chunks, ) -from litellm.router_utils.add_retry_fallback_headers import get_hidden_params from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.openai import ResponsesAPIResponse diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index e8f553cb9d2..b2bb648c852 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -24,6 +24,7 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse from litellm._logging import verbose_logger +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, ) @@ -37,7 +38,6 @@ from litellm.responses.sse_output_recovery import ( record_output_text_chunk, ) from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options -from litellm.router_utils.add_retry_fallback_headers import get_or_create_hidden_params from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionReasoningItem, diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py new file mode 100644 index 00000000000..f70ad701614 --- /dev/null +++ b/litellm/litellm_core_utils/hidden_params.py @@ -0,0 +1,27 @@ +from typing import Final + +_HIDDEN_PARAMS_ATTR: Final = "_hidden_params" +HIDDEN_PARAMS_ATTR: Final = _HIDDEN_PARAMS_ATTR + + +def get_hidden_params(obj: object) -> dict[str, object] | None: + hidden_params: Final = ( + obj.get(_HIDDEN_PARAMS_ATTR) if isinstance(obj, dict) else getattr(obj, _HIDDEN_PARAMS_ATTR, None) + ) + return hidden_params if isinstance(hidden_params, dict) else None + + +def set_hidden_params(obj: object, hidden_params: dict[str, object]) -> None: + if isinstance(obj, dict): + obj[_HIDDEN_PARAMS_ATTR] = hidden_params + else: + setattr(obj, _HIDDEN_PARAMS_ATTR, hidden_params) + + +def get_or_create_hidden_params(obj: object) -> dict[str, object]: + hidden_params: Final = get_hidden_params(obj) + if hidden_params is not None: + return hidden_params + created_hidden_params: Final[dict[str, object]] = {} + set_hidden_params(obj, created_hidden_params) + return created_hidden_params diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index d8a3453260d..f20be842d92 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -8,11 +8,11 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.pass_through.utils import ( is_reasoning_auto_summary_enabled, prompt_cache_key_from_user_id, ) -from litellm.router_utils.add_retry_fallback_headers import get_hidden_params # OpenAI has a 64-character limit for function/tool names # Anthropic does not have this limit, so we need to truncate long names diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 0981713d1f8..27ab6177df8 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -29,12 +29,12 @@ import httpx from typing_extensions import ReadOnly from litellm._logging import verbose_logger +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure_ai.agents.transformation import ( AzureAIAgentsConfig, AzureAIAgentsError, ) -from litellm.router_utils.add_retry_fallback_headers import get_or_create_hidden_params from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionAnnotationURLCitation, diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index 226cd9c13f9..9fb22abbe5e 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -72,13 +72,14 @@ class AzureModelRouterConfig(AzureAIStudioConfig): Also stamps that model onto ``_hidden_params`` so downstream consumers (spend logs, response restamping) can read it instead of guessing the route from the model string. """ + from litellm.litellm_core_utils.hidden_params import ( + get_hidden_params, + set_hidden_params, + ) from litellm.llms.azure_ai.common_utils import ( AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY, AzureFoundryModelInfo, ) - from litellm.router_utils.add_retry_fallback_headers import ( - get_hidden_params_dict, - ) # Get base model for the parent call (strips routing prefixes for API compatibility) base_model: Final[str] = AzureFoundryModelInfo.get_base_model(model) @@ -100,12 +101,14 @@ class AzureModelRouterConfig(AzureAIStudioConfig): ) selected_model: Final = transformed_response.model if selected_model: - # Rebuilt rather than mutated in place: ModelResponseBase declares _hidden_params as a - # class-level dict, so an in-place write can bleed into unrelated responses. - transformed_response._hidden_params = { # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params setter - **get_hidden_params_dict(transformed_response), - AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model, - } + transformed_hidden_params: Final = get_hidden_params(transformed_response) or {} + set_hidden_params( + transformed_response, + { + **transformed_hidden_params, + AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model, + }, + ) return transformed_response def calculate_additional_costs(self, model: str, prompt_tokens: int, completion_tokens: int) -> dict | None: diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py index 79d53389566..f665082a8e2 100644 --- a/litellm/llms/openai/containers/transformation.py +++ b/litellm/llms/openai/containers/transformation.py @@ -5,11 +5,11 @@ import httpx from typing_extensions import ReadOnly, TypedDict import litellm +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment -from litellm.router_utils.add_retry_fallback_headers import get_or_create_hidden_params from litellm.secret_managers.main import get_secret_str from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index 64de73bf614..afc1e1c96fa 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -12,12 +12,12 @@ from typing import Any, Final, cast from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid as uuid_module +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend from litellm.llms.base_llm.files.transformation import BaseFileEndpoints from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.utils import PrismaClient, ProxyLogging -from litellm.router_utils.add_retry_fallback_headers import get_or_create_hidden_params from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose from litellm.types.utils import ExtractedFileData, SpecialEnums diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index 7fcf47141a7..bb04f039ae2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -5,6 +5,7 @@ import httpx import litellm from litellm import stream_chunk_builder +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -17,7 +18,6 @@ from litellm.llms.cohere.common_utils import ( ) from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig from litellm.proxy._types import PassThroughEndpointLoggingTypedDict -from litellm.router_utils.add_retry_fallback_headers import get_or_create_hidden_params from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index ee094efd739..26ebe6b119d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -13,6 +13,7 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -28,7 +29,6 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) -from litellm.router_utils.add_retry_fallback_headers import get_hidden_params, get_or_create_hidden_params from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 9e145c273d7..8494ac39367 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -4,6 +4,7 @@ from collections.abc import Sequence from typing import Any, Final, cast import litellm +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.custom_tools import ( build_tool_call_item_kwargs, @@ -17,7 +18,6 @@ from litellm.responses.litellm_completion_transformation.transformation import ( ) from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.router_utils.add_retry_fallback_headers import get_or_create_hidden_params from litellm.types.llms.openai import ( PART_UNION_TYPES, BaseLiteLLMOpenAIResponseObject, diff --git a/litellm/router.py b/litellm/router.py index b13b9d861d9..0a408445fbb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -88,6 +88,13 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, is_registered_custom_provider, ) +from litellm.litellm_core_utils.hidden_params import ( + HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR, +) +from litellm.litellm_core_utils.hidden_params import ( + get_hidden_params, + set_hidden_params, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES from litellm.litellm_core_utils.ptu_pricing import ( @@ -153,12 +160,10 @@ from litellm.router_utils.add_retry_fallback_headers import ( apply_response_model_id, complexity_router_decision_headers, ensure_response_additional_headers, - get_hidden_params, get_hidden_params_dict, prepare_response_for_header_attachment, replace_complexity_router_headers, response_total_token_count, - set_hidden_params, ) from litellm.router_utils.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, @@ -2901,13 +2906,11 @@ class Router: fallback_item: object, prepared_fallback_hidden_params: tuple[dict[str, object], dict[str, object]], ) -> None: - if fallback_item is None: + if fallback_item is None or not hasattr(fallback_item, _HIDDEN_PARAMS_ATTR): return fallback_hidden_params, fallback_headers = prepared_fallback_hidden_params - item_hidden_params: Final = get_hidden_params(fallback_item) - if item_hidden_params is None: - return + item_hidden_params: Final = get_hidden_params_dict(fallback_item) item_headers = item_hidden_params.get("additional_headers") if not isinstance(item_headers, dict): item_headers = {} diff --git a/litellm/router_utils/add_retry_fallback_headers.py b/litellm/router_utils/add_retry_fallback_headers.py index 226da46b82b..a0795f0ffa7 100644 --- a/litellm/router_utils/add_retry_fallback_headers.py +++ b/litellm/router_utils/add_retry_fallback_headers.py @@ -6,6 +6,13 @@ from typing import Any, Final, TypedDict, cast from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.litellm_core_utils.hidden_params import ( + HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR, +) +from litellm.litellm_core_utils.hidden_params import ( + set_hidden_params, +) + class FallbackErrorInfo(TypedDict): message: str @@ -14,7 +21,6 @@ class FallbackErrorInfo(TypedDict): code: str | None -_HIDDEN_PARAMS_ATTR: Final = "_hidden_params" _EMPTY_OBJECT_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _ROUTING_HEADER_MAPPING: Final = TypeAdapter(Mapping[str, object]) _COMPLEXITY_ROUTER_HEADER_PREFIX: Final = "x-litellm-complexity-router-" @@ -206,29 +212,6 @@ def get_hidden_params_dict( return hidden_params -def get_hidden_params(obj: object) -> dict[str, object] | None: - hidden_params: Final = ( - obj.get(_HIDDEN_PARAMS_ATTR) if isinstance(obj, dict) else getattr(obj, _HIDDEN_PARAMS_ATTR, None) - ) - return hidden_params if isinstance(hidden_params, dict) else None - - -def set_hidden_params(obj: object, hidden_params: dict[str, object]) -> None: - if isinstance(obj, dict): - obj[_HIDDEN_PARAMS_ATTR] = hidden_params - else: - setattr(obj, _HIDDEN_PARAMS_ATTR, hidden_params) - - -def get_or_create_hidden_params(obj: object) -> dict[str, object]: - hidden_params: Final = get_hidden_params(obj) - if hidden_params is not None: - return hidden_params - created_hidden_params: Final[dict[str, object]] = {} - set_hidden_params(obj, created_hidden_params) - return created_hidden_params - - def _write_hidden_params(response: object, hidden_params: dict[str, object]) -> None: if isinstance(response, dict) or hasattr(response, _HIDDEN_PARAMS_ATTR): set_hidden_params(response, hidden_params) diff --git a/tests/unit/litellm_core_utils/test_hidden_params.py b/tests/unit/litellm_core_utils/test_hidden_params.py new file mode 100644 index 00000000000..c1c82752528 --- /dev/null +++ b/tests/unit/litellm_core_utils/test_hidden_params.py @@ -0,0 +1,52 @@ +from typing import Final + +from litellm.litellm_core_utils.hidden_params import ( + get_hidden_params, + get_or_create_hidden_params, + set_hidden_params, +) +from litellm.types.decisions import DecisionsResponse +from litellm.types.utils import ModelResponse + + +def test_get_and_set_hidden_params_on_plain_object() -> None: + class PlainResponse: + def __init__(self) -> None: + self._hidden_params = {"existing": True} + + response: Final = PlainResponse() + stored: Final = get_hidden_params(response) + assert stored is response._hidden_params + + replacement: Final = {"replacement": True} + set_hidden_params(response, replacement) + + assert response._hidden_params is replacement + assert get_hidden_params(response) is replacement + + +def test_get_or_create_hidden_params_sets_empty_dict_on_plain_object() -> None: + class PlainResponse: + pass + + response: Final = PlainResponse() + hidden_params: Final = get_or_create_hidden_params(response) + + assert response._hidden_params is hidden_params + assert hidden_params == {} + + +def test_get_hidden_params_preserves_model_response_identity() -> None: + response: Final = ModelResponse() + + assert get_hidden_params(response) is response.hidden_params + + +def test_set_hidden_params_replaces_frozen_decisions_response_private_attr() -> None: + response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + replacement: Final = {"replacement": True} + + set_hidden_params(response, replacement) + + assert response.hidden_params is replacement + assert response._hidden_params is replacement diff --git a/tests/unit/router_utils/test_add_retry_fallback_headers.py b/tests/unit/router_utils/test_add_retry_fallback_headers.py index 414da4453ed..d6b09e766fb 100644 --- a/tests/unit/router_utils/test_add_retry_fallback_headers.py +++ b/tests/unit/router_utils/test_add_retry_fallback_headers.py @@ -11,13 +11,10 @@ from litellm.router_utils.add_retry_fallback_headers import ( complexity_router_decision_headers, ensure_response_additional_headers, get_fallback_errors_from_headers, - get_hidden_params, get_hidden_params_dict, replace_complexity_router_headers, - set_hidden_params, ) from litellm.types.decisions import DecisionsResponse -from litellm.types.utils import ModelResponse class StreamingWrapper: @@ -223,38 +220,6 @@ def test_get_hidden_params_dict_with_no_hidden_params(): assert get_hidden_params_dict(PlainResponse()) == {} -def test_get_and_set_hidden_params_on_plain_object() -> None: - class PlainResponse: - def __init__(self) -> None: - self._hidden_params = {"existing": True} - - response: Final = PlainResponse() - stored: Final = get_hidden_params(response) - assert stored is response._hidden_params - - replacement: Final = {"replacement": True} - set_hidden_params(response, replacement) - - assert response._hidden_params is replacement - assert get_hidden_params(response) is replacement - - -def test_get_hidden_params_preserves_model_response_identity() -> None: - response: Final = ModelResponse() - - assert get_hidden_params(response) is response.hidden_params - - -def test_set_hidden_params_replaces_frozen_decisions_response_private_attr() -> None: - response: Final = DecisionsResponse(model="decider", answers={}, usage=None) - replacement: Final = {"replacement": True} - - set_hidden_params(response, replacement) - - assert response.hidden_params is replacement - assert response._hidden_params is replacement - - def test_add_fallback_headers_when_no_existing_additional_headers(): class NoHeadersWrapper: def __init__(self): diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 484c5a6bb99..81bce59dae3 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -8,6 +8,7 @@ import litellm from litellm.proxy.proxy_server import _should_include_fallback_errors from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.types.llms.base import HiddenParams def test_apply_fallback_hidden_params_copies_from_fallback_response(): @@ -86,6 +87,33 @@ def test_apply_fallback_hidden_params_updates_plain_duck_chunk(): } +def test_apply_fallback_hidden_params_normalizes_hidden_params_on_plain_duck_chunk(): + class PlainChunk: + def __init__(self) -> None: + self._hidden_params = HiddenParams( + api_base="https://original.example", + model_id="original-model-id", + additional_headers={"x-existing-chunk-header": "keep"}, + ) + + chunk: Final = PlainChunk() + Router._apply_fallback_hidden_params_to_item( + fallback_item=chunk, + prepared_fallback_hidden_params=( + {"api_base": "https://fallback.example", "model_id": "fallback-model-id"}, + {"x-litellm-attempted-fallbacks": 1}, + ), + ) + + assert isinstance(chunk._hidden_params, dict) + assert chunk._hidden_params["api_base"] == "https://fallback.example" + assert chunk._hidden_params["model_id"] == "fallback-model-id" + assert chunk._hidden_params["additional_headers"] == { + "x-existing-chunk-header": "keep", + "x-litellm-attempted-fallbacks": 1, + } + + def _two_group_fallback_router() -> Router: return litellm.Router( model_list=[