diff --git a/litellm/router.py b/litellm/router.py index c934b1e9a8c..92174adee39 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -339,9 +339,9 @@ class Router: ) # names of models under litellm_params. ex. azure/chatgpt-v-2 self.deployment_latency_map = {} ### CACHING ### - cache_type: Literal[ - "local", "redis", "redis-semantic", "s3", "disk" - ] = "local" # default to an in-memory cache + cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = ( + "local" # default to an in-memory cache + ) redis_cache = None cache_config: Dict[str, Any] = {} @@ -562,9 +562,9 @@ class Router: ) ) - self.model_group_retry_policy: Optional[ - Dict[str, RetryPolicy] - ] = model_group_retry_policy + self.model_group_retry_policy: Optional[Dict[str, RetryPolicy]] = ( + model_group_retry_policy + ) self.allowed_fails_policy: Optional[AllowedFailsPolicy] = None if allowed_fails_policy is not None: @@ -1104,17 +1104,21 @@ class Router: ) -> None: """ Adds default litellm params to kwargs, if set. + + Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name """ - self.default_litellm_params[ - metadata_variable_name - ] = self.default_litellm_params.pop("metadata", {}) - for k, v in self.default_litellm_params.items(): - if ( - k not in kwargs and v is not None - ): # prioritize model-specific params > default router params - kwargs[k] = v - elif k == metadata_variable_name: - kwargs[metadata_variable_name].update(v) + # 1) copy your defaults and pull out metadata + defaults = self.default_litellm_params.copy() + metadata_defaults = defaults.pop("metadata", {}) or {} + + # 2) add any non-metadata defaults that aren't already in kwargs + for key, value in defaults.items(): + if value is None: + continue + kwargs.setdefault(key, value) + + # 3) merge in metadata, this handles inserting this as either "metadata" or "litellm_metadata" + kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults) def _handle_clientside_credential( self, deployment: dict, kwargs: dict @@ -3243,11 +3247,11 @@ class Router: if isinstance(e, litellm.ContextWindowExceededError): if context_window_fallbacks is not None: - fallback_model_group: Optional[ - List[str] - ] = self._get_fallback_model_group_from_fallbacks( - fallbacks=context_window_fallbacks, - model_group=model_group, + fallback_model_group: Optional[List[str]] = ( + self._get_fallback_model_group_from_fallbacks( + fallbacks=context_window_fallbacks, + model_group=model_group, + ) ) if fallback_model_group is None: raise original_exception @@ -3279,11 +3283,11 @@ class Router: e.message += "\n{}".format(error_message) elif isinstance(e, litellm.ContentPolicyViolationError): if content_policy_fallbacks is not None: - fallback_model_group: Optional[ - List[str] - ] = self._get_fallback_model_group_from_fallbacks( - fallbacks=content_policy_fallbacks, - model_group=model_group, + fallback_model_group: Optional[List[str]] = ( + self._get_fallback_model_group_from_fallbacks( + fallbacks=content_policy_fallbacks, + model_group=model_group, + ) ) if fallback_model_group is None: raise original_exception diff --git a/tests/litellm/test_router.py b/tests/litellm/test_router.py new file mode 100644 index 00000000000..66a420e79d4 --- /dev/null +++ b/tests/litellm/test_router.py @@ -0,0 +1,54 @@ +import copy +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + + +import litellm + + +def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): + # initialize a real Router (env‑vars can be empty) + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/chatgpt-v-3", + "api_key": os.getenv("AZURE_API_KEY"), + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + }, + } + ], + ) + + # override to known defaults for the test + router.default_litellm_params = { + "foo": "bar", + "metadata": {"baz": 123}, + } + original = copy.deepcopy(router.default_litellm_params) + kwargs = {} + + # invoke the helper + router._update_kwargs_with_default_litellm_params( + kwargs=kwargs, + metadata_variable_name="litellm_metadata", + ) + + # 1) router.defaults must be unchanged + assert router.default_litellm_params == original + + # 2) non‑metadata keys get merged + assert kwargs["foo"] == "bar" + + # 3) metadata lands under "metadata" + assert kwargs["litellm_metadata"] == {"baz": 123}