diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 62e706a0cf5..c5237993641 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import safe_deep_copy from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, ) @@ -123,21 +124,31 @@ async def run_async_fallback( if mg == original_model_group: continue try: + # Deep copy kwargs so each fallback attempt starts from the + # original, unmodified parameters. Without this, provider- + # specific handlers (e.g. Bedrock's converse_handler) can + # mutate kwargs via .pop() calls, leaving subsequent fallback + # providers with missing keys (e.g. `tool_choice` present but + # `tools` removed). See https://github.com/BerriAI/litellm/issues/24764 + fallback_kwargs = safe_deep_copy(kwargs) + # LOGGING - kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception) + fallback_kwargs = litellm_router.log_retry( + kwargs=fallback_kwargs, e=original_exception + ) verbose_router_logger.info(f"Falling back to model_group = {mg}") if isinstance(mg, str): - kwargs["model"] = mg + fallback_kwargs["model"] = mg elif isinstance(mg, dict): - kwargs.update(mg) - kwargs.setdefault("metadata", {}).update( - {"model_group": kwargs.get("model", None)} + fallback_kwargs.update(mg) + fallback_kwargs.setdefault("metadata", {}).update( + {"model_group": fallback_kwargs.get("model", None)} ) # update model_group used, if fallbacks are done fallback_depth = fallback_depth + 1 - kwargs["fallback_depth"] = fallback_depth - kwargs["max_fallbacks"] = max_fallbacks + fallback_kwargs["fallback_depth"] = fallback_depth + fallback_kwargs["max_fallbacks"] = max_fallbacks response = await litellm_router.async_function_with_fallbacks( - *args, **kwargs + *args, **fallback_kwargs ) verbose_router_logger.info("Successful fallback b/w models.") response = add_fallback_headers_to_response( @@ -147,7 +158,7 @@ async def run_async_fallback( # callback for successfull_fallback_event(): await log_success_fallback_event( original_model_group=original_model_group, - kwargs=kwargs, + kwargs=fallback_kwargs, original_exception=original_exception, ) return response @@ -155,7 +166,7 @@ async def run_async_fallback( error_from_fallbacks = e await log_failure_fallback_event( original_model_group=original_model_group, - kwargs=kwargs, + kwargs=fallback_kwargs, original_exception=original_exception, ) raise error_from_fallbacks diff --git a/tests/local_testing/test_router_fallback_handlers.py b/tests/local_testing/test_router_fallback_handlers.py index 29387d70c8d..4f42f53c835 100644 --- a/tests/local_testing/test_router_fallback_handlers.py +++ b/tests/local_testing/test_router_fallback_handlers.py @@ -316,3 +316,121 @@ async def test_multiple_fallbacks(function_name): result._hidden_params["api_base"] == "https://exampleopenaiendpoint-production.up.railway.app/" ) + + +@pytest.mark.asyncio +async def test_fallback_kwargs_not_mutated(): + """ + Verify that each fallback attempt receives a fresh copy of kwargs. + + Regression test for https://github.com/BerriAI/litellm/issues/24764: + When a provider handler mutates kwargs (e.g. Bedrock pops `tools`), + subsequent fallback attempts should still see the original parameters. + """ + call_received_kwargs: List[Dict[str, Any]] = [] + + router = Router( + model_list=[ + { + "model_name": "primary-model", + "litellm_params": { + "model": "openai/primary", + "api_key": "fake-key", + "api_base": "http://localhost:1/", + }, + }, + { + "model_name": "fallback-a", + "litellm_params": { + "model": "openai/fallback-a", + "api_key": "fake-key", + "api_base": "http://localhost:2/", + }, + }, + { + "model_name": "fallback-b", + "litellm_params": { + "model": "openai/fallback-b", + "api_key": "fake-key", + "api_base": "http://localhost:3/", + }, + }, + ], + ) + + original_fn = router.async_function_with_fallbacks + + async def mock_async_function_with_fallbacks(*args, **kwargs): + """ + Capture kwargs snapshot, then simulate the first provider mutating + them (like Bedrock popping 'tools') before raising an error. + """ + import copy + + call_received_kwargs.append(copy.deepcopy(kwargs)) + + # Simulate a provider handler mutating kwargs — pop 'tools' + kwargs.pop("tools", None) + kwargs.pop("stream", None) + + raise litellm.exceptions.ServiceUnavailableError( + message="simulated timeout", + model="test", + llm_provider="openai", + ) + + router.async_function_with_fallbacks = mock_async_function_with_fallbacks + + original_tools = [ + { + "type": "function", + "function": { + "name": "classify", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + original_tool_choice = { + "type": "function", + "function": {"name": "classify"}, + } + + request_kwargs: Dict[str, Any] = { + "messages": [{"role": "user", "content": "test"}], + "tools": original_tools, + "tool_choice": original_tool_choice, + "stream": True, + "metadata": {}, + } + + with pytest.raises(Exception): + await run_async_fallback( + litellm_router=router, + original_function=router._acompletion, + num_retries=0, + fallback_model_group=["fallback-a", "fallback-b"], + original_model_group="primary-model", + original_exception=Exception("primary failed"), + max_fallbacks=5, + fallback_depth=0, + **request_kwargs, + ) + + # Both fallback attempts should have received 'tools' and 'stream' + assert len(call_received_kwargs) == 2, ( + f"Expected 2 fallback attempts, got {len(call_received_kwargs)}" + ) + + for i, received in enumerate(call_received_kwargs): + assert "tools" in received, ( + f"Fallback attempt {i} missing 'tools' — kwargs were mutated by previous attempt" + ) + assert "stream" in received, ( + f"Fallback attempt {i} missing 'stream' — kwargs were mutated by previous attempt" + ) + assert "tool_choice" in received, ( + f"Fallback attempt {i} missing 'tool_choice' — kwargs were mutated by previous attempt" + ) + assert received["tools"] == original_tools, ( + f"Fallback attempt {i} has modified 'tools' value" + )