From c08aebe6f8331e7170f87e166c2a22ccfb556ee5 Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Thu, 1 Oct 2026 10:02:06 +0300 Subject: [PATCH] fix(router): keep per-chunk response_cost after a streaming fallback --- litellm/router.py | 4 ++- ...test_router_streaming_fallback_metadata.py | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 24554e61516..e02bc7f559f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2842,7 +2842,9 @@ class Router: def _prepare_fallback_hidden_params( fallback_response: object, ) -> tuple[dict[str, object], dict[str, object]]: - fallback_hidden_params: Final = get_hidden_params_dict(fallback_response) + fallback_hidden_params: Final = { + key: value for key, value in get_hidden_params_dict(fallback_response).items() if key != "response_cost" + } fallback_headers: Final = fallback_hidden_params.get("additional_headers") if not isinstance(fallback_headers, dict): return fallback_hidden_params, {} diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 6ed70dc7cfe..293cd85680e 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -39,9 +39,7 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response(): Router._apply_fallback_hidden_params_to_item( fallback_item=chunk, - prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params( - fallback_response - ), + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) assert chunk._hidden_params["api_base"] == "https://fallback.example" @@ -118,11 +116,24 @@ def test_prepare_fallback_hidden_params_no_additional_headers(): assert headers == {} -def test_apply_fallback_hidden_params_to_item_none_item(): +def test_apply_fallback_hidden_params_keeps_chunk_response_cost(): + chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[]) + chunk._hidden_params = {"response_cost": None} + fallback_response = MagicMock() + fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"} + Router._apply_fallback_hidden_params_to_item( - None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}) + fallback_item=chunk, + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) + assert chunk._hidden_params["response_cost"] is None + assert chunk._hidden_params["api_base"] == "https://fallback.example" + + +def test_apply_fallback_hidden_params_to_item_none_item(): + Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})) + def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers(): class FakeChunk: @@ -139,9 +150,7 @@ def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers(): assert chunk._hidden_params["api_base"] == "http://fallback.example" assert chunk._hidden_params["model_id"] == "test-id" - assert chunk._hidden_params["additional_headers"] == { - "x-litellm-attempted-fallbacks": 1 - } + assert chunk._hidden_params["additional_headers"] == {"x-litellm-attempted-fallbacks": 1} @pytest.mark.asyncio