From c08aebe6f8331e7170f87e166c2a22ccfb556ee5 Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Thu, 1 Oct 2026 10:02:06 +0300 Subject: [PATCH 1/7] fix(router): keep per-chunk response_cost after a streaming fallback --- litellm/router.py | 4 ++- ...test_router_streaming_fallback_metadata.py | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 24554e61516..e02bc7f559f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2842,7 +2842,9 @@ class Router: def _prepare_fallback_hidden_params( fallback_response: object, ) -> tuple[dict[str, object], dict[str, object]]: - fallback_hidden_params: Final = get_hidden_params_dict(fallback_response) + fallback_hidden_params: Final = { + key: value for key, value in get_hidden_params_dict(fallback_response).items() if key != "response_cost" + } fallback_headers: Final = fallback_hidden_params.get("additional_headers") if not isinstance(fallback_headers, dict): return fallback_hidden_params, {} diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 6ed70dc7cfe..293cd85680e 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -39,9 +39,7 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response(): Router._apply_fallback_hidden_params_to_item( fallback_item=chunk, - prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params( - fallback_response - ), + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) assert chunk._hidden_params["api_base"] == "https://fallback.example" @@ -118,11 +116,24 @@ def test_prepare_fallback_hidden_params_no_additional_headers(): assert headers == {} -def test_apply_fallback_hidden_params_to_item_none_item(): +def test_apply_fallback_hidden_params_keeps_chunk_response_cost(): + chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[]) + chunk._hidden_params = {"response_cost": None} + fallback_response = MagicMock() + fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"} + Router._apply_fallback_hidden_params_to_item( - None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}) + fallback_item=chunk, + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) + assert chunk._hidden_params["response_cost"] is None + assert chunk._hidden_params["api_base"] == "https://fallback.example" + + +def test_apply_fallback_hidden_params_to_item_none_item(): + Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})) + def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers(): class FakeChunk: @@ -139,9 +150,7 @@ def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers(): assert chunk._hidden_params["api_base"] == "http://fallback.example" assert chunk._hidden_params["model_id"] == "test-id" - assert chunk._hidden_params["additional_headers"] == { - "x-litellm-attempted-fallbacks": 1 - } + assert chunk._hidden_params["additional_headers"] == {"x-litellm-attempted-fallbacks": 1} @pytest.mark.asyncio From a559a0c55b0080902e9a41fb054386c08581d280 Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Thu, 1 Oct 2026 10:24:38 +0300 Subject: [PATCH 2/7] fix(router): preserve chunk response_cost without a new mutable comprehension --- litellm/router.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index e02bc7f559f..0c5ff1e86c7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2842,9 +2842,7 @@ class Router: def _prepare_fallback_hidden_params( fallback_response: object, ) -> tuple[dict[str, object], dict[str, object]]: - fallback_hidden_params: Final = { - key: value for key, value in get_hidden_params_dict(fallback_response).items() if key != "response_cost" - } + fallback_hidden_params: Final = get_hidden_params_dict(fallback_response) fallback_headers: Final = fallback_hidden_params.get("additional_headers") if not isinstance(fallback_headers, dict): return fallback_hidden_params, {} @@ -2879,6 +2877,7 @@ class Router: cast(_HiddenParamsHost, fallback_item)._hidden_params = { **item_hidden_params, **fallback_hidden_params, + "response_cost": item_hidden_params.get("response_cost"), "additional_headers": {**item_headers, **fallback_headers}, } From d1f58c492d36d443b9741790076a42aa9df3462f Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Thu, 1 Oct 2026 10:48:33 +0300 Subject: [PATCH 3/7] ci: retrigger checks From d981d5af6724b84081308e7ef80d7480be30a085 Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Thu, 1 Oct 2026 10:50:59 +0300 Subject: [PATCH 4/7] fix(router): null response_cost on the adopted fallback stream wrapper --- litellm/router.py | 2 +- tests/unit/test_router/test_router.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 0c5ff1e86c7..703fc542c7f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -738,12 +738,12 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper): if fallback_hidden_params: self._hidden_params = { **fallback_hidden_params, + "response_cost": None, # dict() because add_retry_fallback_headers mutates additional_headers in place "additional_headers": dict(fallback_headers), } self._base_hidden_params = { **self._hidden_params, - "response_cost": None, } self.fallback_headers_adopted = True diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 96dddf15869..f811ad3f561 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2993,6 +2993,7 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges(): fallback._response_headers = {"x-request-id": "req-FALLBACK"} fallback._hidden_params = { "model_id": "fallback-deployment", + "response_cost": 0.0, "additional_headers": {"llm_provider-x-request-id": "req-FALLBACK"}, } @@ -3003,6 +3004,8 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges(): assert wrapper._response_headers == {"x-request-id": "req-FALLBACK"} assert wrapper._hidden_params["model_id"] == "fallback-deployment" assert "only_on_failed_attempt" not in wrapper._hidden_params + assert wrapper._hidden_params["response_cost"] is None + assert wrapper._base_hidden_params["response_cost"] is None assert wrapper._hidden_params is not fallback._hidden_params # the snapshot CustomStreamWrapper caches at init has to follow, or a chunk built # from it would still be stamped with the deployment that failed From 842cf83653eedc7efca16d5071e53d2a013a5e1e Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Thu, 1 Oct 2026 11:11:23 +0300 Subject: [PATCH 5/7] test(router): use a real response in the fallback response_cost test --- tests/unit/test_router_streaming_fallback_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 293cd85680e..b91ffaa427e 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -119,7 +119,7 @@ def test_prepare_fallback_hidden_params_no_additional_headers(): def test_apply_fallback_hidden_params_keeps_chunk_response_cost(): chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[]) chunk._hidden_params = {"response_cost": None} - fallback_response = MagicMock() + fallback_response = litellm.ModelResponse() fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"} Router._apply_fallback_hidden_params_to_item( From a8d2d6887dc053d2b3b12618d20ee6db73331024 Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Mon, 12 Oct 2026 21:28:12 +0300 Subject: [PATCH 6/7] ci: retrigger checks From 799a3d1db127702f5c3ad3c8c6b9f47535dedced Mon Sep 17 00:00:00 2001 From: Tom Ron Date: Sun, 25 Oct 2026 20:29:38 +0200 Subject: [PATCH 7/7] test(router): cover positive chunk cost and zero-cost stream after fallback Signed-off-by: Tom Ron --- tests/unit/test_router/test_router.py | 3 - ...test_router_streaming_fallback_metadata.py | 69 +++++++++++++++++-- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index fe7a3fdbcd4..4a7a23ff57b 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3237,7 +3237,6 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges(): fallback._response_headers = {"x-request-id": "req-FALLBACK"} fallback._hidden_params = { "model_id": "fallback-deployment", - "response_cost": 0.0, "additional_headers": {"llm_provider-x-request-id": "req-FALLBACK"}, } @@ -3248,8 +3247,6 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges(): assert wrapper._response_headers == {"x-request-id": "req-FALLBACK"} assert wrapper._hidden_params["model_id"] == "fallback-deployment" assert "only_on_failed_attempt" not in wrapper._hidden_params - assert wrapper._hidden_params["response_cost"] is None - assert wrapper._base_hidden_params["response_cost"] is None assert wrapper._hidden_params is not fallback._hidden_params # the snapshot CustomStreamWrapper caches at init has to follow, or a chunk built # from it would still be stamped with the deployment that failed diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 6ee9ce80c0e..f9f518b2d06 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock import pytest import litellm +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy.proxy_server import _should_include_fallback_errors from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict @@ -41,7 +42,9 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response(): Router._apply_fallback_hidden_params_to_item( fallback_item=chunk, - prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params( + fallback_response + ), ) assert chunk._hidden_params["api_base"] == "https://fallback.example" @@ -190,10 +193,11 @@ def test_prepare_fallback_hidden_params_no_additional_headers(): assert headers == {} -def test_apply_fallback_hidden_params_keeps_chunk_response_cost(): - chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[]) - chunk._hidden_params = {"response_cost": None} - fallback_response = litellm.ModelResponse() +@pytest.mark.parametrize("chunk_cost", [0.0042, None]) +def test_apply_fallback_hidden_params_keeps_chunk_response_cost(chunk_cost): + chunk: Final = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[]) + chunk._hidden_params = {"response_cost": chunk_cost} + fallback_response: Final = litellm.ModelResponse() fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"} Router._apply_fallback_hidden_params_to_item( @@ -201,12 +205,14 @@ def test_apply_fallback_hidden_params_keeps_chunk_response_cost(): prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) - assert chunk._hidden_params["response_cost"] is None + assert chunk._hidden_params["response_cost"] == chunk_cost assert chunk._hidden_params["api_base"] == "https://fallback.example" def test_apply_fallback_hidden_params_to_item_none_item(): - Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})) + Router._apply_fallback_hidden_params_to_item( + None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}) + ) def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers(): @@ -272,3 +278,52 @@ def test_should_include_fallback_errors_gated_by_operator_setting(): assert _should_include_fallback_errors({}) is False finally: ps.general_settings = original + + +async def _provider_stream_failing_before_first_chunk(): + raise litellm.InternalServerError(message="provider failed", llm_provider="openai", model="gpt-fake") + yield + + +@pytest.mark.asyncio +async def test_streaming_fallback_chunks_do_not_report_zero_response_cost(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "primary-model", + "litellm_params": {"model": "openai/gpt-fake", "api_key": "sk-fake"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "openai/gpt-fake-2", + "api_key": "sk-fake", + "mock_response": "hello from the fallback", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.01, + }, + }, + ], + fallbacks=[{"primary-model": ["fallback-model"]}], + ) + messages: Final = [{"role": "user", "content": "Hello"}] + # a real stream only to borrow a real logging object for the primary's wrapper + donor: Final = await litellm.acompletion( + model="openai/gpt-fake", messages=messages, stream=True, mock_response="x", api_key="sk-fake" + ) + + stream: Final = await router._acompletion_streaming_iterator( + model_response=CustomStreamWrapper( + completion_stream=_provider_stream_failing_before_first_chunk(), + model="gpt-fake", + custom_llm_provider="openai", + logging_obj=donor.logging_obj, + ), + messages=messages, + initial_kwargs={"model": "primary-model", "stream": True, "stream_options": {"include_usage": True}}, + ) + chunks: Final = [chunk async for chunk in stream] + + # the fallback is priced, so the stream must never claim the call was free + assert chunks[-1].usage.cost > 0 + assert all(chunk._hidden_params.get("response_cost") != 0.0 for chunk in chunks)