diff --git a/litellm/router.py b/litellm/router.py index 705accbcf8d..8c87b85f66f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1088,12 +1088,12 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper): if fallback_hidden_params: self._hidden_params = { **fallback_hidden_params, + "response_cost": None, # dict() because add_retry_fallback_headers mutates additional_headers in place "additional_headers": dict(fallback_headers), } self._base_hidden_params = { **self._hidden_params, - "response_cost": None, } self.fallback_headers_adopted = True @@ -3476,6 +3476,7 @@ class Router: { **item_hidden_params, **fallback_hidden_params, + "response_cost": item_hidden_params.get("response_cost"), "additional_headers": {**item_headers, **fallback_headers}, }, ) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index f06f9e08f7e..47020b580aa 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -4613,6 +4613,7 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges(): fallback._response_headers = {"x-request-id": "req-FALLBACK"} fallback._hidden_params = { "model_id": "fallback-deployment", + "response_cost": 0.0, "additional_headers": {"llm_provider-x-request-id": "req-FALLBACK"}, } @@ -4623,6 +4624,8 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges(): assert wrapper._response_headers == {"x-request-id": "req-FALLBACK"} assert wrapper._hidden_params["model_id"] == "fallback-deployment" assert "only_on_failed_attempt" not in wrapper._hidden_params + assert wrapper._hidden_params["response_cost"] is None + assert wrapper._base_hidden_params["response_cost"] is None assert wrapper._hidden_params is not fallback._hidden_params # the snapshot CustomStreamWrapper caches at init has to follow, or a chunk built # from it would still be stamped with the deployment that failed diff --git a/tests/unit/test_router_streaming_fallback_metadata.py b/tests/unit/test_router_streaming_fallback_metadata.py index 7e0cb30f1e3..6ee9ce80c0e 100644 --- a/tests/unit/test_router_streaming_fallback_metadata.py +++ b/tests/unit/test_router_streaming_fallback_metadata.py @@ -41,9 +41,7 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response(): Router._apply_fallback_hidden_params_to_item( fallback_item=chunk, - prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params( - fallback_response - ), + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) assert chunk._hidden_params["api_base"] == "https://fallback.example" @@ -192,11 +190,24 @@ def test_prepare_fallback_hidden_params_no_additional_headers(): assert headers == {} -def test_apply_fallback_hidden_params_to_item_none_item(): +def test_apply_fallback_hidden_params_keeps_chunk_response_cost(): + chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[]) + chunk._hidden_params = {"response_cost": None} + fallback_response = litellm.ModelResponse() + fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"} + Router._apply_fallback_hidden_params_to_item( - None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}) + fallback_item=chunk, + prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response), ) + assert chunk._hidden_params["response_cost"] is None + assert chunk._hidden_params["api_base"] == "https://fallback.example" + + +def test_apply_fallback_hidden_params_to_item_none_item(): + Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})) + def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers(): chunk: Final = litellm.ModelResponseStream(