mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(router): keep per-chunk response_cost after a streaming fallback
This commit is contained in:
parent
d131c43782
commit
c08aebe6f8
2 changed files with 20 additions and 9 deletions
|
|
@ -2842,7 +2842,9 @@ class Router:
|
|||
def _prepare_fallback_hidden_params(
|
||||
fallback_response: object,
|
||||
) -> tuple[dict[str, object], dict[str, object]]:
|
||||
fallback_hidden_params: Final = get_hidden_params_dict(fallback_response)
|
||||
fallback_hidden_params: Final = {
|
||||
key: value for key, value in get_hidden_params_dict(fallback_response).items() if key != "response_cost"
|
||||
}
|
||||
fallback_headers: Final = fallback_hidden_params.get("additional_headers")
|
||||
if not isinstance(fallback_headers, dict):
|
||||
return fallback_hidden_params, {}
|
||||
|
|
|
|||
|
|
@ -39,9 +39,7 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response():
|
|||
|
||||
Router._apply_fallback_hidden_params_to_item(
|
||||
fallback_item=chunk,
|
||||
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(
|
||||
fallback_response
|
||||
),
|
||||
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response),
|
||||
)
|
||||
|
||||
assert chunk._hidden_params["api_base"] == "https://fallback.example"
|
||||
|
|
@ -118,11 +116,24 @@ def test_prepare_fallback_hidden_params_no_additional_headers():
|
|||
assert headers == {}
|
||||
|
||||
|
||||
def test_apply_fallback_hidden_params_to_item_none_item():
|
||||
def test_apply_fallback_hidden_params_keeps_chunk_response_cost():
|
||||
chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[])
|
||||
chunk._hidden_params = {"response_cost": None}
|
||||
fallback_response = MagicMock()
|
||||
fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"}
|
||||
|
||||
Router._apply_fallback_hidden_params_to_item(
|
||||
None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})
|
||||
fallback_item=chunk,
|
||||
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response),
|
||||
)
|
||||
|
||||
assert chunk._hidden_params["response_cost"] is None
|
||||
assert chunk._hidden_params["api_base"] == "https://fallback.example"
|
||||
|
||||
|
||||
def test_apply_fallback_hidden_params_to_item_none_item():
|
||||
Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}))
|
||||
|
||||
|
||||
def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers():
|
||||
class FakeChunk:
|
||||
|
|
@ -139,9 +150,7 @@ def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers():
|
|||
|
||||
assert chunk._hidden_params["api_base"] == "http://fallback.example"
|
||||
assert chunk._hidden_params["model_id"] == "test-id"
|
||||
assert chunk._hidden_params["additional_headers"] == {
|
||||
"x-litellm-attempted-fallbacks": 1
|
||||
}
|
||||
assert chunk._hidden_params["additional_headers"] == {"x-litellm-attempted-fallbacks": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue