mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
Merge a8d2d6887d into e04e89c701
This commit is contained in:
commit
fe796b487a
3 changed files with 21 additions and 6 deletions
|
|
@ -1088,12 +1088,12 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper):
|
|||
if fallback_hidden_params:
|
||||
self._hidden_params = {
|
||||
**fallback_hidden_params,
|
||||
"response_cost": None,
|
||||
# dict() because add_retry_fallback_headers mutates additional_headers in place
|
||||
"additional_headers": dict(fallback_headers),
|
||||
}
|
||||
self._base_hidden_params = {
|
||||
**self._hidden_params,
|
||||
"response_cost": None,
|
||||
}
|
||||
self.fallback_headers_adopted = True
|
||||
|
||||
|
|
@ -3476,6 +3476,7 @@ class Router:
|
|||
{
|
||||
**item_hidden_params,
|
||||
**fallback_hidden_params,
|
||||
"response_cost": item_hidden_params.get("response_cost"),
|
||||
"additional_headers": {**item_headers, **fallback_headers},
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4613,6 +4613,7 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges():
|
|||
fallback._response_headers = {"x-request-id": "req-FALLBACK"}
|
||||
fallback._hidden_params = {
|
||||
"model_id": "fallback-deployment",
|
||||
"response_cost": 0.0,
|
||||
"additional_headers": {"llm_provider-x-request-id": "req-FALLBACK"},
|
||||
}
|
||||
|
||||
|
|
@ -4623,6 +4624,8 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges():
|
|||
assert wrapper._response_headers == {"x-request-id": "req-FALLBACK"}
|
||||
assert wrapper._hidden_params["model_id"] == "fallback-deployment"
|
||||
assert "only_on_failed_attempt" not in wrapper._hidden_params
|
||||
assert wrapper._hidden_params["response_cost"] is None
|
||||
assert wrapper._base_hidden_params["response_cost"] is None
|
||||
assert wrapper._hidden_params is not fallback._hidden_params
|
||||
# the snapshot CustomStreamWrapper caches at init has to follow, or a chunk built
|
||||
# from it would still be stamped with the deployment that failed
|
||||
|
|
|
|||
|
|
@ -41,9 +41,7 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response():
|
|||
|
||||
Router._apply_fallback_hidden_params_to_item(
|
||||
fallback_item=chunk,
|
||||
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(
|
||||
fallback_response
|
||||
),
|
||||
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response),
|
||||
)
|
||||
|
||||
assert chunk._hidden_params["api_base"] == "https://fallback.example"
|
||||
|
|
@ -192,11 +190,24 @@ def test_prepare_fallback_hidden_params_no_additional_headers():
|
|||
assert headers == {}
|
||||
|
||||
|
||||
def test_apply_fallback_hidden_params_to_item_none_item():
|
||||
def test_apply_fallback_hidden_params_keeps_chunk_response_cost():
|
||||
chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[])
|
||||
chunk._hidden_params = {"response_cost": None}
|
||||
fallback_response = litellm.ModelResponse()
|
||||
fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"}
|
||||
|
||||
Router._apply_fallback_hidden_params_to_item(
|
||||
None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})
|
||||
fallback_item=chunk,
|
||||
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response),
|
||||
)
|
||||
|
||||
assert chunk._hidden_params["response_cost"] is None
|
||||
assert chunk._hidden_params["api_base"] == "https://fallback.example"
|
||||
|
||||
|
||||
def test_apply_fallback_hidden_params_to_item_none_item():
|
||||
Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}))
|
||||
|
||||
|
||||
def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers():
|
||||
chunk: Final = litellm.ModelResponseStream(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue