This commit is contained in:
Tom Ron 2026-10-24 08:30:44 +00:00 • committed by GitHub
commit fe796b487a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 21 additions and 6 deletions

View file

@ -1088,12 +1088,12 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper):
if fallback_hidden_params:
self._hidden_params = {
**fallback_hidden_params,
"response_cost": None,
# dict() because add_retry_fallback_headers mutates additional_headers in place
"additional_headers": dict(fallback_headers),
}
self._base_hidden_params = {
**self._hidden_params,
"response_cost": None,
}
self.fallback_headers_adopted = True
@ -3476,6 +3476,7 @@ class Router:
{
**item_hidden_params,
**fallback_hidden_params,
"response_cost": item_hidden_params.get("response_cost"),
"additional_headers": {**item_headers, **fallback_headers},
},
)

View file

@ -4613,6 +4613,7 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges():
fallback._response_headers = {"x-request-id": "req-FALLBACK"}
fallback._hidden_params = {
"model_id": "fallback-deployment",
"response_cost": 0.0,
"additional_headers": {"llm_provider-x-request-id": "req-FALLBACK"},
}
@ -4623,6 +4624,8 @@ def test_adopt_fallback_response_headers_replaces_rather_than_merges():
assert wrapper._response_headers == {"x-request-id": "req-FALLBACK"}
assert wrapper._hidden_params["model_id"] == "fallback-deployment"
assert "only_on_failed_attempt" not in wrapper._hidden_params
assert wrapper._hidden_params["response_cost"] is None
assert wrapper._base_hidden_params["response_cost"] is None
assert wrapper._hidden_params is not fallback._hidden_params
# the snapshot CustomStreamWrapper caches at init has to follow, or a chunk built
# from it would still be stamped with the deployment that failed

View file

@ -41,9 +41,7 @@ def test_apply_fallback_hidden_params_copies_from_fallback_response():
Router._apply_fallback_hidden_params_to_item(
fallback_item=chunk,
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(
fallback_response
),
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response),
)
assert chunk._hidden_params["api_base"] == "https://fallback.example"
@ -192,11 +190,24 @@ def test_prepare_fallback_hidden_params_no_additional_headers():
assert headers == {}
def test_apply_fallback_hidden_params_to_item_none_item():
def test_apply_fallback_hidden_params_keeps_chunk_response_cost():
chunk = litellm.ModelResponseStream(id="test", model="openai/fallback", choices=[])
chunk._hidden_params = {"response_cost": None}
fallback_response = litellm.ModelResponse()
fallback_response._hidden_params = {"response_cost": 0.0, "api_base": "https://fallback.example"}
Router._apply_fallback_hidden_params_to_item(
None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"})
fallback_item=chunk,
prepared_fallback_hidden_params=Router._prepare_fallback_hidden_params(fallback_response),
)
assert chunk._hidden_params["response_cost"] is None
assert chunk._hidden_params["api_base"] == "https://fallback.example"
def test_apply_fallback_hidden_params_to_item_none_item():
Router._apply_fallback_hidden_params_to_item(None, ({"api_base": "http://fallback.example"}, {"x-custom": "value"}))
def test_apply_fallback_hidden_params_to_item_no_existing_additional_headers():
chunk: Final = litellm.ModelResponseStream(