mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix(proxy): resolve model_id before post_call_failure_hook drops it
post_call_failure_hook pops litellm_logging_obj off request_data before callbacks iterate, so reading it afterwards yielded None and dropped x-litellm-model-id from error responses. Resolve model_id up front. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
1968fbf2b5
commit
a0af33cbb6
2 changed files with 48 additions and 7 deletions
|
|
@ -2605,6 +2605,13 @@ class ProxyBaseLLMRequestProcessing:
|
|||
):
|
||||
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
|
||||
_log_llm_api_exception(e)
|
||||
_litellm_logging_obj: Optional[LiteLLMLoggingObj] = self.data.get("litellm_logging_obj", None)
|
||||
|
||||
# Attempt to get model_id from logging object
|
||||
#
|
||||
# Note: We check the direct model_info path first (not nested in metadata) because that's where the router sets it.
|
||||
# The nested metadata path is only a fallback for cases where model_info wasn't set at the top level.
|
||||
model_id = self.maybe_get_model_id(_litellm_logging_obj)
|
||||
# Allow callbacks to transform the error response
|
||||
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -2624,13 +2631,6 @@ class ProxyBaseLLMRequestProcessing:
|
|||
timeout = getattr(
|
||||
e, "timeout", None
|
||||
) # returns the timeout set by the wrapper. Used for testing if model-specific timeout are set correctly
|
||||
_litellm_logging_obj: Optional[LiteLLMLoggingObj] = self.data.get("litellm_logging_obj", None)
|
||||
|
||||
# Attempt to get model_id from logging object
|
||||
#
|
||||
# Note: We check the direct model_info path first (not nested in metadata) because that's where the router sets it.
|
||||
# The nested metadata path is only a fallback for cases where model_info wasn't set at the top level.
|
||||
model_id = self.maybe_get_model_id(_litellm_logging_obj)
|
||||
|
||||
custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
|
|||
|
|
@ -2948,6 +2948,47 @@ class TestHandleLLMApiExceptionRetryAfter:
|
|||
assert proxy_exc.headers["x-custom"] == "1"
|
||||
|
||||
|
||||
class TestHandleLLMApiExceptionModelIdHeader:
|
||||
"""post_call_failure_hook pops litellm_logging_obj off request_data, so the
|
||||
error handler must resolve model_id before invoking it or x-litellm-model-id
|
||||
is dropped from error responses (the reported embeddings 429 bug)."""
|
||||
|
||||
class _FakeLoggingObj:
|
||||
def __init__(self, model_id: str):
|
||||
self.litellm_params = {"model_info": {"id": model_id}}
|
||||
self.litellm_call_id = "call-123"
|
||||
self.kwargs: dict = {}
|
||||
|
||||
async def test_model_id_survives_failure_hook_that_pops_logging_obj(self):
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
|
||||
data = {
|
||||
"model": "text-embedding-ada-002",
|
||||
"litellm_call_id": "call-123",
|
||||
"litellm_logging_obj": self._FakeLoggingObj("deployment-xyz"),
|
||||
}
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
|
||||
async def _popping_failure_hook(request_data, **kwargs):
|
||||
request_data.pop("litellm_logging_obj", None)
|
||||
return None
|
||||
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(
|
||||
side_effect=_popping_failure_hook
|
||||
)
|
||||
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await processor._handle_llm_api_exception(
|
||||
e=ValueError("429 rate limit"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
assert exc_info.value.headers["x-litellm-model-id"] == "deployment-xyz"
|
||||
|
||||
|
||||
class TestAsyncStreamingDataGeneratorFastPath:
|
||||
"""Fast/slow path branching in async_streaming_data_generator."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue